TSShape loader refactor

final setup for assimp

Added assimp importers and exporters, removed defaulting to all
Added compression to tshape
added dts version to tshape and cmake

Update assimpShapeLoader.cpp

quick fix

fix previewing dsq ground work

automatically export dsq files for animations

Groundwork

Adds the same sort of model for registering loaders and exporters as is set out on gbitmap
Added a bit more safety around the assimp matrix fix to convert incoming models to torques coordinate system.

Engine/lib/CMakeLists.txt

@@ -143,8 +143,15 @@ IF(MSVC)
     advanced_option( ASSIMP_INSTALL_PDB "Install MSVC debug files." ON )
 endif()
 advanced_option(ASSIMP_RAPIDJSON_NO_MEMBER_ITERATOR "Suppress rapidjson warning on MSVC (NOTE: breaks android build)" ON )
-advanced_option(ASSIMP_BUILD_ALL_IMPORTERS_BY_DEFAULT "default value of all ASSIMP_BUILD_XXX_IMPORTER values" ON)
+advanced_option(ASSIMP_BUILD_ALL_IMPORTERS_BY_DEFAULT "default value of all ASSIMP_BUILD_XXX_IMPORTER values" OFF)
-advanced_option(ASSIMP_BUILD_ALL_EXPORTERS_BY_DEFAULT "default value of all ASSIMP_BUILD_XXX_EXPORTER values" ON)
+advanced_option(ASSIMP_BUILD_FBX_IMPORTER "FBX Importer" ON)
+advanced_option(ASSIMP_BUILD_GLTF_IMPORTER "GLTF Importer" ON)
+advanced_option(ASSIMP_BUILD_OBJ_IMPORTER "OBJ Importer" ON)
+advanced_option(ASSIMP_BUILD_BLEND_IMPORTER "BLENDER Importer" ON)
+advanced_option(ASSIMP_BUILD_ALL_EXPORTERS_BY_DEFAULT "default value of all ASSIMP_BUILD_XXX_EXPORTER values" OFF)
+advanced_option(ASSIMP_BUILD_FBX_EXPORTER "FBX Importer" ON)
+advanced_option(ASSIMP_BUILD_GLTF_EXPORTER "GLTF Importer" ON)
+advanced_option(ASSIMP_BUILD_OBJ_EXPORTER "OBJ Importer" ON)
 mark_as_advanced(ASSIMP_ARCHIVE_OUTPUT_DIRECTORY)
 mark_as_advanced(ASSIMP_BIN_INSTALL_DIR)
 mark_as_advanced(ASSIMP_LIB_INSTALL_DIR)

Engine/source/T3D/assets/assetImporter.cpp

@@ -2012,7 +2012,7 @@ void AssetImporter::processShapeAsset(AssetImportObject* assetItem)
       {
          enumColladaForImport(filePath, shapeInfo, false);
       }
-      else if ((fileExt.compare("dts") == 0) || (fileExt.compare("dsq") == 0))
+      else if ((fileExt.compare("dts") == 0))
       {
          enumDTSForImport(filePath, shapeInfo);
       }

Engine/source/gui/editor/guiShapeEdPreview.cpp

@@ -405,6 +405,53 @@ bool GuiShapeEdPreview::setObjectModel(const char* modelName)
    return true;
 }
 
+bool GuiShapeEdPreview::findCompanionShape(const Torque::Path& dsqPath, Torque::Path& outShapePath)
+{
+   // AssimpLoader and ColladaLoader exports as "modelname_sequencename.dsq" alongside "modelname.cached.dts"
+   // so strip everything from the last underscore to find the base name
+   String fileName = dsqPath.getFileName();
+   String::SizeType sep = fileName.find('_');
+
+   if (sep != String::NPos)
+   {
+      Torque::Path candidate(dsqPath);
+      candidate.setFileName(fileName.substr(0, sep));
+
+      candidate.setExtension("cached.dts");
+      if (Torque::FS::IsFile(candidate.getFullPath()))
+      {
+         outShapePath = candidate;
+         return true;
+      }
+
+      candidate.setExtension("dts");
+      if (Torque::FS::IsFile(candidate.getFullPath()))
+      {
+         outShapePath = candidate;
+         return true;
+      }
+   }
+
+   // fallback: same filename, just swap extension
+   Torque::Path direct(dsqPath);
+
+   direct.setExtension("cached.dts");
+   if (Torque::FS::IsFile(direct.getFullPath()))
+   {
+      outShapePath = direct;
+      return true;
+   }
+
+   direct.setExtension("dts");
+   if (Torque::FS::IsFile(direct.getFullPath()))
+   {
+      outShapePath = direct;
+      return true;
+   }
+
+   return false;
+}
+
 bool GuiShapeEdPreview::setObjectShapeAsset(const char* assetId)
 {
    SAFE_DELETE(mModel);
@@ -422,16 +469,68 @@ bool GuiShapeEdPreview::setObjectShapeAsset(const char* assetId)
          ShapeAsset* asset = AssetDatabase.acquireAsset<ShapeAsset>(id);
          modelName = asset->getShapeFile();
          AssetDatabase.releaseAsset(id);
+         return setObjectModel(modelName);
       }
       else if (assetType == StringTable->insert("ShapeAnimationAsset"))
       {
          ShapeAnimationAsset* asset = AssetDatabase.acquireAsset<ShapeAnimationAsset>(id);
-         modelName = asset->getAnimationPath();
+         StringTableEntry animPath = asset->getAnimationPath();
          AssetDatabase.releaseAsset(id);
+
+         Torque::Path dsqPath(animPath);
+         Torque::Path shapePath;
+         if (!findCompanionShape(dsqPath, shapePath))
+         {
+            Con::warnf("GuiShapeEdPreview::setObjectShapeAsset - "
+               "No companion shape found for '%s'", animPath);
+            return false;
+         }
+
+         if (!setObjectModel(shapePath.getFullPath()))
+         {
+            Con::warnf("GuiShapeEdPreview::setObjectShapeAsset - "
+               "Could not load companion shape for '%s'", animPath);
+            return false;
+         }
+
+         FileStream dsqStream;
+         if (!dsqStream.open(animPath, Torque::FS::File::Read))
+         {
+            Con::warnf("GuiShapeEdPreview::setObjectShapeAsset - "
+               "Could not open '%s'", animPath);
+            SAFE_DELETE(mModel);
+            return false;
+         }
+
+         TSShape* shape = mModel->getShape();
+
+         bool ok = shape->importSequences(&dsqStream, String(animPath));
+         dsqStream.close();
+
+         if (!ok)
+         {
+            Con::warnf("GuiShapeEdPreview::setObjectShapeAsset - "
+               "importSequences failed for '%s'", animPath);
+            SAFE_DELETE(mModel);
+            return false;
+         }
+
+         setAllMeshesHidden(true);
+         mRenderNodes = true;
+         if (shape->sequences.size() > 0)
+         {
+            // importSequences appends, so the new one is always last
+            const String& seqName = shape->getSequenceName(shape->sequences.size() - 1);
+            addThread();
+            mActiveThread = 0;
+            setActiveThreadSequence(seqName.c_str(), 0.0f, 0.0f, true);
+         }
+
+         return true;
       }
    }
 
-   return setObjectModel(modelName);
+   return false;
 }
 
 void GuiShapeEdPreview::_onResourceChanged(const Torque::Path& path)

Engine/source/gui/editor/guiShapeEdPreview.h

@@ -199,6 +199,7 @@ public:
 
    void setCurrentDetail(S32 dl);
    bool setObjectModel(const char * modelName);
+   bool findCompanionShape(const Torque::Path& dsqPath, Torque::Path& outShapePath);
    bool setObjectShapeAsset(const char* assetId);
 
    void _onResourceChanged(const Torque::Path& path);

Engine/source/ts/assimp/assimpAppMesh.cpp

@@ -44,6 +44,7 @@
 
 bool AssimpAppMesh::fixedSizeEnabled = false;
 S32 AssimpAppMesh::fixedSize = 2;
+Vector<S32> AssimpAppMesh::sMaterialRemap;
 
 //------------------------------------------------------------------------------
 
@@ -164,7 +165,8 @@ void AssimpAppMesh::lockMesh(F32 t, const MatrixF& objOffset)
    primitives.increment();
    TSDrawPrimitive& primitive = primitives.last();
    primitive.start = 0;
-   primitive.matIndex = (TSDrawPrimitive::Triangles | TSDrawPrimitive::Indexed) | (S32)mMeshData->mMaterialIndex;
+   S32 mappedMat = (mMeshData->mMaterialIndex < (U32)AssimpAppMesh::sMaterialRemap.size()) ? AssimpAppMesh::sMaterialRemap[mMeshData->mMaterialIndex] : TSDrawPrimitive::NoMaterial;
+   primitive.matIndex = (TSDrawPrimitive::Triangles | TSDrawPrimitive::Indexed) | mappedMat;
    primitive.numElements = indicesCount;
 
    for ( U32 n = 0; n < mMeshData->mNumFaces; ++n)

Engine/source/ts/assimp/assimpAppMesh.h

@@ -119,6 +119,8 @@ public:
    /// @return The mesh transform at the specified time
    MatrixF getMeshTransform(F32 time) override;
    F32 getVisValue(F32 t) override;
+
+   static Vector<S32> sMaterialRemap;
 };
 
 #endif // _COLLADA_APPMESH_H_

Engine/source/ts/assimp/assimpShapeLoader.cpp

@@ -62,6 +62,7 @@
 #include <assimp/postprocess.h>
 #include <assimp/types.h>
 #include <assimp/config.h>
+#include <assimp/Exporter.hpp>
 #include <exception>
 
 #if !defined(TORQUE_DISABLE_MEMORY_MANAGER)
@@ -69,56 +70,68 @@
 #  define new  _new
 #endif
 
-extern bool gTryUseDSQs;
+static bool sReadAssimp(const Torque::Path& path, TSShape*& shape);
 
-MODULE_BEGIN( AssimpShapeLoader )
+static struct _privateRegisterAssimp
-   MODULE_INIT_AFTER( ShapeLoader )
-   MODULE_INIT
 {
-      TSShapeLoader::addFormat("DirectX X", "x");
+   _privateRegisterAssimp()
-      TSShapeLoader::addFormat("Autodesk FBX", "fbx");
+   {
-      TSShapeLoader::addFormat("Blender 3D", "blend" );
+      TSShape::ShapeRegistration reg;
-      TSShapeLoader::addFormat("3ds Max 3DS", "3ds");
+
-      TSShapeLoader::addFormat("3ds Max ASE", "ase");
+      Assimp::Importer importer;
-      TSShapeLoader::addFormat("Wavefront Object", "obj");
+      for (U32 i = 0; i < importer.GetImporterCount(); i++)
-      TSShapeLoader::addFormat("Industry Foundation Classes (IFC/Step)", "ifc");
+      {
-      TSShapeLoader::addFormat("Stanford Polygon Library", "ply");
+         const aiImporterDesc* desc = importer.GetImporterInfo(i);
-      TSShapeLoader::addFormat("AutoCAD DXF", "dxf");
+
-      TSShapeLoader::addFormat("LightWave", "lwo");
+         String extensions(desc->mFileExtensions);
-      TSShapeLoader::addFormat("LightWave Scene", "lws");
+
-      TSShapeLoader::addFormat("Modo", "lxo");
+         Vector<String> tokens;
-      TSShapeLoader::addFormat("Stereolithography", "stl");
+         extensions.split(" ", tokens);
-      TSShapeLoader::addFormat("AC3D", "ac");
+         for (U32 t = 0; t < tokens.size(); ++t)
-      TSShapeLoader::addFormat("Milkshape 3D", "ms3d");
+         {
-      TSShapeLoader::addFormat("TrueSpace COB", "cob");
+            const String& ext = tokens[t];
-      TSShapeLoader::addFormat("TrueSpace SCN", "scn");
+            if (ext.isEmpty() ||
-      TSShapeLoader::addFormat("Ogre XML", "xml");
+                  ext.equal("dae", String::NoCase) || // filter out collada importer formats (for now).
-      TSShapeLoader::addFormat("Irrlicht Mesh", "irrmesh");
+                  ext.equal("zae", String::NoCase) ||
-      TSShapeLoader::addFormat("Irrlicht Scene", "irr");
+                  ext.equal("xml", String::NoCase)
-      TSShapeLoader::addFormat("Quake I", "mdl" );
+               )
-      TSShapeLoader::addFormat("Quake II", "md2" );
+               continue;
-      TSShapeLoader::addFormat("Quake III Mesh", "md3");
+
-      TSShapeLoader::addFormat("Quake III Map/BSP", "pk3");
+            reg.extensions.push_back({
-      TSShapeLoader::addFormat("Return to Castle Wolfenstein", "mdc");
+                String(desc->mName), // convert from const char*
-      TSShapeLoader::addFormat("Doom 3", "md5" );
+                ext
-      TSShapeLoader::addFormat("Valve SMD", "smd");
+               });
-      TSShapeLoader::addFormat("Valve VTA", "vta");
-      TSShapeLoader::addFormat("Starcraft II M3", "m3");
-      TSShapeLoader::addFormat("Unreal", "3d");
-      TSShapeLoader::addFormat("BlitzBasic 3D", "b3d" );
-      TSShapeLoader::addFormat("Quick3D Q3D", "q3d");
-      TSShapeLoader::addFormat("Quick3D Q3S", "q3s");
-      TSShapeLoader::addFormat("Neutral File Format", "nff");
-      TSShapeLoader::addFormat("Object File Format", "off");
-      TSShapeLoader::addFormat("PovRAY Raw", "raw");
-      TSShapeLoader::addFormat("Terragen Terrain", "ter");
-      TSShapeLoader::addFormat("3D GameStudio (3DGS)", "mdl");
-      TSShapeLoader::addFormat("3D GameStudio (3DGS) Terrain", "hmp");
-      TSShapeLoader::addFormat("Izware Nendo", "ndo");
-      TSShapeLoader::addFormat("gltf", "gltf");
-      TSShapeLoader::addFormat("gltf binary", "glb");
          }
-MODULE_END;
+
+      }
+
+      Assimp::Exporter exporter;
+
+      for (U32 i = 0; i < exporter.GetExportFormatCount(); ++i)
+      {
+         const aiExportFormatDesc* desc = exporter.GetExportFormatDescription(i);
+         String ext(desc->fileExtension);
+
+         if (ext.isEmpty() ||
+            ext.equal("dae", String::NoCase) || // filter out collada importer formats (for now).
+            ext.equal("zae", String::NoCase) ||
+            ext.equal("xml", String::NoCase)
+            )
+            continue;
+
+         reg.export_extensions.push_back({
+             String(desc->description),
+             ext
+            });
+      }
+
+      reg.readFunc = sReadAssimp;
+      reg.writeFunc = NULL;
+
+      TSShape::sRegisterFormat(reg);
+   }
+} sStaticRegisterAssimp;
+
 
 //-----------------------------------------------------------------------------
 
@@ -272,7 +285,16 @@ void AssimpShapeLoader::enumerateScene()
 
    // Load all materials
    AssimpAppMaterial::sDefaultMatNumber = 0;
-   for (U32 i = 0; i < mScene->mNumMaterials; ++i) {
+   AssimpAppMesh::sMaterialRemap.setSize(mScene->mNumMaterials);
+   for (U32 i = 0; i < mScene->mNumMaterials; ++i)
+   {
+      if (FindMatch::isMatchMultipleExprs(ColladaUtils::getOptions().neverImportMat,
+         mScene->mMaterials[i]->GetName().C_Str(), false))
+      {
+         AssimpAppMesh::sMaterialRemap[i] = TSDrawPrimitive::NoMaterial; // TSDrawPrimitive::NoMaterial
+         continue;
+      }
+      AssimpAppMesh::sMaterialRemap[i] = AppMesh::appMaterials.size();
       AppMesh::appMaterials.push_back(new AssimpAppMaterial(mScene->mMaterials[i]));
    }
 
@@ -391,54 +413,34 @@ void AssimpShapeLoader::getRootAxisTransform()
 
    MatrixF rot(true);
 
-   // ===== Y-UP SOURCE =====
+   // Build source basis
-   if (upAxis == 1)
+   auto axisToVector = [](int axis, int sign) -> Point3F
    {
-      if (frontAxis == 2)
+      Point3F v(0, 0, 0);
-      {
+      v[axis] = (F32)sign;
-         // Y-up, Z-forward  → Z-up, Y-forward
+      return v;
-         // Rotate 180° Y, then 90° X
+   };
-         rot(0, 0) = -1.0f;
-         rot(1, 1) = 0.0f;  rot(2, 1) = 1.0f;
-         rot(1, 2) = 1.0f;  rot(2, 2) = 0.0f;
-      }
-      else if (frontAxis == 0)
-      {
-         // Y-up, X-forward → Z-up, Y-forward
-         // Rotate -90° around Z then 90° around X
-         rot(0, 0) = 0.0f;   rot(0, 1) = -1.0f;
-         rot(1, 0) = 1.0f;   rot(1, 1) = 0.0f;
-         rot(2, 2) = 1.0f;
-      }
-   }
 
-   // ===== Z-UP SOURCE =====
+   Point3F forward = axisToVector(frontAxis, -frontSign);
-   if (upAxis == 2)
+   Point3F up = axisToVector(upAxis, upSign);
-   {
+   Point3F right = mCross(forward, up);
-      if (frontAxis == 1)
-      {
-         // Already Z-up, Y-forward → no change
-      }
-      else if (frontAxis == 0)
-      {
-         // Z-up, X-forward → rotate -90° around Z
-         rot(0, 0) = 0.0f;  rot(0, 1) = -1.0f;
-         rot(1, 0) = 1.0f;  rot(1, 1) = 0.0f;
-      }
-   }
 
-   // ===== X-UP SOURCE =====
+   // Recompute forward
-   if (upAxis == 0)
+   forward = mCross(up, right);
-   {
+
-      if (frontAxis == 2)
+   // Normalize (defensive, though they should already be unit)
-      {
+   right.normalize();
-         // X-up, Z-forward → Z-up, Y-forward
+   forward.normalize();
-         // Rotate -90° around Y then -90° around Z
+   up.normalize();
-         rot(0, 0) = 0.0f;  rot(0, 1) = 0.0f;  rot(0, 2) = -1.0f;
+
-         rot(1, 0) = 1.0f;  rot(1, 1) = 0.0f;  rot(1, 2) = 0.0f;
+   MatrixF srcBasis(true);
-         rot(2, 0) = 0.0f;  rot(2, 1) = -1.0f; rot(2, 2) = 0.0f;
+   srcBasis.setColumn(0, right);
-      }
+   srcBasis.setColumn(1, forward);
-   }
+   srcBasis.setColumn(2, up);
+
+   // Convert to Torque space
+   rot = srcBasis;
+   rot.inverse();
 
    ColladaUtils::getOptions().axisCorrectionMat = rot;
 }
@@ -689,30 +691,6 @@ bool AssimpShapeLoader::canLoadCachedDTS(const Torque::Path& path)
    return false;
 }
 
-/// Check if an up-to-date cached DSQ is available for this file
-bool AssimpShapeLoader::canLoadCachedDSQ(const Torque::Path& path)
-{
-   // Generate the cached filename
-   Torque::Path cachedPath(path);
-   cachedPath.setExtension("dsq");
-
-   // Check if a cached DTS newer than this file is available
-   FileTime cachedModifyTime;
-   if (Platform::getFileTimes(cachedPath.getFullPath(), NULL, &cachedModifyTime))
-   {
-      bool forceLoad = Con::getBoolVariable("$assimp::forceLoad", false);
-
-      FileTime daeModifyTime;
-      if (!Platform::getFileTimes(path.getFullPath(), NULL, &daeModifyTime) ||
-         (!forceLoad && (Platform::compareFileTimes(cachedModifyTime, daeModifyTime) >= 0)))
-      {
-         // Original file not found, or cached DTS is newer
-         return true;
-      }
-   }
-   return false;
-}
-
 void AssimpShapeLoader::assimpLogCallback(const char* message, char* user)
 {
    Con::printf("[Assimp log message] %s", StringUnit::getUnit(message, 0, "\n"));
@@ -992,65 +970,12 @@ bool AssimpShapeLoader::getMetaString(const char* key, String& stringVal)
 }
 //-----------------------------------------------------------------------------
 /// This function is invoked by the resource manager based on file extension.
-TSShape* assimpLoadShape(const Torque::Path &path)
+static bool sReadAssimp(const Torque::Path &path, TSShape*& res_shape)
 {
-   // TODO: add .cached.dts generation.
-   // Generate the cached filename
-   Torque::Path cachedPath(path);
-   bool canLoadCached = false;
-   bool canLoadDSQ = false;
-
-   // Check if an up-to-date cached DTS version of this file exists, and
-   // if so, use that instead.
-   if (AssimpShapeLoader::canLoadCachedDTS(path))
-   {
-      cachedPath.setExtension("cached.dts");
-      canLoadCached = true;
-   }
-   else if (gTryUseDSQs && AssimpShapeLoader::canLoadCachedDSQ(path))
-   {
-      cachedPath.setExtension("dsq");
-      canLoadDSQ = true;
-   }
-   if (canLoadCached || canLoadDSQ)
-   {
-      FileStream cachedStream;
-      cachedStream.open(cachedPath.getFullPath(), Torque::FS::File::Read);
-      if (cachedStream.getStatus() == Stream::Ok)
-      {
-         TSShape *shape = new TSShape;
-         bool readSuccess = false;
-         if (canLoadCached)
-         {
-            readSuccess = shape->read(&cachedStream);
-         }
-         else
-         {
-            readSuccess = shape->importSequences(&cachedStream, cachedPath);
-         }
-         cachedStream.close();
-
-         if (readSuccess)
-         {
-         #ifdef TORQUE_DEBUG
-            Con::printf("Loaded cached shape from %s", cachedPath.getFullPath().c_str());
-         #endif
-            return shape;
-         }
-         else
-         {
-         #ifdef TORQUE_DEBUG
-            Con::errorf("assimpLoadShape: Load sequence file '%s' failed", cachedPath.getFullPath().c_str());
-         #endif
-            delete shape;
-         }
-      }
-   }
-
    if (!Torque::FS::IsFile(path))
    {
       // File does not exist, bail.
-      return NULL;
+      return false;
    }
 
    // Allow TSShapeConstructor object to override properties
@@ -1068,27 +993,7 @@ TSShape* assimpLoadShape(const Torque::Path &path)
       TSShapeLoader::updateProgress(TSShapeLoader::Load_Complete, "Import complete");
       Con::printf("[ASSIMP] Shape created successfully.");
 
-      bool realMesh = false;
+      Torque::Path cachedPath(path);
-      for (U32 i = 0; i < tss->meshes.size(); ++i)
-      {
-         if (tss->meshes[i] && tss->meshes[i]->getMeshType() != TSMesh::NullMeshType)
-            realMesh = true;
-      }
-
-      if (!realMesh && gTryUseDSQs)
-      {
-         Torque::Path dsqPath(cachedPath);
-         dsqPath.setExtension("dsq");
-         FileStream animOutStream;
-         dsqPath.setFileName(cachedPath.getFileName());
-         if (animOutStream.open(dsqPath.getFullPath(), Torque::FS::File::Write))
-         {
-            Con::printf("Writing DSQ Animation File for '%s'", dsqPath.getFileName().c_str());
-            tss->exportSequences(&animOutStream);
-         }
-      }
-      else
-      {
       // Cache the model to a DTS file for faster loading next time.
       cachedPath.setExtension("cached.dts");
       // Cache the model to a DTS file for faster loading next time.
@@ -1098,10 +1003,30 @@ TSShape* assimpLoadShape(const Torque::Path &path)
          Con::printf("Writing cached shape to %s", cachedPath.getFullPath().c_str());
          tss->write(&dtsStream);
       }
+
+      if (tss->sequences.size() > 0)
+      {
+         Torque::Path dsqPath(cachedPath);
+         dsqPath.setExtension("dsq");
+         FileStream animOutStream;
+
+         for (S32 i = 0; i < tss->sequences.size(); i++)
+         {
+            const String& seqName = tss->getName(tss->sequences[i].nameIndex);
+            Con::printf("Writing DSQ Animation File for sequence '%s'", seqName.c_str());
+
+            dsqPath.setFileName(cachedPath.getFileName() + "_" + seqName);
+            if (animOutStream.open(dsqPath.getFullPath(), Torque::FS::File::Write))
+            {
+               tss->exportSequence(&animOutStream, tss->sequences[i], false);
+               animOutStream.close();
+            }
+         }
       }
    }
    loader.releaseImport();
-   return tss;
+   res_shape = tss;
+   return true;
 }
 
 DefineEngineFunction(GetShapeInfo, bool, (const char* shapePath, const char* ctrl, bool loadCachedDts), ("", "", true),

Engine/source/ts/assimp/assimpShapeLoader.h

@@ -41,8 +41,6 @@ struct aiMetadata;
 //-----------------------------------------------------------------------------
 class AssimpShapeLoader : public TSShapeLoader
 {
-   friend TSShape* assimpLoadShape(const Torque::Path &path);
-
 protected:
    Assimp::Importer mImporter;
    const aiScene* mScene;
@@ -79,7 +77,6 @@ public:
    bool fillGuiTreeView(const char* shapePath, GuiTreeViewCtrl* tree);
 
    static bool canLoadCachedDTS(const Torque::Path& path);
-   static bool canLoadCachedDSQ(const Torque::Path& path);
    static void assimpLogCallback(const char* message, char* user);
 };
 

Engine/source/ts/collada/colladaShapeLoader.cpp

@@ -50,15 +50,23 @@
 #include "core/util/zip/zipVolume.h"
 #include "gfx/bitmap/gBitmap.h"
 
-extern bool gTryUseDSQs;
+static bool sReadCollada(const Torque::Path& path, TSShape*& shape);
-MODULE_BEGIN( ColladaShapeLoader )
+
-   MODULE_INIT_AFTER( ShapeLoader )
+static struct _privateRegisterCollada
-   MODULE_INIT
 {
-      TSShapeLoader::addFormat("Collada", "dae");
+   _privateRegisterCollada()
-      TSShapeLoader::addFormat("Google Earth", "kmz");
+   {
+      TSShape::ShapeRegistration reg;
+      reg.extensions.push_back({ "Collada", "dae" });
+      reg.export_extensions.push_back({ "Collada", "dae" });
+      reg.extensions.push_back({ "Google Earth", "kmz" });
+
+      reg.readFunc = sReadCollada;
+      reg.writeFunc = NULL;
+
+      TSShape::sRegisterFormat(reg);
    }
-MODULE_END;
+} sStaticRegisterCollada;
 
 // 
 static DAE sDAE;                 // Collada model database (holds the last loaded file)
@@ -549,39 +557,6 @@ bool ColladaShapeLoader::canLoadCachedDTS(const Torque::Path& path)
      
    return false;
 }
-bool ColladaShapeLoader::canLoadCachedDSQ(const Torque::Path& path)
-{
-   // Generate the cached filename
-   Torque::Path cachedPath(path);
-   cachedPath.setExtension("dsq");
-
-   // Check if a cached DSQ newer than this file is available
-   FileTime cachedModifyTime;
-   if (Platform::getFileTimes(cachedPath.getFullPath(), NULL, &cachedModifyTime))
-   {
-      bool forceLoadDAE = Con::getBoolVariable("$collada::forceLoadDAE", false);
-
-      FileTime daeModifyTime;
-      if (!Platform::getFileTimes(path.getFullPath(), NULL, &daeModifyTime) ||
-         (!forceLoadDAE && (Platform::compareFileTimes(cachedModifyTime, daeModifyTime) >= 0)))
-      {
-         // DAE not found, or cached DTS is newer
-         return true;
-      }
-   }
-
-   //assume the dts is good since it was zipped on purpose
-   Torque::FS::FileSystemRef ref = Torque::FS::GetFileSystem(cachedPath);
-   if (ref && !String::compare("Zip", ref->getTypeStr().c_str()))
-   {
-      bool forceLoadDAE = Con::getBoolVariable("$collada::forceLoadDAE", false);
-
-      if (!forceLoadDAE && Torque::FS::IsFile(cachedPath))
-         return true;
-   }
-
-   return false;
-}
 bool ColladaShapeLoader::checkAndMountSketchup(const Torque::Path& path, String& mountPoint, Torque::Path& daePath)
 {
    bool isSketchup = path.getExtension().equal("kmz", String::NoCase);
@@ -683,61 +658,8 @@ domCOLLADA* ColladaShapeLoader::readColladaFile(const String& path)
 
 //-----------------------------------------------------------------------------
 /// This function is invoked by the resource manager based on file extension.
-TSShape* loadColladaShape(const Torque::Path &path)
+static bool sReadCollada(const Torque::Path& path, TSShape*& res_shape)
 {
-#ifndef DAE2DTS_TOOL
-   // Generate the cached filename
-   Torque::Path cachedPath(path);
-   bool canLoadCached = false;
-   bool canLoadDSQ = false;
-   // Check if an up-to-date cached DTS version of this file exists, and
-   // if so, use that instead.
-   if (ColladaShapeLoader::canLoadCachedDTS(path))
-   {
-      cachedPath.setExtension("cached.dts");
-      canLoadCached = true;
-   }
-   else if (gTryUseDSQs && ColladaShapeLoader::canLoadCachedDSQ(path))
-   {
-      cachedPath.setExtension("dsq");
-      canLoadDSQ = true;
-   }
-   if (canLoadCached || canLoadDSQ)
-   {
-      FileStream cachedStream;
-      cachedStream.open(cachedPath.getFullPath(), Torque::FS::File::Read);
-      if (cachedStream.getStatus() == Stream::Ok)
-      {
-         TSShape *shape = new TSShape;
-         bool readSuccess = false;
-         if (canLoadCached)
-         {
-            readSuccess = shape->read(&cachedStream);
-         }
-         else
-         {
-            readSuccess = shape->importSequences(&cachedStream, cachedPath);
-         }
-         cachedStream.close();
-
-         if (readSuccess)
-         {
-         #ifdef TORQUE_DEBUG
-            Con::printf("Loaded cached Collada shape from %s", cachedPath.getFullPath().c_str());
-         #endif
-            return shape;
-         }
-         else
-         {
-         #ifdef TORQUE_DEBUG
-            Con::errorf("loadColladaShape: Load sequence file '%s' failed", cachedPath.getFullPath().c_str());
-         #endif
-            delete shape;
-         }
-      }
-   }
-#endif // DAE2DTS_TOOL
-
    if (!Torque::FS::IsFile(path))
    {
       // DAE file does not exist, bail.
@@ -777,52 +699,44 @@ TSShape* loadColladaShape(const Torque::Path &path)
       tss = loader.generateShape(daePath);
       if (tss)
       {
-#ifndef DAE2DTS_TOOL
+         TSShapeLoader::updateProgress(TSShapeLoader::Load_Complete, "Import complete");
+         Con::printf("[COLLADA] Shape created successfully.");
 
-         bool realMesh = false;
+         Torque::Path cachedPath(path);
-         for (U32 i = 0; i < tss->meshes.size(); ++i)
+         // Cache the model to a DTS file for faster loading next time.
+         cachedPath.setExtension("cached.dts");
+         // Cache the model to a DTS file for faster loading next time.
+         FileStream dtsStream;
+         if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write))
          {
-            if (tss->meshes[i] && tss->meshes[i]->getMeshType() != TSMesh::NullMeshType)
+            Con::printf("Writing cached shape to %s", cachedPath.getFullPath().c_str());
-               realMesh = true;
+            tss->write(&dtsStream);
          }
 
-         if (!realMesh && gTryUseDSQs)
+         // Add collada materials to materials.tscript
+         updateMaterialsScript(path, isSketchup);
+
+         if (tss->sequences.size() > 0)
          {
             Torque::Path dsqPath(cachedPath);
             dsqPath.setExtension("dsq");
             FileStream animOutStream;
-            dsqPath.setFileName(cachedPath.getFileName());
+
+            for (S32 i = 0; i < tss->sequences.size(); i++)
+            {
+               const String& seqName = tss->getName(tss->sequences[i].nameIndex);
+               Con::printf("Writing DSQ Animation File for sequence '%s'", seqName.c_str());
+
+               dsqPath.setFileName(cachedPath.getFileName() + "_" + seqName);
                if (animOutStream.open(dsqPath.getFullPath(), Torque::FS::File::Write))
                {
-               Con::printf("Writing DSQ Animation File for '%s'", dsqPath.getFileName().c_str());
+                  tss->exportSequence(&animOutStream, tss->sequences[i], false);
-               tss->exportSequences(&animOutStream);
                   animOutStream.close();
                }
             }
-         else
-         {
-            // Cache the Collada model to a DTS file for faster loading next time.
-            cachedPath.setExtension("cached.dts");
-            FileStream dtsStream;
-            if (dtsStream.open(cachedPath.getFullPath(), Torque::FS::File::Write))
-            {
-               Torque::FS::FileSystemRef ref = Torque::FS::GetFileSystem(daePath);
-               if (ref && !String::compare("Zip", ref->getTypeStr().c_str()))
-                  Con::errorf("No cached dts file found in archive for %s. Forcing cache to disk.", daePath.getFullFileName().c_str());
-
-               Con::printf("Writing cached COLLADA shape to %s", cachedPath.getFullPath().c_str());
-               tss->write(&dtsStream);
          }
       }
-#endif // DAE2DTS_TOOL
-
-         // Add collada materials to materials.tscript
-         updateMaterialsScript(path, isSketchup);
    }
-   }
-
-   // Close progress dialog
-   TSShapeLoader::updateProgress(TSShapeLoader::Load_Complete, "Import complete");
 
    if (isSketchup)
    {
@@ -830,5 +744,6 @@ TSShape* loadColladaShape(const Torque::Path &path)
       Torque::FS::Unmount(mountPoint);
    }
 
-   return tss;
+   res_shape = tss;
+   return true;
 }

Engine/source/ts/collada/colladaShapeLoader.h

@@ -34,8 +34,6 @@ struct AnimChannels;
 //-----------------------------------------------------------------------------
 class ColladaShapeLoader : public TSShapeLoader
 {
-   friend TSShape* loadColladaShape(const Torque::Path &path);
-
    domCOLLADA*             root;
    Vector<AnimChannels*>   animations;       ///< Holds all animation channels for deletion after loading
 
@@ -53,7 +51,6 @@ public:
    void computeBounds(Box3F& bounds) override;
 
    static bool canLoadCachedDTS(const Torque::Path& path);
-   static bool canLoadCachedDSQ(const Torque::Path& path);
    static bool checkAndMountSketchup(const Torque::Path& path, String& mountPoint, Torque::Path& daePath);
    static domCOLLADA* getDomCOLLADA(const Torque::Path& path);
    static domCOLLADA* readColladaFile(const String& path);

Engine/source/ts/loader/tsShapeLoader.cpp

@@ -36,11 +36,9 @@ MODULE_BEGIN( ShapeLoader )
    MODULE_INIT
    {
       TSShapeLoader::addFormat("Torque DTS", "dts");
-      TSShapeLoader::addFormat("Torque DSQ", "dsq");
    }
 MODULE_END;
 
-bool gTryUseDSQs = false;
 const F32 TSShapeLoader::DefaultTime = -1.0f;
 const F64 TSShapeLoader::MinFrameRate = 15.0f;
 const F64 TSShapeLoader::MaxFrameRate = 60.0f;
@@ -1304,12 +1302,17 @@ String TSShapeLoader::getFormatExtensions()
 {
    // "*.dsq TAB *.dae TAB
    StringBuilder output;
-   for(U32 n = 0; n < smFormats.size(); ++n)
+   for(U32 n = 0; n < TSShape::sRegistrations.size(); ++n)
    {
+      TSShape::ShapeRegistration reg = TSShape::sRegistrations[n];
+      for (U32 i = 0; i < reg.extensions.size(); i++)
+      {
+         TSShape::ShapeFormat format = reg.extensions[i];
          output.append("*.");
-      output.append(smFormats[n].mExtension);
+         output.append(format.mExtension);
          output.append("\t");
       }
+   }
    return output.end();
 }
 
@@ -1317,13 +1320,18 @@ String TSShapeLoader::getFormatFilters()
 {
    // "DSQ Files|*.dsq|COLLADA Files|*.dae|"
    StringBuilder output;
-   for(U32 n = 0; n < smFormats.size(); ++n)
+   for (U32 n = 0; n < TSShape::sRegistrations.size(); ++n)
    {
-      output.append(smFormats[n].mName);
+      TSShape::ShapeRegistration reg = TSShape::sRegistrations[n];
+      for (U32 i = 0; i < reg.extensions.size(); i++)
+      {
+         TSShape::ShapeFormat format = reg.extensions[i];
+         output.append(format.mName);
          output.append("|*.");
-      output.append(smFormats[n].mExtension);
+         output.append(format.mExtension);
          output.append("|");
       }
+   }
    return output.end();
 }
 

Engine/source/ts/tsShape.cpp

@@ -38,16 +38,33 @@
 #include "core/stream/fileStream.h"
 #include "core/fileObject.h"
 
-#ifdef TORQUE_COLLADA
+Vector<TSShape::ShapeRegistration>   TSShape::sRegistrations(__FILE__, __LINE__);
-extern TSShape* loadColladaShape(const Torque::Path &path);
-#endif
 
-#ifdef TORQUE_ASSIMP
+void TSShape::sRegisterFormat(const ShapeRegistration& reg)
-extern TSShape* assimpLoadShape(const Torque::Path &path);
+{
-#endif
+   U32 insert = sRegistrations.size();
+   sRegistrations.insert(insert, reg);
+}
+
+const TSShape::ShapeRegistration* TSShape::sFindRegInfo(const String& extension, bool exporting)
+{
+   for (U32 i = 0; i < TSShape::sRegistrations.size(); i++)
+   {
+      const TSShape::ShapeRegistration& reg = TSShape::sRegistrations[i];
+      const Vector<ShapeFormat>& extensions = exporting ? reg.export_extensions : reg.extensions;
+
+      for (U32 j = 0; j < extensions.size(); j++)
+      {
+         if (extensions[j].mExtension.equal(extension, String::NoCase))
+            return &reg;
+      }
+   }
+
+   return NULL;
+}
 
 /// most recent version -- this is the version we write
-S32 TSShape::smVersion = 28;
+S32 TSShape::smVersion = TORQUE_DTS_VERSION;
 /// the version currently being read...valid only during a read
 S32 TSShape::smReadVersion = -1;
 const U32 TSShape::smMostRecentExporterVersion = DTS_EXPORTER_CURRENT_VERSION;
@@ -168,6 +185,34 @@ TSShape::~TSShape()
       delete[] mShapeData;
 }
 
+void TSShape::compressionKey(U8* keyOut, U32 keyLen)
+{
+   // Concatenate app name and zip password at compile time
+   static const char kSeed[] = TORQUE_APP_NAME DEFAULT_ZIP_PASSWORD;
+
+   // djb2 hash to collapse the seed string into a 32-bit value
+   U32 state = 5381u;
+   for (const char* c = kSeed; *c; ++c)
+      state = ((state << 5) + state) ^ (U8)*c;
+
+   // Expand into a key stream via a linear-congruential generator
+   // (Numerical Recipes coefficients)
+   for (U32 i = 0; i < keyLen; ++i)
+   {
+      state = state * 1664525u + 1013904223u;
+      keyOut[i] = (U8)(state >> 16);
+   }
+}
+
+void TSShape::xorBufferAtOffset(void* data, U32 byteCount,
+   U32 startOffset,
+   const U8* key, U32 keyLen)
+{
+   U8* p = (U8*)data;
+   for (U32 i = 0; i < byteCount; ++i)
+      p[i] ^= key[(startOffset + i) % keyLen];
+}
+
 const String& TSShape::getName( S32 nameIndex ) const
 {
    AssertFatal(nameIndex>=0 && nameIndex<names.size(),"TSShape::getName");
@@ -1892,6 +1937,20 @@ void TSShape::write(Stream * s, bool saveOldFormat)
       size8 += 4;
    size8 >>= 2;
 
+   if (smVersion >= 29)
+   {
+      const U32 KEY_LEN = 256;
+      U8 key[KEY_LEN];
+      compressionKey(key, KEY_LEN);
+
+      U32 offset = 0;
+      xorBufferAtOffset(buffer32, size32 * 4, offset, key, KEY_LEN);
+      offset += size32 * 4;
+      xorBufferAtOffset(buffer16, size16 * 4, offset, key, KEY_LEN);
+      offset += size16 * 4;
+      xorBufferAtOffset(buffer8, size8 * 4, offset, key, KEY_LEN);
+   }
+
    S32 sizeMemBuffer, start16, start8;
    sizeMemBuffer = size32 + size16 + size8;
    start16 = size32;
@@ -1972,6 +2031,17 @@ bool TSShape::read(Stream * s)
 
       S32 * tmp = new S32[sizeMemBuffer];
       s->read(sizeof(S32) * sizeMemBuffer, (U8*)tmp);
+
+      if (smReadVersion >= 29)
+      {
+         const U32 KEY_LEN = 256;
+         U8 key[KEY_LEN];
+         compressionKey(key, KEY_LEN);
+
+         // The whole tmp block is one contiguous encrypted region
+         xorBufferAtOffset(tmp, sizeMemBuffer * 4, 0, key, KEY_LEN);
+      }
+
       memBuffer32 = tmp;
       memBuffer16 = (S16*)(tmp+startU16);
       memBuffer8  = (S8*)(tmp+startU8);
@@ -2166,11 +2236,41 @@ template<> void *Resource<TSShape>::create(const Torque::Path &path)
    TSShape * ret = 0;
    bool readSuccess = false;
    const String extension = path.getExtension();
+   bool canLoadCached = false;
 
-   if ( extension.equal( "dts", String::NoCase ) )
+   // Generate the cached filename
+   Torque::Path cachedPath(path);
+   cachedPath.setExtension("cached.dts");
+
+   // Check if a cached DTS newer than this file is available
+   FileTime cachedModifyTime;
+   if (Platform::getFileTimes(cachedPath.getFullPath(), NULL, &cachedModifyTime))
+   {
+      bool forceLoadDAE = Con::getBoolVariable("$collada::forceLoadDAE", false);
+
+      FileTime daeModifyTime;
+      if (!Platform::getFileTimes(path.getFullPath(), NULL, &daeModifyTime) ||
+         (!forceLoadDAE && (Platform::compareFileTimes(cachedModifyTime, daeModifyTime) >= 0)))
+      {
+         // Non DTS not found, or cached DTS is newer
+         canLoadCached = true;
+      }
+   }
+
+   //assume the dts is good since it was zipped on purpose
+   Torque::FS::FileSystemRef ref = Torque::FS::GetFileSystem(cachedPath);
+   if (ref && !String::compare("Zip", ref->getTypeStr().c_str()))
+   {
+      bool forceLoadDAE = Con::getBoolVariable("$collada::forceLoadDAE", false);
+
+      if (!forceLoadDAE && Torque::FS::IsFile(cachedPath))
+         canLoadCached = true;
+   }
+
+   if (extension.equal("dts", String::NoCase) || canLoadCached)
    {
       FileStream stream;
-      stream.open( path.getFullPath(), Torque::FS::File::Read );
+      stream.open(canLoadCached ? cachedPath.getFullPath() : path.getFullPath(), Torque::FS::File::Read);
       if ( stream.getStatus() != Stream::Ok )
       {
          Con::errorf( "Resource<TSShape>::create - Could not open '%s'", path.getFullPath().c_str() );
@@ -2180,46 +2280,16 @@ template<> void *Resource<TSShape>::create(const Torque::Path &path)
       ret = new TSShape;
       readSuccess = ret->read(&stream);
    }
-   else if ( extension.equal( "dae", String::NoCase ) || extension.equal( "kmz", String::NoCase ) )
+   else
    {
-#ifdef TORQUE_COLLADA
+      const TSShape::ShapeRegistration* regInfo = TSShape::sFindRegInfo(extension);
-      // Attempt to load the DAE file
+      if (regInfo == NULL)
-      ret = loadColladaShape(path);
-      readSuccess = (ret != NULL);
-#else
-      // No COLLADA support => attempt to load the cached DTS file instead
-      Torque::Path cachedPath = path;
-      cachedPath.setExtension("cached.dts");
-       
-      FileStream stream;
-      stream.open( cachedPath.getFullPath(), Torque::FS::File::Read );
-      if ( stream.getStatus() != Stream::Ok )
       {
-         Con::errorf( "Resource<TSShape>::create - Could not open '%s'", cachedPath.getFullPath().c_str() );
+         readSuccess = false;
-         return NULL;
-      }
-      ret = new TSShape;
-      readSuccess = ret->read(&stream);
-#endif
       }
       else
       {
-      //Con::errorf( "Resource<TSShape>::create - '%s' has an unknown file format", path.getFullPath().c_str() );
+         readSuccess = regInfo->readFunc(path, ret);
-      //delete ret;
-      //return NULL;
-
-      // andrewmac: Open Asset Import Library
-#ifdef TORQUE_ASSIMP
-      ret = assimpLoadShape(path);
-      readSuccess = (ret != NULL);
-#endif
-
-      // andrewmac : I could have used another conditional macro but I think this is suffice:
-      if (!readSuccess)
-      {
-         Con::errorf("Resource<TSShape>::create - '%s' has an unknown file format", path.getFullPath().c_str());
-         delete ret;
-         return NULL;
       }
    }
 

Engine/source/ts/tsShape.h

@@ -96,6 +96,35 @@ class TSShape
       AnyScale       = UniformScale | AlignedScale | ArbitraryScale
    };
 
+   struct ShapeFormat
+   {
+      String mName;
+      String mExtension;
+   };
+
+   struct ShapeRegistration
+   {
+      typedef bool(*ReadShape)(const Torque::Path& path, TSShape*& shape);
+      typedef bool(*WriteShape)(const Torque::Path& path, TSShape* shape);
+      Vector<ShapeFormat>   extensions;         ///< the list of file extensions for this Loader [these should be lower case]
+      Vector<ShapeFormat>   export_extensions;  ///< the list of file extensions for this Loader [these should be lower case]
+
+      ReadShape    readFunc;            ///< the read function to read from a file.
+      WriteShape   writeFunc;           ///< the write function to write to a file.
+
+      ShapeRegistration()
+      {
+         readFunc = NULL;
+         writeFunc = NULL;
+         VECTOR_SET_ASSOCIATION(extensions);
+         VECTOR_SET_ASSOCIATION(export_extensions);
+      }
+   };
+
+   static void sRegisterFormat(const ShapeRegistration& reg);
+   static const ShapeRegistration* sFindRegInfo(const String& extension, bool exporting = false);
+   static Vector<ShapeRegistration>   sRegistrations;
+
    /// Nodes hold the transforms in the shape's tree.  They are the bones of the skeleton.
    struct Node
    {
@@ -422,6 +451,8 @@ class TSShape
    // constructor/destructor
    TSShape();
    ~TSShape();
+   void compressionKey(U8* keyOut, U32 keyLen);
+   void xorBufferAtOffset(void* data, U32 byteCount, U32 startOffset, const U8* key, U32 keyLen);
    void init();
    void initMaterialList();    ///< you can swap in a new material list, but call this if you do
    void finalizeEditable();

Engine/source/ts/tsShapeEdit.cpp

@@ -1361,16 +1361,7 @@ bool TSShape::isShapeFileType(Torque::Path filePath)
 {
    String fileExt = filePath.getExtension();
 
-   if (
+   if (TSShape::sFindRegInfo(fileExt))
-      fileExt.equal("dts", String::NoCase) ||
-      fileExt.equal("dsq", String::NoCase) ||
-      fileExt.equal("dae", String::NoCase) ||
-      fileExt.equal("fbx", String::NoCase) ||
-      fileExt.equal("blend", String::NoCase) ||
-      fileExt.equal("obj", String::NoCase) ||
-      fileExt.equal("gltf", String::NoCase) ||
-      fileExt.equal("glb", String::NoCase)
-      )
       return true;
 
    return false;

Tools/CMake/torqueConfig.h.in

@@ -231,3 +231,4 @@
 
 /// Password to use when opening encrypted zip files. Change this to whatever the password is for your zips.
 #define DEFAULT_ZIP_PASSWORD     "@TORQUE_APP_PASSWORD@"
+#define TORQUE_DTS_VERSION @TORQUE_DTS_VERSION@

Tools/CMake/torque_configs.cmake

@@ -114,6 +114,12 @@ advanced_option(TORQUE_DISABLE_MEMORY_MANAGER "Disable memory manager" ON)
 set(TORQUE_ENTRY_FUNCTION "" CACHE STRING "Specify a console function to execute instead of looking for a main.tscript file")
 mark_as_advanced(TORQUE_ENTRY_FUNCTION)
 
+set(TORQUE_DTS_VERSION "29" CACHE STRING "Device Tree Source version")
+
+set_property(CACHE TORQUE_DTS_VERSION PROPERTY STRINGS
+    28 29
+)
+
 #fileIO
 set(TORQUE_APP_PASSWORD "changeme" CACHE STRING "zip file password")
 advanced_option(TORQUE_DISABLE_VIRTUAL_MOUNT_SYSTEM "Disable virtual mount system" OFF)