Merge pull request #534 from Areloch/ImporterSanitizing

Importer sanitizing

Engine/source/T3D/assets/assetImporter.cpp

@@ -2381,11 +2381,25 @@ void AssetImporter::resolveAssetItemIssues(AssetImportObject* assetItem)
       }
       else if (activeImportConfig->DuplicateAutoResolution == String("FolderPrefix"))
       {
-         //Set trailing number
          String renamedAssetName = assetItem->assetName;
-         String owningFolder = assetItem->filePath.getDirectory(assetItem->filePath.getDirectoryCount() - 1);
 
-         renamedAssetName = owningFolder + "_" + renamedAssetName;
+         //Set trailing number
+         S32 dirIndex = assetItem->filePath.getDirectoryCount() - 1;
+         while (dirIndex > -1)
+         {
+            renamedAssetName = assetItem->assetName;
+            String owningFolder = assetItem->filePath.getDirectory(dirIndex);
+
+            renamedAssetName = owningFolder + "_" + renamedAssetName;
+
+            if (AssetDatabase.isDeclaredAsset(renamedAssetName))
+            {
+               dirIndex--;
+               continue;
+            }
+
+            break;
+         }
 
          //Log it's renaming
          dSprintf(importLogBuffer, sizeof(importLogBuffer), "Asset %s was renamed due to %s as part of the Import Configuration", assetItem->assetName.c_str(), humanReadableReason.c_str());

Engine/source/console/consoleFunctions.cpp

@@ -574,6 +574,40 @@ DefineEngineFunction( stripChars, const char*, ( const char* str, const char* ch
    return( ret );
 }
 
+//-----------------------------------------------------------------------------
+DefineEngineFunction(sanitizeString, const char*, (const char* str), ,
+   "Sanitizes a string of common name issues, such as:\n"
+   "Starting with numbers, replacing spaces with _, and removing any name un-compliant characters such as .,- etc\n"
+   "@param str The string to sanitize.\n"
+   "@return A version of @a str with all occurrences of invalid characters removed.\n\n"
+   "@tsexample\n"
+   "cleanString( \"123 .-_abc\"); // Returns \"__abc\"."
+   "@endtsexample\n"
+   "@ingroup Strings")
+{
+   String processedString = str;
+
+   U32 start;
+   U32 end;
+   String firstNumber = String::GetFirstNumber(processedString, start, end);
+   if (!firstNumber.isEmpty() && processedString.startsWith(firstNumber.c_str()))
+      processedString = processedString.replace(firstNumber, "");
+
+   processedString = processedString.replace(" ", "_");
+
+   U32 len = processedString.length() + 1;
+   char* ret = Con::getReturnBuffer(len);
+   dStrcpy(ret, processedString.c_str(), len);
+
+   U32 pos = dStrcspn(ret, "-+*/%$&<26>=()[].?\\\"#,;!~<>|<7C>^{}");
+   while (pos < dStrlen(ret))
+   {
+      dStrcpy(ret + pos, ret + pos + 1, len - pos);
+      pos = dStrcspn(ret, "-+*/%$&<26>=()[].?\\\"#,;!~<>|<7C>^{}");
+   }
+   return(ret);
+}
+
 //-----------------------------------------------------------------------------
 
 DefineEngineFunction( strlwr, const char*, ( const char* str ),,

Engine/source/ts/loader/tsShapeLoader.cpp

@@ -125,7 +125,8 @@ void TSShapeLoader::updateProgress(S32 major, const char* msg, S32 numMinor, S32
       progressMsg = avar("%s (%d of %d)", msg, minor + 1, numMinor);
    }
 
-   Con::executef("updateTSShapeLoadProgress", Con::getFloatArg(progress), progressMsg);
+   if(Con::isFunction("updateTSShapeLoadProgress"))
+      Con::executef("updateTSShapeLoadProgress", Con::getFloatArg(progress), progressMsg);
 }
 
 //-----------------------------------------------------------------------------
@@ -1353,4 +1354,4 @@ DefineEngineFunction( getFormatFilters, const char*, ( ),,
 DefineEngineFunction(isSupportedFormat, bool, (const char* extension), , "")
 {
    return TSShapeLoader::isSupportedFormat(extension);
-}
+}

Templates/BaseGame/game/tools/projectImporter/scripts/pre40/T3Dpre4ProjectImporter.tscript

@@ -77,8 +77,15 @@ function T3Dpre4ProjectImporter::copyFiles(%this)
    {
       %filePath = filePath(%file);
       %fileName = fileName(%file);
+      %fileBase = fileBase(%file);
       %fileExt = fileExt(%file);
       
+      if(endsWith(%fileName, ".asset.taml"))
+      {
+         %fileBase = strreplace(%fileBase, ".asset", "");
+         %fileExt = ".asset.taml";
+      }
+      
       if(%fileExt $= ".dll" || %fileExt $= ".log" || %fileExt $= ".exe" || %fileExt $= ".manifest"|| %fileExt $= ".h" ||
          %fileExt $= ".cpp" || %fileExt $= ".so" || %fileExt $= ".do" || %fileExt $= ".lib" ||%fileExt $= ".exp")
       {
@@ -99,6 +106,17 @@ function T3Dpre4ProjectImporter::copyFiles(%this)
       }
       
       %targetFilePath = strReplace(%file, $ProjectImporter::sourceContentFolder, $ProjectImporter::modulePath);
+      
+      %sanitizedFilename = sanitizeString(%fileBase);
+      if(startsWith(%sanitizedFilename, "_"))
+      {
+         %sanitizedFilename = substr(%sanitizedFilename, 1, -1);
+      }
+      if(%sanitizedFilename !$= %fileBase)
+      {
+         %targetFilePath = filePath(%targetFilePath) @ "/" @ %sanitizedFilename @ %fileExt;
+      }
+      
       %targetFolder = filePath(%targetFilePath);
       
       if(!isDirectory(%targetFolder))
@@ -214,7 +232,7 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this)
             %line = $ProjectImporter::fileObject.readLine();
             %trimmedLine = trim(%line);
             
-            if(strIsMatchExpr("*new*(*)*", %line))
+            if(strIsMatchExpr("*new*(*)*", %line) && strpos(%line, "::") == -1)
             {
                %className = findObjectClass(%line, "new");
 
@@ -246,6 +264,18 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this)
                   %fileWasChanged = true;
                }
                   
+               %sanitizedName = sanitizeString(%objectName);
+               if(startsWith(%sanitizedName, "_"))
+               {
+                  %sanitizedName = substr(%sanitizedName, 1, -1);
+               }
+               if(%sanitizedName !$= %objectName)
+               {
+                  %line = strReplace(%line, %objectName, %sanitizedName);
+                  
+                  %fileWasChanged = true;
+               }
+                  
                   if(%objectClassStack.count() == 1)
                   {
                   %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1);
@@ -278,6 +308,18 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this)
                
                %objectName = findObjectName(%line, "singleton");
                   
+               %sanitizedName = sanitizeString(%objectName);
+               if(startsWith(%sanitizedName, "_"))
+               {
+                  %sanitizedName = substr(%sanitizedName, 1, -1);
+               }
+               if(%sanitizedName !$= %objectName)
+               {
+                  %line = strReplace(%line, %objectName, %sanitizedName);
+                  
+                  %fileWasChanged = true;
+               }
+               
                   if(%objectClassStack.count() == 1)
                   {
                   %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1);
@@ -418,7 +460,7 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this)
             %line = $ProjectImporter::fileObject.readLine();
             %trimmedLine = trim(%line);
             
-            if(strIsMatchExpr("*new*(*)*", %line))
+            if(strIsMatchExpr("*new*(*)*", %line) && strpos(%line, "::") == -1)
             {
                %className = findObjectClass(%line, "new");
                
@@ -428,6 +470,18 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this)
                
                if(%objectName !$= "")
                {
+                  %sanitizedName = sanitizeString(%objectName);
+                  if(startsWith(%sanitizedName, "_"))
+                  {
+                     %sanitizedName = substr(%sanitizedName, 1, -1);
+                  }
+                  if(%sanitizedName !$= %objectName)
+                  {
+                     %line = strReplace(%line, %objectName, %sanitizedName);
+                     
+                     %fileWasChanged = true;
+                  }
+                  
                   if(%objectClassStack.count() == 1)
                   {
                      %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1);
@@ -513,6 +567,18 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this)
                
                if(%objectName !$= "")
                {
+                  %sanitizedName = sanitizeString(%objectName);
+                  if(startsWith(%sanitizedName, "_"))
+                  {
+                     %sanitizedName = substr(%sanitizedName, 1, -1);
+                  }
+                  if(%sanitizedName !$= %objectName)
+                  {
+                     %line = strReplace(%line, %objectName, %sanitizedName);
+                     
+                     %fileWasChanged = true;
+                  }
+               
                   if(%objectClassStack.count() == 1)
                   {
                      %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1);
@@ -764,6 +830,7 @@ T3Dpre4ProjectImporter::genProcessor("GroundPlane", "material materialAsset");
 T3Dpre4ProjectImporter::genProcessor("LevelInfo", "accuTexture accuTextureAsset");
 T3Dpre4ProjectImporter::genProcessor("TSStatic", "shape shapeAsset shapeName shapeAsset");
 T3Dpre4ProjectImporter::genProcessor("TSForestItemData", "shape shapeAsset shapeName shapeAsset shapeFile shapeAsset");
+T3Dpre4ProjectImporter::genProcessor("TerrainBlock", "terrainFile terrainAsset");
 //==============================================================================
 // Levels
 //==============================================================================
@@ -1034,57 +1101,8 @@ function processGuiBitmapButtonCtrlField(%line, %originalFieldName, %newFieldNam
          //These are going to be texture/render targets, and we can leave them alone
          return %line;
       }
-      //find any assets with that filename
-      else if(startsWith(%value, "./"))
-      {
-         %targetFilename = strReplace(%value, "./", $ProjectImporter::currentFilePath @ "/");
-      }
-      else if(startsWith(%value, "../"))
-      {
-         %slashPos = strposr($ProjectImporter::currentFilePath, "/");
-         if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up
-         {
-            %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2);
-         }
-         
-         %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos);
-         %targetFilename = strReplace(%value, "../", %parentPath @ "/");
-      }
-      else if(startsWith(%value, "~"))
-      {
-         %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/");
-         if(!isFile(%targetFilename))
-         {
-            %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/main/");
-         }
-      }
-      else if ((strpos(%value,"/") == -1)&&(strpos(%value,"\\") == -1))
-      {
-         %targetFilename = $ProjectImporter::currentFilePath @ %value;
-      }
-      else if(!startsWith(%value, $ProjectImporter::modulePath @ "/"))
-      {
-         %targetFilename = $ProjectImporter::modulePath @ "/" @ %value;
-      }
-      else
-      {
-         %targetFilename = %value;
-      }
       
-      %targetFilename = strReplace(%targetFilename, "//", "/");
-      %targetFilename = testFilenameExtensions(%targetFilename);
-      
-      if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out
-      {
-         //Now we test for a suffix
-         %targetPath = filePath(%targetFilename);
-         %targetName = fileBase(%targetFilename);
-         %targetExt = fileExt(%targetFilename);
-         
-         %targetFilename = %targetPath @ "/" @ %targetName @ "_n" @ %targetExt;
-         %targetFilename = strReplace(%targetFilename, "//", "/");
-         %targetFilename = testFilenameExtensions(%targetFilename);
-      }
+      %targetFilename = sanitizeFilename(%value);
       
       //If we still have nothing, then we fail it out
       if(!isFile(%targetFilename))

Templates/BaseGame/game/tools/projectImporter/scripts/projectImporter.tscript

@@ -220,10 +220,7 @@ function ProjectImportWizardPage3::openPage(%this)
          %topFolder = getToken($ProjectImporter::sourceContentFolder, "/", %slashCount-2);
          
       //clean up invalid characters and stuff
-      %topFolder = strReplace(%topFolder, " ", "");
-      %topFolder = strReplace(%topFolder, "!", "");
-      %topFolder = strReplace(%topFolder, "-", "");
-      %topFolder = strReplace(%topFolder, ".", "");
+      %topFolder = sanitizeString(%topFolder);
      
       $ProjectImporter::useExistingModule = false;
       $ProjectImporter::moduleName = %topFolder; //preseed the module name
@@ -366,6 +363,91 @@ function beginProjectImport()
    AssetBrowser.refresh(); //update the AB just in case
 }
 
+function sanitizeFilename(%file)
+{
+   if(startsWith(%file, "./"))
+   {
+      %targetFilename = strReplace(%file, "./", $ProjectImporter::currentFilePath @ "/");
+   }
+   else if(startsWith(%file, "../"))
+   {
+      %slashPos = strposr($ProjectImporter::currentFilePath, "/");
+      if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up
+      {
+         %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2);
+      }
+      
+      %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos);
+      %targetFilename = strReplace(%file, "../", %parentPath @ "/");
+   }
+   else if(startsWith(%file, "~"))
+   {
+      %targetFilename = strReplace(%file, "~", $ProjectImporter::modulePath @ "/");
+      if(!isFile(%targetFilename))
+      {
+         %targetFilename = strReplace(%file, "~", $ProjectImporter::modulePath @ "/main/");
+      }
+   }
+   else if ((strpos(%file,"/") == -1)&&(strpos(%file,"\\") == -1))
+   {
+      %targetFilename = $ProjectImporter::currentFilePath @ %file;
+   }
+   else if(!startsWith(%file, $ProjectImporter::modulePath @ "/"))
+   {
+      %targetFilename = $ProjectImporter::modulePath @ "/" @ %file;
+   }
+   else
+   {
+      %targetFilename = %file;
+   }
+   
+   //Now we test for a suffix
+   %flName = fileName(%targetFilename);
+   %targetPath = filePath(%targetFilename);
+   %targetName = fileBase(%targetFilename);
+   %targetExt = fileExt(%targetFilename);
+   
+   if(endsWith(%flName, ".asset.taml"))
+   {
+      %targetName = strreplace(%targetName, ".asset", "");
+      %targetExt = ".asset.taml";
+   }
+   
+   %sanitizedName = sanitizeString(%targetName);
+   if(startsWith(%sanitizedName, "_"))
+   {
+      %sanitizedName = substr(%sanitizedName, 1, -1);
+   }
+   if(%sanitizedName !$= %targetName)
+   {
+      %targetName = %sanitizedName;
+      %targetFilename = %targetPath @ "/" @ %targetName @ %targetExt;
+   }
+   
+   if(!isFile(%targetFilename))
+   {
+      %bitmapFile = %targetPath @ "/" @ %targetName @ "_n" @ %targetExt;
+      if(isFile(%bitmapFile))
+         %targetFilename = %bitmapFile;
+   }
+   
+   if(!isFile(%targetFilename))
+   {
+      %targetFilename = testFilenameExtensions(%targetFilename);
+   }
+   
+   %targetFilename = strReplace(%targetFilename, "//", "/");
+   
+   if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out
+   {
+      return %file;
+   }
+   else
+   {
+      return %targetFilename;
+   }
+}
+
 function testFilenameExtensions(%filename)
 {
    %ext = fileExt(%filename);  
@@ -432,47 +514,17 @@ function processLegacyField(%line, %originalFieldName, %newFieldName)
          //These are going to be texture/render targets, and we can leave them alone
          return %line;
       }
-      //find any assets with that filename
-      else if(startsWith(%value, "./"))
-      {
-         %targetFilename = strReplace(%value, "./", $ProjectImporter::currentFilePath @ "/");
-      }
-      else if(startsWith(%value, "../"))
-      {
-         %slashPos = strposr($ProjectImporter::currentFilePath, "/");
-         if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up
-         {
-            %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2);
-         }
          
-         %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos);
-         %targetFilename = strReplace(%value, "../", %parentPath @ "/");
-      }
-      else if(startsWith(%value, "~"))
-      {
-         %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/");
-         if(!isFile(%targetFilename))
+      %targetFilename = sanitizeFilename(%value);
+      
+      if(isObject(%targetFilename))
          {
-            %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/main/");
-         }
-      }
-      else if ((strpos(%value,"/") == -1)&&(strpos(%value,"\\") == -1))
-      {
-         %targetFilename = $ProjectImporter::currentFilePath @ %value;
-      }
-      else if(!startsWith(%value, $ProjectImporter::modulePath @ "/"))
-      {
-         %targetFilename = $ProjectImporter::modulePath @ "/" @ %value;
+         //likely a material name, so handle it that way
+         %assetId = MaterialAsset::getAssetIdByMaterialName(%targetFilename);
       }
       else
       {
-         %targetFilename = %value;
-      }
-      
-      %targetFilename = strReplace(%targetFilename, "//", "/");
-      %targetFilename = testFilenameExtensions(%targetFilename);
-      
-      if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out
+         if(!isFile(%targetFilename))
       {
          error("Legacy Project Importer - file described in line could not be found/is not valid");
          return %line;
@@ -483,11 +535,12 @@ function processLegacyField(%line, %originalFieldName, %newFieldName)
       if(%foundAssets != 0)
       {
          %assetId = $ProjectImporter::assetQuery.getAsset(0);
-         echo("Legacy Project Importer - processing of legacy field line's value: " @ %value @ " has found a matching AssetId: " @ %assetId);
+         }
       }
      
       if(%assetId !$= "" && AssetDatabase.isDeclaredAsset(%assetId))
       {
+         echo("Legacy Project Importer - processing of legacy field line's value: " @ %value @ " has found a matching AssetId: " @ %assetId);
          //if (%assetId.getStatusString() $= "Ok")
          %outLine = strReplace(%outLine, %value, %assetId);
          //else
@@ -523,45 +576,7 @@ function processLegacyShapeConstructorField(%line)
    if(strPos(%animSourcePath, ":") != -1)
       return %line;
                
-   //otherwise, try and see if we've got an animation source file here
-   if(startsWith(%animSourcePath, "./"))
-   {
-      %targetFilename = strReplace(%animSourcePath, "./", $ProjectImporter::currentFilePath @ "/");
-   }
-   else if(startsWith(%animSourcePath, "../"))
-   {
-      %slashPos = strposr($ProjectImporter::currentFilePath, "/");
-      if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up
-      {
-         %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2);
-      }
-      
-      %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos);
-      %targetFilename = strReplace(%animSourcePath, "../", %parentPath @ "/");
-   }
-   else if(startsWith(%animSourcePath, "~"))
-   {
-      %targetFilename = strReplace(%animSourcePath, "~", $ProjectImporter::modulePath @ "/");
-      if(!isFile(%targetFilename))
-      {
-         %targetFilename = strReplace(%animSourcePath, "~", $ProjectImporter::modulePath @ "/main/");
-      }
-   }
-   else if ((strpos(%animSourcePath,"/") == -1)&&(strpos(%animSourcePath,"\\") == -1))
-   {
-      %targetFilename = $ProjectImporter::currentFilePath @ %animSourcePath;
-   }
-   else if(!startsWith(%animSourcePath, $ProjectImporter::modulePath @ "/"))
-   {
-      %targetFilename = $ProjectImporter::modulePath @ "/" @ %animSourcePath;
-   }
-   else
-   {
-      %targetFilename = %animSourcePath;
-   }
-   
-   %targetFilename = strReplace(%targetFilename, "//", "/");
-   %targetFilename = testFilenameExtensions(%targetFilename);
+   %targetFilename = sanitizeFilename(%animSourcePath);
    
    if(!isFile(%targetFilename))
    {
@@ -828,6 +843,14 @@ function beginTerrainImport()
       %filePath = filePath(%file);
       if(%fileExt $= ".ter")
       {
+         %sanitizedFile = sanitizeFilename(%file);
+         if(%sanitizedFile !$= %file)
+         {
+            %file = %sanitizedFile;
+            %fileName = fileName(%file); 
+            %filePath = filePath(%file);
+         }
+            
          $ProjectImporter::assetQuery.clear();
          %assetsFound = AssetDatabase.findAssetLooseFile($ProjectImporter::assetQuery, %file);
          if(%assetsFound == 0)