From 48e994f7bddf4f5060d89741918d03c3ac7957c9 Mon Sep 17 00:00:00 2001 From: Areloch Date: Sun, 15 Aug 2021 03:07:40 -0500 Subject: [PATCH] Shift file handling in importer to a standardized function Added santizing of strings to filenames, object names and asset names for the importer process Added sanitizeString console function for above Added processing of terrainBlock object's fields for importer Added check to avoid updateTSShapeLoadProgress spam Adjusted folderPrefix logic to walk up directory to find assetName that isn't already in use. --- Engine/source/T3D/assets/assetImporter.cpp | 20 +- Engine/source/console/consoleFunctions.cpp | 34 ++++ Engine/source/ts/loader/tsShapeLoader.cpp | 5 +- .../pre40/T3Dpre4ProjectImporter.tscript | 122 +++++++----- .../scripts/projectImporter.tscript | 183 ++++++++++-------- 5 files changed, 227 insertions(+), 137 deletions(-) diff --git a/Engine/source/T3D/assets/assetImporter.cpp b/Engine/source/T3D/assets/assetImporter.cpp index f333e62c4..b0497631e 100644 --- a/Engine/source/T3D/assets/assetImporter.cpp +++ b/Engine/source/T3D/assets/assetImporter.cpp @@ -2381,11 +2381,25 @@ void AssetImporter::resolveAssetItemIssues(AssetImportObject* assetItem) } else if (activeImportConfig->DuplicateAutoResolution == String("FolderPrefix")) { - //Set trailing number String renamedAssetName = assetItem->assetName; - String owningFolder = assetItem->filePath.getDirectory(assetItem->filePath.getDirectoryCount() - 1); - renamedAssetName = owningFolder + "_" + renamedAssetName; + //Set trailing number + S32 dirIndex = assetItem->filePath.getDirectoryCount() - 1; + while (dirIndex > -1) + { + renamedAssetName = assetItem->assetName; + String owningFolder = assetItem->filePath.getDirectory(dirIndex); + + renamedAssetName = owningFolder + "_" + renamedAssetName; + + if (AssetDatabase.isDeclaredAsset(renamedAssetName)) + { + dirIndex--; + continue; + } + + break; + } //Log it's renaming dSprintf(importLogBuffer, sizeof(importLogBuffer), "Asset %s was renamed due to %s as part of the Import Configuration", assetItem->assetName.c_str(), humanReadableReason.c_str()); diff --git a/Engine/source/console/consoleFunctions.cpp b/Engine/source/console/consoleFunctions.cpp index 236b17434..88bbc14e3 100644 --- a/Engine/source/console/consoleFunctions.cpp +++ b/Engine/source/console/consoleFunctions.cpp @@ -574,6 +574,40 @@ DefineEngineFunction( stripChars, const char*, ( const char* str, const char* ch return( ret ); } +//----------------------------------------------------------------------------- +DefineEngineFunction(sanitizeString, const char*, (const char* str), , + "Sanitizes a string of common name issues, such as:\n" + "Starting with numbers, replacing spaces with _, and removing any name un-compliant characters such as .,- etc\n" + "@param str The string to sanitize.\n" + "@return A version of @a str with all occurrences of invalid characters removed.\n\n" + "@tsexample\n" + "cleanString( \"123 .-_abc\"); // Returns \"__abc\"." + "@endtsexample\n" + "@ingroup Strings") +{ + String processedString = str; + + U32 start; + U32 end; + String firstNumber = String::GetFirstNumber(processedString, start, end); + if (!firstNumber.isEmpty() && processedString.startsWith(firstNumber.c_str())) + processedString = processedString.replace(firstNumber, ""); + + processedString = processedString.replace(" ", "_"); + + U32 len = processedString.length() + 1; + char* ret = Con::getReturnBuffer(len); + dStrcpy(ret, processedString.c_str(), len); + + U32 pos = dStrcspn(ret, "-+*/%$&�=()[].?\\\"#,;!~<>|�^{}"); + while (pos < dStrlen(ret)) + { + dStrcpy(ret + pos, ret + pos + 1, len - pos); + pos = dStrcspn(ret, "-+*/%$&�=()[].?\\\"#,;!~<>|�^{}"); + } + return(ret); +} + //----------------------------------------------------------------------------- DefineEngineFunction( strlwr, const char*, ( const char* str ),, diff --git a/Engine/source/ts/loader/tsShapeLoader.cpp b/Engine/source/ts/loader/tsShapeLoader.cpp index 49567d7e9..98c1bc7e7 100644 --- a/Engine/source/ts/loader/tsShapeLoader.cpp +++ b/Engine/source/ts/loader/tsShapeLoader.cpp @@ -125,7 +125,8 @@ void TSShapeLoader::updateProgress(S32 major, const char* msg, S32 numMinor, S32 progressMsg = avar("%s (%d of %d)", msg, minor + 1, numMinor); } - Con::executef("updateTSShapeLoadProgress", Con::getFloatArg(progress), progressMsg); + if(Con::isFunction("updateTSShapeLoadProgress")) + Con::executef("updateTSShapeLoadProgress", Con::getFloatArg(progress), progressMsg); } //----------------------------------------------------------------------------- @@ -1353,4 +1354,4 @@ DefineEngineFunction( getFormatFilters, const char*, ( ),, DefineEngineFunction(isSupportedFormat, bool, (const char* extension), , "") { return TSShapeLoader::isSupportedFormat(extension); -} \ No newline at end of file +} diff --git a/Templates/BaseGame/game/tools/projectImporter/scripts/pre40/T3Dpre4ProjectImporter.tscript b/Templates/BaseGame/game/tools/projectImporter/scripts/pre40/T3Dpre4ProjectImporter.tscript index 4f795bee3..9c0b1691f 100644 --- a/Templates/BaseGame/game/tools/projectImporter/scripts/pre40/T3Dpre4ProjectImporter.tscript +++ b/Templates/BaseGame/game/tools/projectImporter/scripts/pre40/T3Dpre4ProjectImporter.tscript @@ -77,8 +77,15 @@ function T3Dpre4ProjectImporter::copyFiles(%this) { %filePath = filePath(%file); %fileName = fileName(%file); + %fileBase = fileBase(%file); %fileExt = fileExt(%file); + if(endsWith(%fileName, ".asset.taml")) + { + %fileBase = strreplace(%fileBase, ".asset", ""); + %fileExt = ".asset.taml"; + } + if(%fileExt $= ".dll" || %fileExt $= ".log" || %fileExt $= ".exe" || %fileExt $= ".manifest"|| %fileExt $= ".h" || %fileExt $= ".cpp" || %fileExt $= ".so" || %fileExt $= ".do" || %fileExt $= ".lib" ||%fileExt $= ".exp") { @@ -99,6 +106,17 @@ function T3Dpre4ProjectImporter::copyFiles(%this) } %targetFilePath = strReplace(%file, $ProjectImporter::sourceContentFolder, $ProjectImporter::modulePath); + + %sanitizedFilename = sanitizeString(%fileBase); + if(startsWith(%sanitizedFilename, "_")) + { + %sanitizedFilename = substr(%sanitizedFilename, 1, -1); + } + if(%sanitizedFilename !$= %fileBase) + { + %targetFilePath = filePath(%targetFilePath) @ "/" @ %sanitizedFilename @ %fileExt; + } + %targetFolder = filePath(%targetFilePath); if(!isDirectory(%targetFolder)) @@ -214,7 +232,7 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this) %line = $ProjectImporter::fileObject.readLine(); %trimmedLine = trim(%line); - if(strIsMatchExpr("*new*(*)*", %line)) + if(strIsMatchExpr("*new*(*)*", %line) && strpos(%line, "::") == -1) { %className = findObjectClass(%line, "new"); @@ -246,6 +264,18 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this) %fileWasChanged = true; } + %sanitizedName = sanitizeString(%objectName); + if(startsWith(%sanitizedName, "_")) + { + %sanitizedName = substr(%sanitizedName, 1, -1); + } + if(%sanitizedName !$= %objectName) + { + %line = strReplace(%line, %objectName, %sanitizedName); + + %fileWasChanged = true; + } + if(%objectClassStack.count() == 1) { %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1); @@ -278,6 +308,18 @@ function T3Dpre4ProjectImporter::beginMaterialFilesImport(%this) %objectName = findObjectName(%line, "singleton"); + %sanitizedName = sanitizeString(%objectName); + if(startsWith(%sanitizedName, "_")) + { + %sanitizedName = substr(%sanitizedName, 1, -1); + } + if(%sanitizedName !$= %objectName) + { + %line = strReplace(%line, %objectName, %sanitizedName); + + %fileWasChanged = true; + } + if(%objectClassStack.count() == 1) { %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1); @@ -418,7 +460,7 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this) %line = $ProjectImporter::fileObject.readLine(); %trimmedLine = trim(%line); - if(strIsMatchExpr("*new*(*)*", %line)) + if(strIsMatchExpr("*new*(*)*", %line) && strpos(%line, "::") == -1) { %className = findObjectClass(%line, "new"); @@ -428,6 +470,18 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this) if(%objectName !$= "") { + %sanitizedName = sanitizeString(%objectName); + if(startsWith(%sanitizedName, "_")) + { + %sanitizedName = substr(%sanitizedName, 1, -1); + } + if(%sanitizedName !$= %objectName) + { + %line = strReplace(%line, %objectName, %sanitizedName); + + %fileWasChanged = true; + } + if(%objectClassStack.count() == 1) { %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1); @@ -513,6 +567,18 @@ function T3Dpre4ProjectImporter::beginCodeFilesImport(%this) if(%objectName !$= "") { + %sanitizedName = sanitizeString(%objectName); + if(startsWith(%sanitizedName, "_")) + { + %sanitizedName = substr(%sanitizedName, 1, -1); + } + if(%sanitizedName !$= %objectName) + { + %line = strReplace(%line, %objectName, %sanitizedName); + + %fileWasChanged = true; + } + if(%objectClassStack.count() == 1) { %currentObjClass = %objectClassStack.getKey(%objectClassStack.count()-1); @@ -764,6 +830,7 @@ T3Dpre4ProjectImporter::genProcessor("GroundPlane", "material materialAsset"); T3Dpre4ProjectImporter::genProcessor("LevelInfo", "accuTexture accuTextureAsset"); T3Dpre4ProjectImporter::genProcessor("TSStatic", "shape shapeAsset shapeName shapeAsset"); T3Dpre4ProjectImporter::genProcessor("TSForestItemData", "shape shapeAsset shapeName shapeAsset shapeFile shapeAsset"); +T3Dpre4ProjectImporter::genProcessor("TerrainBlock", "terrainFile terrainAsset"); //============================================================================== // Levels //============================================================================== @@ -1034,57 +1101,8 @@ function processGuiBitmapButtonCtrlField(%line, %originalFieldName, %newFieldNam //These are going to be texture/render targets, and we can leave them alone return %line; } - //find any assets with that filename - else if(startsWith(%value, "./")) - { - %targetFilename = strReplace(%value, "./", $ProjectImporter::currentFilePath @ "/"); - } - else if(startsWith(%value, "../")) - { - %slashPos = strposr($ProjectImporter::currentFilePath, "/"); - if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up - { - %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2); - } - - %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos); - %targetFilename = strReplace(%value, "../", %parentPath @ "/"); - } - else if(startsWith(%value, "~")) - { - %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/"); - if(!isFile(%targetFilename)) - { - %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/main/"); - } - } - else if ((strpos(%value,"/") == -1)&&(strpos(%value,"\\") == -1)) - { - %targetFilename = $ProjectImporter::currentFilePath @ %value; - } - else if(!startsWith(%value, $ProjectImporter::modulePath @ "/")) - { - %targetFilename = $ProjectImporter::modulePath @ "/" @ %value; - } - else - { - %targetFilename = %value; - } - %targetFilename = strReplace(%targetFilename, "//", "/"); - %targetFilename = testFilenameExtensions(%targetFilename); - - if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out - { - //Now we test for a suffix - %targetPath = filePath(%targetFilename); - %targetName = fileBase(%targetFilename); - %targetExt = fileExt(%targetFilename); - - %targetFilename = %targetPath @ "/" @ %targetName @ "_n" @ %targetExt; - %targetFilename = strReplace(%targetFilename, "//", "/"); - %targetFilename = testFilenameExtensions(%targetFilename); - } + %targetFilename = sanitizeFilename(%value); //If we still have nothing, then we fail it out if(!isFile(%targetFilename)) diff --git a/Templates/BaseGame/game/tools/projectImporter/scripts/projectImporter.tscript b/Templates/BaseGame/game/tools/projectImporter/scripts/projectImporter.tscript index eff77c7d4..05a0847f4 100644 --- a/Templates/BaseGame/game/tools/projectImporter/scripts/projectImporter.tscript +++ b/Templates/BaseGame/game/tools/projectImporter/scripts/projectImporter.tscript @@ -220,10 +220,7 @@ function ProjectImportWizardPage3::openPage(%this) %topFolder = getToken($ProjectImporter::sourceContentFolder, "/", %slashCount-2); //clean up invalid characters and stuff - %topFolder = strReplace(%topFolder, " ", ""); - %topFolder = strReplace(%topFolder, "!", ""); - %topFolder = strReplace(%topFolder, "-", ""); - %topFolder = strReplace(%topFolder, ".", ""); + %topFolder = sanitizeString(%topFolder); $ProjectImporter::useExistingModule = false; $ProjectImporter::moduleName = %topFolder; //preseed the module name @@ -366,6 +363,91 @@ function beginProjectImport() AssetBrowser.refresh(); //update the AB just in case } +function sanitizeFilename(%file) +{ + if(startsWith(%file, "./")) + { + %targetFilename = strReplace(%file, "./", $ProjectImporter::currentFilePath @ "/"); + } + else if(startsWith(%file, "../")) + { + %slashPos = strposr($ProjectImporter::currentFilePath, "/"); + if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up + { + %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2); + } + + %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos); + %targetFilename = strReplace(%file, "../", %parentPath @ "/"); + } + else if(startsWith(%file, "~")) + { + %targetFilename = strReplace(%file, "~", $ProjectImporter::modulePath @ "/"); + if(!isFile(%targetFilename)) + { + %targetFilename = strReplace(%file, "~", $ProjectImporter::modulePath @ "/main/"); + } + } + else if ((strpos(%file,"/") == -1)&&(strpos(%file,"\\") == -1)) + { + %targetFilename = $ProjectImporter::currentFilePath @ %file; + } + else if(!startsWith(%file, $ProjectImporter::modulePath @ "/")) + { + %targetFilename = $ProjectImporter::modulePath @ "/" @ %file; + } + else + { + %targetFilename = %file; + } + + //Now we test for a suffix + %flName = fileName(%targetFilename); + %targetPath = filePath(%targetFilename); + %targetName = fileBase(%targetFilename); + %targetExt = fileExt(%targetFilename); + + if(endsWith(%flName, ".asset.taml")) + { + %targetName = strreplace(%targetName, ".asset", ""); + %targetExt = ".asset.taml"; + } + + %sanitizedName = sanitizeString(%targetName); + if(startsWith(%sanitizedName, "_")) + { + %sanitizedName = substr(%sanitizedName, 1, -1); + } + if(%sanitizedName !$= %targetName) + { + %targetName = %sanitizedName; + %targetFilename = %targetPath @ "/" @ %targetName @ %targetExt; + } + + if(!isFile(%targetFilename)) + { + %bitmapFile = %targetPath @ "/" @ %targetName @ "_n" @ %targetExt; + if(isFile(%bitmapFile)) + %targetFilename = %bitmapFile; + } + + if(!isFile(%targetFilename)) + { + %targetFilename = testFilenameExtensions(%targetFilename); + } + + %targetFilename = strReplace(%targetFilename, "//", "/"); + + if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out + { + return %file; + } + else + { + return %targetFilename; + } +} + function testFilenameExtensions(%filename) { %ext = fileExt(%filename); @@ -432,47 +514,17 @@ function processLegacyField(%line, %originalFieldName, %newFieldName) //These are going to be texture/render targets, and we can leave them alone return %line; } - //find any assets with that filename - else if(startsWith(%value, "./")) - { - %targetFilename = strReplace(%value, "./", $ProjectImporter::currentFilePath @ "/"); - } - else if(startsWith(%value, "../")) - { - %slashPos = strposr($ProjectImporter::currentFilePath, "/"); - if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up - { - %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2); - } - %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos); - %targetFilename = strReplace(%value, "../", %parentPath @ "/"); - } - else if(startsWith(%value, "~")) - { - %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/"); - if(!isFile(%targetFilename)) + %targetFilename = sanitizeFilename(%value); + + if(isObject(%targetFilename)) { - %targetFilename = strReplace(%value, "~", $ProjectImporter::modulePath @ "/main/"); - } - } - else if ((strpos(%value,"/") == -1)&&(strpos(%value,"\\") == -1)) - { - %targetFilename = $ProjectImporter::currentFilePath @ %value; - } - else if(!startsWith(%value, $ProjectImporter::modulePath @ "/")) - { - %targetFilename = $ProjectImporter::modulePath @ "/" @ %value; + //likely a material name, so handle it that way + %assetId = MaterialAsset::getAssetIdByMaterialName(%targetFilename); } else { - %targetFilename = %value; - } - - %targetFilename = strReplace(%targetFilename, "//", "/"); - %targetFilename = testFilenameExtensions(%targetFilename); - - if(!isFile(%targetFilename)) //if our presumed file target is bad, just bail out + if(!isFile(%targetFilename)) { error("Legacy Project Importer - file described in line could not be found/is not valid"); return %line; @@ -483,11 +535,12 @@ function processLegacyField(%line, %originalFieldName, %newFieldName) if(%foundAssets != 0) { %assetId = $ProjectImporter::assetQuery.getAsset(0); - echo("Legacy Project Importer - processing of legacy field line's value: " @ %value @ " has found a matching AssetId: " @ %assetId); + } } if(%assetId !$= "" && AssetDatabase.isDeclaredAsset(%assetId)) { + echo("Legacy Project Importer - processing of legacy field line's value: " @ %value @ " has found a matching AssetId: " @ %assetId); //if (%assetId.getStatusString() $= "Ok") %outLine = strReplace(%outLine, %value, %assetId); //else @@ -523,45 +576,7 @@ function processLegacyShapeConstructorField(%line) if(strPos(%animSourcePath, ":") != -1) return %line; - //otherwise, try and see if we've got an animation source file here - if(startsWith(%animSourcePath, "./")) - { - %targetFilename = strReplace(%animSourcePath, "./", $ProjectImporter::currentFilePath @ "/"); - } - else if(startsWith(%animSourcePath, "../")) - { - %slashPos = strposr($ProjectImporter::currentFilePath, "/"); - if(%slashPos == strlen($ProjectImporter::currentFilePath)-1) //if it's right at the end, we'll get the next one up - { - %slashPos = strposr($ProjectImporter::currentFilePath, "/", 2); - } - - %parentPath = getSubStr($ProjectImporter::currentFilePath, 0, %slashPos); - %targetFilename = strReplace(%animSourcePath, "../", %parentPath @ "/"); - } - else if(startsWith(%animSourcePath, "~")) - { - %targetFilename = strReplace(%animSourcePath, "~", $ProjectImporter::modulePath @ "/"); - if(!isFile(%targetFilename)) - { - %targetFilename = strReplace(%animSourcePath, "~", $ProjectImporter::modulePath @ "/main/"); - } - } - else if ((strpos(%animSourcePath,"/") == -1)&&(strpos(%animSourcePath,"\\") == -1)) - { - %targetFilename = $ProjectImporter::currentFilePath @ %animSourcePath; - } - else if(!startsWith(%animSourcePath, $ProjectImporter::modulePath @ "/")) - { - %targetFilename = $ProjectImporter::modulePath @ "/" @ %animSourcePath; - } - else - { - %targetFilename = %animSourcePath; - } - - %targetFilename = strReplace(%targetFilename, "//", "/"); - %targetFilename = testFilenameExtensions(%targetFilename); + %targetFilename = sanitizeFilename(%animSourcePath); if(!isFile(%targetFilename)) { @@ -828,6 +843,14 @@ function beginTerrainImport() %filePath = filePath(%file); if(%fileExt $= ".ter") { + %sanitizedFile = sanitizeFilename(%file); + if(%sanitizedFile !$= %file) + { + %file = %sanitizedFile; + %fileName = fileName(%file); + %filePath = filePath(%file); + } + $ProjectImporter::assetQuery.clear(); %assetsFound = AssetDatabase.findAssetLooseFile($ProjectImporter::assetQuery, %file); if(%assetsFound == 0)