From 77ea34d2b2ebb46c9c2eed55099e4d3bbc2b2cac Mon Sep 17 00:00:00 2001 From: Robert MacGregor Date: Sun, 10 Oct 2021 04:19:27 -0400 Subject: [PATCH 1/7] * Adjustment: Minor tweaks to allow case insensitivity on Unix systems. --- Engine/source/platformPOSIX/posixVolume.cpp | 83 ++++++++++++++++++++- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/Engine/source/platformPOSIX/posixVolume.cpp b/Engine/source/platformPOSIX/posixVolume.cpp index 271df1e18..656a69335 100644 --- a/Engine/source/platformPOSIX/posixVolume.cpp +++ b/Engine/source/platformPOSIX/posixVolume.cpp @@ -149,18 +149,95 @@ PosixFileSystem::~PosixFileSystem() { } +static void MungeCase(char* pathName, S32 pathNameSize) +{ + const int MaxPath = PATH_MAX; + + char tempBuf[MaxPath]; + dStrncpy(tempBuf, pathName, pathNameSize); + + AssertFatal(pathName[0] == '/', "PATH must be absolute"); + + struct stat filestat; + const int MaxPathEl = 200; + char *currChar = pathName; + char testPath[MaxPath]; + char pathEl[MaxPathEl]; + bool done = false; + + dStrncpy(tempBuf, "/", MaxPath); + currChar++; + + while (!done) + { + char* termChar = dStrchr(currChar, '/'); + if (termChar == NULL) + termChar = dStrchr(currChar, '\0'); + AssertFatal(termChar, "Can't find / or NULL terminator"); + + S32 pathElLen = (termChar - currChar); + dStrncpy(pathEl, currChar, pathElLen); + pathEl[pathElLen] = '\0'; + dStrncpy(testPath, tempBuf, MaxPath); + dStrcat(testPath, pathEl, MaxPath); + if (stat(testPath, &filestat) != -1) + { + dStrncpy(tempBuf, testPath, MaxPath); + } + else + { + DIR *dir = opendir(tempBuf); + struct dirent* ent; + bool foundMatch = false; + while (dir != NULL && (ent = readdir(dir)) != NULL) + { + if (dStricmp(pathEl, ent->d_name) == 0) + { + foundMatch = true; + dStrcat(tempBuf, ent->d_name, MaxPath); + break; + } + } + + if (!foundMatch) + dStrncpy(tempBuf, testPath, MaxPath); + if (dir) + closedir(dir); + } + if (*termChar == '/') + { + dStrcat(tempBuf, "/", MaxPath); + termChar++; + currChar = termChar; + } + else + done = true; + } + + dStrncpy(pathName, tempBuf, pathNameSize); +} + +// static void MungeCase(char* pathName, S32 pathNameSize) FileNodeRef PosixFileSystem::resolve(const Path& path) { String file = buildFileName(_volume,path); struct stat info; - if (stat(file.c_str(),&info) == 0) + + UTF8 testPath[1024]; + dMemcpy(testPath, file.c_str(), file.length()); + testPath[file.length()] = 0x00; + MungeCase(testPath, file.length()); + + String realFile(testPath); + + if (stat(realFile.c_str(),&info) == 0) { // Construct the appropriate object if (S_ISREG(info.st_mode)) - return new PosixFile(path,file); + return new PosixFile(path,realFile); if (S_ISDIR(info.st_mode)) - return new PosixDirectory(path,file); + return new PosixDirectory(path,realFile); } return 0; From 497a94f884badc354dc34e8c616c5ff380eadb72 Mon Sep 17 00:00:00 2001 From: Robert MacGregor Date: Mon, 11 Oct 2021 10:24:24 -0400 Subject: [PATCH 2/7] * Adjustment: Cleanups to the case insensitivity code for POSIX systems and add case insensitivty to directory dumping code. --- Engine/source/platformPOSIX/posixVolume.cpp | 93 +++---------------- .../source/platformX86UNIX/x86UNIXFileio.cpp | 20 +++- Tools/CMake/torque3d.cmake | 3 + Tools/CMake/torqueConfig.h.in | 3 + 4 files changed, 37 insertions(+), 82 deletions(-) diff --git a/Engine/source/platformPOSIX/posixVolume.cpp b/Engine/source/platformPOSIX/posixVolume.cpp index 656a69335..411559ffa 100644 --- a/Engine/source/platformPOSIX/posixVolume.cpp +++ b/Engine/source/platformPOSIX/posixVolume.cpp @@ -42,7 +42,7 @@ //#define DEBUG_SPEW - +extern void ResolvePathCaseInsensitive(char* pathName, S32 pathNameSize); namespace Torque { @@ -149,95 +149,32 @@ PosixFileSystem::~PosixFileSystem() { } -static void MungeCase(char* pathName, S32 pathNameSize) -{ - const int MaxPath = PATH_MAX; - - char tempBuf[MaxPath]; - dStrncpy(tempBuf, pathName, pathNameSize); - - AssertFatal(pathName[0] == '/', "PATH must be absolute"); - - struct stat filestat; - const int MaxPathEl = 200; - char *currChar = pathName; - char testPath[MaxPath]; - char pathEl[MaxPathEl]; - bool done = false; - - dStrncpy(tempBuf, "/", MaxPath); - currChar++; - - while (!done) - { - char* termChar = dStrchr(currChar, '/'); - if (termChar == NULL) - termChar = dStrchr(currChar, '\0'); - AssertFatal(termChar, "Can't find / or NULL terminator"); - - S32 pathElLen = (termChar - currChar); - dStrncpy(pathEl, currChar, pathElLen); - pathEl[pathElLen] = '\0'; - dStrncpy(testPath, tempBuf, MaxPath); - dStrcat(testPath, pathEl, MaxPath); - if (stat(testPath, &filestat) != -1) - { - dStrncpy(tempBuf, testPath, MaxPath); - } - else - { - DIR *dir = opendir(tempBuf); - struct dirent* ent; - bool foundMatch = false; - while (dir != NULL && (ent = readdir(dir)) != NULL) - { - if (dStricmp(pathEl, ent->d_name) == 0) - { - foundMatch = true; - dStrcat(tempBuf, ent->d_name, MaxPath); - break; - } - } - - if (!foundMatch) - dStrncpy(tempBuf, testPath, MaxPath); - if (dir) - closedir(dir); - } - if (*termChar == '/') - { - dStrcat(tempBuf, "/", MaxPath); - termChar++; - currChar = termChar; - } - else - done = true; - } - - dStrncpy(pathName, tempBuf, pathNameSize); -} - -// static void MungeCase(char* pathName, S32 pathNameSize) FileNodeRef PosixFileSystem::resolve(const Path& path) { String file = buildFileName(_volume,path); struct stat info; - UTF8 testPath[1024]; - dMemcpy(testPath, file.c_str(), file.length()); - testPath[file.length()] = 0x00; - MungeCase(testPath, file.length()); +#ifdef TORQUE_POSIX_PATH_CASE_INSENSITIVE + // Resolve the case sensitive filepath + String::SizeType fileLength = file.length(); + UTF8 caseSensitivePath[fileLength + 1]; + dMemcpy(caseSensitivePath, file.c_str(), fileLength); + caseSensitivePath[fileLength] = 0x00; + ResolvePathCaseInsensitive(caseSensitivePath, fileLength); - String realFile(testPath); + String caseSensitiveFile(caseSensitivePath); +#else + String caseSensitiveFile = file; +#endif - if (stat(realFile.c_str(),&info) == 0) + if (stat(caseSensitiveFile.c_str(),&info) == 0) { // Construct the appropriate object if (S_ISREG(info.st_mode)) - return new PosixFile(path,realFile); + return new PosixFile(path,caseSensitiveFile); if (S_ISDIR(info.st_mode)) - return new PosixDirectory(path,realFile); + return new PosixDirectory(path,caseSensitiveFile); } return 0; diff --git a/Engine/source/platformX86UNIX/x86UNIXFileio.cpp b/Engine/source/platformX86UNIX/x86UNIXFileio.cpp index eee06a84e..26a43ee7e 100644 --- a/Engine/source/platformX86UNIX/x86UNIXFileio.cpp +++ b/Engine/source/platformX86UNIX/x86UNIXFileio.cpp @@ -33,7 +33,7 @@ directory. Files are never created or modified in the game directory. For case-sensitivity, the MungePath code will test whether a given path - specified by the engine exists. If not, it will use the MungeCase function + specified by the engine exists. If not, it will use the ResolvePathCaseInsensitive function which will try to determine if an actual filesystem path matches the specified path case insensitive. If one is found, the actual path transparently (we hope) replaces the one requested by the engine. @@ -183,7 +183,7 @@ bool dPathCopy(const char *fromName, const char *toName, bool nooverwrite) // munge the case of the specified pathName. This means try to find the actual // filename in with case-insensitive matching on the specified pathName, and // store the actual found name. - static void MungeCase(char* pathName, S32 pathNameSize) + void ResolvePathCaseInsensitive(char* pathName, S32 pathNameSize) { char tempBuf[MaxPath]; dStrncpy(tempBuf, pathName, pathNameSize); @@ -296,7 +296,7 @@ bool dPathCopy(const char *fromName, const char *toName, bool nooverwrite) return; // otherwise munge the case of the path - MungeCase(dest, destSize); + ResolvePathCaseInsensitive(dest, destSize); } //----------------------------------------------------------------------------- @@ -1284,7 +1284,19 @@ bool dPathCopy(const char *fromName, const char *toName, bool nooverwrite) bool Platform::dumpDirectories(const char *path, Vector &directoryVector, S32 depth, bool noBasePath) { - bool retVal = recurseDumpDirectories(path, "", directoryVector, -1, depth, noBasePath); +#ifdef TORQUE_POSIX_PATH_CASE_INSENSITIVE + dsize_t pathLength = dStrlen(path); + char caseSensitivePath[pathLength + 1]; + + // Load path into temporary buffer + dMemcpy(caseSensitivePath, path, pathLength); + caseSensitivePath[pathLength] = 0x00; + ResolvePathCaseInsensitive(caseSensitivePath, pathLength); +#else + const char* caseSensitivePath = path; +#endif + + bool retVal = recurseDumpDirectories(caseSensitivePath, "", directoryVector, -1, depth, noBasePath); clearExcludedDirectories(); return retVal; } diff --git a/Tools/CMake/torque3d.cmake b/Tools/CMake/torque3d.cmake index 987a094b8..460d32e51 100644 --- a/Tools/CMake/torque3d.cmake +++ b/Tools/CMake/torque3d.cmake @@ -163,6 +163,9 @@ endif() option(TORQUE_MULTITHREAD "Multi Threading" ON) mark_as_advanced(TORQUE_MULTITHREAD) +option(TORQUE_POSIX_PATH_CASE_INSENSITIVE ON) +mark_as_advanced(TORQUE_POSIX_PATH_CASE_INSENSITIVE) + option(TORQUE_DISABLE_MEMORY_MANAGER "Disable memory manager" ON) mark_as_advanced(TORQUE_DISABLE_MEMORY_MANAGER) diff --git a/Tools/CMake/torqueConfig.h.in b/Tools/CMake/torqueConfig.h.in index 13ce58b6c..6465012c4 100644 --- a/Tools/CMake/torqueConfig.h.in +++ b/Tools/CMake/torqueConfig.h.in @@ -44,6 +44,9 @@ /// Define me if you want to enable Arcane FX support. #cmakedefine TORQUE_AFX_ENABLED +/// Define me if you want path case insensitivity support on POSIX systems. Does nothing on Windows. +#cmakedefine TORQUE_POSIX_PATH_CASE_INSENSITIVE + /// Define me if you want to enable multithreading support. #cmakedefine TORQUE_MULTITHREAD From 444c9dcf41fc3c9fcd358abb29da776ca659dd01 Mon Sep 17 00:00:00 2001 From: Robert MacGregor Date: Mon, 11 Oct 2021 10:45:02 -0400 Subject: [PATCH 3/7] * BugFix: Correct non-constant array allocations in the POSIX case insensitivity code. --- Engine/source/platformPOSIX/posixVolume.cpp | 14 +++++++++----- Engine/source/platformX86UNIX/x86UNIXFileio.cpp | 6 +++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Engine/source/platformPOSIX/posixVolume.cpp b/Engine/source/platformPOSIX/posixVolume.cpp index 411559ffa..b8b070217 100644 --- a/Engine/source/platformPOSIX/posixVolume.cpp +++ b/Engine/source/platformPOSIX/posixVolume.cpp @@ -157,7 +157,7 @@ FileNodeRef PosixFileSystem::resolve(const Path& path) #ifdef TORQUE_POSIX_PATH_CASE_INSENSITIVE // Resolve the case sensitive filepath String::SizeType fileLength = file.length(); - UTF8 caseSensitivePath[fileLength + 1]; + UTF8* caseSensitivePath = new UTF8[fileLength + 1]; dMemcpy(caseSensitivePath, file.c_str(), fileLength); caseSensitivePath[fileLength] = 0x00; ResolvePathCaseInsensitive(caseSensitivePath, fileLength); @@ -167,17 +167,21 @@ FileNodeRef PosixFileSystem::resolve(const Path& path) String caseSensitiveFile = file; #endif + FileNodeRef result = 0; if (stat(caseSensitiveFile.c_str(),&info) == 0) { // Construct the appropriate object if (S_ISREG(info.st_mode)) - return new PosixFile(path,caseSensitiveFile); + result = new PosixFile(path,caseSensitiveFile); if (S_ISDIR(info.st_mode)) - return new PosixDirectory(path,caseSensitiveFile); + result = new PosixDirectory(path,caseSensitiveFile); } - - return 0; + +#ifdef TORQUE_POSIX_PATH_CASE_INSENSITIVE + delete[] caseSensitivePath; +#endif + return result; } FileNodeRef PosixFileSystem::create(const Path& path, FileNode::Mode mode) diff --git a/Engine/source/platformX86UNIX/x86UNIXFileio.cpp b/Engine/source/platformX86UNIX/x86UNIXFileio.cpp index 26a43ee7e..9669638ae 100644 --- a/Engine/source/platformX86UNIX/x86UNIXFileio.cpp +++ b/Engine/source/platformX86UNIX/x86UNIXFileio.cpp @@ -1286,7 +1286,7 @@ bool dPathCopy(const char *fromName, const char *toName, bool nooverwrite) { #ifdef TORQUE_POSIX_PATH_CASE_INSENSITIVE dsize_t pathLength = dStrlen(path); - char caseSensitivePath[pathLength + 1]; + char* caseSensitivePath = new char[pathLength + 1]; // Load path into temporary buffer dMemcpy(caseSensitivePath, path, pathLength); @@ -1298,6 +1298,10 @@ bool dPathCopy(const char *fromName, const char *toName, bool nooverwrite) bool retVal = recurseDumpDirectories(caseSensitivePath, "", directoryVector, -1, depth, noBasePath); clearExcludedDirectories(); + +#ifdef TORQUE_POSIX_PATH_CASE_INSENSITIVE + delete[] caseSensitivePath; +#endif return retVal; } From 8d0128698a9476f1fd8f79587bf42b63aa00d052 Mon Sep 17 00:00:00 2001 From: Robert MacGregor Date: Mon, 11 Oct 2021 19:02:58 -0400 Subject: [PATCH 4/7] * [ZIPVolume] BugFix: Add the capability of resolving ZIP prefix paths with case insensitivity and a CMake option to control this behavior. --- Engine/source/core/util/zip/zipVolume.cpp | 9 ++++++++- Tools/CMake/torque3d.cmake | 3 +++ Tools/CMake/torqueConfig.h.in | 3 +++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Engine/source/core/util/zip/zipVolume.cpp b/Engine/source/core/util/zip/zipVolume.cpp index 5dcb8dee8..a3a0ac7c9 100644 --- a/Engine/source/core/util/zip/zipVolume.cpp +++ b/Engine/source/core/util/zip/zipVolume.cpp @@ -407,8 +407,15 @@ FileNodeRef ZipFileSystem::resolve(const Path& path) if(mZipNameIsDir) { // Remove the fake root from the name so things can be found +#ifdef TORQUE_ZIP_PATH_CASE_INSENSITIVE + String lowerFakeRoot = String::ToLower(mFakeRoot); + String lowerName = String::ToLower(name); + if(lowerName.find(lowerFakeRoot) == 0) + name = name.substr(mFakeRoot.length()); +#else if(name.find(mFakeRoot) == 0) - name = name.substr(mFakeRoot.length()); + name = name.substr(mFakeRoot.length()); +#endif #ifdef TORQUE_DISABLE_FIND_ROOT_WITHIN_ZIP else diff --git a/Tools/CMake/torque3d.cmake b/Tools/CMake/torque3d.cmake index 460d32e51..cd09fabd6 100644 --- a/Tools/CMake/torque3d.cmake +++ b/Tools/CMake/torque3d.cmake @@ -166,6 +166,9 @@ mark_as_advanced(TORQUE_MULTITHREAD) option(TORQUE_POSIX_PATH_CASE_INSENSITIVE ON) mark_as_advanced(TORQUE_POSIX_PATH_CASE_INSENSITIVE) +option(TORQUE_ZIP_PATH_CASE_INSENSITIVE ON) +mark_as_advanced(TORQUE_ZIP_PATH_CASE_INSENSITIVE) + option(TORQUE_DISABLE_MEMORY_MANAGER "Disable memory manager" ON) mark_as_advanced(TORQUE_DISABLE_MEMORY_MANAGER) diff --git a/Tools/CMake/torqueConfig.h.in b/Tools/CMake/torqueConfig.h.in index 6465012c4..9e045b1d5 100644 --- a/Tools/CMake/torqueConfig.h.in +++ b/Tools/CMake/torqueConfig.h.in @@ -47,6 +47,9 @@ /// Define me if you want path case insensitivity support on POSIX systems. Does nothing on Windows. #cmakedefine TORQUE_POSIX_PATH_CASE_INSENSITIVE +/// Define me if you want path case insensitivity support in ZIP files. +#cmakedefine TORQUE_ZIP_PATH_CASE_INSENSITIVE + /// Define me if you want to enable multithreading support. #cmakedefine TORQUE_MULTITHREAD From c376bc3f9cf12b00aaf7aeb437e426b30ef00e6d Mon Sep 17 00:00:00 2001 From: Robert MacGregor Date: Mon, 11 Oct 2021 19:31:37 -0400 Subject: [PATCH 5/7] * [ZIP] BugFix: Correct another codepath that may fail due to case sensitive string searches. --- Engine/source/core/util/zip/zipVolume.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Engine/source/core/util/zip/zipVolume.cpp b/Engine/source/core/util/zip/zipVolume.cpp index a3a0ac7c9..d56c9471e 100644 --- a/Engine/source/core/util/zip/zipVolume.cpp +++ b/Engine/source/core/util/zip/zipVolume.cpp @@ -487,8 +487,15 @@ FileNodeRef ZipFileSystem::resolveLoose(const Path& path) if(mZipNameIsDir) { // Remove the fake root from the name so things can be found +#ifdef TORQUE_ZIP_PATH_CASE_INSENSITIVE + String lowerFakeRoot = String::ToLower(mFakeRoot); + String lowerName = String::ToLower(name); + if(lowerName.find(lowerFakeRoot) == 0) + name = name.substr(mFakeRoot.length()); +#else if(name.find(mFakeRoot) == 0) - name = name.substr(mFakeRoot.length()); + name = name.substr(mFakeRoot.length()); +#endif #ifdef TORQUE_DISABLE_FIND_ROOT_WITHIN_ZIP else From 66cfd34b7127d37a21665bba41ba5ecabfc9cc12 Mon Sep 17 00:00:00 2001 From: Robert MacGregor Date: Tue, 12 Oct 2021 13:09:12 -0400 Subject: [PATCH 6/7] * [CMake] BugFix: Correctly set the description and default values of TORQUE_POSIX_PATH_CASE_INSENSITIVE and TORQUE_ZIP_PATH_CASE_INSENSITIVE. --- Tools/CMake/torque3d.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/CMake/torque3d.cmake b/Tools/CMake/torque3d.cmake index cd09fabd6..97b57260e 100644 --- a/Tools/CMake/torque3d.cmake +++ b/Tools/CMake/torque3d.cmake @@ -163,10 +163,10 @@ endif() option(TORQUE_MULTITHREAD "Multi Threading" ON) mark_as_advanced(TORQUE_MULTITHREAD) -option(TORQUE_POSIX_PATH_CASE_INSENSITIVE ON) +option(TORQUE_POSIX_PATH_CASE_INSENSITIVE "POSIX Pathing Case Insensitivity" ON) mark_as_advanced(TORQUE_POSIX_PATH_CASE_INSENSITIVE) -option(TORQUE_ZIP_PATH_CASE_INSENSITIVE ON) +option(TORQUE_ZIP_PATH_CASE_INSENSITIVE "ZIP Pathing Case Insensitivity" ON) mark_as_advanced(TORQUE_ZIP_PATH_CASE_INSENSITIVE) option(TORQUE_DISABLE_MEMORY_MANAGER "Disable memory manager" ON) From bd9b3fd55ba27c596653e30cb9b894620c95b8ff Mon Sep 17 00:00:00 2001 From: Robert MacGregor Date: Wed, 13 Oct 2021 11:40:54 -0400 Subject: [PATCH 7/7] * [Win32] Adjustment: Only trigger the Linux compat warning when TORQUE_POSIX_PATH_CASE_INSENSITIVE is disabled. --- Engine/source/platformWin32/winVolume.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Engine/source/platformWin32/winVolume.cpp b/Engine/source/platformWin32/winVolume.cpp index af0bf96bc..5d168ac95 100644 --- a/Engine/source/platformWin32/winVolume.cpp +++ b/Engine/source/platformWin32/winVolume.cpp @@ -238,10 +238,12 @@ Win32FileSystem::~Win32FileSystem() void Win32FileSystem::verifyCompatibility(const Path& _path, WIN32_FIND_DATAW _info) { +#ifndef TORQUE_POSIX_PATH_CASE_INSENSITIVE if (_path.getFullFileName().isNotEmpty() && _path.getFullFileName().compare(String(_info.cFileName)) != 0) { Con::warnf("Linux Compatibility Warning: %s != %s", String(_info.cFileName).c_str(), _path.getFullFileName().c_str()); } +#endif } FileNodeRef Win32FileSystem::resolve(const Path& path)