TorqueGameEngines/Torque3D
1
0
Fork 0
mirror of https://github.com/TorqueGameEngines/Torque3D.git synced 2026-07-09 21:54:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Engine directory for ticket #1

Browse source
This commit is contained in:
DavidWyand-GG 2012-09-19 11:15:01 -04:00
parent 352279af7a
commit 7dbfe6994d
3795 changed files with 1363358 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

163
Engine/source/core/strings/findMatch.cpp Normal file
View file

@ -0,0 +1,163 @@
//-----------------------------------------------------------------------------
// Copyright (c) 2012 GarageGames, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
#include "core/strings/findMatch.h"
#include "core/strings/stringFunctions.h"
//--------------------------------------------------------------------------------
// NAME
// FindMatch::FindMatch( const char *_expression, S32 maxNumMatches )
//
// DESCRIPTION
// Class to match regular expressions (file names)
// only works with '*','?', and 'chars'
//
// ARGUMENTS
// _expression - The regular expression you intend to match (*.??abc.bmp)
// _maxMatches - The maximum number of strings you wish to match.
//
// RETURNS
//
// NOTES
//
//--------------------------------------------------------------------------------
FindMatch::FindMatch( U32 _maxMatches )
{
VECTOR_SET_ASSOCIATION(matchList);
expression = NULL;
maxMatches = _maxMatches;
matchList.reserve( maxMatches );
}
FindMatch::FindMatch( char *_expression, U32 _maxMatches )
{
VECTOR_SET_ASSOCIATION(matchList);
expression = NULL;
setExpression( _expression );
maxMatches = _maxMatches;
matchList.reserve( maxMatches );
}
FindMatch::~FindMatch()
{
delete [] expression;
matchList.clear();
}
void FindMatch::setExpression( const char *_expression )
{
delete [] expression;
expression = new char[dStrlen(_expression) + 1];
dStrcpy(expression, _expression);
dStrupr(expression);
}
bool FindMatch::findMatch( const char *str, bool caseSensitive )
{
if ( isFull() )
return false;
char nstr[512];
dStrcpy( nstr,str );
dStrupr(nstr);
if ( isMatch( expression, nstr, caseSensitive ) )
{
matchList.push_back( (char*)str );
return true;
}
return false;
}
inline bool IsCharMatch( char e, char s, bool caseSensitive )
{
return ( ( e == '?' ) || ( caseSensitive && e == s ) || ( dToupper(e) == dToupper(s) ) );
}
bool FindMatch::isMatch( const char *exp, const char *str, bool caseSensitive )
{
while ( *str && ( *exp != '*' ) )
{
if ( !IsCharMatch( *exp++, *str++, caseSensitive ) )
return false;
}
const char* cp = NULL;
const char* mp = NULL;
while ( *str )
{
if ( *exp == '*' )
{
if ( !*++exp )
return true;
mp = exp;
cp = str+1;
}
else if ( IsCharMatch( *exp, *str, caseSensitive ) )
{
exp++;
str++;
}
else
{
exp = mp;
str = cp++;
}
}
while ( *exp == '*' )
exp++;
return !*exp;
}
bool FindMatch::isMatchMultipleExprs( const char *exps, const char *str, bool caseSensitive )
{
char *tok = 0;
int len = dStrlen(exps);
char *e = new char[len+1];
dStrcpy(e,exps);
// [tom, 12/18/2006] This no longer supports space separated expressions as
// they don't work when the paths have spaces in.
// search for each expression. return true soon as we see one.
for( tok = dStrtok(e,"\t"); tok != NULL; tok = dStrtok(NULL,"\t"))
{
if( isMatch( tok, str, caseSensitive) )
{
delete []e;
return true;
}
}
delete []e;
return false;
}

63
Engine/source/core/strings/findMatch.h Normal file
View file

@ -0,0 +1,63 @@
//-----------------------------------------------------------------------------
// Copyright (c) 2012 GarageGames, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
#ifndef _FINDMATCH_H_
#define _FINDMATCH_H_
#ifndef _TVECTOR_H_
#include "core/util/tVector.h"
#endif
class FindMatch
{
char* expression;
U32 maxMatches;
public:
static bool isMatch( const char *exp, const char *string, bool caseSensitive = false );
static bool isMatchMultipleExprs( const char *exps, const char *str, bool caseSensitive );
Vector<char *> matchList;
FindMatch( U32 _maxMatches = 256 );
FindMatch( char *_expression, U32 _maxMatches = 256 );
~FindMatch();
bool findMatch(const char *string, bool caseSensitive = false);
void setExpression( const char *_expression );
S32 numMatches() const
{
return(matchList.size());
}
bool isFull() const
{
return (matchList.size() >= S32(maxMatches));
}
void clear()
{
matchList.clear();
}
};
#endif // _FINDMATCH_H_

534
Engine/source/core/strings/stringFunctions.cpp Normal file
View file

@ -0,0 +1,534 @@
//-----------------------------------------------------------------------------
// Copyright (c) 2012 GarageGames, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
#include <stdarg.h>
#include <stdio.h>
#include "core/strings/stringFunctions.h"
#include "platform/platform.h"
#if defined(TORQUE_OS_WIN32) || defined(TORQUE_OS_XBOX) || defined(TORQUE_OS_XENON)
// This standard function is not defined when compiling with VC7...
#define vsnprintf _vsnprintf
#endif
//-----------------------------------------------------------------------------
// Original code from: http://sourcefrog.net/projects/natsort/
// Somewhat altered here.
//TODO: proper UTF8 support; currently only working for single-byte characters
/* -*- mode: c; c-file-style: "k&r" -*-
strnatcmp.c -- Perform 'natural order' comparisons of strings in C.
Copyright (C) 2000, 2004 by Martin Pool <mbp sourcefrog net>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
*/
/* partial change history:
*
* 2004-10-10 mbp: Lift out character type dependencies into macros.
*
* Eric Sosman pointed out that ctype functions take a parameter whose
* value must be that of an unsigned int, even on platforms that have
* negative chars in their default char type.
*/
typedef char nat_char;
/* These are defined as macros to make it easier to adapt this code to
* different characters types or comparison functions. */
static inline int
nat_isdigit( nat_char a )
{
return dIsdigit( a );
}
static inline int
nat_isspace( nat_char a )
{
return dIsspace( a );
}
static inline nat_char
nat_toupper( nat_char a )
{
return dToupper( a );
}
static int
compare_right(const nat_char* a, const nat_char* b)
{
int bias = 0;
/* The longest run of digits wins. That aside, the greatest
value wins, but we can't know that it will until we've scanned
both numbers to know that they have the same magnitude, so we
remember it in BIAS. */
for (;; a++, b++) {
if (!nat_isdigit(*a) && !nat_isdigit(*b))
return bias;
else if (!nat_isdigit(*a))
return -1;
else if (!nat_isdigit(*b))
return +1;
else if (*a < *b) {
if (!bias)
bias = -1;
} else if (*a > *b) {
if (!bias)
bias = +1;
} else if (!*a && !*b)
return bias;
}
return 0;
}
static int
compare_left(const nat_char* a, const nat_char* b)
{
/* Compare two left-aligned numbers: the first to have a
different value wins. */
for (;; a++, b++) {
if (!nat_isdigit(*a) && !nat_isdigit(*b))
return 0;
else if (!nat_isdigit(*a))
return -1;
else if (!nat_isdigit(*b))
return +1;
else if (*a < *b)
return -1;
else if (*a > *b)
return +1;
}
return 0;
}
static int strnatcmp0(const nat_char* a, const nat_char* b, int fold_case)
{
int ai, bi;
nat_char ca, cb;
int fractional, result;
ai = bi = 0;
while (1) {
ca = a[ai]; cb = b[bi];
/* skip over leading spaces or zeros */
while (nat_isspace(ca))
ca = a[++ai];
while (nat_isspace(cb))
cb = b[++bi];
/* process run of digits */
if (nat_isdigit(ca) && nat_isdigit(cb)) {
fractional = (ca == '0' || cb == '0');
if (fractional) {
if ((result = compare_left(a+ai, b+bi)) != 0)
return result;
} else {
if ((result = compare_right(a+ai, b+bi)) != 0)
return result;
}
}
if (!ca && !cb) {
/* The strings compare the same. Perhaps the caller
will want to call strcmp to break the tie. */
return 0;
}
if (fold_case) {
ca = nat_toupper(ca);
cb = nat_toupper(cb);
}
if (ca < cb)
return -1;
else if (ca > cb)
return +1;
++ai; ++bi;
}
}
int dStrnatcmp(const nat_char* a, const nat_char* b) {
return strnatcmp0(a, b, 0);
}
/* Compare, recognizing numeric string and ignoring case. */
int dStrnatcasecmp(const nat_char* a, const nat_char* b) {
return strnatcmp0(a, b, 1);
}
//------------------------------------------------------------------------------
// non-standard string functions
char *dStrdup_r(const char *src, const char *fileName, dsize_t lineNumber)
{
char *buffer = (char *) dMalloc_r(dStrlen(src) + 1, fileName, lineNumber);
dStrcpy(buffer, src);
return buffer;
}
char* dStrichr( char* str, char ch )
{
AssertFatal( str != NULL, "dStrichr - NULL string" );
if( !ch )
return dStrchr( str, ch );
char c = dToupper( ch );
while( *str )
{
if( dToupper( *str ) == c )
return str;
++ str;
}
return NULL;
}
const char* dStrichr( const char* str, char ch )
{
AssertFatal( str != NULL, "dStrichr - NULL string" );
if( !ch )
return dStrchr( str, ch );
char c = dToupper( ch );
while( *str )
{
if( dToupper( *str ) == c )
return str;
++ str;
}
return NULL;
}
// concatenates a list of src's onto the end of dst
// the list of src's MUST be terminated by a NULL parameter
// dStrcatl(dst, sizeof(dst), src1, src2, NULL);
char* dStrcatl(char *dst, dsize_t dstSize, ...)
{
const char* src = NULL;
char *p = dst;
AssertFatal(dstSize > 0, "dStrcatl: destination size is set zero");
dstSize--; // leave room for string termination
// find end of dst
while (dstSize && *p++)
dstSize--;
va_list args;
va_start(args, dstSize);
// concatenate each src to end of dst
while ( (src = va_arg(args, const char*)) != NULL )
{
while( dstSize && *src )
{
*p++ = *src++;
dstSize--;
}
}
va_end(args);
// make sure the string is terminated
*p = 0;
return dst;
}
// copy a list of src's into dst
// the list of src's MUST be terminated by a NULL parameter
// dStrccpyl(dst, sizeof(dst), src1, src2, NULL);
char* dStrcpyl(char *dst, dsize_t dstSize, ...)
{
const char* src = NULL;
char *p = dst;
AssertFatal(dstSize > 0, "dStrcpyl: destination size is set zero");
dstSize--; // leave room for string termination
va_list args;
va_start(args, dstSize);
// concatenate each src to end of dst
while ( (src = va_arg(args, const char*)) != NULL )
{
while( dstSize && *src )
{
*p++ = *src++;
dstSize--;
}
}
va_end(args);
// make sure the string is terminated
*p = 0;
return dst;
}
int dStrcmp( const UTF16 *str1, const UTF16 *str2)
{
#if defined(TORQUE_OS_WIN32) || defined(TORQUE_OS_XBOX) || defined(TORQUE_OS_XENON)
return wcscmp( reinterpret_cast<const wchar_t *>( str1 ), reinterpret_cast<const wchar_t *>( str2 ) );
#else
int ret;
const UTF16 *a, *b;
a = str1;
b = str2;
while( ((ret = *a - *b) == 0) && *a && *b )
a++, b++;
return ret;
#endif
}
char* dStrupr(char *str)
{
#if defined(TORQUE_OS_WIN32) || defined(TORQUE_OS_XBOX) || defined(TORQUE_OS_XENON)
return _strupr(str);
#else
if (str == NULL)
return(NULL);
char* saveStr = str;
while (*str)
{
*str = toupper(*str);
str++;
}
return saveStr;
#endif
}
char* dStrlwr(char *str)
{
#if defined(TORQUE_OS_WIN32) || defined(TORQUE_OS_XBOX) || defined(TORQUE_OS_XENON)
return _strlwr(str);
#else
if (str == NULL)
return(NULL);
char* saveStr = str;
while (*str)
{
*str = tolower(*str);
str++;
}
return saveStr;
#endif
}
//------------------------------------------------------------------------------
// standard I/O functions
void dPrintf(const char *format, ...)
{
va_list args;
va_start(args, format);
vprintf(format, args);
}
S32 dVprintf(const char *format, void *arglist)
{
return vprintf(format, (char*)arglist);
}
S32 dSprintf(char *buffer, U32 bufferSize, const char *format, ...)
{
va_list args;
va_start(args, format);
S32 len = vsnprintf(buffer, bufferSize, format, args);
AssertWarn( len < bufferSize, "Buffer too small in call to dSprintf!" );
return (len);
}
S32 dVsprintf(char *buffer, U32 bufferSize, const char *format, void *arglist)
{
S32 len = vsnprintf(buffer, bufferSize, format, (char*)arglist);
AssertWarn( len < bufferSize, "Buffer too small in call to dVsprintf!" );
return (len);
}
S32 dSscanf(const char *buffer, const char *format, ...)
{
#if defined(TORQUE_OS_WIN32) || defined(TORQUE_OS_XBOX) || defined(TORQUE_OS_XENON)
va_list args;
va_start(args, format);
// Boy is this lame. We have to scan through the format string, and find out how many
// arguments there are. We'll store them off as void*, and pass them to the sscanf
// function through specialized calls. We're going to have to put a cap on the number of args that
// can be passed, 8 for the moment. Sigh.
static void* sVarArgs[20];
U32 numArgs = 0;
for (const char* search = format; *search != '\0'; search++) {
if (search[0] == '%' && search[1] != '%')
numArgs++;
}
AssertFatal(numArgs <= 20, "Error, too many arguments to lame implementation of dSscanf. Fix implmentation");
// Ok, we have the number of arguments...
for (U32 i = 0; i < numArgs; i++)
sVarArgs[i] = va_arg(args, void*);
va_end(args);
switch (numArgs) {
case 0: return 0;
case 1: return sscanf(buffer, format, sVarArgs[0]);
case 2: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1]);
case 3: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2]);
case 4: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3]);
case 5: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4]);
case 6: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5]);
case 7: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6]);
case 8: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7]);
case 9: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8]);
case 10: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9]);
case 11: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10]);
case 12: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11]);
case 13: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11], sVarArgs[12]);
case 14: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11], sVarArgs[12], sVarArgs[13]);
case 15: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11], sVarArgs[12], sVarArgs[13], sVarArgs[14]);
case 16: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11], sVarArgs[12], sVarArgs[13], sVarArgs[14], sVarArgs[15]);
case 17: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11], sVarArgs[12], sVarArgs[13], sVarArgs[14], sVarArgs[15], sVarArgs[16]);
case 18: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11], sVarArgs[12], sVarArgs[13], sVarArgs[14], sVarArgs[15], sVarArgs[16], sVarArgs[17]);
case 19: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11], sVarArgs[12], sVarArgs[13], sVarArgs[14], sVarArgs[15], sVarArgs[16], sVarArgs[17], sVarArgs[18]);
case 20: return sscanf(buffer, format, sVarArgs[0], sVarArgs[1], sVarArgs[2], sVarArgs[3], sVarArgs[4], sVarArgs[5], sVarArgs[6], sVarArgs[7], sVarArgs[8], sVarArgs[9], sVarArgs[10], sVarArgs[11], sVarArgs[12], sVarArgs[13], sVarArgs[14], sVarArgs[15], sVarArgs[16], sVarArgs[17], sVarArgs[18], sVarArgs[19]);
}
return 0;
#else
va_list args;
va_start(args, format);
return vsscanf(buffer, format, args);
#endif
}
/// Safe form of dStrcmp: checks both strings for NULL before comparing
bool dStrEqual(const char* str1, const char* str2)
{
if (!str1 || !str2)
return false;
else
return (dStrcmp(str1, str2) == 0);
}
/// Check if one string starts with another
bool dStrStartsWith(const char* str1, const char* str2)
{
return !dStrnicmp(str1, str2, dStrlen(str2));
}
/// Check if one string ends with another
bool dStrEndsWith(const char* str1, const char* str2)
{
const char *p = str1 + dStrlen(str1) - dStrlen(str2);
return ((p >= str1) && !dStricmp(p, str2));
}
/// Strip the path from the input filename
char* dStripPath(const char* filename)
{
const char* itr = filename + dStrlen(filename);
while(--itr != filename) {
if (*itr == '/' || *itr == '\\') {
itr++;
break;
}
}
return dStrdup(itr);
}
char* dStristr( char* str1, const char* str2 )
{
if( !str1 || !str2 )
return NULL;
// Slow but at least we have it.
U32 str2len = strlen( str2 );
while( *str1 )
{
if( strncasecmp( str1, str2, str2len ) == 0 )
return str1;
++ str1;
}
return NULL;
}
const char* dStristr( const char* str1, const char* str2 )
{
return dStristr( const_cast< char* >( str1 ), str2 );
}

227
Engine/source/core/strings/stringFunctions.h Normal file
View file

@ -0,0 +1,227 @@
//-----------------------------------------------------------------------------
// Copyright (c) 2012 GarageGames, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
#ifndef _STRINGFUNCTIONS_H_
#define _STRINGFUNCTIONS_H_
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#ifndef _TORQUE_TYPES_H_
#include "platform/types.h"
#endif
#if defined(TORQUE_OS_WIN32) || defined(TORQUE_OS_XBOX) || defined(TORQUE_OS_XENON)
// These standard functions are not defined on Win32 and other Microsoft platforms...
#define strcasecmp _stricmp
#define strncasecmp _strnicmp
#define strtof (float)strtod
#endif
//------------------------------------------------------------------------------
// standard string functions [defined in platformString.cpp]
inline char *dStrcat(char *dst, const char *src)
{
return strcat(dst,src);
}
inline char *dStrncat(char *dst, const char *src, dsize_t len)
{
return strncat(dst,src,len);
}
inline int dStrcmp(const char *str1, const char *str2)
{
return strcmp(str1, str2);
}
inline int dStrncmp(const char *str1, const char *str2, dsize_t len)
{
return strncmp(str1, str2, len);
}
inline int dStricmp(const char *str1, const char *str2)
{
return strcasecmp( str1, str2 );
}
inline int dStrnicmp(const char *str1, const char *str2, dsize_t len)
{
return strncasecmp( str1, str2, len );
}
inline char *dStrcpy(char *dst, const char *src)
{
return strcpy(dst,src);
}
inline char *dStrncpy(char *dst, const char *src, dsize_t len)
{
return strncpy(dst,src,len);
}
inline dsize_t dStrlen(const char *str)
{
return strlen(str);
}
inline char *dStrchr(char *str, int c)
{
return strchr(str,c);
}
inline const char *dStrchr(const char *str, int c)
{
return strchr(str,c);
}
inline char *dStrrchr(char *str, int c)
{
return strrchr(str,c);
}
inline const char *dStrrchr(const char *str, int c)
{
return strrchr(str,c);
}
inline dsize_t dStrspn(const char *str, const char *set)
{
return strspn(str, set);
}
inline dsize_t dStrcspn(const char *str, const char *set)
{
return strcspn(str, set);
}
inline char *dStrstr(const char *str1, const char *str2)
{
return strstr((char *)str1,str2);
}
const char* dStristr( const char* str1, const char* str2 );
char* dStristr( char* str1, const char* str2 );
inline char *dStrtok(char *str, const char *sep)
{
return strtok(str, sep);
}
inline S32 dAtoi(const char *str)
{
return strtol(str, NULL, 10);
}
inline U32 dAtoui(const char *str, U32 base = 10)
{
return strtoul(str, NULL, base);
}
inline F32 dAtof(const char *str)
{
return strtof(str, NULL);
}
inline char dToupper(const char c)
{
return toupper( c );
}
inline char dTolower(const char c)
{
return tolower( c );
}
inline bool dIsalnum(const char c)
{
return isalnum(c);
}
inline bool dIsalpha(const char c)
{
return isalpha(c);
}
inline bool dIsspace(const char c)
{
return isspace(c);
}
inline bool dIsdigit(const char c)
{
return isdigit(c);
}
inline bool dIsquote(const char c)
{
return ( c == '\"' );
}
//------------------------------------------------------------------------------
// non-standard string functions [defined in stringFunctions.cpp]
#define dStrdup(x) dStrdup_r(x, __FILE__, __LINE__)
extern char *dStrdup_r(const char *src, const char*, dsize_t);
extern char *dStrcpyl(char *dst, dsize_t dstSize, ...);
extern char *dStrcatl(char *dst, dsize_t dstSize, ...);
extern char *dStrupr(char *str);
extern char *dStrlwr(char *str);
extern char* dStrichr( char* str, char ch );
extern const char* dStrichr( const char* str, char ch );
extern int dStrcmp(const UTF16 *str1, const UTF16 *str2);
extern int dStrnatcmp( const char* str1, const char* str2 );
extern int dStrnatcasecmp( const char* str1, const char* str2 );
inline bool dAtob(const char *str)
{
return !dStricmp(str, "true") || dAtof(str);
}
bool dStrEqual(const char* str1, const char* str2);
bool dStrStartsWith(const char* str1, const char* str2);
bool dStrEndsWith(const char* str1, const char* str2);
char* dStripPath(const char* filename);
//------------------------------------------------------------------------------
// standard I/O functions [defined in platformString.cpp]
extern void dPrintf(const char *format, ...);
extern int dVprintf(const char *format, void *arglist);
extern int dSprintf(char *buffer, U32 bufferSize, const char *format, ...);
extern int dVsprintf(char *buffer, U32 bufferSize, const char *format, void *arglist);
extern int dSscanf(const char *buffer, const char *format, ...);
#endif

217
Engine/source/core/strings/stringUnit.cpp Normal file
View file

@ -0,0 +1,217 @@
//-----------------------------------------------------------------------------
// Copyright (c) 2012 GarageGames, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
#include "core/strings/stringFunctions.h"
#include "core/strings/stringUnit.h"
#include "console/console.h"
namespace StringUnit
{
static char _returnBuffer[ 2048 ];
const char *getUnit(const char *string, U32 index, const char *set, char* buffer, U32 bufferSize)
{
if( !buffer )
{
buffer = _returnBuffer;
bufferSize = sizeof( _returnBuffer );
}
AssertFatal( bufferSize, "StringUnit::getUnit - bufferSize cannot be zero!" );
if( !bufferSize )
return "";
buffer[0] = 0;
U32 sz;
while(index--)
{
if(!*string)
return buffer;
sz = dStrcspn(string, set);
if (string[sz] == 0)
return buffer;
string += (sz + 1);
}
sz = dStrcspn(string, set);
if (sz == 0)
return buffer;
AssertWarn( sz + 1 < bufferSize, "Size of returned string too large for return buffer" );
sz = getMin( sz, bufferSize - 1 );
dStrncpy(buffer, string, sz);
buffer[sz] = '\0';
return buffer;
}
const char *getUnits(const char *string, S32 startIndex, S32 endIndex, const char *set)
{
if( startIndex > endIndex )
return "";
S32 sz;
S32 index = startIndex;
while(index--)
{
if(!*string)
return "";
sz = dStrcspn(string, set);
if (string[sz] == 0)
return "";
string += (sz + 1);
}
const char *startString = string;
for( U32 i = startIndex; i <= endIndex && *string; i ++ )
{
sz = dStrcspn(string, set);
string += sz;
if( i < endIndex )
string ++;
}
S32 totalSize = ( S32 )( string - startString );
AssertWarn( totalSize + 1 < sizeof( _returnBuffer ), "Size of returned string too large for return buffer" );
totalSize = getMin( totalSize, S32( sizeof( _returnBuffer ) - 1 ) );
if( totalSize > 0 )
{
char *ret = &_returnBuffer[0];
dStrncpy( ret, startString, totalSize );
ret[ totalSize ] = '\0';
return ret;
}
return "";
}
U32 getUnitCount(const char *string, const char *set)
{
U32 count = 0;
U8 last = 0;
while(*string)
{
last = *string++;
for(U32 i =0; set[i]; i++)
{
if(last == set[i])
{
count++;
last = 0;
break;
}
}
}
if(last)
count++;
return count;
}
const char* setUnit(const char *string, U32 index, const char *replace, const char *set)
{
U32 sz;
const char *start = string;
AssertFatal( dStrlen(string) + dStrlen(replace) + 1 < sizeof( _returnBuffer ), "Size of returned string too large for return buffer" );
char *ret = &_returnBuffer[0];
ret[0] = '\0';
U32 padCount = 0;
while(index--)
{
sz = dStrcspn(string, set);
if(string[sz] == 0)
{
string += sz;
padCount = index + 1;
break;
}
else
string += (sz + 1);
}
// copy first chunk
sz = string-start;
dStrncpy(ret, start, sz);
for(U32 i = 0; i < padCount; i++)
ret[sz++] = set[0];
// replace this unit
ret[sz] = '\0';
dStrcat(ret, replace);
// copy remaining chunks
sz = dStrcspn(string, set); // skip chunk we're replacing
if(!sz && !string[sz])
return ret;
string += sz;
dStrcat(ret, string);
return ret;
}
const char* removeUnit(const char *string, U32 index, const char *set)
{
U32 sz;
const char *start = string;
AssertFatal( dStrlen(string) + 1 < sizeof( _returnBuffer ), "Size of returned string too large for return buffer" );
char *ret = &_returnBuffer[0];
ret[0] = '\0';
while(index--)
{
sz = dStrcspn(string, set);
// if there was no unit out there... return the original string
if(string[sz] == 0)
return start;
else
string += (sz + 1);
}
// copy first chunk
sz = string-start;
dStrncpy(ret, start, sz);
ret[sz] = 0;
// copy remaining chunks
sz = dStrcspn(string, set); // skip chunk we're removing
if(string[sz] == 0) { // if that was the last...
if(string != start) {
ret[string - start - 1] = 0; // then kill any trailing delimiter
}
return ret; // and bail
}
string += sz + 1; // skip the extra field delimiter
dStrcat(ret, string);
return ret;
}
}

39
Engine/source/core/strings/stringUnit.h Normal file
View file

@ -0,0 +1,39 @@
//-----------------------------------------------------------------------------
// Copyright (c) 2012 GarageGames, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
#ifndef _STRINGUNIT_H_
#define _STRINGUNIT_H_
#include "platform/types.h"
/// These functions are used for chunking up strings by delimiter.
/// Especially useful for handling TorqueScript space-delimited fields
namespace StringUnit
{
const char *getUnit(const char *string, U32 index, const char *set, char* buffer = 0, U32 bufferSize = 0);
const char *getUnits(const char *string, S32 startIndex, S32 endIndex, const char *set);
U32 getUnitCount(const char *string, const char *set);
const char* setUnit(const char *string, U32 index, const char *replace, const char *set);
const char* removeUnit(const char *string, U32 index, const char *set);
};
#endif

663
Engine/source/core/strings/unicode.cpp Normal file
View file

@ -0,0 +1,663 @@
//-----------------------------------------------------------------------------
// Copyright (c) 2012 GarageGames, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
#include <stdio.h>
#include "core/frameAllocator.h"
#include "core/strings/unicode.h"
#include "core/strings/stringFunctions.h"
#include "platform/profiler.h"
#include "console/console.h"
#define TORQUE_ENABLE_UTF16_CACHE
#ifdef TORQUE_ENABLE_UTF16_CACHE
#include "core/util/tDictionary.h"
#include "core/util/hashFunction.h"
#endif
//-----------------------------------------------------------------------------
/// replacement character. Standard correct value is 0xFFFD.
#define kReplacementChar 0xFFFD
/// Look up table. Shift a byte >> 1, then look up how many bytes to expect after it.
/// Contains -1's for illegal values.
static const U8 sgFirstByteLUT[128] =
{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x0F // single byte ascii
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x1F // single byte ascii
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x2F // single byte ascii
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x3F // single byte ascii
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4F // trailing utf8
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x5F // trailing utf8
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // 0x6F // first of 2
3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 6, 0, // 0x7F // first of 3,4,5,illegal in utf-8
};
/// Look up table. Shift a 16-bit word >> 10, then look up whether it is a surrogate,
/// and which part. 0 means non-surrogate, 1 means 1st in pair, 2 means 2nd in pair.
static const U8 sgSurrogateLUT[64] =
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x1F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x2F
0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, // 0x3F
};
/// Look up table. Feed value from firstByteLUT in, gives you
/// the mask for the data bits of that UTF-8 code unit.
static const U8 sgByteMask8LUT[] = { 0x3f, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 }; // last 0=6, 1=7, 2=5, 4, 3, 2, 1 bits
/// Mask for the data bits of a UTF-16 surrogate.
static const U16 sgByteMaskLow10 = 0x03ff;
//-----------------------------------------------------------------------------
#ifdef TORQUE_ENABLE_UTF16_CACHE
/// Cache data for UTF16 strings. This is wrapped in a class so that data is
/// automatically freed when the hash table is deleted.
struct UTF16Cache
{
UTF16 *mString;
U32 mLength;
UTF16Cache()
{
mString = NULL;
mLength = 0;
}
UTF16Cache(UTF16 *str, U32 len)
{
mLength = len;
mString = new UTF16[mLength];
dMemcpy(mString, str, mLength * sizeof(UTF16));
}
UTF16Cache(const UTF16Cache &other)
{
mLength = other.mLength;
mString = new UTF16[mLength];
dMemcpy(mString, other.mString, mLength * sizeof(UTF16));
}
void operator =(const UTF16Cache &other)
{
delete [] mString;
mLength = other.mLength;
mString = new UTF16[mLength];
dMemcpy(mString, other.mString, mLength * sizeof(UTF16));
}
~UTF16Cache()
{
delete [] mString;
}
void copyToBuffer(UTF16 *outBuffer, U32 lenToCopy, bool nullTerminate = true) const
{
U32 copy = getMin(mLength, lenToCopy);
if(mString && copy > 0)
dMemcpy(outBuffer, mString, copy * sizeof(UTF16));
if(nullTerminate)
outBuffer[copy] = 0;
}
};
/// Cache for UTF16 strings
typedef HashTable<U32, UTF16Cache> UTF16CacheTable;
static UTF16CacheTable sgUTF16Cache;
#endif // TORQUE_ENABLE_UTF16_CACHE
//-----------------------------------------------------------------------------
inline bool isSurrogateRange(U32 codepoint)
{
return ( 0xd800 < codepoint && codepoint < 0xdfff );
}
inline bool isAboveBMP(U32 codepoint)
{
return ( codepoint > 0xFFFF );
}
//-----------------------------------------------------------------------------
U32 convertUTF8toUTF16(const UTF8 *unistring, UTF16 *outbuffer, U32 len)
{
AssertFatal(len >= 1, "Buffer for unicode conversion must be large enough to hold at least the null terminator.");
PROFILE_SCOPE(convertUTF8toUTF16);
#ifdef TORQUE_ENABLE_UTF16_CACHE
// If we have cached this conversion already, don't do it again
U32 hashKey = Torque::hash((const U8 *)unistring, dStrlen(unistring), 0);
UTF16CacheTable::Iterator cacheItr = sgUTF16Cache.find(hashKey);
if(cacheItr != sgUTF16Cache.end())
{
const UTF16Cache &cache = (*cacheItr).value;
cache.copyToBuffer(outbuffer, len);
outbuffer[len-1] = '\0';
return getMin(cache.mLength,len - 1);
}
#endif
U32 walked, nCodepoints;
UTF32 middleman;
nCodepoints=0;
while(*unistring != '\0' && nCodepoints < len)
{
walked = 1;
middleman = oneUTF8toUTF32(unistring,&walked);
outbuffer[nCodepoints] = oneUTF32toUTF16(middleman);
unistring+=walked;
nCodepoints++;
}
nCodepoints = getMin(nCodepoints,len - 1);
outbuffer[nCodepoints] = '\0';
#ifdef TORQUE_ENABLE_UTF16_CACHE
// Cache the results.
// FIXME As written, this will result in some unnecessary memory copying due to copy constructor calls.
UTF16Cache cache(outbuffer, nCodepoints);
sgUTF16Cache.insertUnique(hashKey, cache);
#endif
return nCodepoints;
}
//-----------------------------------------------------------------------------
U32 convertUTF16toUTF8( const UTF16 *unistring, UTF8 *outbuffer, U32 len)
{
AssertFatal(len >= 1, "Buffer for unicode conversion must be large enough to hold at least the null terminator.");
PROFILE_START(convertUTF16toUTF8);
U32 walked, nCodeunits, codeunitLen;
UTF32 middleman;
nCodeunits=0;
while( *unistring != '\0' && nCodeunits + 3 < len )
{
walked = 1;
middleman = oneUTF16toUTF32(unistring,&walked);
codeunitLen = oneUTF32toUTF8(middleman, &outbuffer[nCodeunits]);
unistring += walked;
nCodeunits += codeunitLen;
}
nCodeunits = getMin(nCodeunits,len - 1);
outbuffer[nCodeunits] = '\0';
PROFILE_END();
return nCodeunits;
}
U32 convertUTF16toUTF8DoubleNULL( const UTF16 *unistring, UTF8 *outbuffer, U32 len)
{
AssertFatal(len >= 1, "Buffer for unicode conversion must be large enough to hold at least the null terminator.");
PROFILE_START(convertUTF16toUTF8DoubleNULL);
U32 walked, nCodeunits, codeunitLen;
UTF32 middleman;
nCodeunits=0;
while( ! (*unistring == '\0' && *(unistring + 1) == '\0') && nCodeunits + 3 < len )
{
walked = 1;
middleman = oneUTF16toUTF32(unistring,&walked);
codeunitLen = oneUTF32toUTF8(middleman, &outbuffer[nCodeunits]);
unistring += walked;
nCodeunits += codeunitLen;
}
nCodeunits = getMin(nCodeunits,len - 1);
outbuffer[nCodeunits] = NULL;
outbuffer[nCodeunits+1] = NULL;
PROFILE_END();
return nCodeunits;
}
//-----------------------------------------------------------------------------
// Functions that convert buffers of unicode code points
//-----------------------------------------------------------------------------
UTF16* convertUTF8toUTF16( const UTF8* unistring)
{
PROFILE_SCOPE(convertUTF8toUTF16_create);
// allocate plenty of memory.
U32 nCodepoints, len = dStrlen(unistring) + 1;
FrameTemp<UTF16> buf(len);
// perform conversion
nCodepoints = convertUTF8toUTF16( unistring, buf, len);
// add 1 for the NULL terminator the converter promises it included.
nCodepoints++;
// allocate the return buffer, copy over, and return it.
UTF16 *ret = new UTF16[nCodepoints];
dMemcpy(ret, buf, nCodepoints * sizeof(UTF16));
return ret;
}
//-----------------------------------------------------------------------------
UTF8* convertUTF16toUTF8( const UTF16* unistring)
{
PROFILE_SCOPE(convertUTF16toUTF8_create);
// allocate plenty of memory.
U32 nCodeunits, len = dStrlen(unistring) * 3 + 1;
FrameTemp<UTF8> buf(len);
// perform conversion
nCodeunits = convertUTF16toUTF8( unistring, buf, len);
// add 1 for the NULL terminator the converter promises it included.
nCodeunits++;
// allocate the return buffer, copy over, and return it.
UTF8 *ret = new UTF8[nCodeunits];
dMemcpy(ret, buf, nCodeunits * sizeof(UTF8));
return ret;
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
// Functions that converts one unicode codepoint at a time
//-----------------------------------------------------------------------------
UTF32 oneUTF8toUTF32( const UTF8* codepoint, U32 *unitsWalked)
{
PROFILE_SCOPE(oneUTF8toUTF32);
// codepoints 6 codeunits long are read, but do not convert correctly,
// and are filtered out anyway.
// early out for ascii
if(!(*codepoint & 0x0080))
{
if (unitsWalked != NULL)
*unitsWalked = 1;
return (UTF32)*codepoint;
}
U32 expectedByteCount;
UTF32 ret = 0;
U8 codeunit;
// check the first byte ( a.k.a. codeunit ) .
unsigned char c = codepoint[0];
c = c >> 1;
expectedByteCount = sgFirstByteLUT[c];
if(expectedByteCount > 0) // 0 or negative is illegal to start with
{
// process 1st codeunit
ret |= sgByteMask8LUT[expectedByteCount] & codepoint[0]; // bug?
// process trailing codeunits
for(U32 i=1;i<expectedByteCount; i++)
{
codeunit = codepoint[i];
if( sgFirstByteLUT[codeunit>>1] == 0 )
{
ret <<= 6; // shift up 6
ret |= (codeunit & 0x3f); // mask in the low 6 bits of this codeunit byte.
}
else
{
// found a bad codepoint - did not get a medial where we wanted one.
// Dump the replacement, and claim to have parsed only 1 char,
// so that we'll dump a slew of replacements, instead of eating the next char.
ret = kReplacementChar;
expectedByteCount = 1;
break;
}
}
}
else
{
// found a bad codepoint - got a medial or an illegal codeunit.
// Dump the replacement, and claim to have parsed only 1 char,
// so that we'll dump a slew of replacements, instead of eating the next char.
ret = kReplacementChar;
expectedByteCount = 1;
}
if(unitsWalked != NULL)
*unitsWalked = expectedByteCount;
// codepoints in the surrogate range are illegal, and should be replaced.
if(isSurrogateRange(ret))
ret = kReplacementChar;
// codepoints outside the Basic Multilingual Plane add complexity to our UTF16 string classes,
// we've read them correctly so they won't foul the byte stream,
// but we kill them here to make sure they wont foul anything else
if(isAboveBMP(ret))
ret = kReplacementChar;
return ret;
}
//-----------------------------------------------------------------------------
UTF32 oneUTF16toUTF32(const UTF16* codepoint, U32 *unitsWalked)
{
PROFILE_START(oneUTF16toUTF32);
U8 expectedType;
U32 unitCount;
UTF32 ret = 0;
UTF16 codeunit1,codeunit2;
codeunit1 = codepoint[0];
expectedType = sgSurrogateLUT[codeunit1 >> 10];
switch(expectedType)
{
case 0: // simple
ret = codeunit1;
unitCount = 1;
break;
case 1: // 2 surrogates
codeunit2 = codepoint[1];
if( sgSurrogateLUT[codeunit2 >> 10] == 2)
{
ret = ((codeunit1 & sgByteMaskLow10 ) << 10) | (codeunit2 & sgByteMaskLow10);
unitCount = 2;
break;
}
// else, did not find a trailing surrogate where we expected one,
// so fall through to the error
case 2: // error
// found a trailing surrogate where we expected a codepoint or leading surrogate.
// Dump the replacement.
ret = kReplacementChar;
unitCount = 1;
break;
default:
// unexpected return
AssertFatal(false, "oneUTF16toUTF323: unexpected type");
ret = kReplacementChar;
unitCount = 1;
break;
}
if(unitsWalked != NULL)
*unitsWalked = unitCount;
// codepoints in the surrogate range are illegal, and should be replaced.
if(isSurrogateRange(ret))
ret = kReplacementChar;
// codepoints outside the Basic Multilingual Plane add complexity to our UTF16 string classes,
// we've read them correctly so they wont foul the byte stream,
// but we kill them here to make sure they wont foul anything else
// NOTE: these are perfectly legal codepoints, we just dont want to deal with them.
if(isAboveBMP(ret))
ret = kReplacementChar;
PROFILE_END();
return ret;
}
//-----------------------------------------------------------------------------
UTF16 oneUTF32toUTF16(const UTF32 codepoint)
{
// found a codepoint outside the encodable UTF-16 range!
// or, found an illegal codepoint!
if(codepoint >= 0x10FFFF || isSurrogateRange(codepoint))
return kReplacementChar;
// these are legal, we just don't want to deal with them.
if(isAboveBMP(codepoint))
return kReplacementChar;
return (UTF16)codepoint;
}
//-----------------------------------------------------------------------------
U32 oneUTF32toUTF8(const UTF32 codepoint, UTF8 *threeByteCodeunitBuf)
{
PROFILE_START(oneUTF32toUTF8);
U32 bytecount = 0;
UTF8 *buf;
U32 working = codepoint;
buf = threeByteCodeunitBuf;
//-----------------
if(isSurrogateRange(working)) // found an illegal codepoint!
working = kReplacementChar;
if(isAboveBMP(working)) // these are legal, we just dont want to deal with them.
working = kReplacementChar;
//-----------------
if( working < (1 << 7)) // codeable in 7 bits
bytecount = 1;
else if( working < (1 << 11)) // codeable in 11 bits
bytecount = 2;
else if( working < (1 << 16)) // codeable in 16 bits
bytecount = 3;
AssertISV( bytecount > 0, "Error converting to UTF-8 in oneUTF32toUTF8(). isAboveBMP() should have caught this!");
//-----------------
U8 mask = sgByteMask8LUT[0]; // 0011 1111
U8 marker = ( ~mask << 1); // 1000 0000
// Process the low order bytes, shifting the codepoint down 6 each pass.
for( int i = bytecount-1; i > 0; i--)
{
threeByteCodeunitBuf[i] = marker | (working & mask);
working >>= 6;
}
// Process the 1st byte. filter based on the # of expected bytes.
mask = sgByteMask8LUT[bytecount];
marker = ( ~mask << 1 );
threeByteCodeunitBuf[0] = marker | working & mask;
PROFILE_END();
return bytecount;
}
//-----------------------------------------------------------------------------
U32 dStrlen(const UTF16 *unistring)
{
if(!unistring)
return 0;
U32 i = 0;
while(unistring[i] != '\0')
i++;
// AssertFatal( wcslen(unistring) == i, "Incorrect length" );
return i;
}
//-----------------------------------------------------------------------------
U32 dStrlen(const UTF32 *unistring)
{
U32 i = 0;
while(unistring[i] != '\0')
i++;
return i;
}
//-----------------------------------------------------------------------------
U32 dStrncmp(const UTF16* unistring1, const UTF16* unistring2, U32 len)
{
UTF16 c1, c2;
for(U32 i = 0; i<len; i++)
{
c1 = *unistring1++;
c2 = *unistring2++;
if(c1 < c2) return -1;
if(c1 > c2) return 1;
if(!c1) return 0;
}
return 0;
}
//-----------------------------------------------------------------------------
const UTF16* dStrrchr(const UTF16* unistring, U32 c)
{
if(!unistring) return NULL;
const UTF16* tmp = unistring + dStrlen(unistring);
while( tmp >= unistring)
{
if(*tmp == c)
return tmp;
tmp--;
}
return NULL;
}
UTF16* dStrrchr(UTF16* unistring, U32 c)
{
const UTF16* str = unistring;
return const_cast<UTF16*>(dStrrchr(str, c));
}
const UTF16* dStrchr(const UTF16* unistring, U32 c)
{
if(!unistring) return NULL;
const UTF16* tmp = unistring;
while ( *tmp && *tmp != c)
tmp++;
return (*tmp == c) ? tmp : NULL;
}
UTF16* dStrchr(UTF16* unistring, U32 c)
{
const UTF16* str = unistring;
return const_cast<UTF16*>(dStrchr(str, c));
}
//-----------------------------------------------------------------------------
const UTF8* getNthCodepoint(const UTF8 *unistring, const U32 n)
{
const UTF8* ret = unistring;
U32 charsseen = 0;
while( *ret && charsseen < n)
{
ret++;
if((*ret & 0xC0) != 0x80)
charsseen++;
}
return ret;
}
/* alternate utf-8 decode impl for speed, no error checking,
left here for your amusement:
U32 codeunit = codepoint + expectedByteCount - 1;
U32 i = 0;
switch(expectedByteCount)
{
case 6: ret |= ( *(codeunit--) & 0x3f ); i++;
case 5: ret |= ( *(codeunit--) & 0x3f ) << (6 * i++);
case 4: ret |= ( *(codeunit--) & 0x3f ) << (6 * i++);
case 3: ret |= ( *(codeunit--) & 0x3f ) << (6 * i++);
case 2: ret |= ( *(codeunit--) & 0x3f ) << (6 * i++);
case 1: ret |= *(codeunit) & byteMask8LUT[expectedByteCount] << (6 * i);
}
*/
//------------------------------------------------------------------------------
// Byte Order Mark functions
bool chompUTF8BOM( const char *inString, char **outStringPtr )
{
*outStringPtr = const_cast<char *>( inString );
U8 bom[4];
dMemcpy( bom, inString, 4 );
bool valid = isValidUTF8BOM( bom );
// This is hackey, but I am not sure the best way to do it at the present.
// The only valid BOM is a UTF8 BOM, which is 3 bytes, even though we read
// 4 bytes because it could possibly be a UTF32 BOM, and we want to provide
// an accurate error message. Perhaps this could be re-worked when more UTF
// formats are supported to have isValidBOM return the size of the BOM, in
// bytes.
if( valid )
(*outStringPtr) += 3; // SEE ABOVE!! -pw
return valid;
}
bool isValidUTF8BOM( U8 bom[4] )
{
// Is it a BOM?
if( bom[0] == 0 )
{
// Could be UTF32BE
if( bom[1] == 0 && bom[2] == 0xFE && bom[3] == 0xFF )
{
Con::warnf( "Encountered a UTF32 BE BOM in this file; Torque does NOT support this file encoding. Use UTF8!" );
return false;
}
return false;
}
else if( bom[0] == 0xFF )
{
// It's little endian, either UTF16 or UTF32
if( bom[1] == 0xFE )
{
if( bom[2] == 0 && bom[3] == 0 )
Con::warnf( "Encountered a UTF32 LE BOM in this file; Torque does NOT support this file encoding. Use UTF8!" );
else
Con::warnf( "Encountered a UTF16 LE BOM in this file; Torque does NOT support this file encoding. Use UTF8!" );
}
return false;
}
else if( bom[0] == 0xFE && bom[1] == 0xFF )
{
Con::warnf( "Encountered a UTF16 BE BOM in this file; Torque does NOT support this file encoding. Use UTF8!" );
return false;
}
else if( bom[0] == 0xEF && bom[1] == 0xBB && bom[2] == 0xBF )
{
// Can enable this if you want -pw
//Con::printf("Encountered a UTF8 BOM. Torque supports this.");
return true;
}
// Don't print out an error message here, because it will try this with
// every script. -pw
return false;
}

131
Engine/source/core/strings/unicode.h Normal file
View file

@ -0,0 +1,131 @@
//-----------------------------------------------------------------------------
// Copyright (c) 2012 GarageGames, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//-----------------------------------------------------------------------------
#ifndef _UNICODE_H_
#define _UNICODE_H_
#ifndef _TORQUE_TYPES_H_
#include "platform/types.h"
#endif
/// Unicode conversion utility functions
///
/// Some definitions first:
/// - <b>Code Point</b>: a single character of Unicode text. Used to disabmiguate from C char type.
/// - <b>UTF-32</b>: a Unicode encoding format where one code point is always 32 bits wide.
/// This format can in theory contain any Unicode code point that will ever be needed, now or in the future. 4billion+ code points should be enough, right?
/// - <b>UTF-16</b>: a variable length Unicode encoding format where one code point can be
/// either one or two 16-bit code units long.
/// - <b>UTF-8</b>: a variable length Unicode endocing format where one code point can be
/// up to four 8-bit code units long. The first bit of a single byte UTF-8 code point is 0.
/// The first few bits of a multi-byte code point determine the length of the code point.
/// @see http://en.wikipedia.org/wiki/UTF-8
/// - <b>Surrogate Pair</b>: a pair of special UTF-16 code units, that encode a code point
/// that is too large to fit into 16 bits. The surrogate values sit in a special reserved range of Unicode.
/// - <b>Code Unit</b>: a single unit of a variable length Unicode encoded code point.
/// UTF-8 has 8 bit wide code units. UTF-16 has 16 bit wide code units.
/// - <b>BMP</b>: "Basic Multilingual Plane". Unicode values U+0000 - U+FFFF. This range
/// of Unicode contains all the characters for all the languages of the world, that one would
/// usually be interested in. All code points in the BMP are 16 bits wide or less.
/// The current implementation of these conversion functions deals only with the BMP.
/// Any code points above 0xFFFF, the top of the BMP, are replaced with the
/// standard unicode replacement character: 0xFFFD.
/// Any UTF16 surrogates are read correctly, but replaced.
/// UTF-8 code points up to 6 code units wide will be read, but 5+ is illegal,
/// and 4+ is above the BMP, and will be replaced.
/// This means that UTF-8 output is clamped to 3 code units ( bytes ) per code point.
//-----------------------------------------------------------------------------
/// Functions that convert buffers of unicode code points, allocating a buffer.
/// - These functions allocate their own return buffers. You are responsible for
/// calling delete[] on these buffers.
/// - Because they allocate memory, do not use these functions in a tight loop.
/// - These are useful when you need a new long term copy of a string.
UTF16* convertUTF8toUTF16( const UTF8 *unistring);
UTF8* convertUTF16toUTF8( const UTF16 *unistring);
//-----------------------------------------------------------------------------
/// Functions that convert buffers of unicode code points, into a provided buffer.
/// - These functions are useful for working on existing buffers.
/// - These cannot convert a buffer in place. If unistring is the same memory as
/// outbuffer, the behavior is undefined.
/// - The converter clamps output to the BMP (Basic Multilingual Plane) .
/// - Conversion to UTF-8 requires a buffer of 3 bytes (U8's) per character, + 1.
/// - Conversion to UTF-16 requires a buffer of 1 U16 (2 bytes) per character, + 1.
/// - Conversion to UTF-32 requires a buffer of 1 U32 (4 bytes) per character, + 1.
/// - UTF-8 only requires 3 bytes per character in the worst case.
/// - Output is null terminated. Be sure to provide 1 extra byte, U16 or U32 for
/// the null terminator, or you will see truncated output.
/// - If the provided buffer is too small, the output will be truncated.
U32 convertUTF8toUTF16(const UTF8 *unistring, UTF16 *outbuffer, U32 len);
U32 convertUTF16toUTF8( const UTF16 *unistring, UTF8 *outbuffer, U32 len);
//-----------------------------------------------------------------------------
/// Functions that converts one unicode codepoint at a time
/// - Since these functions are designed to be used in tight loops, they do not
/// allocate buffers.
/// - oneUTF8toUTF32() and oneUTF16toUTF32() return the converted Unicode code point
/// in *codepoint, and set *unitsWalked to the \# of code units *codepoint took up.
/// The next Unicode code point should start at *(codepoint + *unitsWalked).
/// - oneUTF32toUTF8() requires a 3 byte buffer, and returns the \# of bytes used.
UTF32 oneUTF8toUTF32( const UTF8 *codepoint, U32 *unitsWalked = NULL);
UTF32 oneUTF16toUTF32(const UTF16 *codepoint, U32 *unitsWalked = NULL);
UTF16 oneUTF32toUTF16(const UTF32 codepoint);
U32 oneUTF32toUTF8( const UTF32 codepoint, UTF8 *threeByteCodeunitBuf);
//-----------------------------------------------------------------------------
/// Functions that calculate the length of unicode strings.
/// - Since calculating the length of a UTF8 string is nearly as expensive as
/// converting it to another format, a dStrlen for UTF8 is not provided here.
/// - If *unistring does not point to a null terminated string of the correct type,
/// the behavior is undefined.
U32 dStrlen(const UTF16 *unistring);
U32 dStrlen(const UTF32 *unistring);
//-----------------------------------------------------------------------------
/// Comparing unicode strings
U32 dStrncmp(const UTF16* unistring1, const UTF16* unistring2, U32 len);
//-----------------------------------------------------------------------------
/// Scanning for characters in unicode strings
UTF16* dStrrchr(UTF16* unistring, U32 c);
const UTF16* dStrrchr(const UTF16* unistring, U32 c);
UTF16* dStrchr(UTF16* unistring, U32 c);
const UTF16* dStrchr(const UTF16* unistring, U32 c);
//-----------------------------------------------------------------------------
/// Functions that scan for characters in a utf8 string.
/// - this is useful for getting a character-wise offset into a UTF8 string,
/// as opposed to a byte-wise offset into a UTF8 string: foo[i]
const UTF8* getNthCodepoint(const UTF8 *unistring, const U32 n);
//------------------------------------------------------------------------------
/// Functions to read and validate UTF BOMs (Byte Order Marker)
/// For reference: http://en.wikipedia.org/wiki/Byte_Order_Mark
bool chompUTF8BOM( const char *inString, char **outStringPtr );
bool isValidUTF8BOM( U8 bom[4] );
#endif // _UNICODE_H_