Engine directory for ticket #1

Engine/source/core/stream/bitStream.h

@@ -0,0 +1,385 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#ifndef _BITSTREAM_H_
+#define _BITSTREAM_H_
+
+#ifndef _STREAM_H_
+#include "core/stream/stream.h"
+#endif
+#ifndef _MPOINT3_H_
+#include "math/mPoint3.h"
+#endif
+#ifndef _CRC_H_
+#include "core/crc.h"
+#endif
+
+//-------------------------------------- Some caveats when using this class:
+//                                        - Get/setPosition semantics are changed
+//                                         to indicate bit position rather than
+//                                         byte position.
+//
+
+class Point3F;
+class MatrixF;
+class HuffmanProcessor;
+class BitVector;
+class QuatF;
+
+class BitStream : public Stream
+{
+protected:
+   U8 *dataPtr;
+   S32  bitNum;
+   S32  bufSize;
+   bool error;
+   S32  maxReadBitNum;
+   S32  maxWriteBitNum;
+   char *stringBuffer;
+   Point3F mCompressPoint;
+
+   friend class HuffmanProcessor;
+public:
+   static BitStream *getPacketStream(U32 writeSize = 0);
+   static void sendPacketStream(const NetAddress *addr);
+
+   void setBuffer(void *bufPtr, S32 bufSize, S32 maxSize = 0);
+   U8*  getBuffer() { return dataPtr; }
+   U8*  getBytePtr();
+
+   U32 getReadByteSize();
+   U32 getWriteByteSize();
+
+   S32  getCurPos() const;
+   void setCurPos(const U32);
+
+   // HACK: We reverted BitStream to this previous version
+   // because it was crashing the build.
+   //
+   // These are just here so that we don't have to revert
+   // the changes from the rest of the code.
+   //
+   // 9/11/2008 - Tom Spilman
+   //
+   S32 getBitPosition() const { return getCurPos(); }
+   void clearStringBuffer();
+
+   BitStream(void *bufPtr, S32 bufSize, S32 maxWriteSize = -1) { setBuffer(bufPtr, bufSize,maxWriteSize); stringBuffer = NULL; }
+   void clear();
+
+   void setStringBuffer(char buffer[256]);
+   void writeInt(S32 value, S32 bitCount);
+   S32  readInt(S32 bitCount);
+
+   /// Use this method to write out values in a concise but ass backwards way...
+   /// Good for values you expect to be frequently zero, often small. Worst case
+   /// this will bloat values by nearly 20% (5 extra bits!) Best case you'll get
+   /// one bit (if it's zero).
+   ///
+   /// This is not so much for efficiency's sake, as to make life painful for
+   /// people that want to reverse engineer our network or file formats.
+   void writeCussedU32(U32 val)
+   {
+      // Is it zero?
+      if(writeFlag(val == 0))
+         return;
+
+      if(writeFlag(val <= 0xF)) // 4 bit
+         writeRangedU32(val, 0, 0xF);
+      else if(writeFlag(val <= 0xFF)) // 8 bit
+         writeRangedU32(val, 0, 0xFF);
+      else if(writeFlag(val <= 0xFFFF)) // 16 bit
+         writeRangedU32(val, 0, 0xFFFF);
+      else if(writeFlag(val <= 0xFFFFFF)) // 24 bit
+         writeRangedU32(val, 0, 0xFFFFFF);
+      else
+         writeRangedU32(val, 0, 0xFFFFFFFF);
+   }
+
+   U32 readCussedU32()
+   {
+      if(readFlag())
+         return 0;
+
+      if(readFlag())
+         return readRangedU32(0, 0xF);
+      else if(readFlag())
+         return readRangedU32(0, 0xFF);
+      else if(readFlag())
+         return readRangedU32(0, 0xFFFF);
+      else if(readFlag())
+         return readRangedU32(0, 0xFFFFFF);
+      else
+         return readRangedU32(0, 0xFFFFFFFF);
+   }
+
+   void writeSignedInt(S32 value, S32 bitCount);
+   S32  readSignedInt(S32 bitCount);
+
+   void writeRangedU32(U32 value, U32 rangeStart, U32 rangeEnd);
+   U32  readRangedU32(U32 rangeStart, U32 rangeEnd);
+   
+   /// Writes a clamped signed integer to the stream using 
+   /// an optimal amount of bits for the range.
+   void writeRangedS32( S32 value, S32 min, S32 max );
+
+   /// Reads a ranged signed integer written with writeRangedS32.
+   S32 readRangedS32( S32 min, S32 max );
+
+   // read and write floats... floats are 0 to 1 inclusive, signed floats are -1 to 1 inclusive
+
+   F32  readFloat(S32 bitCount);
+   F32  readSignedFloat(S32 bitCount);
+
+   void writeFloat(F32 f, S32 bitCount);
+   void writeSignedFloat(F32 f, S32 bitCount);
+
+   /// Writes a clamped floating point value to the 
+   /// stream with the desired bits of precision.
+   void writeRangedF32( F32 value, F32 min, F32 max, U32 numBits );
+
+   /// Reads a ranged floating point value written with writeRangedF32.
+   F32 readRangedF32( F32 min, F32 max, U32 numBits );
+
+   void writeClassId(U32 classId, U32 classType, U32 classGroup);
+   S32 readClassId(U32 classType, U32 classGroup); // returns -1 if the class type is out of range
+
+   // writes a normalized vector
+   void writeNormalVector(const Point3F& vec, S32 bitCount);
+   void readNormalVector(Point3F *vec, S32 bitCount);
+
+   void clearCompressionPoint();
+   void setCompressionPoint(const Point3F& p);
+
+   // Matching calls to these compression methods must, of course,
+   // have matching scale values.
+   void writeCompressedPoint(const Point3F& p,F32 scale = 0.001f);
+   void readCompressedPoint(Point3F* p,F32 scale = 0.001f);
+
+   // Uses the above method to reduce the precision of a normal vector so the server can
+   //  determine exactly what is on the client.  (Pre-dumbing the vector before sending
+   //  to the client can result in precision errors...)
+   static Point3F dumbDownNormal(const Point3F& vec, S32 bitCount);
+
+   /// Writes a compressed vector as separate magnitude and
+   /// normal components.  The final space used depends on the 
+   /// content of the vector.
+   ///
+   ///  - 1 bit is used to skip over zero length vectors.
+   ///  - 1 bit is used to mark if the magnitude exceeds max.
+   ///  - The magnitude as:
+   ///     a. magBits if less than maxMag.
+   ///     b. a full 32bit value if greater than maxMag.   
+   ///  - The normal as a phi and theta sized normalBits+1 and normalBits.
+   ///
+   void writeVector( Point3F vec, F32 maxMag, S32 magBits, S32 normalBits );
+
+   /// Reads a compressed vector.
+   /// @see writeVector
+   void readVector( Point3F *outVec, F32 maxMag, S32 magBits, S32 normalBits );
+
+   // writes an affine transform (full precision version)
+   void writeAffineTransform(const MatrixF&);
+   void readAffineTransform(MatrixF*);
+
+   /// Writes a quaternion in a lossy compressed format that
+   /// is ( bitCount * 3 ) + 1 bits in size.
+   ///
+   /// @param quat The normalized quaternion to write.
+   /// @param bitCount The the storage space for the xyz component of
+   ///                 the quaternion.
+   ///
+   void writeQuat( const QuatF& quat, U32 bitCount = 9 );
+
+   /// Reads a quaternion written with writeQuat.
+   ///
+   /// @param quat The normalized quaternion to write.
+   /// @param bitCount The the storage space for the xyz component of
+   ///                 the quaternion.  Must match the bitCount at write.
+   /// @see writeQuat
+   ///
+   void readQuat( QuatF *outQuat, U32 bitCount = 9 );
+
+   virtual void writeBits(S32 bitCount, const void *bitPtr);
+   virtual void readBits(S32 bitCount, void *bitPtr);
+   virtual bool writeFlag(bool val);
+   
+   inline bool writeFlag(U32 val)
+   {
+      return writeFlag(val != 0);
+   }
+
+   inline bool writeFlag(void *val)
+   {
+      return writeFlag(val != 0);
+   }
+
+   virtual bool readFlag();
+
+   void writeBits(const BitVector &bitvec);
+   void readBits(BitVector *bitvec);
+
+   void setBit(S32 bitCount, bool set);
+   bool testBit(S32 bitCount);
+
+   bool isFull() { return bitNum > (bufSize << 3); }
+   bool isValid() { return !error; }
+
+   bool _read (const U32 size,void* d);
+   bool _write(const U32 size,const void* d);
+
+   void readString(char stringBuf[256]);
+   void writeString(const char *stringBuf, S32 maxLen=255);
+
+   bool hasCapability(const Capability) const { return true; }
+   U32  getPosition() const;
+   bool setPosition(const U32 in_newPosition);
+   U32  getStreamSize();
+};
+
+class ResizeBitStream : public BitStream
+{
+protected:
+   U32 mMinSpace;
+public:
+   ResizeBitStream(U32 minSpace = 1500, U32 initialSize = 0);
+   void validate();
+   ~ResizeBitStream();
+};
+
+/// This class acts to provide an "infinitely extending" stream.
+///
+/// Basically, it does what ResizeBitStream does, but it validates
+/// on every write op, so that you never have to worry about overwriting
+/// the buffer.
+class InfiniteBitStream : public ResizeBitStream
+{
+public:
+   InfiniteBitStream();
+   ~InfiniteBitStream();
+
+   /// Ensure we have space for at least upcomingBytes more bytes in the stream.
+   void validate(U32 upcomingBytes);
+
+   /// Reset the stream to zero length (but don't clean memory).
+   void reset();
+
+   /// Shrink the buffer down to match the actual size of the data.
+   void compact();
+
+   /// Write us out to a stream... Results in last byte getting padded!
+   void writeToStream(Stream &s);
+
+   virtual void writeBits(S32 bitCount, const void *bitPtr)
+   {
+      validate((bitCount >> 3) + 1); // Add a little safety.
+      BitStream::writeBits(bitCount, bitPtr);
+   }
+
+   virtual bool writeFlag(bool val)
+   {
+      validate(1); // One bit will at most grow our buffer by a byte.
+      return BitStream::writeFlag(val);
+   }
+
+   const U32 getCRC()
+   {
+      // This could be kinda inefficient - BJG
+      return CRC::calculateCRC(getBuffer(), getStreamSize());
+   }
+};
+
+//------------------------------------------------------------------------------
+//-------------------------------------- INLINES
+//
+inline S32 BitStream::getCurPos() const
+{
+   return bitNum;
+}
+
+inline void BitStream::setCurPos(const U32 in_position)
+{
+   AssertFatal(in_position < (U32)(bufSize << 3), "Out of range bitposition");
+   bitNum = S32(in_position);
+}
+
+inline bool BitStream::readFlag()
+{
+   if(bitNum > maxReadBitNum)
+   {
+      error = true;
+      AssertFatal(false, "Out of range read");
+      return false;
+   }
+   S32 mask = 1 << (bitNum & 0x7);
+   bool ret = (*(dataPtr + (bitNum >> 3)) & mask) != 0;
+   bitNum++;
+   return ret;
+}
+
+inline void BitStream::writeRangedU32(U32 value, U32 rangeStart, U32 rangeEnd)
+{
+   AssertFatal(value >= rangeStart && value <= rangeEnd, "Out of bounds value!");
+   AssertFatal(rangeEnd >= rangeStart, "error, end of range less than start");
+
+   U32 rangeSize = rangeEnd - rangeStart + 1;
+   U32 rangeBits = getBinLog2(getNextPow2(rangeSize));
+
+   writeInt(S32(value - rangeStart), S32(rangeBits));
+}
+
+inline U32 BitStream::readRangedU32(U32 rangeStart, U32 rangeEnd)
+{
+   AssertFatal(rangeEnd >= rangeStart, "error, end of range less than start");
+
+   U32 rangeSize = rangeEnd - rangeStart + 1;
+   U32 rangeBits = getBinLog2(getNextPow2(rangeSize));
+
+   U32 val = U32(readInt(S32(rangeBits)));
+   return val + rangeStart;
+}
+
+inline void BitStream::writeRangedS32( S32 value, S32 min, S32 max )
+{
+   value = mClamp( value, min, max );
+   writeRangedU32( ( value - min ), 0, ( max - min ) );
+}
+
+inline S32 BitStream::readRangedS32( S32 min, S32 max )
+{
+   return readRangedU32( 0, ( max - min ) ) + min;
+}
+
+inline void BitStream::writeRangedF32( F32 value, F32 min, F32 max, U32 numBits )
+{
+   value = ( mClampF( value, min, max ) - min ) / ( max - min );
+   writeInt( (S32)mFloor(value * F32( (1 << numBits) - 1 )), numBits );
+}
+
+inline F32 BitStream::readRangedF32( F32 min, F32 max, U32 numBits )
+{
+   F32 value = (F32)readInt( numBits );
+   value /= F32( ( 1 << numBits ) - 1 );
+   return min + value * ( max - min );
+}
+
+#endif //_BITSTREAM_H_

Engine/source/core/stream/fileStream.cpp

@@ -0,0 +1,579 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#include "platform/platform.h"
+#include "core/stream/fileStream.h"
+
+
+//-----------------------------------------------------------------------------
+// FileStream methods...
+//-----------------------------------------------------------------------------
+
+//-----------------------------------------------------------------------------
+FileStream::FileStream()
+{
+   // initialize the file stream
+   init();
+}
+
+FileStream *FileStream::createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode)
+{
+   FileStream  *newStream = new FileStream;
+
+   if ( newStream )
+   {
+      bool success = newStream->open( inFileName, inMode );
+
+      if ( !success )
+      {
+         delete newStream;
+         newStream = NULL;
+      }
+   }
+
+   return newStream;
+}
+
+//-----------------------------------------------------------------------------
+FileStream::~FileStream()
+{
+   // make sure the file stream is closed
+   close();
+}
+
+//-----------------------------------------------------------------------------
+bool FileStream::hasCapability(const Capability i_cap) const
+{
+   return(0 != (U32(i_cap) & mStreamCaps));
+}
+
+//-----------------------------------------------------------------------------
+U32 FileStream::getPosition() const
+{
+   AssertFatal(0 != mStreamCaps, "FileStream::getPosition: the stream isn't open");
+   //AssertFatal(true == hasCapability(StreamPosition), "FileStream::getPosition(): lacks positioning capability");
+
+   // return the position inside the buffer if its valid, otherwise return the underlying file position
+   return((BUFFER_INVALID != mBuffHead) ? mBuffPos : mFile->getPosition());
+}
+
+//-----------------------------------------------------------------------------
+bool FileStream::setPosition(const U32 i_newPosition)
+{
+   AssertFatal(0 != mStreamCaps, "FileStream::setPosition: the stream isn't open");
+   AssertFatal(hasCapability(StreamPosition), "FileStream::setPosition: lacks positioning capability");
+
+   // if the buffer is valid, test the new position against the bounds of the buffer
+   if ((BUFFER_INVALID != mBuffHead) && (i_newPosition >= mBuffHead) && (i_newPosition <= mBuffTail))
+   {
+      // set the position and return
+      mBuffPos = i_newPosition;
+      
+      // FIXME [tom, 9/5/2006] This needs to be checked. Basically, when seeking within
+      // the buffer, if the stream has an EOS status before the seek then if you try to
+      // read immediately after seeking, you'll incorrectly get an EOS.
+      //
+      // I am not 100% sure if this fix is correct, but it seems to be working for the undo system.
+      if(mBuffPos < mBuffTail)
+         Stream::setStatus(Ok);
+      
+      return(true);
+   }
+   // otherwise the new position lies in some block not in memory
+   else
+   {
+      if (mDirty)
+         flush();
+
+      clearBuffer();
+
+      mFile->setPosition(i_newPosition, Torque::FS::File::Begin);
+
+      setStatus();
+      
+      if (mFile->getStatus() == Torque::FS::FileNode::EndOfFile)
+         mEOF = true;
+
+      return(Ok == getStatus() || EOS == getStatus());
+   }
+}
+
+//-----------------------------------------------------------------------------
+U32 FileStream::getStreamSize()
+{
+   AssertWarn(0 != mStreamCaps, "FileStream::getStreamSize: the stream isn't open");
+   AssertFatal((BUFFER_INVALID != mBuffHead && true == mDirty) || false == mDirty, "FileStream::getStreamSize: buffer must be valid if its dirty");
+
+   // the stream size may not match the size on-disk if its been written to...
+   if (mDirty)
+      return(getMax((U32)(mFile->getSize()), mBuffTail + 1));  ///<@todo U64 vs U32 issue
+   // otherwise just get the size on disk...
+   else
+      return(mFile->getSize());
+}
+
+//-----------------------------------------------------------------------------
+bool FileStream::open(const String &inFileName, Torque::FS::File::AccessMode inMode)
+{
+   AssertWarn(0 == mStreamCaps, "FileStream::setPosition: the stream is already open");
+   AssertFatal(inFileName.isNotEmpty(), "FileStream::open: empty filename");
+
+   // make sure the file stream's state is clean
+   clearBuffer();
+
+   Torque::Path   filePath(inFileName);
+
+   // IF we are writing, make sure the path exists
+   if( inMode == Torque::FS::File::Write || inMode == Torque::FS::File::WriteAppend || inMode == Torque::FS::File::ReadWrite )
+      Torque::FS::CreatePath(filePath);
+
+   mFile = Torque::FS::OpenFile(filePath, inMode);
+
+   if (mFile != NULL)
+   {
+      setStatus();
+      switch (inMode)
+      {
+         case Torque::FS::File::Read:
+            mStreamCaps = U32(StreamRead) |
+                          U32(StreamPosition);
+            break;
+         case Torque::FS::File::Write:
+         case Torque::FS::File::WriteAppend:
+            mStreamCaps = U32(StreamWrite) |
+                          U32(StreamPosition);
+            break;
+         case Torque::FS::File::ReadWrite:
+            mStreamCaps = U32(StreamRead)  |
+                          U32(StreamWrite) |
+                          U32(StreamPosition);
+            break;
+         default:
+            AssertFatal(false, String::ToString( "FileStream::open: bad access mode on %s", inFileName.c_str() ));
+      }
+   }
+   else
+   {
+      Stream::setStatus(IOError);
+      return(false);
+   }
+
+   return getStatus() == Ok;
+}
+
+//-----------------------------------------------------------------------------
+void FileStream::close()
+{
+   if (getStatus() == Closed)
+      return;
+
+   if (mFile != NULL )
+   {
+      // make sure nothing in the buffer differs from what is on disk
+      if (mDirty)
+         flush();
+
+      // and close the file
+      mFile->close();
+
+      AssertFatal(mFile->getStatus() == Torque::FS::FileNode::Closed, "FileStream::close: close failed");
+
+      mFile = NULL;
+   }
+
+   // clear the file stream's state
+   init();
+}
+
+//-----------------------------------------------------------------------------
+bool FileStream::flush()
+{
+   AssertWarn(0 != mStreamCaps, "FileStream::flush: the stream isn't open");
+   AssertFatal(false == mDirty || BUFFER_INVALID != mBuffHead, "FileStream::flush: buffer must be valid if its dirty");
+
+   // if the buffer is dirty
+   if (mDirty)
+   {
+      AssertFatal(hasCapability(StreamWrite), "FileStream::flush: a buffer without write-capability should never be dirty");
+      
+      // align the file pointer to the buffer head
+      if (mBuffHead != mFile->getPosition())
+      {
+         mFile->setPosition(mBuffHead, Torque::FS::File::Begin);
+         if (mFile->getStatus() != Torque::FS::FileNode::Open && mFile->getStatus() != Torque::FS::FileNode::EndOfFile)
+            return(false);
+      }
+
+      // write contents of the buffer to disk
+      U32 blockHead;
+      calcBlockHead(mBuffHead, &blockHead);
+      mFile->write((char *)mBuffer + (mBuffHead - blockHead), mBuffTail - mBuffHead + 1);
+      // and update the file stream's state
+      setStatus();
+      if (EOS == getStatus())
+         mEOF = true;
+
+      if (Ok == getStatus() || EOS == getStatus())
+         // and update the status of the buffer
+         mDirty = false;
+      else
+         return(false);
+   }
+   return(true);
+}
+
+//-----------------------------------------------------------------------------
+bool FileStream::_read(const U32 i_numBytes, void *o_pBuffer)
+{
+   AssertFatal(0 != mStreamCaps, "FileStream::_read: the stream isn't open");
+   AssertFatal(NULL != o_pBuffer || i_numBytes == 0, "FileStream::_read: NULL destination pointer with non-zero read request");
+
+   if (!hasCapability(Stream::StreamRead))
+   {
+      AssertFatal(false, "FileStream::_read: file stream lacks capability");
+      Stream::setStatus(IllegalCall);
+      return(false);
+   }
+
+   // exit on pre-existing errors
+   if (Ok != getStatus())
+      return(false);
+
+   // if a request of non-zero length was made
+   if (0 != i_numBytes)
+   {
+      U8 *pDst = (U8 *)o_pBuffer;
+      U32 readSize;
+      U32 remaining = i_numBytes;
+      U32 bytesRead;
+      U32 blockHead;
+      U32 blockTail;
+
+      // check if the buffer has some data in it
+      if (BUFFER_INVALID != mBuffHead)
+      {
+         // copy as much as possible from the buffer into the destination
+         readSize = ((mBuffTail + 1) >= mBuffPos) ? (mBuffTail + 1 - mBuffPos) : 0;
+         readSize = getMin(readSize, remaining);
+         calcBlockHead(mBuffPos, &blockHead);
+         dMemcpy(pDst, mBuffer + (mBuffPos - blockHead), readSize);
+         // reduce the remaining amount to read
+         remaining -= readSize;
+         // advance the buffer pointers
+         mBuffPos += readSize;
+         pDst += readSize;
+
+         if (mBuffPos > mBuffTail && remaining != 0)
+         {
+            flush();
+            mBuffHead = BUFFER_INVALID;
+            if (mEOF == true)
+               Stream::setStatus(EOS);
+         }
+      }
+
+      // if the request wasn't satisfied by the buffer and the file has more data
+      if (false == mEOF && 0 < remaining)
+      {
+         // flush the buffer if its dirty, since we now need to go to disk
+         if (true == mDirty)
+            flush();
+
+         // make sure we know the current read location in the underlying file
+         mBuffPos = mFile->getPosition();
+         calcBlockBounds(mBuffPos, &blockHead, &blockTail);
+
+         // check if the data to be read falls within a single block
+         if ((mBuffPos + remaining) <= blockTail)
+         {
+            // fill the buffer from disk
+            if (true == fillBuffer(mBuffPos))
+            {
+               // copy as much as possible from the buffer to the destination
+               remaining = getMin(remaining, mBuffTail - mBuffPos + 1);
+               dMemcpy(pDst, mBuffer + (mBuffPos - blockHead), remaining);
+               // advance the buffer pointer
+               mBuffPos += remaining;
+            }
+            else
+               return(false);
+         }
+         // otherwise the remaining spans multiple blocks
+         else
+         {
+            clearBuffer();
+            // read from disk directly into the destination
+            bytesRead = mFile->read((char *)pDst, remaining);
+            setStatus();
+            // check to make sure we read as much as expected
+            if (Ok == getStatus() || EOS == getStatus())
+            {
+               // if not, update the end-of-file status
+               if (0 != bytesRead && EOS == getStatus())
+               {
+                  Stream::setStatus(Ok);
+                  mEOF = true;
+               }
+            }
+            else
+               return(false);
+         }
+      }
+   }
+   return(true);
+}
+
+//-----------------------------------------------------------------------------
+bool FileStream::_write(const U32 i_numBytes, const void *i_pBuffer)
+{
+   AssertFatal(0 != mStreamCaps, "FileStream::_write: the stream isn't open");
+   AssertFatal(NULL != i_pBuffer || i_numBytes == 0, "FileStream::_write: NULL source buffer pointer on non-zero write request");
+
+   if (!hasCapability(Stream::StreamWrite))
+   {
+      AssertFatal(false, "FileStream::_write: file stream lacks capability");
+      Stream::setStatus(IllegalCall);
+      return(false);
+   }
+
+   // exit on pre-existing errors
+   if (Ok != getStatus() && EOS != getStatus())
+      return(false);
+
+   // if a request of non-zero length was made
+   if (0 != i_numBytes)
+   {
+      U8 *pSrc = (U8 *)i_pBuffer;
+      U32 writeSize;
+      U32 remaining = i_numBytes;
+      U32 bytesWrit;
+      U32 blockHead;
+      U32 blockTail;
+
+      // check if the buffer is valid
+      if (BUFFER_INVALID != mBuffHead)
+      {
+         // copy as much as possible from the source to the buffer
+         calcBlockBounds(mBuffHead, &blockHead, &blockTail);
+         writeSize = (mBuffPos > blockTail) ? 0 : blockTail - mBuffPos + 1;
+         writeSize = getMin(writeSize, remaining);
+
+         AssertFatal(0 == writeSize || (mBuffPos - blockHead) < BUFFER_SIZE, "FileStream::_write: out of bounds buffer position");
+         dMemcpy(mBuffer + (mBuffPos - blockHead), pSrc, writeSize);
+         // reduce the remaining amount to be written
+         remaining -= writeSize;
+         // advance the buffer pointers
+         mBuffPos += writeSize;
+         mBuffTail = getMax(mBuffTail, mBuffPos - 1);
+         pSrc += writeSize;
+         // mark the buffer dirty
+         if (0 < writeSize)
+            mDirty = true;
+      }
+
+      // if the request wasn't satisfied by the buffer
+      if (0 < remaining)
+      {
+         // flush the buffer if its dirty, since we now need to go to disk
+         if (mDirty)
+            flush();
+
+         // make sure we know the current write location in the underlying file
+         mBuffPos = mFile->getPosition();
+         calcBlockBounds(mBuffPos, &blockHead, &blockTail);
+
+         // check if the data to be written falls within a single block
+         if ((mBuffPos + remaining) <= blockTail)
+         {
+            // write the data to the buffer
+            dMemcpy(mBuffer + (mBuffPos - blockHead), pSrc, remaining);
+            // update the buffer pointers
+            mBuffHead = mBuffPos;
+            mBuffPos += remaining;
+            mBuffTail = mBuffPos - 1;
+            // mark the buffer dirty
+            mDirty = true;
+         }
+         // otherwise the remaining spans multiple blocks
+         else
+         {
+            clearBuffer();
+            // write to disk directly from the source
+            bytesWrit = mFile->write((char *)pSrc, remaining);
+            setStatus();
+            return(Ok == getStatus() || EOS == getStatus());
+         }
+      }
+   }
+   return(true);
+}
+
+//-----------------------------------------------------------------------------
+void FileStream::init()
+{
+   mStreamCaps = 0;
+   Stream::setStatus(Closed);
+   clearBuffer();
+}
+
+//-----------------------------------------------------------------------------
+bool FileStream::fillBuffer(const U32 i_startPosition)
+{
+   AssertFatal(0 != mStreamCaps, "FileStream::fillBuffer: the stream isn't open");
+   AssertFatal(false == mDirty, "FileStream::fillBuffer: buffer must be clean to fill");
+
+   // make sure start position and file pointer jive
+   if (i_startPosition != mFile->getPosition())
+   {
+      mFile->setPosition(i_startPosition, Torque::FS::File::Begin);
+      if (mFile->getStatus() != Torque::FS::FileNode::Open && mFile->getStatus() != Torque::FS::FileNode::EndOfFile)
+      {
+         setStatus();
+         return(false);
+      }
+      else
+         // update buffer pointer
+         mBuffPos = i_startPosition;
+   }
+
+   // check if file pointer is at end-of-file
+   if (EOS == getStatus())
+   {
+      // invalidate the buffer
+      mBuffHead = BUFFER_INVALID;
+      // set the status to end-of-stream
+      mEOF = true;
+   }
+   // otherwise
+   else
+   {
+      U32 blockHead;
+      // locate bounds of buffer containing current position
+      calcBlockHead(mBuffPos, &blockHead);
+      // read as much as possible from input file
+      U32 bytesRead = mFile->read((char *)mBuffer + (i_startPosition - blockHead), BUFFER_SIZE - (i_startPosition - blockHead));
+      setStatus();
+      if (Ok == getStatus() || EOS == getStatus())
+      {
+         // update buffer pointers
+         mBuffHead = i_startPosition;
+         mBuffPos = i_startPosition;
+         mBuffTail = i_startPosition + bytesRead - 1;
+         // update end-of-file status
+         if (0 != bytesRead && EOS == getStatus())
+         {
+            Stream::setStatus(Ok);
+            mEOF = true;
+         }
+      }
+      else
+      {
+         mBuffHead = BUFFER_INVALID;
+         return(false);
+      }
+   }
+   return(true);
+}
+
+//-----------------------------------------------------------------------------
+void FileStream::clearBuffer()
+{
+   mBuffHead = BUFFER_INVALID;
+   mBuffPos  = 0;
+   mBuffTail = 0;
+   mDirty = false;
+   mEOF = false;
+}
+
+//-----------------------------------------------------------------------------
+void FileStream::calcBlockHead(const U32 i_position, U32 *o_blockHead)
+{
+   AssertFatal(NULL != o_blockHead, "FileStream::calcBlockHead: NULL pointer passed for block head");
+
+   *o_blockHead = i_position/BUFFER_SIZE * BUFFER_SIZE;
+}
+
+//-----------------------------------------------------------------------------
+void FileStream::calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail)
+{
+   AssertFatal(NULL != o_blockHead, "FileStream::calcBlockBounds: NULL pointer passed for block head");
+   AssertFatal(NULL != o_blockTail, "FileStream::calcBlockBounds: NULL pointer passed for block tail");
+
+   *o_blockHead = i_position/BUFFER_SIZE * BUFFER_SIZE;
+   *o_blockTail = *o_blockHead + BUFFER_SIZE - 1;
+}
+
+//-----------------------------------------------------------------------------
+void FileStream::setStatus()
+{
+   switch (mFile->getStatus())
+   {
+      case Torque::FS::FileNode::Open:
+         Stream::setStatus(Ok);
+         break;
+
+      case Torque::FS::FileNode::Closed:
+         Stream::setStatus(Closed);
+         break;
+
+      case Torque::FS::FileNode::EndOfFile:
+         Stream::setStatus(EOS);
+         break;
+
+      case Torque::FS::FileNode::FileSystemFull:
+      case Torque::FS::FileNode::NoSuchFile:
+      case Torque::FS::FileNode::AccessDenied:
+      case Torque::FS::FileNode::NoDisk:
+      case Torque::FS::FileNode::SharingViolation:
+         Stream::setStatus(IOError);
+         break;
+
+      case Torque::FS::FileNode::IllegalCall:
+         Stream::setStatus(IllegalCall);
+         break;
+
+      case Torque::FS::FileNode::UnknownError:
+         Stream::setStatus(UnknownError);
+         break;
+
+      default:
+         AssertFatal(false, "FileStream::setStatus: invalid error mode");
+   }
+}
+
+FileStream* FileStream::clone() const
+{
+   Torque::FS::File::AccessMode mode;
+   if( hasCapability( StreamWrite ) && hasCapability( StreamRead ) )
+      mode = Torque::FS::File::ReadWrite;
+   else if( hasCapability( StreamWrite ) )
+      mode = Torque::FS::File::Write;
+   else
+      mode = Torque::FS::File::Read;
+
+   FileStream* copy = createAndOpen( mFile->getName(), mode );
+   if( copy && copy->setPosition( getPosition() ) )
+      return copy;
+
+   delete copy;
+   return NULL;
+}

Engine/source/core/stream/fileStream.h

@@ -0,0 +1,93 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#ifndef _FILESTREAM_H_
+#define _FILESTREAM_H_
+
+#ifndef _VOLUME_H_
+#include "core/volume.h"
+#endif
+#ifndef _STREAM_H_
+#include "core/stream/stream.h"
+#endif
+
+class FileStream : public Stream
+{
+public:
+   enum
+   {
+      BUFFER_SIZE = 8 * 1024,       // this can be changed to anything appropriate [in k]
+      BUFFER_INVALID = 0xffffffff   // file offsets must all be less than this
+   };
+
+public:
+   FileStream();                       // default constructor
+   virtual ~FileStream();              // destructor
+
+   // This function will allocate a new FileStream and open it.
+   //  If it fails to allocate or fails to open, it will return NULL.
+   //  The caller is responsible for deleting the instance.
+   static FileStream *createAndOpen(const String &inFileName, Torque::FS::File::AccessMode inMode);
+
+   // mandatory methods from Stream base class...
+   virtual bool hasCapability(const Capability i_cap) const;
+
+   virtual U32  getPosition() const;
+   virtual bool setPosition(const U32 i_newPosition);
+   virtual U32  getStreamSize();
+
+   // additional methods needed for a file stream...
+   virtual bool open(const String &inFileName, Torque::FS::File::AccessMode inMode);
+   virtual void close();
+
+   bool flush();
+   FileStream* clone() const;
+
+protected:
+   // more mandatory methods from Stream base class...
+   virtual bool _read(const U32 i_numBytes, void *o_pBuffer);
+   virtual bool _write(const U32 i_numBytes, const void* i_pBuffer);
+
+   void init();
+   bool fillBuffer(const U32 i_startPosition);
+   void clearBuffer();
+   static void calcBlockHead(const U32 i_position, U32 *o_blockHead);
+   static void calcBlockBounds(const U32 i_position, U32 *o_blockHead, U32 *o_blockTail);
+   void setStatus();
+
+   U32  mStreamCaps;                   // dependent on access mode
+
+private:
+   Torque::FS::FileRef mFile;          // file being streamed
+
+   U8   mBuffer[BUFFER_SIZE];
+   U32  mBuffHead;                     // first valid position of buffer (from start-of-file)
+   U32  mBuffPos;                      // next read or write will occur here
+   U32  mBuffTail;                     // last valid position in buffer (inclusive)
+   bool mDirty;                        // whether buffer has been written to
+   bool mEOF;                          // whether disk reads have reached the end-of-file
+
+   FileStream(const FileStream &i_fileStrm);             // disable copy constructor
+   FileStream& operator=(const FileStream &i_fileStrm);  // disable assignment operator
+};
+
+#endif // _FILE_STREAM_H

Engine/source/core/stream/fileStreamObject.cpp

@@ -0,0 +1,175 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#include "core/stream/fileStreamObject.h"
+#include "console/engineAPI.h"
+
+//-----------------------------------------------------------------------------
+// Local Globals
+//-----------------------------------------------------------------------------
+
+static const struct
+{
+   const char *strMode;
+   Torque::FS::File::AccessMode mode;
+} gModeMap[]=
+{
+   { "read", Torque::FS::File::Read },
+   { "write", Torque::FS::File::Write },
+   { "readwrite", Torque::FS::File::ReadWrite },
+   { "writeappend", Torque::FS::File::WriteAppend },
+   { NULL, (Torque::FS::File::AccessMode)0 }
+};
+
+//-----------------------------------------------------------------------------
+// Constructor/Destructor
+//-----------------------------------------------------------------------------
+
+FileStreamObject::FileStreamObject()
+{
+}
+
+FileStreamObject::~FileStreamObject()
+{
+   close();
+}
+
+IMPLEMENT_CONOBJECT(FileStreamObject);
+
+ConsoleDocClass( FileStreamObject,
+   "@brief A wrapper around StreamObject for parsing text and data from files.\n\n"
+
+   "FileStreamObject inherits from StreamObject and provides some unique methods for working "
+   "with strings.  If you're looking for general file handling, you may want to use FileObject.\n\n"
+
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Get the status and print it\n"
+   "%status = %fsObject.getStatus();\n"
+   "echo(%status);\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@see StreamObject for the list of inherited functions variables\n"
+   "@see FileObject for general file handling.\n"
+
+   "@ingroup FileSystem\n"
+);
+
+//-----------------------------------------------------------------------------
+
+bool FileStreamObject::onAdd()
+{
+   // [tom, 2/12/2007] Skip over StreamObject's onAdd() so that we can
+   // be instantiated from script.
+   return SimObject::onAdd();
+}
+
+//-----------------------------------------------------------------------------
+// Public Methods
+//-----------------------------------------------------------------------------
+
+bool FileStreamObject::open(const char *filename, Torque::FS::File::AccessMode mode)
+{
+   close();
+
+   if(! mFileStream.open(filename, mode))
+      return false;
+
+   mStream = &mFileStream;
+   return true;
+}
+
+void FileStreamObject::close()
+{
+   mFileStream.close();
+   mStream = NULL;
+}
+
+//-----------------------------------------------------------------------------
+// Console Methods
+//-----------------------------------------------------------------------------
+
+DefineEngineMethod( FileStreamObject, open, bool, (const char* filename, const char* openMode),,
+   "@brief Open a file for reading, writing, reading and writing, or appending\n\n"
+   
+   "Using \"Read\" for the open mode allows you to parse the contents of file, but not making modifications. \"Write\" will create a new "
+   "file if it does not exist, or erase the contents of an existing file when opened. Write also allows you to modify the contents of the file.\n\n"
+
+   "\"ReadWrite\" will provide the ability to parse data (read it in) and manipulate data (write it out) interchangeably. Keep in mind the stream can "
+   "move during each operation. Finally, \"WriteAppend\" will open a file if it exists, but will not clear the contents. You can write new data starting "
+   " at the end of the files existing contents.\n\n"
+
+   "@param filename Name of file to open\n"
+   "@param openMode One of \"Read\", \"Write\", \"ReadWrite\" or \"WriteAppend\"\n\n"
+
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Get the status and print it\n"
+   "%status = %fsObject.getStatus();\n"
+   "echo(%status);\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@return True if the file was successfully opened, false if something went wrong"
+   
+   "@see close()")
+{
+	for(S32 i = 0;gModeMap[i].strMode;++i)
+   {
+      if(dStricmp(gModeMap[i].strMode, openMode) == 0)
+      {
+         Torque::FS::File::AccessMode mode = gModeMap[i].mode;
+         char buffer[1024];
+         Con::expandScriptFilename(buffer, sizeof(buffer), filename);
+         return object->open(buffer, mode);
+      }
+   }
+
+   Con::errorf("FileStreamObject::open - Mode must be one of Read, Write, ReadWrite or WriteAppend.");
+   return false;
+}
+
+DefineEngineMethod( FileStreamObject, close, void, (),,
+   "@brief Close the file. You can no longer read or write to it unless you open it again.\n\n"
+   
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+   
+   "@see open()")
+{
+	object->close();
+}

Engine/source/core/stream/fileStreamObject.h

@@ -0,0 +1,67 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#ifndef _FILESTREAMOBJECT_H_
+#define _FILESTREAMOBJECT_H_
+
+#ifndef _STREAMOBJECT_H_
+#include "core/stream/streamObject.h"
+#endif
+
+#ifndef _FILESTREAM_H_
+#include "core/stream/fileStream.h"
+#endif
+
+
+class FileStreamObject : public StreamObject
+{
+   typedef StreamObject Parent;
+
+protected:
+   FileStream mFileStream;
+
+public:
+   FileStreamObject();
+   virtual ~FileStreamObject();
+   DECLARE_CONOBJECT(FileStreamObject);
+
+   virtual bool onAdd();
+
+   //-----------------------------------------------------------------------------
+   /// @brief Open a file
+   /// 
+   /// @param filename Name of file to open
+   /// @param mode One of #Torque::FS::File::Read, #Torque::FS::File::Write, #Torque::FS::File::ReadWrite or #Torque::FS::File::WriteAppend
+   /// @return true for success, false for failure
+   /// @see close()
+   //-----------------------------------------------------------------------------
+   bool open(const char *filename, Torque::FS::File::AccessMode mode);
+
+   //-----------------------------------------------------------------------------
+   /// @brief Close the file
+   /// 
+   /// @see open()
+   //-----------------------------------------------------------------------------
+   void close();
+};
+
+#endif // _FILESTREAMOBJECT_H_

Engine/source/core/stream/ioHelper.h

@@ -0,0 +1,115 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#ifndef _UTIL_IOHELPER_H_
+#define _UTIL_IOHELPER_H_
+
+#ifndef _CORE_STREAM_H_
+#include "core/stream/stream.h"
+#endif
+
+/// Helper templates to aggregate IO operations - generally used in
+/// template expansion.
+namespace IOHelper
+{
+   template<class A,class B,class C,class D,class E,class F,class G,class H,class I,class J>
+      inline bool reads(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f,G& g,H& h,I& i,J& j)
+   { s->read(&a); s->read(&b); s->read(&c); s->read(&d); s->read(&e); s->read(&f); s->read(&g); s->read(&h); s->read(&i); return s->read(&j); }
+
+   template<class A,class B,class C,class D,class E,class F,class G,class H,class I>
+      inline bool reads(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f,G& g,H& h,I& i)
+   { s->read(&a); s->read(&b); s->read(&c); s->read(&d); s->read(&e); s->read(&f); s->read(&g); s->read(&h); return s->read(&i); }
+
+   template<class A,class B,class C,class D,class E,class F,class G,class H>
+      inline bool reads(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f,G& g,H& h)
+   { s->read(&a); s->read(&b); s->read(&c); s->read(&d); s->read(&e); s->read(&f); s->read(&g); return s->read(&h); }
+
+   template<class A,class B,class C,class D,class E,class F,class G>
+      inline bool reads(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f,G& g)
+   { s->read(&a); s->read(&b); s->read(&c); s->read(&d); s->read(&e); s->read(&f); return s->read(&g); }
+
+   template<class A,class B,class C,class D,class E,class F>
+      inline bool reads(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f)
+   { s->read(&a); s->read(&b); s->read(&c); s->read(&d); s->read(&e); return s->read(&f); }
+
+   template<class A,class B,class C,class D,class E>
+      inline bool reads(Stream *s,A& a,B& b,C& c,D& d,E& e)
+   { s->read(&a); s->read(&b); s->read(&c); s->read(&d); return s->read(&e); }
+
+   template<class A,class B,class C,class D>
+      inline bool reads(Stream *s,A& a,B& b,C& c,D& d)
+   { s->read(&a); s->read(&b); s->read(&c); return s->read(&d); }
+
+   template<class A,class B,class C>
+      inline bool reads(Stream *s,A& a,B& b,C& c)
+   { s->read(&a); s->read(&b); return s->read(&c); }
+
+   template<class A,class B>
+      inline bool reads(Stream *s,A& a,B& b)
+   { s->read(&a); return s->read(&b); }
+
+   template<class A>
+      inline bool reads(Stream *s,A& a)
+   { return s->read(&a); }
+
+   template<class A,class B,class C,class D,class E,class F,class G,class H,class I,class J>
+      inline bool writes(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f,G& g,H& h,I& i,J& j)
+   { s->write(a); s->write(b); s->write(c); s->write(d); s->write(e); s->write(f); s->write(g); s->write(h); s->write(i); return s->write(j); }
+
+   template<class A,class B,class C,class D,class E,class F,class G,class H,class I>
+      inline bool writes(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f,G& g,H& h,I& i)
+   { s->write(a); s->write(b); s->write(c); s->write(d); s->write(e); s->write(f); s->write(g); s->write(h); return s->write(i); }
+
+   template<class A,class B,class C,class D,class E,class F,class G,class H>
+      inline bool writes(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f,G& g,H& h)
+   { s->write(a); s->write(b); s->write(c); s->write(d); s->write(e); s->write(f); s->write(g); return s->write(h); }
+
+   template<class A,class B,class C,class D,class E,class F,class G>
+      inline bool writes(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f,G& g)
+   { s->write(a); s->write(b); s->write(c); s->write(d); s->write(e); s->write(f); return s->write(g); }
+
+   template<class A,class B,class C,class D,class E,class F>
+      inline bool writes(Stream *s,A& a,B& b,C& c,D& d,E& e,F& f)
+   { s->write(a); s->write(b); s->write(c); s->write(d); s->write(e); return s->write(f); }
+
+   template<class A,class B,class C,class D,class E>
+      inline bool writes(Stream *s,A& a,B& b,C& c,D& d,E& e)
+   { s->write(a); s->write(b); s->write(c); s->write(d); return s->write(e); }
+
+   template<class A,class B,class C,class D>
+      inline bool writes(Stream *s,A& a,B& b,C& c,D& d)
+   { s->write(a); s->write(b); s->write(c); return s->write(d); }
+
+   template<class A,class B,class C>
+      inline bool writes(Stream *s,A& a,B& b,C& c)
+   { s->write(a); s->write(b); return s->write(c); }
+
+   template<class A,class B>
+      inline bool writes(Stream *s,A& a,B& b)
+   { s->write(a); return s->write(b); }
+
+   template<class A>
+      inline bool writes(Stream *s,A& a)
+   { return s->write(a); }
+}
+
+#endif

Engine/source/core/stream/memStream.cpp

@@ -0,0 +1,233 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#include "platform/platform.h"
+#include "core/stream/memStream.h"
+
+
+MemStream::MemStream( U32 growSize,
+                      bool allowRead,
+                      bool allowWrite )
+   :  mGrowSize( growSize ),
+      mBufferSize( 0 ),
+      mStreamSize( 0 ),
+      mBufferBase( NULL ),
+      mInstCaps( 0 ),
+      mCurrentPosition( 0 )
+{
+   AssertFatal( allowRead || allowWrite, "Either write or read must be allowed" );
+
+   if ( allowRead )
+      mInstCaps |= Stream::StreamRead;
+   if ( allowWrite )
+      mInstCaps |= Stream::StreamWrite;
+
+   mOwnsMemory = true;
+   setStatus( mGrowSize > 0 ? Ok : EOS );
+}
+
+MemStream::MemStream( U32 bufferSize,
+                      void *buffer,
+                      bool allowRead,
+                      bool allowWrite)
+ : mGrowSize( 0 ),
+   mBufferSize( bufferSize ),
+   mStreamSize( bufferSize ),
+   mBufferBase( buffer ),
+   mInstCaps( 0 ),
+   mCurrentPosition( 0 ),
+   mOwnsMemory( false )
+{
+   AssertFatal( bufferSize > 0,  "Invalid buffer size");
+   AssertFatal( allowRead || allowWrite, "Either write or read must be allowed");
+
+   if ( allowRead )
+      mInstCaps |= Stream::StreamRead;
+   if ( allowWrite )
+      mInstCaps |= Stream::StreamWrite;
+
+   if ( !buffer )
+   {
+      mOwnsMemory = true;
+      mBufferBase = dMalloc( mBufferSize );
+   }
+
+   setStatus( Ok );
+}
+
+MemStream::~MemStream()
+{
+   if ( mOwnsMemory )
+      dFree( mBufferBase );
+
+   mBufferBase     = NULL;
+   mCurrentPosition = 0;
+
+   setStatus( Closed );
+}
+
+U32 MemStream::getStreamSize()
+{
+   AssertFatal( getStatus() != Closed, "Stream not open, size undefined" );
+
+   return mStreamSize;
+}
+
+bool MemStream::hasCapability(const Capability in_cap) const
+{
+   // Closed streams can't do anything
+   //
+   if (getStatus() == Closed)
+      return false;
+
+   U32 totalCaps = U32(Stream::StreamPosition) | mInstCaps;
+
+   return (U32(in_cap) & totalCaps) != 0;
+}
+
+U32 MemStream::getPosition() const
+{
+   AssertFatal(getStatus() != Closed, "Position of a closed stream is undefined");
+
+   return mCurrentPosition;
+}
+
+bool MemStream::setPosition(const U32 in_newPosition)
+{
+   AssertFatal(getStatus() != Closed, "SetPosition of a closed stream is not allowed");
+   AssertFatal(in_newPosition <= mStreamSize, "Invalid position");
+
+   mCurrentPosition = in_newPosition;
+   if (mCurrentPosition > mStreamSize) {
+      // Never gets here in debug version, this is for the release builds...
+      //
+      setStatus(UnknownError);
+      return false;
+   } else if (mCurrentPosition == mStreamSize) {
+      setStatus(EOS);
+   } else {
+      setStatus(Ok);
+   }
+
+   return true;
+}
+
+bool MemStream::_read(const U32 in_numBytes, void *out_pBuffer)
+{
+   AssertFatal(getStatus() != Closed, "Attempted read from a closed stream");
+
+   if (in_numBytes == 0)
+      return true;
+
+   AssertFatal(out_pBuffer != NULL, "Invalid output buffer");
+
+   if (hasCapability(StreamRead) == false) {
+      AssertWarn(false, "Reading is disallowed on this stream");
+      setStatus(IllegalCall);
+      return false;
+   }
+
+   bool success     = true;
+   U32  actualBytes = in_numBytes;
+   if ((mCurrentPosition + in_numBytes) > mStreamSize) {
+      success = false;
+      actualBytes = mStreamSize - mCurrentPosition;
+   }
+
+   // Obtain a current pointer, and do the copy
+   const void* pCurrent = (const void*)((const U8*)mBufferBase + mCurrentPosition);
+   dMemcpy(out_pBuffer, pCurrent, actualBytes);
+
+   // Advance the stream position
+   mCurrentPosition += actualBytes;
+
+   if (!success)
+      setStatus(EOS);
+   else
+      setStatus(Ok);
+
+   return success;
+}
+
+bool MemStream::_write(const U32 in_numBytes, const void *in_pBuffer)
+{
+   AssertFatal(getStatus() != Closed, "Attempted write to a closed stream");
+
+   if (in_numBytes == 0)
+      return true;
+
+   if (hasCapability(StreamWrite) == false) 
+   {
+      AssertWarn(0, "Writing is disallowed on this stream");
+      setStatus(IllegalCall);
+      return false;
+   }
+
+   bool success     = true;
+   U32  actualBytes = in_numBytes;
+   if ((mCurrentPosition + in_numBytes) > mBufferSize) 
+   {
+      if ( mGrowSize > 0 ) 
+      {
+         U32 growSize = (mCurrentPosition + in_numBytes) - mBufferSize;
+         mBufferSize += growSize + ( mGrowSize - ( growSize % mGrowSize ) );
+         mBufferBase = dRealloc( mBufferBase, mBufferSize );
+      } 
+      else
+      {
+         success = false;
+         actualBytes = mBufferSize - mCurrentPosition;
+      }
+   }
+
+   AssertFatal(in_pBuffer != NULL, "Invalid input buffer");
+
+   // Obtain a current pointer, and do the copy
+   void* pCurrent = (void*)((U8*)mBufferBase + mCurrentPosition);
+   dMemcpy(pCurrent, in_pBuffer, actualBytes);
+
+   // Advance the stream position
+   mCurrentPosition += actualBytes;
+   if (mCurrentPosition > mStreamSize)
+      mStreamSize = mCurrentPosition;
+
+   if (mCurrentPosition == mStreamSize)
+      setStatus(EOS);
+   else
+      setStatus(Ok);
+
+   return success;
+}
+
+void *MemStream::takeBuffer()
+{
+   void *buffer = mBufferBase;
+   
+   mBufferBase = NULL;
+   mBufferSize = 0;
+   mStreamSize = 0;
+   mCurrentPosition = 0;
+
+   setStatus(EOS);
+
+   return buffer;
+}

Engine/source/core/stream/memStream.h

@@ -0,0 +1,104 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#ifndef _MEMSTREAM_H_
+#define _MEMSTREAM_H_
+
+#ifndef _STREAM_H_
+#include "core/stream/stream.h"
+#endif
+
+
+/// The MemStream class is used to read and write to a memory buffer.
+class MemStream : public Stream
+{
+   typedef Stream Parent;
+
+   protected:
+
+      /// The amount to grow the buffer when we run out of
+      /// space writing to the stream.
+      U32 mGrowSize;
+
+      /// The actual size of the memory buffer.  It is always
+      /// greater than or equal to the mStreamSize.
+      U32 mBufferSize;
+
+      /// The size of the data in the stream. It is always
+      /// less than or equal to the mBufferSize.
+      U32 mStreamSize;
+
+      /// The memory buffer.
+      void *mBufferBase;
+
+      /// If true the memory is owned by the steam and it
+      /// will be deleted in the destructor.
+      bool mOwnsMemory;
+
+      ///
+      U32 mInstCaps;
+
+      /// Our current read/write position within the buffer.
+      U32 mCurrentPosition;
+
+   public:
+
+      /// This constructs an empty memory stream that will grow 
+      /// in increments as needed.
+      MemStream(  U32 growSize,
+                  bool allowRead  = true,
+                  bool allowWrite = true );
+
+      /// This constructs the stream with a fixed size memory buffer.  If
+      /// buffer is null then it will be allocated for you.
+      MemStream(  U32 bufferSize,
+                  void *buffer,
+                  bool allowRead  = true,
+                  bool allowWrite = true );
+
+      /// The destructor.
+      virtual ~MemStream();
+
+   protected:
+
+      // Stream
+      bool _read( const U32 in_numBytes,  void *out_pBuffer );
+      bool _write( const U32 in_numBytes, const void *in_pBuffer );
+
+   public:
+
+      // Stream
+      bool hasCapability( const Capability caps ) const;
+      U32 getPosition() const;
+      bool setPosition( const U32 in_newPosition );
+      U32 getStreamSize();
+
+      /// Returns the memory buffer.
+      void *getBuffer() { return mBufferBase; }
+      const void *getBuffer() const { return mBufferBase; }
+
+      /// Takes the memory buffer reseting the stream.
+      void *takeBuffer();
+
+};
+
+#endif //_MEMSTREAM_H_

Engine/source/core/stream/stream.cpp

@@ -0,0 +1,378 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#include "core/color.h"
+#include "core/util/rawData.h"
+#include "core/frameAllocator.h"
+#include "platform/platformNet.h"
+
+#include "core/stream/stream.h"
+
+#include "core/stringTable.h"
+#include "core/strings/stringFunctions.h"
+
+#include "core/util/byteBuffer.h"
+#include "core/util/endian.h"
+#include "core/util/str.h"
+
+
+#define IMPLEMENT_OVERLOADED_READ(type)      \
+   bool Stream::read(type* out_read)         \
+   {                                         \
+      return read(sizeof(type), out_read);   \
+   }
+
+#define IMPLEMENT_OVERLOADED_WRITE(type)        \
+   bool Stream::write(type in_write)            \
+   {                                            \
+      return write(sizeof(type), &in_write);    \
+   }
+
+#define IMPLEMENT_ENDIAN_OVERLOADED_READ(type)  \
+   bool Stream::read(type* out_read)            \
+   {                                            \
+      type temp;                                \
+      bool success = read(sizeof(type), &temp); \
+      *out_read = convertLEndianToHost(temp);   \
+      return success;                           \
+   }
+
+#define IMPLEMENT_ENDIAN_OVERLOADED_WRITE(type)    \
+   bool Stream::write(type in_write)               \
+   {                                               \
+      type temp = convertHostToLEndian(in_write);  \
+      return write(sizeof(type), &temp);           \
+   }
+
+IMPLEMENT_OVERLOADED_WRITE(S8)
+IMPLEMENT_OVERLOADED_WRITE(U8)
+
+IMPLEMENT_ENDIAN_OVERLOADED_WRITE(S16)
+IMPLEMENT_ENDIAN_OVERLOADED_WRITE(S32)
+IMPLEMENT_ENDIAN_OVERLOADED_WRITE(U16)
+IMPLEMENT_ENDIAN_OVERLOADED_WRITE(U32)
+IMPLEMENT_ENDIAN_OVERLOADED_WRITE(U64)
+IMPLEMENT_ENDIAN_OVERLOADED_WRITE(F32)
+IMPLEMENT_ENDIAN_OVERLOADED_WRITE(F64)
+
+IMPLEMENT_OVERLOADED_READ(S8)
+IMPLEMENT_OVERLOADED_READ(U8)
+
+IMPLEMENT_ENDIAN_OVERLOADED_READ(S16)
+IMPLEMENT_ENDIAN_OVERLOADED_READ(S32)
+IMPLEMENT_ENDIAN_OVERLOADED_READ(U16)
+IMPLEMENT_ENDIAN_OVERLOADED_READ(U32)
+IMPLEMENT_ENDIAN_OVERLOADED_READ(U64)
+IMPLEMENT_ENDIAN_OVERLOADED_READ(F32)
+IMPLEMENT_ENDIAN_OVERLOADED_READ(F64)
+
+
+Stream::Stream()
+ : m_streamStatus(Closed)
+{
+}
+
+const char* Stream::getStatusString(const Status in_status)
+{
+   switch (in_status) {
+      case Ok:
+         return "StreamOk";
+      case IOError:
+         return "StreamIOError";
+      case EOS:
+         return "StreamEOS";
+      case IllegalCall:
+         return "StreamIllegalCall";
+      case Closed:
+         return "StreamClosed";
+      case UnknownError:
+         return "StreamUnknownError";
+
+     default:
+      return "Invalid Stream::Status";
+   }
+}
+
+void Stream::writeString(const char *string, S32 maxLen)
+{
+   S32 len = string ? dStrlen(string) : 0;
+   if(len > maxLen)
+      len = maxLen;
+
+   write(U8(len));
+   if(len)
+      write(len, string);
+}
+
+void Stream::readString(char buf[256])
+{
+   U8 len;
+   read(&len);
+   read(S32(len), buf);
+   buf[len] = 0;
+}
+
+const char *Stream::readSTString(bool casesens)
+{
+   char buf[256];
+   readString(buf);
+   return StringTable->insert(buf, casesens);
+}
+
+void Stream::readLongString(U32 maxStringLen, char *stringBuf)
+{
+   U32 len;
+   read(&len);
+   if(len > maxStringLen)
+   {
+      m_streamStatus = IOError;
+      return;
+   }
+   read(len, stringBuf);
+   stringBuf[len] = 0;
+}
+
+void Stream::writeLongString(U32 maxStringLen, const char *string)
+{
+   U32 len = dStrlen(string);
+   if(len > maxStringLen)
+      len = maxStringLen;
+   write(len);
+   write(len, string);
+}
+
+void Stream::readLine(U8 *buffer, U32 bufferSize)
+{
+   bufferSize--;  // account for NULL terminator
+   U8 *buff = buffer;
+   U8 *buffEnd = buff + bufferSize;
+   *buff = '\r';
+
+   // strip off preceding white space
+   while ( *buff == '\r' )
+   {
+      if ( !read(buff) || *buff == '\n' )
+      {
+         *buff = 0;
+         return;
+      }
+   }
+
+   // read line
+   while ( buff != buffEnd && read(++buff) && *buff != '\n' )
+   {
+      if ( *buff == '\r' )
+      {
+
+#if defined(TORQUE_OS_MAC)
+      U32 pushPos = getPosition(); // in case we need to back up.
+      if (read(buff)) // feeling free to overwrite the \r as the NULL below will overwrite again...
+	      if (*buff != '\n') // then push our position back.
+	         setPosition(pushPos);
+	   break; // we're always done after seeing the CR...
+#else
+      buff--; // 'erases' the CR of a CRLF
+#endif
+
+      }
+   }
+   *buff = 0;
+}
+
+void Stream::writeText(const char *text)
+{
+   if (text && text[0])
+      write(dStrlen(text), text);
+}
+
+void Stream::writeLine(const U8 *buffer)
+{
+   write(dStrlen((const char *)buffer), buffer);
+   write(2, "\r\n");
+}
+
+void Stream::_write(const String & str)
+{
+   U32 len = str.length();
+
+   if (len<255)
+      write(U8(len));
+   else
+   {
+      // longer string, write full length
+      write(U8(255));
+
+      // fail if longer than 16 bits (will truncate string modulo 2^16)
+      AssertFatal(len < (1<<16),"String too long");
+
+      len &= (1<<16)-1;
+      write(U16(len));
+   }
+
+   write(len,str.c_str());
+}
+
+void Stream::_read(String * str)
+{
+   U16 len;
+
+   U8 len8;
+   read(&len8);
+   if (len8==255)
+      read(&len);
+   else
+      len = len8;
+
+   char * buffer = (char*)FrameAllocator::alloc(len);
+   read(len, buffer);
+   *str = String(buffer,len);
+}
+
+
+bool Stream::write(const ColorI& rColor)
+{
+   bool success = write(rColor.red);
+   success     |= write(rColor.green);
+   success     |= write(rColor.blue);
+   success     |= write(rColor.alpha);
+
+   return success;
+}
+
+bool Stream::write(const ColorF& rColor)
+{
+   ColorI temp = rColor;
+   return write(temp);
+}
+
+bool Stream::read(ColorI* pColor)
+{
+   bool success = read(&pColor->red);
+   success     |= read(&pColor->green);
+   success     |= read(&pColor->blue);
+   success     |= read(&pColor->alpha);
+
+   return success;
+}
+
+bool Stream::read(ColorF* pColor)
+{
+   ColorI temp;
+   bool success = read(&temp);
+
+   *pColor = temp;
+   return success;
+}
+
+bool Stream::write(const NetAddress &na)
+{
+   bool success = write(na.type);
+   success &= write(na.port);
+   success &= write(na.netNum[0]);
+   success &= write(na.netNum[1]);
+   success &= write(na.netNum[2]);
+   success &= write(na.netNum[3]);
+   success &= write(na.nodeNum[0]);
+   success &= write(na.nodeNum[1]);
+   success &= write(na.nodeNum[2]);
+   success &= write(na.nodeNum[3]);
+   success &= write(na.nodeNum[4]);
+   success &= write(na.nodeNum[5]);
+   return success;
+}
+
+bool Stream::read(NetAddress *na)
+{
+   bool success = read(&na->type);
+   success &= read(&na->port);
+   success &= read(&na->netNum[0]);
+   success &= read(&na->netNum[1]);
+   success &= read(&na->netNum[2]);
+   success &= read(&na->netNum[3]);
+   success &= read(&na->nodeNum[0]);
+   success &= read(&na->nodeNum[1]);
+   success &= read(&na->nodeNum[2]);
+   success &= read(&na->nodeNum[3]);
+   success &= read(&na->nodeNum[4]);
+   success &= read(&na->nodeNum[5]);
+   return success;
+}
+
+bool Stream::write(const RawData &rd)
+{
+   bool s = write(rd.size);
+   s &= write(rd.size, rd.data);
+   return s;
+}
+
+bool Stream::read(RawData *rd)
+{
+   U32 size = 0;
+   bool s = read(&size);
+
+   rd->alloc(size);
+   s &= read(rd->size, rd->data);
+
+   return s;
+}
+
+bool Stream::write(const Torque::ByteBuffer &rd)
+{
+   bool s = write(rd.getBufferSize());
+   s &= write(rd.getBufferSize(), rd.getBuffer());
+   return s;
+}
+
+bool Stream::read(Torque::ByteBuffer *rd)
+{
+   U32 size = 0;
+   bool s = read(&size);
+
+   rd->resize(size);
+   s &= read(rd->getBufferSize(), rd->getBuffer());
+
+   return s;
+}
+
+bool Stream::copyFrom(Stream *other)
+{
+   U8 buffer[1024];
+   U32 numBytes = other->getStreamSize() - other->getPosition();
+   while((other->getStatus() != Stream::EOS) && numBytes > 0)
+   {
+      U32 numRead = numBytes > sizeof(buffer) ? sizeof(buffer) : numBytes;
+      if(! other->read(numRead, buffer))
+         return false;
+
+      if(! write(numRead, buffer))
+         return false;
+
+      numBytes -= numRead;
+   }
+
+   return true;
+}
+
+Stream* Stream::clone() const
+{
+   return NULL;
+}

Engine/source/core/stream/stream.h

@@ -0,0 +1,242 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#ifndef _STREAM_H_
+#define _STREAM_H_
+
+#ifndef _TORQUE_TYPES_H_
+#include "platform/types.h"
+#endif
+#ifndef _ENDIAN_H_
+#include "core/util/endian.h"
+#endif
+
+
+/// @defgroup stream_overload Primitive Type Stream Operation Overloads
+/// These macros declare the read and write functions for all primitive types.
+/// @{
+#define DECLARE_OVERLOADED_READ(type)  bool read(type* out_read);
+#define DECLARE_OVERLOADED_WRITE(type) bool write(type in_write);
+/// @}
+
+class ColorI;
+class ColorF;
+struct NetAddress;
+class RawData;
+class String;
+
+namespace Torque {
+   class ByteBuffer;
+}
+
+//------------------------------------------------------------------------------
+//-------------------------------------- Base Stream class
+//
+/// Base stream class for streaming data across a specific media
+class Stream
+{
+   // Public structs and enumerations...
+public:
+     /// Status constants for the stream
+   enum Status {
+      Ok = 0,           ///< Ok!
+      IOError,          ///< Read or Write error
+      EOS,              ///< End of Stream reached (mostly for reads)
+      IllegalCall,      ///< An unsupported operation used.  Always w/ accompanied by AssertWarn
+      Closed,           ///< Tried to operate on a closed stream (or detached filter)
+      UnknownError      ///< Catchall
+   };
+
+   enum Capability {
+      StreamWrite    = BIT(0), ///< Can this stream write?
+      StreamRead     = BIT(1), ///< Can this stream read?
+      StreamPosition = BIT(2)  ///< Can this stream position?
+   };
+
+   // Accessible only through inline accessors
+private:
+   Status m_streamStatus;
+
+   // Derived accessible data modifiers...
+protected:
+   void setStatus(const Status in_newStatus) { m_streamStatus = in_newStatus; }
+
+public:
+   Stream();
+   virtual ~Stream() {}
+
+   /// Gets the status of the stream
+   Stream::Status getStatus() const { return m_streamStatus; }
+   /// Gets a printable string form of the status
+   static const char* getStatusString(const Status in_status);
+
+   // Derived classes must override these...
+protected:
+   virtual bool _read(const U32 in_numBytes,  void* out_pBuffer)      = 0;
+   virtual bool _write(const U32 in_numBytes, const void* in_pBuffer) = 0;
+
+   virtual void _write(const String & str);
+   virtual void _read(String * str);
+
+
+public:
+     /// Checks to see if this stream has the capability of a given function
+   virtual bool hasCapability(const Capability caps) const = 0;
+
+   /// Gets the position in the stream
+   virtual U32  getPosition() const                      = 0;
+   /// Sets the position of the stream.  Returns if the new position is valid or not
+   virtual bool setPosition(const U32 in_newPosition) = 0;
+   /// Gets the size of the stream
+   virtual U32  getStreamSize() = 0;
+
+   /// Reads a line from the stream.
+   /// @param buffer buffer to be read into
+   /// @param bufferSize max size of the buffer.  Will not read more than the "bufferSize"
+   void readLine(U8 *buffer, U32 bufferSize);
+   /// writes a line to the stream
+   void writeLine(const U8 *buffer);
+
+   /// Reads a string and inserts it into the StringTable
+   /// @see StringTable
+   const char *readSTString(bool casesens = false);
+   /// Reads a string of maximum 255 characters long
+   virtual void readString(char stringBuf[256]);
+   /// Reads a string that could potentially be more than 255 characters long.
+   /// @param maxStringLen Maximum length to read.  If the string is longer than maxStringLen, only maxStringLen bytes will be read.
+   /// @param stringBuf buffer where data is read into
+   void readLongString(U32 maxStringLen, char *stringBuf);
+   /// Writes a string to the stream.  This function is slightly unstable.
+   /// Only use this if you have a valid string that is not empty.
+   /// writeString is safer.
+   void writeLongString(U32 maxStringLen, const char *string);
+
+   /// Write raw text to the stream
+   void writeText(const char *text);
+
+   /// Writes a string to the stream.
+   virtual void writeString(const char *stringBuf, S32 maxLen=255);
+
+   // read/write real strings
+   void write(const String & str) { _write(str); }
+   void read(String * str) { _read(str); }
+
+   /// Write an integral color to the stream.
+   bool write(const ColorI&);
+   /// Write a floating point color to the stream.
+   bool write(const ColorF&);
+   /// Read an integral color from the stream.
+   bool read(ColorI*);
+   /// Read a floating point color from the stream.
+   bool read(ColorF*);
+
+   /// Write a network address to the stream.
+   bool write(const NetAddress &);
+   /// Read a network address from the stream.
+   bool read(NetAddress*);
+
+   /// Write some raw data onto the stream.
+   bool write(const RawData &);
+   /// Read some raw data from the stream.
+   bool read(RawData *);
+
+   /// Write some raw data onto the stream.
+   bool write(const Torque::ByteBuffer &);
+   /// Read some raw data from the stream.
+   bool read(Torque::ByteBuffer *);
+
+   // Overloaded write and read ops..
+  public:
+   bool read(const U32 in_numBytes,  void* out_pBuffer) {
+      return _read(in_numBytes, out_pBuffer);
+   }
+   bool write(const U32 in_numBytes, const void* in_pBuffer) {
+      return _write(in_numBytes, in_pBuffer);
+   }
+
+   DECLARE_OVERLOADED_WRITE(S8)
+   DECLARE_OVERLOADED_WRITE(U8)
+
+   DECLARE_OVERLOADED_WRITE(S16)
+   DECLARE_OVERLOADED_WRITE(S32)
+   DECLARE_OVERLOADED_WRITE(U16)
+   DECLARE_OVERLOADED_WRITE(U32)
+   DECLARE_OVERLOADED_WRITE(U64)
+   DECLARE_OVERLOADED_WRITE(F32)
+   DECLARE_OVERLOADED_WRITE(F64)
+
+   DECLARE_OVERLOADED_READ(S8)
+   DECLARE_OVERLOADED_READ(U8)
+
+   DECLARE_OVERLOADED_READ(S16)
+   DECLARE_OVERLOADED_READ(S32)
+   DECLARE_OVERLOADED_READ(U16)
+   DECLARE_OVERLOADED_READ(U32)
+   DECLARE_OVERLOADED_READ(U64)
+   DECLARE_OVERLOADED_READ(F32)
+   DECLARE_OVERLOADED_READ(F64)
+
+   // We have to do the bool's by hand, since they are different sizes
+   //  on different compilers...
+   //
+   bool read(bool* out_pRead) {
+      U8 translate;
+      bool success = read(&translate);
+      if (success == false)
+         return false;
+
+      *out_pRead = translate != 0;
+      return true;
+   }
+   bool write(const bool& in_rWrite) {
+      U8 translate = in_rWrite ? U8(1) : U8(0);
+      return write(translate);
+   }
+
+   /// Copy the contents of another stream into this one
+   bool copyFrom(Stream *other);
+
+   /// Write a number of tabs to this stream
+   void writeTabs(U32 count)
+   {
+      char tab[] = "   ";
+      while(count--)
+         write(3, (void*)tab);
+   }
+
+   /// Create an exact replica of this stream.
+   /// Return NULL if the cloning fails.
+   virtual Stream* clone() const;
+};
+
+// This interface is used to provide the amount of bytes actually written/read when using the stream as a 
+// file.  The Stream interface does not provide this information.  This is a lame workaround.  
+class IStreamByteCount
+{
+public:
+   virtual ~IStreamByteCount() {}
+
+   virtual U32 getLastBytesRead() = 0;
+   virtual U32 getLastBytesWritten() = 0;
+};
+
+#endif //_STREAM_H_

Engine/source/core/stream/streamObject.cpp

@@ -0,0 +1,476 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#include "core/stream/streamObject.h"
+#include "console/engineAPI.h"
+
+//-----------------------------------------------------------------------------
+// Constructor/Destructor
+//-----------------------------------------------------------------------------
+
+// This entire class has some major issues. Lots of bad characters when reading and
+// writing. Stream doesn't seem to jump correctly. All in all this either needs
+// to be deprecated, fixed, or someone explain to me what I'm doing wrong.
+// Hard to document when it's not working. -Mich 7/16/2010
+
+StreamObject::StreamObject()
+{
+   mStream = NULL;
+}
+
+StreamObject::StreamObject(Stream *stream)
+{
+   mStream = stream;
+}
+
+StreamObject::~StreamObject()
+{
+}
+
+IMPLEMENT_CONOBJECT(StreamObject);
+
+ConsoleDocClass( StreamObject,
+   "@brief Base class for working with streams.\n\n"
+
+   "You do not instantiate a StreamObject directly.  Instead, it is used as part of a FileStreamObject and ZipObject to "
+   "support working with uncompressed and compressed files respectively."
+
+   "@tsexample\n"
+   "// You cannot actually declare a StreamObject\n"
+   "// Instead, use the derived class \"FileStreamObject\"\n"
+   "%fsObject = FileStreamObject();\n"
+   "@endtsexample\n\n"
+
+   "@see FileStreamObject for the derived class usable in script.\n"
+   "@see ZipObject where StreamObject is used to read and write to files within a zip archive.\n"
+
+   "@ingroup FileSystem\n\n"
+);
+//-----------------------------------------------------------------------------
+
+bool StreamObject::onAdd()
+{
+   if(mStream == NULL)
+   {
+      Con::errorf("StreamObject::onAdd - StreamObject can not be instantiated from script.");
+      return false;
+   }
+   return Parent::onAdd();
+}
+
+//-----------------------------------------------------------------------------
+// Public Methods
+//-----------------------------------------------------------------------------
+
+const char * StreamObject::getStatus()
+{
+   if(mStream == NULL)
+      return "";
+
+   switch(mStream->getStatus())
+   {
+      case Stream::Ok:
+         return "Ok";
+      case Stream::IOError:
+         return "IOError";
+      case Stream::EOS:
+         return "EOS";
+      case Stream::IllegalCall:
+         return "IllegalCall";
+      case Stream::Closed:
+         return "Closed";
+      case Stream::UnknownError:
+         return "UnknownError";
+
+      default:
+         return "Invalid";
+   }
+}
+
+//-----------------------------------------------------------------------------
+
+const char * StreamObject::readLine()
+{
+   if(mStream == NULL)
+      return NULL;
+
+   char *buffer = Con::getReturnBuffer(256);
+   mStream->readLine((U8 *)buffer, 256);
+   return buffer;
+}
+
+const char * StreamObject::readString()
+{
+   if(mStream == NULL)
+      return NULL;
+
+   char *buffer = Con::getReturnBuffer(256);
+   mStream->readString(buffer);
+   return buffer;
+}
+
+const char *StreamObject::readLongString(U32 maxStringLen)
+{
+   if(mStream == NULL)
+      return NULL;
+
+   char *buffer = Con::getReturnBuffer(maxStringLen + 1);
+   mStream->readLongString(maxStringLen, buffer);
+   return buffer;
+}
+
+//-----------------------------------------------------------------------------
+// Console Methods
+//-----------------------------------------------------------------------------
+
+DefineEngineMethod( StreamObject, getStatus, const char*, (),,
+   "@brief Gets a printable string form of the stream's status\n\n"
+   
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Get the status and print it\n"
+   "%status = %fsObject.getStatus();\n"
+   "echo(%status);\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@return String containing status constant, one of the following:\n\n"
+
+   "	OK - Stream is active and no file errors\n\n"
+
+   "	IOError - Something went wrong during read or writing the stream\n\n"
+
+   "	EOS - End of Stream reached (mostly for reads)\n\n"
+
+   "	IllegalCall - An unsupported operation used.  Always w/ accompanied by AssertWarn\n\n"
+
+   "  Closed - Tried to operate on a closed stream (or detached filter)\n\n"
+
+   "	UnknownError - Catch all for an error of some kind\n\n"
+
+   "	Invalid - Entire stream is invalid\n\n")
+{
+	return object->getStatus();
+}
+
+DefineEngineMethod( StreamObject, isEOS, bool, (),,
+   "@brief Tests if the stream has reached the end of the file\n\n"
+   
+   "This is an alternative name for isEOF. Both functions are interchangeable. This simply exists "
+   "for those familiar with some C++ file I/O standards.\n\n"
+
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Keep reading until we reach the end of the file\n"
+   "while( !%fsObject.isEOS() )\n"
+   "{\n"
+   "   %line = %fsObject.readLine();\n"
+   "   echo(%line);\n"
+   "}\n"
+   "// Made it to the end\n"
+   "echo(\"Finished reading file\");\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@return True if the parser has reached the end of the file, false otherwise\n"
+
+   "@see isEOF()")
+{
+	return object->isEOS();
+}
+
+DefineEngineMethod( StreamObject, isEOF, bool, (),,
+   "@brief Tests if the stream has reached the end of the file\n\n"
+   
+   "This is an alternative name for isEOS. Both functions are interchangeable. This simply exists "
+   "for those familiar with some C++ file I/O standards.\n\n"
+   
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Keep reading until we reach the end of the file\n"
+   "while( !%fsObject.isEOF() )\n"
+   "{\n"
+   "   %line = %fsObject.readLine();\n"
+   "   echo(%line);\n"
+   "}\n"
+   "// Made it to the end\n"
+   "echo(\"Finished reading file\");\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@return True if the parser has reached the end of the file, false otherwise\n"
+   
+   "@see isEOS()")
+{
+	return object->isEOS();
+}
+
+//-----------------------------------------------------------------------------
+
+DefineEngineMethod( StreamObject, getPosition, S32, (),,
+   "@brief Gets the position in the stream\n\n"
+   
+   "The easiest way to visualize this is to think of a cursor in a text file. If you have moved the cursor by "
+   "five characters, the current position is 5. If you move ahead 10 more characters, the position is now 15. "
+   "For StreamObject, when you read in the line the position is increased by the number of characters parsed, "
+   "the null terminator, and a newline.\n\n"
+   
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "// This file contains two lines of text repeated:\n"
+   "// Hello World\n"
+   "// Hello World\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Read in the first line\n"
+   "%line = %fsObject.readLine();\n\n"
+   "// Get the position of the stream\n"
+   "%position = %fsObject.getPosition();\n\n"
+   "// Print the current position\n"
+   "// Should be 13, 10 for the words, 1 for the space, 1 for the null terminator, and 1 for the newline\n"
+   "echo(%position);\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@return Number of bytes which stream has parsed so far, null terminators and newlines are included\n"
+   
+   "@see setPosition()")
+{
+	return object->getPosition();
+}
+
+DefineEngineMethod( StreamObject, setPosition, bool, (S32 newPosition),,
+   "@brief Gets the position in the stream\n\n"
+   
+   "The easiest way to visualize this is to think of a cursor in a text file. If you have moved the cursor by "
+   "five characters, the current position is 5. If you move ahead 10 more characters, the position is now 15. "
+   "For StreamObject, when you read in the line the position is increased by the number of characters parsed, "
+   "the null terminator, and a newline. Using setPosition allows you to skip to specific points of the file.\n\n"
+   
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "// This file contains the following two lines:\n"
+   "// 11111111111\n"
+   "// Hello World\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Skip ahead by 12, which will bypass the first line entirely\n"
+   "%fsObject.setPosition(12);\n\n"
+   "// Read in the next line\n"
+   "%line = %fsObject.readLine();\n\n"
+   "// Print the line just read in, should be \"Hello World\"\n"
+   "echo(%line);\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@return Number of bytes which stream has parsed so far, null terminators and newlines are included\n"
+   
+   "@see getPosition()")
+{
+	return object->setPosition(newPosition);
+}
+
+DefineEngineMethod( StreamObject, getStreamSize, S32, (),,
+   "@brief Gets the size of the stream\n\n"
+   
+   "The size is dependent on the type of stream being used. If it is a file stream, returned value will "
+   "be the size of the file. If it is a memory stream, it will be the size of the allocated buffer.\n\n"
+      
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open a file for reading\n"
+   "// This file contains the following two lines:\n"
+   "// HelloWorld\n"
+   "// HelloWorld\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Found out how large the file stream is\n"
+   "// Then print it to the console\n"
+   "// Should be 22\n"
+   "%streamSize = %fsObject.getStreamSize();\n"
+   "echo(%streamSize);\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@return Size of stream, in bytes\n")
+{
+	return object->getStreamSize();
+}
+
+//-----------------------------------------------------------------------------
+DefineEngineMethod( StreamObject, readLine, const char*, (),,
+   "@brief Read a line from the stream.\n\n"
+   
+   "Emphasis on *line*, as in you cannot parse individual characters or chunks of data. "
+   "There is no limitation as to what kind of data you can read.\n\n"
+   
+   "@tsexample\n"
+   "// Create a file stream object for reading\n"
+   "// This file contains the following two lines:\n"
+   "// HelloWorld\n"
+   "// HelloWorld\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "%fsObject.open(\"./test.txt\", \"read\");\n\n"
+   "// Read in the first line\n"
+   "%line = %fsObject.readLine();\n\n"
+   "// Print the line we just read\n"
+   "echo(%line);\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+
+   "@return String containing the line of data that was just read\n"
+   
+   "@see writeLine()")
+{
+	const char *str = object->readLine();
+	return str ? str : "";
+}
+
+DefineEngineMethod( StreamObject, writeLine, void, ( const char* line ),,
+   "@brief Write a line to the stream, if it was opened for writing.\n\n"
+   
+   "There is no limit as to what kind of data you can write. Any format and data is allowable, not just text. "
+   "Be careful of what you write, as whitespace, current values, and literals will be preserved.\n\n"
+
+   "@param line The data we are writing out to file."
+   
+   "@tsexample\n"
+   "// Create a file stream\n"
+   "%fsObject = new FileStreamObject();\n\n"
+   "// Open the file for writing\n"
+   "// If it does not exist, it is created. If it does exist, the file is cleared\n"
+   "%fsObject.open(\"./test.txt\", \"write\");\n\n"
+   "// Write a line to the file\n"
+   "%fsObject.writeLine(\"Hello World\");\n\n"
+   "// Write another line to the file\n"
+   "%fsObject.writeLine(\"Documentation Rocks!\");\n\n"
+   "// Always remember to close a file stream when finished\n"
+   "%fsObject.close();\n"
+   "@endtsexample\n\n"
+   
+   "@see readLine()")
+{
+	object->writeLine((U8 *)line);
+}
+
+//-----------------------------------------------------------------------------
+/* readSTString, readString, and readLongString need to be fixed or deprecated */
+DefineEngineMethod(StreamObject, readSTString, String, ( bool caseSensitive ), ( false ),
+   "@brief Read in a string and place it on the string table.\n\n"
+   "@param caseSensitive If false then case will not be taken into account when attempting "
+   "to match the read in string with what is already in the string table.\n"
+   "@return The string that was read from the stream.\n"
+   "@see writeString()"
+   
+   "@note When working with these particular string reading and writing methods, the stream "
+   "begins with the length of the string followed by the string itself, and does not include "
+   "a NULL terminator.")
+{
+   const char *str = object->readSTString(caseSensitive);
+   return str ? str : "";
+}
+
+DefineEngineMethod(StreamObject, readString, String, (),,
+   "@brief Read a string up to a maximum of 256 characters"
+   "@return The string that was read from the stream.\n"
+   "@see writeString()"
+   
+   "@note When working with these particular string reading and writing methods, the stream "
+   "begins with the length of the string followed by the string itself, and does not include "
+   "a NULL terminator.")
+{
+   const char *str = object->readString();
+   return str ? str : "";
+}
+
+DefineEngineMethod(StreamObject, readLongString, String, ( S32 maxLength ),,
+   "@brief Read in a string up to the given maximum number of characters.\n\n"
+   "@param maxLength The maximum number of characters to read in.\n"
+   "@return The string that was read from the stream.\n"
+   "@see writeLongString()"
+   
+   "@note When working with these particular string reading and writing methods, the stream "
+   "begins with the length of the string followed by the string itself, and does not include "
+   "a NULL terminator.")
+{
+   const char *str = object->readLongString(maxLength);
+   return str ? str : "";
+}
+
+DefineEngineMethod(StreamObject, writeLongString, void, ( S32 maxLength, const char* string ),,
+   "@brief Write out a string up to the maximum number of characters.\n\n"
+   "@param maxLength The maximum number of characters that will be written.\n"
+   "@param string The string to write out to the stream.\n"
+   "@see readLongString()"
+   
+   "@note When working with these particular string reading and writing methods, the stream "
+   "begins with the length of the string followed by the string itself, and does not include "
+   "a NULL terminator.")
+{
+   object->writeLongString(maxLength, string);
+}
+
+DefineEngineMethod(StreamObject, writeString, void, ( const char* string, S32 maxLength ), ( 256 ),
+   "@brief Write out a string with a default maximum length of 256 characters.\n\n"
+   "@param string The string to write out to the stream\n"
+   "@param maxLength The maximum string length to write out with a default of 256 characters.  This "
+   "value should not be larger than 256 as it is written to the stream as a single byte.\n"
+   "@see readString()"
+   
+   "@note When working with these particular string reading and writing methods, the stream "
+   "begins with the length of the string followed by the string itself, and does not include "
+   "a NULL terminator.")
+{
+   object->writeString(string, maxLength);
+}
+
+//-----------------------------------------------------------------------------
+
+DefineEngineMethod(StreamObject, copyFrom, bool, (SimObject* other),,
+   "@brief Copy from another StreamObject into this StreamObject\n\n"
+   "@param other The StreamObject to copy from.\n"
+   "@return True if the copy was successful.\n")
+{
+   StreamObject *so = dynamic_cast<StreamObject *>(other);
+   if(so == NULL)
+      return false;
+
+   return object->copyFrom(so);
+}

Engine/source/core/stream/streamObject.h

@@ -0,0 +1,138 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#ifndef _STREAMOBJECT_H_
+#define _STREAMOBJECT_H_
+
+#ifndef _SIMOBJECT_H_
+#include "console/simObject.h"
+#endif
+
+/// @addtogroup zip_group
+// @{
+
+//-----------------------------------------------------------------------------
+/// @brief Script wrapper for the Stream class
+/// 
+/// It is not possible to instantiate StreamObject in script. Instead,
+/// it is instantiated in C++ code and returned to script.
+/// 
+/// This was mainly intended to allow the \ref zip_group "zip code" to
+/// provide the stream interface to script.
+//-----------------------------------------------------------------------------
+class StreamObject : public SimObject
+{
+   typedef SimObject Parent;
+
+protected:
+   Stream *mStream;
+
+public:
+   StreamObject();
+   StreamObject(Stream *stream);
+   virtual ~StreamObject();
+
+   DECLARE_CONOBJECT(StreamObject);
+
+   virtual bool onAdd();
+
+   /// Set the stream to allow reuse of the object
+   void setStream(Stream *stream)      { mStream = stream; }
+
+   /// Get the underlying stream. Used with setStream() to support object reuse
+   Stream *getStream()           { return mStream; }
+
+   /// Gets a printable string form of the status
+   const char* getStatus();
+
+   bool isEOS()                  { return mStream ? mStream->getStatus() == Stream::EOS : true; }
+
+   /// Gets the position in the stream
+   U32  getPosition() const
+   {
+      return mStream ? mStream->getPosition() : 0;
+   }
+
+   /// Sets the position of the stream.  Returns if the new position is valid or not
+   bool setPosition(const U32 in_newPosition)
+   {
+      return mStream ? mStream->setPosition(in_newPosition) : false;
+   }
+
+   /// Gets the size of the stream
+   U32  getStreamSize()
+   {
+      return mStream ? mStream->getStreamSize() : 0;
+   }
+
+   /// Reads a line from the stream.
+   const char * readLine();
+
+   /// Writes a line to the stream
+   void writeLine(U8 *buffer)
+   {
+      if(mStream)
+         mStream->writeLine(buffer);
+   }
+
+   /// Reads a string and inserts it into the StringTable
+   /// @see StringTable
+   const char *readSTString(bool casesens = false)
+   {
+      return mStream ? mStream->readSTString(casesens) : NULL;
+   }
+
+   /// Reads a string of maximum 255 characters long
+   const char *readString();
+   /// Reads a string that could potentially be more than 255 characters long.
+   /// @param maxStringLen Maximum length to read.  If the string is longer than maxStringLen, only maxStringLen bytes will be read.
+   /// @param stringBuf buffer where data is read into
+   const char * readLongString(U32 maxStringLen);
+   /// Writes a string to the stream.  This function is slightly unstable.
+   /// Only use this if you have a valid string that is not empty.
+   /// writeString is safer.
+   void writeLongString(U32 maxStringLen, const char *string)
+   {
+      if(mStream)
+         mStream->writeLongString(maxStringLen, string);
+   }
+
+   /// Writes a string to the stream.
+   void writeString(const char *stringBuf, S32 maxLen=255)
+   {
+      if(mStream)
+         mStream->writeString(stringBuf, maxLen);
+   }
+
+   /// Copy the contents of another stream into this one
+   bool copyFrom(StreamObject *other)
+   {
+      if(mStream)
+         return mStream->copyFrom(other->getStream());
+
+      return false;
+   }
+};
+
+// @}
+
+#endif // _STREAMOBJECT_H_

Engine/source/core/stream/tStream.h

@@ -0,0 +1,373 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2012 GarageGames, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//-----------------------------------------------------------------------------
+
+#ifndef _TSTREAM_H_
+#define _TSTREAM_H_
+
+#ifndef _PLATFORM_H_
+   #include "platform/platform.h"
+#endif
+#ifndef _TYPETRAITS_H_
+   #include "platform/typetraits.h"
+#endif
+
+
+//TODO: error handling
+
+
+/// @file
+/// Definitions for lightweight componentized streaming.
+///
+/// This file is an assembly of lightweight classes/interfaces that
+/// describe various aspects of streaming classes.  The advantage
+/// over using Torque's Stream class is that very little requirements
+/// are placed on implementations, that specific abilities can be
+/// mixed and matched very selectively, and that complex stream processing
+/// chains can be hidden behind very simple stream interfaces.
+
+
+
+/// Status of an asynchronous I/O operation.
+enum EAsyncIOStatus
+{
+   ASYNC_IO_Pending,       ///< I/O is still in queue or being processed.
+   ASYNC_IO_Complete,      ///< I/O has completed successfully.
+   ASYNC_IO_Error          ///< I/O has aborted with an error.
+};
+
+//-----------------------------------------------------------------------------
+//    Several component interfaces.
+//-----------------------------------------------------------------------------
+
+/// Interface for streams with an explicit position property.
+template< typename P = U32 >
+class IPositionable
+{
+   public:
+   
+      typedef void Parent;
+
+      /// The type used to indicate positions.
+      typedef P PositionType;
+
+      /// @return the current position.
+      virtual PositionType getPosition() const = 0;
+      
+      /// Set the current position to be "pos".
+      /// @param pos The new position.
+      virtual void setPosition( PositionType pos ) = 0;
+};
+
+/// Interface for structures that allow their state to be reset.
+class IResettable
+{
+   public:
+   
+      typedef void Parent;
+   
+      /// Reset to the initial state.
+      virtual void reset() = 0;
+};
+
+/// Interface for structures of finite size.
+template< typename T = U32 >
+class ISizeable
+{
+   public:
+   
+      typedef void Parent;
+
+      /// The type used to indicate the structure's size.
+      typedef T SizeType;
+
+      /// @return the size of the structure in number of elements.
+      virtual SizeType getSize() const = 0;
+};
+
+/// Interface for structures that represent processes.
+class IProcess
+{
+   public:
+   
+      typedef void Parent;
+
+      /// Start the process.
+      virtual void start() = 0;
+
+      /// Stop the process.
+      virtual void stop() = 0;
+
+      /// Pause the process.
+      virtual void pause() = 0;
+};
+
+/// Interface for objects that need continuous explicit updates from
+/// an outside source.
+class IPolled
+{
+   public:
+
+      typedef void Parent;
+   
+      /// Update the object state.
+      virtual bool update() = 0;
+};
+
+
+//-----------------------------------------------------------------------------
+//    IInputStream.
+//-----------------------------------------------------------------------------
+
+/// An input stream delivers a sequence of elements of type "T".
+///
+/// @note This stream has an inherent position property and is thus not
+///   safe for concurrent access.
+template< typename T >
+class IInputStream
+{
+   public:
+
+      typedef void Parent;
+
+      /// The element type of this input stream.
+      typedef T ElementType;
+      
+      /// Read the next "num" elements into "buffer".
+      ///
+      /// @param buffer The buffer into which the elements are stored.
+      /// @param num Number of elements to read.
+      /// @return the number of elements actually read; this may be less than
+      ///   "num" or even zero if no elements are available or reading failed.
+      virtual U32 read( ElementType* buffer, U32 num ) = 0;
+};
+
+/// An input stream over elements of type "T" that reads from
+/// user-specified explicit offsets.
+template< typename T, typename Offset = U32 >
+class IOffsetInputStream
+{
+   public:
+
+      typedef void Parent;
+      typedef Offset OffsetType;
+      typedef T ElementType;
+
+      /// Read the next "num" elements at "offset" into "buffer".
+      ///
+      /// @param offset The offset in the stream from which to read.
+      /// @param buffer The buffer into which the elements are stored.
+      /// @param num Number of elements to read.
+      /// @return the number of elements actually read; this may be less than
+      ///   "num" or even zero if no elements are available or reading failed.
+      virtual U32 readAt( OffsetType offset, T* buffer, U32 num ) = 0;
+};
+
+/// An input stream over elements of type "T" that works in
+/// the background.
+template< typename T, typename Offset = U32 >
+class IAsyncInputStream
+{
+   public:
+
+      typedef void Parent;
+      typedef Offset OffsetType;
+      typedef T ElementType;
+
+      /// Queue a read of "num" elements at "offset" into "buffer".
+      ///
+      /// @param offset The offset in the stream from which to read.
+      /// @param buffer The buffer into which the elements are stored.
+      /// @param num Number of elements to read.
+      /// @return a handle for the asynchronous read operation or NULL if the
+      ///   operation could not be queued.
+      virtual void* issueReadAt( OffsetType offset, T* buffer, U32 num ) = 0;
+
+      /// Try moving the given asynchronous read operation to ASYNC_IO_Complete.
+      ///
+      /// @note This method invalidates the given handle.
+      ///
+      /// @param handle Handle returned by "issueReadAt".
+      /// @param outNumRead Reference that receives the number of bytes actually read in the
+      ///   operation.
+      /// @param wait If true, the method waits until the given operation either fails or
+      ///   completes successfully before returning.
+      /// @return the final operation status.
+      virtual EAsyncIOStatus tryCompleteReadAt( void* handle, U32& outNumRead, bool wait = false ) = 0;
+
+      /// Cancel the given asynchronous read operation.
+      ///
+      /// @note This method invalidates the given handle.
+      ///
+      /// @param handle Handle returned by "issueReadAt".
+      virtual void cancelReadAt( void* handle ) = 0;
+};
+
+//-----------------------------------------------------------------------------
+//    IOutputStream.
+//-----------------------------------------------------------------------------
+
+/// An output stream that writes elements of type "T".
+///
+/// @note This stream has an inherent position property and is thus not
+///   safe for concurrent access.
+template< typename T >
+class IOutputStream
+{
+   public:
+
+      typedef void Parent;      
+
+      /// The element type of this input stream.
+      typedef T ElementType;
+
+      /// Write "num" elements from "buffer" to the stream at its
+      /// current position.
+      ///
+      /// @param buffer The buffer from which to read elements.
+      /// @param num Number of elements to write.
+      virtual void write( const ElementType* buffer, U32 num ) = 0;
+};
+
+/// An output stream that writes elements of type "T" to a
+/// user-specified explicit offset.
+template< typename T, typename Offset = U32 >
+class IOffsetOutputStream
+{
+   public:
+
+      typedef void Parent;
+      typedef Offset OffsetType;
+      typedef T ElementType;
+
+      /// Write "num" elements from "buffer" to the stream at "offset".
+      ///
+      /// @param offset The offset in the stream at which to write the elements.
+      /// @param buffer The buffer from which to read elements.
+      /// @param num Number of elements to write.
+      virtual void writeAt( OffsetType offset, const ElementType* buffer, U32 num ) = 0;
+};
+
+/// An output stream that writes elements of type "T" in the background.
+template< typename T, typename Offset = U32 >
+class IAsyncOutputStream
+{
+   public:
+
+      typedef void Parent;
+      typedef Offset OffsetType;
+      typedef T ElementType;
+
+      /// Queue a write operation of "num" elements from "buffer" to stream position
+      /// "offset".
+      ///
+      /// @param offset The offset in the stream at which to write the elements.
+      /// @param buffer The buffer from which to read elements.
+      /// @param num The number of elements to write.
+      /// @return a handle to the asynchronous write operatior or NULL if the operation
+      ///   could not be queued.
+      virtual void* issueWriteAt( OffsetType offset, const ElementType* buffer, U32 num ) = 0;
+
+      /// Try moving the given asynchronous write operation to ASYNC_IO_Complete.
+      ///
+      /// @note This method invalidates the given handle.
+      ///
+      /// @param handle Handle returned by "issueWriteAt".
+      /// @param wait If true, the method waits until the given operation either fails or
+      ///   completes successfully before returning.
+      /// @return the final operation status.
+      virtual EAsyncIOStatus tryCompleteWriteAt( void* handle, bool wait = false ) = 0;
+
+      /// Cancel the given asynchronous write operation.
+      ///
+      /// @note This method invalidates the given handle.
+      ///
+      /// @param handle Handle return by "issueWriteAt".
+      virtual void cancelWriteAt( void* handle ) = 0;
+};
+
+//-----------------------------------------------------------------------------
+//    IInputStreamFilter.
+//-----------------------------------------------------------------------------
+
+/// An input stream filter takes an input stream "Stream" and processes it
+/// into an input stream over type "To".
+template< typename To, typename Stream >
+class IInputStreamFilter : public IInputStream< To >
+{
+   public:
+
+      typedef IInputStream< To > Parent;
+
+      ///
+      typedef typename TypeTraits< Stream >::BaseType SourceStreamType;
+
+      /// The element type of the source stream.
+      typedef typename SourceStreamType::ElementType SourceElementType;
+
+      /// Construct a filter reading elements from "stream".
+      IInputStreamFilter( const Stream& stream )
+         : mSourceStream( stream ) {}
+
+      /// Return the stream from which this filter is reading its
+      /// source elements.
+      const Stream& getSourceStream() const { return mSourceStream; }
+      Stream& getSourceStream() { return mSourceStream; }
+
+   private:
+
+      Stream mSourceStream;
+};
+
+//-----------------------------------------------------------------------------
+//    IOutputStreamFilter.
+//-----------------------------------------------------------------------------
+
+/// An output stream filter takes an output stream "Stream" and processes it
+/// into an output stream over type "To".
+template< typename To, class Stream >
+class IOutputStreamFilter : public IOutputStream< To >
+{
+   public:
+
+      typedef IOutputStream< To > Parent;
+
+      ///
+      typedef typename TypeTraits< Stream >::BaseType TargetStreamType;
+
+      /// The element type of the target stream.
+      typedef typename TargetStreamType::ElementType TargetElementType;
+
+      /// Construct a filter writing elements to "stream".
+      IOutputStreamFilter( const Stream& stream )
+         : mTargetStream( stream ) {}
+
+      /// Return the stream to which this filter is writing its
+      /// elements.
+      const Stream& getTargetStream() const { return mTargetStream; }
+      Stream& getTargetStream() { return mTargetStream; }
+
+   private:
+
+      Stream mTargetStream;
+};
+
+#endif // _TSTREAM_H_