Merge pull request #483 from Mazo/xtoolsv2

XToolsV2

build.sbt

@@ -48,7 +48,7 @@ lazy val commonSettings = Seq(
     "io.kamon"                   %% "kamon-bundle"  % "2.1.0",
     "io.kamon"                   %% "kamon-apm-reporter" % "2.1.0",
     "org.json4s"                 %% "json4s-native" % "3.6.8",
-    "com.typesafe.akka"          %% "akka-stream"   % "2.6.5",
+    "com.typesafe.akka"          %% "akka-stream"   % "2.6.5"
   )
 )
 
@@ -102,5 +102,14 @@ lazy val common = (project in file("common")).
   ).
   settings(pscryptoSettings: _*)
 
+lazy val decodePackets = (project in file("tools/decode-packets")).
+  settings(commonSettings: _*).
+  settings(
+    libraryDependencies ++= Seq(
+      "org.scala-lang.modules"     %% "scala-parallel-collections" % "0.2.0"
+    )
+  ).
+  dependsOn(common)
+
 // Special test configuration for really quiet tests (used in CI)
 lazy val QuietTest = config("quiet") extend(Test)

tools/decode-packets/src/main/scala/XToolsV2.scala

@@ -0,0 +1,157 @@
+import java.io.{BufferedWriter, File, FileWriter}
+import java.nio.charset.CodingErrorAction
+
+import net.psforever.packet.PacketCoding
+import scodec.bits._
+import scodec.Attempt.{Failure, Successful}
+import java.nio.file.{Files, Paths, StandardCopyOption}
+
+import scala.io.{Codec, Source}
+import util.control.Breaks._
+import scala.collection.parallel.CollectionConverters._
+
+object XToolsV2 {
+  def main(args: Array[String]): Unit = {
+
+    // Replace the below directories with the correct locations before running
+
+    // Directory containing gcapy ASCII output files
+    val dirToProcess = "C:\\xtools\\in"
+
+    // Directory for final decoded packet logs
+    val dirForDecoded = "C:\\xtools\\out"
+
+    // Temporary directory to write current log before moving to final directory
+    val tempDir = "C:\\xtools\\temp"
+
+    val files = new File(dirToProcess).listFiles
+
+    files.par.foreach { f =>
+      val file = new File(f.toString)
+      val FileToWrite = tempDir + "/" + file.getName().split(".gcapy")(0) + ".txt"
+      val FileToMoveTo = dirForDecoded + "/" + file.getName().split(".gcapy")(0) + ".txt"
+
+      if (new File(FileToMoveTo).exists()) {
+        println(s"File ${file.getName} exists - skipping")
+        return
+      } else {
+        println(s"${FileToMoveTo} doesn't exist - Got new file ${file.getName}")
+      }
+
+      val FileToRead = file.toString
+      val fw = new BufferedWriter(new FileWriter(FileToWrite, false))
+      val decoder = Codec.UTF8.decoder.onMalformedInput(CodingErrorAction.REPORT)
+
+      try {
+        var linesToSkip = 0
+        for (line <- Source.fromFile(FileToRead)(decoder).getLines().drop(1)) {
+          breakable {
+            if(linesToSkip > 0) {
+              linesToSkip -= 1
+              break
+            }
+
+            val decodedLine = DecodePacket(line.drop(line.lastIndexOf(' ')))
+            fw.write(s"${ShortGcapyString(line)}")
+            fw.newLine()
+
+            if(!IsNestedPacket(decodedLine)) {
+              // Standard line, output as is with a bit of extra whitespace for readability
+              fw.write(decodedLine.replace(",", ", "))
+              fw.newLine()
+            } else {
+              // Packet with nested packets, including possibly other nested packets within e.g. SlottedMetaPacket containing a MultiPacketEx
+              fw.write(s"${decodedLine.replace(",", ", ")}")
+              fw.newLine()
+              val nestedLinesToSkip = RecursivelyHandleNestedPacket(decodedLine, fw)
+
+              // Gcapy output has duplicated lines for SlottedMetaPackets, so we can skip over those if found to reduce noise
+              // The only difference between the original and duplicate lines is a slight difference in timestamp of when the packet was processed
+              linesToSkip = decodedLine.indexOf("SlottedMetaPacket") match {
+                case pos if pos >= 0 && nestedLinesToSkip > 0 =>
+                  fw.write(s"Skipping $nestedLinesToSkip duplicate lines")
+                  fw.newLine()
+                  nestedLinesToSkip
+                case _ => 0
+              }
+            }
+
+            fw.newLine()
+          }
+        }
+      }
+      catch {
+        case e: Throwable =>
+          println(s"File ${file.getName} threw an exception")
+          e.printStackTrace()
+      }
+      finally {
+        fw.close()
+        MoveFile(FileToWrite, FileToMoveTo)
+      }
+    }
+  }
+
+  /*
+    Traverse down any nested packets such as SlottedMetaPacket, MultiPacket and MultiPacketEx and add indent for each layer down
+    The number of lines to skip will be returned so duplicate lines following SlottedMetaPackets in the gcapy output can be filtered out
+   */
+  def RecursivelyHandleNestedPacket(decodedLine : String, fw : BufferedWriter, depth : Int = 0): Int = {
+    if(decodedLine.indexOf("Failed to parse") >= 0) return depth
+    val regex = "(0x[a-f0-9]+)".r
+    val matches = regex.findAllIn(decodedLine)
+
+    var linesToSkip = 0
+    while(matches.hasNext) {
+      val packet = matches.next
+
+      for(i <- depth to 0 by -1) {
+        if(i == 0) fw.write("> ")
+        else fw.write("-")
+      }
+
+      val nextDecodedLine = DecodePacket(packet)
+      fw.write(s"${nextDecodedLine.replace(",", ", ")}")
+      fw.newLine()
+
+      if(IsNestedPacket(nextDecodedLine)) {
+        linesToSkip += RecursivelyHandleNestedPacket(nextDecodedLine, fw, depth + 1)
+      }
+
+      linesToSkip += 1
+    }
+
+    linesToSkip
+  }
+
+  def ShortGcapyString(line : String): String = {
+    val regex = "Game record ([0-9]+) at ([0-9.]+s) is from ([S|C]).* to ([S|C]).*contents (.*)".r
+    line match {
+      case regex(index, time, from, to, contents) => s"#$index @ $time $from -> $to ($contents)"
+    }
+  }
+
+  def IsNestedPacket(decodedLine : String) : Boolean = {
+    // Also matches MultiPacketEx
+    decodedLine.indexOf("MultiPacket") >= 0 || decodedLine.indexOf("SlottedMetaPacket") >= 0
+  }
+
+  def DecodePacket(hexString: String) : String = {
+    PacketCoding.DecodePacket(ByteVector.fromValidHex(hexString)) match {
+      case Successful(value) => value.toString
+      case Failure(cause) => cause.toString
+    }
+  }
+
+  def MoveFile(sourcePath: String, targetPath: String) : Boolean = {
+    var success = true
+    try
+      Files.move(Paths.get(sourcePath), Paths.get(targetPath), StandardCopyOption.REPLACE_EXISTING)
+    catch {
+      case e: Exception =>
+        success = false
+        e.printStackTrace()
+    }
+    success
+  }
+}

tools/decode-packets/src/main/scala/xtoolspar.scala

@@ -0,0 +1,274 @@
+/**
+  * Created by SouNourS on 20/12/2016.
+  */
+
+// Make sure the input files have UTF8 encoding!
+
+import java.io.{BufferedWriter, File, FileWriter}
+import java.nio.charset.CodingErrorAction
+
+import net.psforever.packet._
+import scodec.Attempt
+import scodec.bits._
+
+import scala.io.{Codec, Source}
+import scala.collection.parallel.CollectionConverters._
+
+object Xtoolspar {
+
+  def main(args: Array[String]): Unit = {
+    val dirToProcess = "C:\\xtools\\in"
+    val dirForDecoded = "C:\\xtools\\out"
+    val tempDir = "C:\\xtools\\temp"
+
+    val files = new File(dirToProcess).listFiles
+
+    // TODO decode packet
+    files.par.foreach { f =>
+      val file = new File(f.toString)
+      val FileToWrite = tempDir + "/" + file.getName().split(".gcapy")(0) + ".txt"
+      val FileToMoveTo = dirForDecoded + "/" + file.getName().split(".gcapy")(0) + ".txt"
+
+      if (new File(FileToMoveTo).exists()) {
+        println(s"File ${file.getName} exists - skipping")
+      } else {
+        println(s"${FileToMoveTo} doesn't exist - Got new file ${file.getName}")
+
+
+        val FileToRead = file.toString
+        val fw = new BufferedWriter(new FileWriter(FileToWrite, false))
+
+        try {
+          val decoder = Codec.UTF8.decoder.onMalformedInput(CodingErrorAction.REPORT)
+          var i = 0
+          for (line <- Source.fromFile(FileToRead)(decoder).getLines()) {
+            val lineTest: String = line.substring(1, 3)
+//            if (!lineTest.equalsIgnoreCase("IF")) {
+              if (i != 0) { // skip first line
+              ////          println(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' '))))
+              ////          println(PacketCoding.DecodePacket(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' ')))))
+              //          handlePkt(PacketCoding.DecodePacket(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' ')))))
+
+              fw.write(System.getProperty("line.separator") + "#" + line + System.getProperty("line.separator"))
+              var isSlotted = -1
+              var isMultiPacketEx = -1
+              var isMultiPacket = -1
+              var isMultiPacketExSlot = -1
+              var isHandleGamePacket = -1
+              val decodedLine = line.drop(line.lastIndexOf(' '))
+              var AfterDecode = Fdecode(decodedLine)
+              var AfterDecode2 = ""
+              var AfterDecode3 = ""
+              var AfterDecode4 = ""
+              var AfterDecode5 = ""
+
+              isMultiPacket = AfterDecode.indexOf("Successful(MultiPacket(")
+              isSlotted = AfterDecode.indexOf("Successful(SlottedMetaPacket(")
+              isMultiPacketEx = AfterDecode.indexOf("Successful(MultiPacketEx(")
+
+              if (isSlotted != 0 && isMultiPacket == -1 && isMultiPacketEx == -1) {
+                fw.write(AfterDecode + System.getProperty("line.separator"))
+                //        println(AfterDecode )
+              }
+
+              if (isMultiPacket != -1) {
+                fw.write(AfterDecode + System.getProperty("line.separator"))
+                //        println(AfterDecode)
+                var xindex1 = 1
+                var zindex1 = 0
+                var boucle1 = 0
+                while (boucle1 != -1) {
+                  AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex1) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex1 + 1)))
+                  xindex1 = AfterDecode.indexOf(" 0x", xindex1) + 1
+                  boucle1 = AfterDecode.indexOf(" 0x", xindex1)
+                  zindex1 = AfterDecode.indexOf(")", zindex1) + 1
+                  isSlotted = AfterDecode2.indexOf("Successful(SlottedMetaPacket(")
+                  if (isSlotted == 0) {
+                    fw.write("> " + AfterDecode2 + System.getProperty("line.separator"))
+                    //            println("> " + AfterDecode2)
+                    AfterDecode3 = Fdecode(AfterDecode2.drop(AfterDecode2.lastIndexOf(" 0x") + 3).dropRight(AfterDecode2.length - AfterDecode2.indexOf(")")))
+                    isMultiPacketExSlot = AfterDecode3.indexOf("Successful(MultiPacketEx(")
+                    if (isMultiPacketExSlot != -1) {
+                      fw.write("-> " + AfterDecode3 + System.getProperty("line.separator"))
+                      //                println("-> " + AfterDecode3)
+                      var xindex2 = 1
+                      var zindex2 = 0
+                      var boucle2 = 0
+                      while (boucle2 != -1) {
+                        AfterDecode4 = Fdecode(AfterDecode3.drop(AfterDecode3.indexOf(" 0x", xindex2) + 3).dropRight(AfterDecode3.length - AfterDecode3.indexOf(")", zindex2 + 1)))
+                        xindex2 = AfterDecode3.indexOf(" 0x", xindex2) + 1
+                        boucle2 = AfterDecode3.indexOf(" 0x", xindex2)
+                        zindex2 = AfterDecode3.indexOf(")", zindex2) + 1
+                        fw.write("--> " + AfterDecode4 + System.getProperty("line.separator"))
+                        //                println("--> " + AfterDecode4 )
+                      }
+                      isMultiPacketEx = -1
+                      isMultiPacketExSlot = -1
+                    } else {
+                      fw.write("-> " + AfterDecode3 + System.getProperty("line.separator"))
+                      //                println("-> " + AfterDecode3 )
+                    }
+                  } else {
+                    fw.write("> " + AfterDecode2 + System.getProperty("line.separator"))
+                    //              println("> " + AfterDecode2 )
+                  }
+                }
+              }
+              if (isSlotted == 0 && isMultiPacket == -1) {
+                fw.write(AfterDecode + System.getProperty("line.separator"))
+                //        println(AfterDecode)
+                AfterDecode = Fdecode(AfterDecode.drop(AfterDecode.lastIndexOf(" 0x") + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")")))
+                isMultiPacketExSlot = AfterDecode.indexOf("Successful(MultiPacketEx(")
+                isHandleGamePacket = AfterDecode.indexOf("Successful(HandleGamePacket(")
+                if (isHandleGamePacket != -1) {
+                  fw.write("> " + AfterDecode + System.getProperty("line.separator"))
+                  //              println("> " + AfterDecode )
+                  if (AfterDecode.lastIndexOf(" 0x") != -1) {
+                    AfterDecode5 = Fdecode(AfterDecode.drop(AfterDecode.lastIndexOf(" 0x") + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")")))
+                    fw.write("-> " + AfterDecode5 + System.getProperty("line.separator"))
+                    //              println("-> " + AfterDecode5 )
+                  }
+                }
+                if (isMultiPacketExSlot == -1 && isHandleGamePacket == -1) {
+                  fw.write("> " + AfterDecode + System.getProperty("line.separator"))
+                  //          println("> " + AfterDecode )
+                }
+                if (isMultiPacketExSlot != -1 && isHandleGamePacket == -1) {
+                  fw.write("> " + AfterDecode + System.getProperty("line.separator"))
+                  //          println("> " + AfterDecode )
+                  var xindex3 = 1
+                  var zindex3 = 0
+                  var boucle3 = 0
+                  while (boucle3 != -1) {
+                    AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex3) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex3 + 1)))
+                    fw.write("-> " + AfterDecode2 + System.getProperty("line.separator"))
+                    //            println("-> " + AfterDecode2)
+                    xindex3 = AfterDecode.indexOf(" 0x", xindex3) + 1
+                    boucle3 = AfterDecode.indexOf(" 0x", xindex3)
+                    zindex3 = AfterDecode.indexOf(")", zindex3) + 1
+                  }
+                }
+              }
+              if ((isMultiPacketEx != -1 || isMultiPacketExSlot != -1) && isSlotted != 0) {
+                fw.write(AfterDecode + System.getProperty("line.separator"))
+                //        println( AfterDecode )
+                var xindex = 1
+                var zindex = 0
+                var boucle = 0
+                while (boucle != -1) {
+                  AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex + 1)))
+                  fw.write("> " + AfterDecode2 + System.getProperty("line.separator"))
+                  //          println("> " + AfterDecode2)
+                  xindex = AfterDecode.indexOf(" 0x", xindex) + 1
+                  boucle = AfterDecode.indexOf(" 0x", xindex)
+                  zindex = AfterDecode.indexOf(")", zindex) + 1
+                }
+              }
+            } else {
+                i += 1
+              }
+          }
+        }
+        catch {
+          case e: Throwable =>
+            println(s"File ${file.getName} threw an exception")
+            e.printStackTrace()
+        }
+        finally {
+          fw.close()
+          moveFile(FileToWrite, FileToMoveTo)
+        }
+      }
+    }
+
+
+
+
+    // TODO : end
+  }
+
+  import java.nio.file.{Files, Paths, StandardCopyOption}
+
+  def moveFile(sourcePath: String, targetPath: String): Boolean = {
+    var flag = true
+    try
+      Files.move(Paths.get(sourcePath), Paths.get(targetPath), StandardCopyOption.REPLACE_EXISTING)
+    catch {
+      case e: Exception =>
+        flag = false
+        e.printStackTrace()
+    }
+    flag
+  }
+
+  def Fdecode(toto: String): String = {
+    val ADecode = PacketCoding.DecodePacket(ByteVector.fromValidHex(toto)).toString;
+    return ADecode
+  }
+
+
+  def handlePkt(pkt : Attempt[PlanetSidePacket]) : Unit = pkt match {
+    case ctrl : PlanetSideControlPacket =>
+      println(ctrl)
+    //      handleControlPkt(ctrl)
+    case game : PlanetSideGamePacket =>
+      println(game)
+    //      handleGamePkt(game)
+    case default => println(s"Invalid packet class received: $default")
+  }
+
+  def handlePktContainer(pkt : PlanetSidePacketContainer) : Unit = pkt match {
+    case ctrl @ ControlPacket(opcode, ctrlPkt) =>
+      //      println(pkt)
+      println(ctrlPkt)
+    //      handleControlPkt(ctrlPkt)
+    case game @ GamePacket(opcode, seq, gamePkt) =>
+      //      println(pkt)
+      println(gamePkt)
+    //      handleGamePkt(gamePkt)
+    case default => println(s"Invalid packet container class received: $default")
+  }
+
+  //  def handleControlPkt(pkt : PlanetSideControlPacket) = {
+  //    //    println(pkt)
+  //    pkt match {
+  //      case SlottedMetaPacket(slot, subslot, innerPacket) =>
+  ////        sendResponse(PacketCoding.CreateControlPacket(SlottedMetaAck(slot, subslot)))
+  //
+  //        PacketCoding.DecodePacket(innerPacket) match {
+  //          case Failure(e) =>
+  //            println(innerPacket.toString)
+  //            println(s"Failed to decode inner packet of SlottedMetaPacket: $e")
+  //          case Successful(v) =>
+  //            handlePkt(v)
+  //        }
+  //      case sync @ ControlSync(diff, unk, f1, f2, f3, f4, fa, fb) =>
+  //        println(s"SYNC: ${sync}")
+  //        val serverTick = Math.abs(System.nanoTime().toInt) // limit the size to prevent encoding error
+  ////        sendResponse(PacketCoding.CreateControlPacket(ControlSyncResp(diff, serverTick, fa, fb, fb, fa)))
+  //      case MultiPacket(packets) =>
+  //        packets.foreach { pkt =>
+  //          PacketCoding.DecodePacket(pkt) match {
+  //            case Failure(e) =>
+  //              println(pkt.toString)
+  //              println(s"Failed to decode inner packet of MultiPacket: $e")
+  //            case Successful(v) =>
+  //              handlePkt(v)
+  //          }
+  //        }
+  //      case MultiPacketEx(packets) =>
+  //        packets.foreach { pkt =>
+  //          PacketCoding.DecodePacket(pkt) match {
+  //            case Failure(e) =>
+  //              println(pkt.toString)
+  //              println(s"Failed to decode inner packet of MultiPacketEx: $e")
+  //            case Successful(v) =>
+  //              handlePkt(v)
+  //          }
+  //        }
+  //      case default =>
+  //        println(s"Unhandled ControlPacket $default")
+  //    }
+  //  }
+
+}