diff --git a/build.sbt b/build.sbt index b825a016..855851fc 100644 --- a/build.sbt +++ b/build.sbt @@ -48,7 +48,7 @@ lazy val commonSettings = Seq( "io.kamon" %% "kamon-bundle" % "2.1.0", "io.kamon" %% "kamon-apm-reporter" % "2.1.0", "org.json4s" %% "json4s-native" % "3.6.8", - "com.typesafe.akka" %% "akka-stream" % "2.6.5", + "com.typesafe.akka" %% "akka-stream" % "2.6.5" ) ) @@ -102,5 +102,14 @@ lazy val common = (project in file("common")). ). settings(pscryptoSettings: _*) +lazy val decodePackets = (project in file("tools/decode-packets")). + settings(commonSettings: _*). + settings( + libraryDependencies ++= Seq( + "org.scala-lang.modules" %% "scala-parallel-collections" % "0.2.0" + ) + ). + dependsOn(common) + // Special test configuration for really quiet tests (used in CI) lazy val QuietTest = config("quiet") extend(Test) diff --git a/tools/decode-packets/src/main/scala/XToolsV2.scala b/tools/decode-packets/src/main/scala/XToolsV2.scala new file mode 100644 index 00000000..a162bd39 --- /dev/null +++ b/tools/decode-packets/src/main/scala/XToolsV2.scala @@ -0,0 +1,157 @@ +import java.io.{BufferedWriter, File, FileWriter} +import java.nio.charset.CodingErrorAction + +import net.psforever.packet.PacketCoding +import scodec.bits._ +import scodec.Attempt.{Failure, Successful} +import java.nio.file.{Files, Paths, StandardCopyOption} + +import scala.io.{Codec, Source} +import util.control.Breaks._ +import scala.collection.parallel.CollectionConverters._ + +object XToolsV2 { + def main(args: Array[String]): Unit = { + + // Replace the below directories with the correct locations before running + + // Directory containing gcapy ASCII output files + val dirToProcess = "C:\\xtools\\in" + + // Directory for final decoded packet logs + val dirForDecoded = "C:\\xtools\\out" + + // Temporary directory to write current log before moving to final directory + val tempDir = "C:\\xtools\\temp" + + val files = new File(dirToProcess).listFiles + + files.par.foreach { f => + val file = new File(f.toString) + val FileToWrite = tempDir + "/" + file.getName().split(".gcapy")(0) + ".txt" + val FileToMoveTo = dirForDecoded + "/" + file.getName().split(".gcapy")(0) + ".txt" + + if (new File(FileToMoveTo).exists()) { + println(s"File ${file.getName} exists - skipping") + return + } else { + println(s"${FileToMoveTo} doesn't exist - Got new file ${file.getName}") + } + + val FileToRead = file.toString + val fw = new BufferedWriter(new FileWriter(FileToWrite, false)) + val decoder = Codec.UTF8.decoder.onMalformedInput(CodingErrorAction.REPORT) + + try { + var linesToSkip = 0 + for (line <- Source.fromFile(FileToRead)(decoder).getLines().drop(1)) { + breakable { + if(linesToSkip > 0) { + linesToSkip -= 1 + break + } + + val decodedLine = DecodePacket(line.drop(line.lastIndexOf(' '))) + fw.write(s"${ShortGcapyString(line)}") + fw.newLine() + + if(!IsNestedPacket(decodedLine)) { + // Standard line, output as is with a bit of extra whitespace for readability + fw.write(decodedLine.replace(",", ", ")) + fw.newLine() + } else { + // Packet with nested packets, including possibly other nested packets within e.g. SlottedMetaPacket containing a MultiPacketEx + fw.write(s"${decodedLine.replace(",", ", ")}") + fw.newLine() + val nestedLinesToSkip = RecursivelyHandleNestedPacket(decodedLine, fw) + + // Gcapy output has duplicated lines for SlottedMetaPackets, so we can skip over those if found to reduce noise + // The only difference between the original and duplicate lines is a slight difference in timestamp of when the packet was processed + linesToSkip = decodedLine.indexOf("SlottedMetaPacket") match { + case pos if pos >= 0 && nestedLinesToSkip > 0 => + fw.write(s"Skipping $nestedLinesToSkip duplicate lines") + fw.newLine() + nestedLinesToSkip + case _ => 0 + } + } + + fw.newLine() + } + } + } + catch { + case e: Throwable => + println(s"File ${file.getName} threw an exception") + e.printStackTrace() + } + finally { + fw.close() + MoveFile(FileToWrite, FileToMoveTo) + } + } + } + + /* + Traverse down any nested packets such as SlottedMetaPacket, MultiPacket and MultiPacketEx and add indent for each layer down + The number of lines to skip will be returned so duplicate lines following SlottedMetaPackets in the gcapy output can be filtered out + */ + def RecursivelyHandleNestedPacket(decodedLine : String, fw : BufferedWriter, depth : Int = 0): Int = { + if(decodedLine.indexOf("Failed to parse") >= 0) return depth + val regex = "(0x[a-f0-9]+)".r + val matches = regex.findAllIn(decodedLine) + + var linesToSkip = 0 + while(matches.hasNext) { + val packet = matches.next + + for(i <- depth to 0 by -1) { + if(i == 0) fw.write("> ") + else fw.write("-") + } + + val nextDecodedLine = DecodePacket(packet) + fw.write(s"${nextDecodedLine.replace(",", ", ")}") + fw.newLine() + + if(IsNestedPacket(nextDecodedLine)) { + linesToSkip += RecursivelyHandleNestedPacket(nextDecodedLine, fw, depth + 1) + } + + linesToSkip += 1 + } + + linesToSkip + } + + def ShortGcapyString(line : String): String = { + val regex = "Game record ([0-9]+) at ([0-9.]+s) is from ([S|C]).* to ([S|C]).*contents (.*)".r + line match { + case regex(index, time, from, to, contents) => s"#$index @ $time $from -> $to ($contents)" + } + } + + def IsNestedPacket(decodedLine : String) : Boolean = { + // Also matches MultiPacketEx + decodedLine.indexOf("MultiPacket") >= 0 || decodedLine.indexOf("SlottedMetaPacket") >= 0 + } + + def DecodePacket(hexString: String) : String = { + PacketCoding.DecodePacket(ByteVector.fromValidHex(hexString)) match { + case Successful(value) => value.toString + case Failure(cause) => cause.toString + } + } + + def MoveFile(sourcePath: String, targetPath: String) : Boolean = { + var success = true + try + Files.move(Paths.get(sourcePath), Paths.get(targetPath), StandardCopyOption.REPLACE_EXISTING) + catch { + case e: Exception => + success = false + e.printStackTrace() + } + success + } +} diff --git a/tools/decode-packets/src/main/scala/xtoolspar.scala b/tools/decode-packets/src/main/scala/xtoolspar.scala new file mode 100644 index 00000000..f54b8248 --- /dev/null +++ b/tools/decode-packets/src/main/scala/xtoolspar.scala @@ -0,0 +1,274 @@ +/** + * Created by SouNourS on 20/12/2016. + */ + +// Make sure the input files have UTF8 encoding! + +import java.io.{BufferedWriter, File, FileWriter} +import java.nio.charset.CodingErrorAction + +import net.psforever.packet._ +import scodec.Attempt +import scodec.bits._ + +import scala.io.{Codec, Source} +import scala.collection.parallel.CollectionConverters._ + +object Xtoolspar { + + def main(args: Array[String]): Unit = { + val dirToProcess = "C:\\xtools\\in" + val dirForDecoded = "C:\\xtools\\out" + val tempDir = "C:\\xtools\\temp" + + val files = new File(dirToProcess).listFiles + + // TODO decode packet + files.par.foreach { f => + val file = new File(f.toString) + val FileToWrite = tempDir + "/" + file.getName().split(".gcapy")(0) + ".txt" + val FileToMoveTo = dirForDecoded + "/" + file.getName().split(".gcapy")(0) + ".txt" + + if (new File(FileToMoveTo).exists()) { + println(s"File ${file.getName} exists - skipping") + } else { + println(s"${FileToMoveTo} doesn't exist - Got new file ${file.getName}") + + + val FileToRead = file.toString + val fw = new BufferedWriter(new FileWriter(FileToWrite, false)) + + try { + val decoder = Codec.UTF8.decoder.onMalformedInput(CodingErrorAction.REPORT) + var i = 0 + for (line <- Source.fromFile(FileToRead)(decoder).getLines()) { + val lineTest: String = line.substring(1, 3) +// if (!lineTest.equalsIgnoreCase("IF")) { + if (i != 0) { // skip first line + //// println(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' ')))) + //// println(PacketCoding.DecodePacket(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' '))))) + // handlePkt(PacketCoding.DecodePacket(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' '))))) + + fw.write(System.getProperty("line.separator") + "#" + line + System.getProperty("line.separator")) + var isSlotted = -1 + var isMultiPacketEx = -1 + var isMultiPacket = -1 + var isMultiPacketExSlot = -1 + var isHandleGamePacket = -1 + val decodedLine = line.drop(line.lastIndexOf(' ')) + var AfterDecode = Fdecode(decodedLine) + var AfterDecode2 = "" + var AfterDecode3 = "" + var AfterDecode4 = "" + var AfterDecode5 = "" + + isMultiPacket = AfterDecode.indexOf("Successful(MultiPacket(") + isSlotted = AfterDecode.indexOf("Successful(SlottedMetaPacket(") + isMultiPacketEx = AfterDecode.indexOf("Successful(MultiPacketEx(") + + if (isSlotted != 0 && isMultiPacket == -1 && isMultiPacketEx == -1) { + fw.write(AfterDecode + System.getProperty("line.separator")) + // println(AfterDecode ) + } + + if (isMultiPacket != -1) { + fw.write(AfterDecode + System.getProperty("line.separator")) + // println(AfterDecode) + var xindex1 = 1 + var zindex1 = 0 + var boucle1 = 0 + while (boucle1 != -1) { + AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex1) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex1 + 1))) + xindex1 = AfterDecode.indexOf(" 0x", xindex1) + 1 + boucle1 = AfterDecode.indexOf(" 0x", xindex1) + zindex1 = AfterDecode.indexOf(")", zindex1) + 1 + isSlotted = AfterDecode2.indexOf("Successful(SlottedMetaPacket(") + if (isSlotted == 0) { + fw.write("> " + AfterDecode2 + System.getProperty("line.separator")) + // println("> " + AfterDecode2) + AfterDecode3 = Fdecode(AfterDecode2.drop(AfterDecode2.lastIndexOf(" 0x") + 3).dropRight(AfterDecode2.length - AfterDecode2.indexOf(")"))) + isMultiPacketExSlot = AfterDecode3.indexOf("Successful(MultiPacketEx(") + if (isMultiPacketExSlot != -1) { + fw.write("-> " + AfterDecode3 + System.getProperty("line.separator")) + // println("-> " + AfterDecode3) + var xindex2 = 1 + var zindex2 = 0 + var boucle2 = 0 + while (boucle2 != -1) { + AfterDecode4 = Fdecode(AfterDecode3.drop(AfterDecode3.indexOf(" 0x", xindex2) + 3).dropRight(AfterDecode3.length - AfterDecode3.indexOf(")", zindex2 + 1))) + xindex2 = AfterDecode3.indexOf(" 0x", xindex2) + 1 + boucle2 = AfterDecode3.indexOf(" 0x", xindex2) + zindex2 = AfterDecode3.indexOf(")", zindex2) + 1 + fw.write("--> " + AfterDecode4 + System.getProperty("line.separator")) + // println("--> " + AfterDecode4 ) + } + isMultiPacketEx = -1 + isMultiPacketExSlot = -1 + } else { + fw.write("-> " + AfterDecode3 + System.getProperty("line.separator")) + // println("-> " + AfterDecode3 ) + } + } else { + fw.write("> " + AfterDecode2 + System.getProperty("line.separator")) + // println("> " + AfterDecode2 ) + } + } + } + if (isSlotted == 0 && isMultiPacket == -1) { + fw.write(AfterDecode + System.getProperty("line.separator")) + // println(AfterDecode) + AfterDecode = Fdecode(AfterDecode.drop(AfterDecode.lastIndexOf(" 0x") + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")"))) + isMultiPacketExSlot = AfterDecode.indexOf("Successful(MultiPacketEx(") + isHandleGamePacket = AfterDecode.indexOf("Successful(HandleGamePacket(") + if (isHandleGamePacket != -1) { + fw.write("> " + AfterDecode + System.getProperty("line.separator")) + // println("> " + AfterDecode ) + if (AfterDecode.lastIndexOf(" 0x") != -1) { + AfterDecode5 = Fdecode(AfterDecode.drop(AfterDecode.lastIndexOf(" 0x") + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")"))) + fw.write("-> " + AfterDecode5 + System.getProperty("line.separator")) + // println("-> " + AfterDecode5 ) + } + } + if (isMultiPacketExSlot == -1 && isHandleGamePacket == -1) { + fw.write("> " + AfterDecode + System.getProperty("line.separator")) + // println("> " + AfterDecode ) + } + if (isMultiPacketExSlot != -1 && isHandleGamePacket == -1) { + fw.write("> " + AfterDecode + System.getProperty("line.separator")) + // println("> " + AfterDecode ) + var xindex3 = 1 + var zindex3 = 0 + var boucle3 = 0 + while (boucle3 != -1) { + AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex3) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex3 + 1))) + fw.write("-> " + AfterDecode2 + System.getProperty("line.separator")) + // println("-> " + AfterDecode2) + xindex3 = AfterDecode.indexOf(" 0x", xindex3) + 1 + boucle3 = AfterDecode.indexOf(" 0x", xindex3) + zindex3 = AfterDecode.indexOf(")", zindex3) + 1 + } + } + } + if ((isMultiPacketEx != -1 || isMultiPacketExSlot != -1) && isSlotted != 0) { + fw.write(AfterDecode + System.getProperty("line.separator")) + // println( AfterDecode ) + var xindex = 1 + var zindex = 0 + var boucle = 0 + while (boucle != -1) { + AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex + 1))) + fw.write("> " + AfterDecode2 + System.getProperty("line.separator")) + // println("> " + AfterDecode2) + xindex = AfterDecode.indexOf(" 0x", xindex) + 1 + boucle = AfterDecode.indexOf(" 0x", xindex) + zindex = AfterDecode.indexOf(")", zindex) + 1 + } + } + } else { + i += 1 + } + } + } + catch { + case e: Throwable => + println(s"File ${file.getName} threw an exception") + e.printStackTrace() + } + finally { + fw.close() + moveFile(FileToWrite, FileToMoveTo) + } + } + } + + + + + // TODO : end + } + + import java.nio.file.{Files, Paths, StandardCopyOption} + + def moveFile(sourcePath: String, targetPath: String): Boolean = { + var flag = true + try + Files.move(Paths.get(sourcePath), Paths.get(targetPath), StandardCopyOption.REPLACE_EXISTING) + catch { + case e: Exception => + flag = false + e.printStackTrace() + } + flag + } + + def Fdecode(toto: String): String = { + val ADecode = PacketCoding.DecodePacket(ByteVector.fromValidHex(toto)).toString; + return ADecode + } + + + def handlePkt(pkt : Attempt[PlanetSidePacket]) : Unit = pkt match { + case ctrl : PlanetSideControlPacket => + println(ctrl) + // handleControlPkt(ctrl) + case game : PlanetSideGamePacket => + println(game) + // handleGamePkt(game) + case default => println(s"Invalid packet class received: $default") + } + + def handlePktContainer(pkt : PlanetSidePacketContainer) : Unit = pkt match { + case ctrl @ ControlPacket(opcode, ctrlPkt) => + // println(pkt) + println(ctrlPkt) + // handleControlPkt(ctrlPkt) + case game @ GamePacket(opcode, seq, gamePkt) => + // println(pkt) + println(gamePkt) + // handleGamePkt(gamePkt) + case default => println(s"Invalid packet container class received: $default") + } + + // def handleControlPkt(pkt : PlanetSideControlPacket) = { + // // println(pkt) + // pkt match { + // case SlottedMetaPacket(slot, subslot, innerPacket) => + //// sendResponse(PacketCoding.CreateControlPacket(SlottedMetaAck(slot, subslot))) + // + // PacketCoding.DecodePacket(innerPacket) match { + // case Failure(e) => + // println(innerPacket.toString) + // println(s"Failed to decode inner packet of SlottedMetaPacket: $e") + // case Successful(v) => + // handlePkt(v) + // } + // case sync @ ControlSync(diff, unk, f1, f2, f3, f4, fa, fb) => + // println(s"SYNC: ${sync}") + // val serverTick = Math.abs(System.nanoTime().toInt) // limit the size to prevent encoding error + //// sendResponse(PacketCoding.CreateControlPacket(ControlSyncResp(diff, serverTick, fa, fb, fb, fa))) + // case MultiPacket(packets) => + // packets.foreach { pkt => + // PacketCoding.DecodePacket(pkt) match { + // case Failure(e) => + // println(pkt.toString) + // println(s"Failed to decode inner packet of MultiPacket: $e") + // case Successful(v) => + // handlePkt(v) + // } + // } + // case MultiPacketEx(packets) => + // packets.foreach { pkt => + // PacketCoding.DecodePacket(pkt) match { + // case Failure(e) => + // println(pkt.toString) + // println(s"Failed to decode inner packet of MultiPacketEx: $e") + // case Successful(v) => + // handlePkt(v) + // } + // } + // case default => + // println(s"Unhandled ControlPacket $default") + // } + // } + +} \ No newline at end of file