Merge pull request #483 from Mazo/xtoolsv2

XToolsV2
This commit is contained in:
Mazo 2020-06-07 21:41:22 +01:00 committed by GitHub
commit e3fd5ff854
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 441 additions and 1 deletions

View file

@ -48,7 +48,7 @@ lazy val commonSettings = Seq(
"io.kamon" %% "kamon-bundle" % "2.1.0",
"io.kamon" %% "kamon-apm-reporter" % "2.1.0",
"org.json4s" %% "json4s-native" % "3.6.8",
"com.typesafe.akka" %% "akka-stream" % "2.6.5",
"com.typesafe.akka" %% "akka-stream" % "2.6.5"
)
)
@ -102,5 +102,14 @@ lazy val common = (project in file("common")).
).
settings(pscryptoSettings: _*)
lazy val decodePackets = (project in file("tools/decode-packets")).
settings(commonSettings: _*).
settings(
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-parallel-collections" % "0.2.0"
)
).
dependsOn(common)
// Special test configuration for really quiet tests (used in CI)
lazy val QuietTest = config("quiet") extend(Test)

View file

@ -0,0 +1,157 @@
import java.io.{BufferedWriter, File, FileWriter}
import java.nio.charset.CodingErrorAction
import net.psforever.packet.PacketCoding
import scodec.bits._
import scodec.Attempt.{Failure, Successful}
import java.nio.file.{Files, Paths, StandardCopyOption}
import scala.io.{Codec, Source}
import util.control.Breaks._
import scala.collection.parallel.CollectionConverters._
object XToolsV2 {
def main(args: Array[String]): Unit = {
// Replace the below directories with the correct locations before running
// Directory containing gcapy ASCII output files
val dirToProcess = "C:\\xtools\\in"
// Directory for final decoded packet logs
val dirForDecoded = "C:\\xtools\\out"
// Temporary directory to write current log before moving to final directory
val tempDir = "C:\\xtools\\temp"
val files = new File(dirToProcess).listFiles
files.par.foreach { f =>
val file = new File(f.toString)
val FileToWrite = tempDir + "/" + file.getName().split(".gcapy")(0) + ".txt"
val FileToMoveTo = dirForDecoded + "/" + file.getName().split(".gcapy")(0) + ".txt"
if (new File(FileToMoveTo).exists()) {
println(s"File ${file.getName} exists - skipping")
return
} else {
println(s"${FileToMoveTo} doesn't exist - Got new file ${file.getName}")
}
val FileToRead = file.toString
val fw = new BufferedWriter(new FileWriter(FileToWrite, false))
val decoder = Codec.UTF8.decoder.onMalformedInput(CodingErrorAction.REPORT)
try {
var linesToSkip = 0
for (line <- Source.fromFile(FileToRead)(decoder).getLines().drop(1)) {
breakable {
if(linesToSkip > 0) {
linesToSkip -= 1
break
}
val decodedLine = DecodePacket(line.drop(line.lastIndexOf(' ')))
fw.write(s"${ShortGcapyString(line)}")
fw.newLine()
if(!IsNestedPacket(decodedLine)) {
// Standard line, output as is with a bit of extra whitespace for readability
fw.write(decodedLine.replace(",", ", "))
fw.newLine()
} else {
// Packet with nested packets, including possibly other nested packets within e.g. SlottedMetaPacket containing a MultiPacketEx
fw.write(s"${decodedLine.replace(",", ", ")}")
fw.newLine()
val nestedLinesToSkip = RecursivelyHandleNestedPacket(decodedLine, fw)
// Gcapy output has duplicated lines for SlottedMetaPackets, so we can skip over those if found to reduce noise
// The only difference between the original and duplicate lines is a slight difference in timestamp of when the packet was processed
linesToSkip = decodedLine.indexOf("SlottedMetaPacket") match {
case pos if pos >= 0 && nestedLinesToSkip > 0 =>
fw.write(s"Skipping $nestedLinesToSkip duplicate lines")
fw.newLine()
nestedLinesToSkip
case _ => 0
}
}
fw.newLine()
}
}
}
catch {
case e: Throwable =>
println(s"File ${file.getName} threw an exception")
e.printStackTrace()
}
finally {
fw.close()
MoveFile(FileToWrite, FileToMoveTo)
}
}
}
/*
Traverse down any nested packets such as SlottedMetaPacket, MultiPacket and MultiPacketEx and add indent for each layer down
The number of lines to skip will be returned so duplicate lines following SlottedMetaPackets in the gcapy output can be filtered out
*/
def RecursivelyHandleNestedPacket(decodedLine : String, fw : BufferedWriter, depth : Int = 0): Int = {
if(decodedLine.indexOf("Failed to parse") >= 0) return depth
val regex = "(0x[a-f0-9]+)".r
val matches = regex.findAllIn(decodedLine)
var linesToSkip = 0
while(matches.hasNext) {
val packet = matches.next
for(i <- depth to 0 by -1) {
if(i == 0) fw.write("> ")
else fw.write("-")
}
val nextDecodedLine = DecodePacket(packet)
fw.write(s"${nextDecodedLine.replace(",", ", ")}")
fw.newLine()
if(IsNestedPacket(nextDecodedLine)) {
linesToSkip += RecursivelyHandleNestedPacket(nextDecodedLine, fw, depth + 1)
}
linesToSkip += 1
}
linesToSkip
}
def ShortGcapyString(line : String): String = {
val regex = "Game record ([0-9]+) at ([0-9.]+s) is from ([S|C]).* to ([S|C]).*contents (.*)".r
line match {
case regex(index, time, from, to, contents) => s"#$index @ $time $from -> $to ($contents)"
}
}
def IsNestedPacket(decodedLine : String) : Boolean = {
// Also matches MultiPacketEx
decodedLine.indexOf("MultiPacket") >= 0 || decodedLine.indexOf("SlottedMetaPacket") >= 0
}
def DecodePacket(hexString: String) : String = {
PacketCoding.DecodePacket(ByteVector.fromValidHex(hexString)) match {
case Successful(value) => value.toString
case Failure(cause) => cause.toString
}
}
def MoveFile(sourcePath: String, targetPath: String) : Boolean = {
var success = true
try
Files.move(Paths.get(sourcePath), Paths.get(targetPath), StandardCopyOption.REPLACE_EXISTING)
catch {
case e: Exception =>
success = false
e.printStackTrace()
}
success
}
}

View file

@ -0,0 +1,274 @@
/**
* Created by SouNourS on 20/12/2016.
*/
// Make sure the input files have UTF8 encoding!
import java.io.{BufferedWriter, File, FileWriter}
import java.nio.charset.CodingErrorAction
import net.psforever.packet._
import scodec.Attempt
import scodec.bits._
import scala.io.{Codec, Source}
import scala.collection.parallel.CollectionConverters._
object Xtoolspar {
def main(args: Array[String]): Unit = {
val dirToProcess = "C:\\xtools\\in"
val dirForDecoded = "C:\\xtools\\out"
val tempDir = "C:\\xtools\\temp"
val files = new File(dirToProcess).listFiles
// TODO decode packet
files.par.foreach { f =>
val file = new File(f.toString)
val FileToWrite = tempDir + "/" + file.getName().split(".gcapy")(0) + ".txt"
val FileToMoveTo = dirForDecoded + "/" + file.getName().split(".gcapy")(0) + ".txt"
if (new File(FileToMoveTo).exists()) {
println(s"File ${file.getName} exists - skipping")
} else {
println(s"${FileToMoveTo} doesn't exist - Got new file ${file.getName}")
val FileToRead = file.toString
val fw = new BufferedWriter(new FileWriter(FileToWrite, false))
try {
val decoder = Codec.UTF8.decoder.onMalformedInput(CodingErrorAction.REPORT)
var i = 0
for (line <- Source.fromFile(FileToRead)(decoder).getLines()) {
val lineTest: String = line.substring(1, 3)
// if (!lineTest.equalsIgnoreCase("IF")) {
if (i != 0) { // skip first line
//// println(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' '))))
//// println(PacketCoding.DecodePacket(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' ')))))
// handlePkt(PacketCoding.DecodePacket(ByteVector.fromValidHex(line.drop(line.lastIndexOf(' ')))))
fw.write(System.getProperty("line.separator") + "#" + line + System.getProperty("line.separator"))
var isSlotted = -1
var isMultiPacketEx = -1
var isMultiPacket = -1
var isMultiPacketExSlot = -1
var isHandleGamePacket = -1
val decodedLine = line.drop(line.lastIndexOf(' '))
var AfterDecode = Fdecode(decodedLine)
var AfterDecode2 = ""
var AfterDecode3 = ""
var AfterDecode4 = ""
var AfterDecode5 = ""
isMultiPacket = AfterDecode.indexOf("Successful(MultiPacket(")
isSlotted = AfterDecode.indexOf("Successful(SlottedMetaPacket(")
isMultiPacketEx = AfterDecode.indexOf("Successful(MultiPacketEx(")
if (isSlotted != 0 && isMultiPacket == -1 && isMultiPacketEx == -1) {
fw.write(AfterDecode + System.getProperty("line.separator"))
// println(AfterDecode )
}
if (isMultiPacket != -1) {
fw.write(AfterDecode + System.getProperty("line.separator"))
// println(AfterDecode)
var xindex1 = 1
var zindex1 = 0
var boucle1 = 0
while (boucle1 != -1) {
AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex1) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex1 + 1)))
xindex1 = AfterDecode.indexOf(" 0x", xindex1) + 1
boucle1 = AfterDecode.indexOf(" 0x", xindex1)
zindex1 = AfterDecode.indexOf(")", zindex1) + 1
isSlotted = AfterDecode2.indexOf("Successful(SlottedMetaPacket(")
if (isSlotted == 0) {
fw.write("> " + AfterDecode2 + System.getProperty("line.separator"))
// println("> " + AfterDecode2)
AfterDecode3 = Fdecode(AfterDecode2.drop(AfterDecode2.lastIndexOf(" 0x") + 3).dropRight(AfterDecode2.length - AfterDecode2.indexOf(")")))
isMultiPacketExSlot = AfterDecode3.indexOf("Successful(MultiPacketEx(")
if (isMultiPacketExSlot != -1) {
fw.write("-> " + AfterDecode3 + System.getProperty("line.separator"))
// println("-> " + AfterDecode3)
var xindex2 = 1
var zindex2 = 0
var boucle2 = 0
while (boucle2 != -1) {
AfterDecode4 = Fdecode(AfterDecode3.drop(AfterDecode3.indexOf(" 0x", xindex2) + 3).dropRight(AfterDecode3.length - AfterDecode3.indexOf(")", zindex2 + 1)))
xindex2 = AfterDecode3.indexOf(" 0x", xindex2) + 1
boucle2 = AfterDecode3.indexOf(" 0x", xindex2)
zindex2 = AfterDecode3.indexOf(")", zindex2) + 1
fw.write("--> " + AfterDecode4 + System.getProperty("line.separator"))
// println("--> " + AfterDecode4 )
}
isMultiPacketEx = -1
isMultiPacketExSlot = -1
} else {
fw.write("-> " + AfterDecode3 + System.getProperty("line.separator"))
// println("-> " + AfterDecode3 )
}
} else {
fw.write("> " + AfterDecode2 + System.getProperty("line.separator"))
// println("> " + AfterDecode2 )
}
}
}
if (isSlotted == 0 && isMultiPacket == -1) {
fw.write(AfterDecode + System.getProperty("line.separator"))
// println(AfterDecode)
AfterDecode = Fdecode(AfterDecode.drop(AfterDecode.lastIndexOf(" 0x") + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")")))
isMultiPacketExSlot = AfterDecode.indexOf("Successful(MultiPacketEx(")
isHandleGamePacket = AfterDecode.indexOf("Successful(HandleGamePacket(")
if (isHandleGamePacket != -1) {
fw.write("> " + AfterDecode + System.getProperty("line.separator"))
// println("> " + AfterDecode )
if (AfterDecode.lastIndexOf(" 0x") != -1) {
AfterDecode5 = Fdecode(AfterDecode.drop(AfterDecode.lastIndexOf(" 0x") + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")")))
fw.write("-> " + AfterDecode5 + System.getProperty("line.separator"))
// println("-> " + AfterDecode5 )
}
}
if (isMultiPacketExSlot == -1 && isHandleGamePacket == -1) {
fw.write("> " + AfterDecode + System.getProperty("line.separator"))
// println("> " + AfterDecode )
}
if (isMultiPacketExSlot != -1 && isHandleGamePacket == -1) {
fw.write("> " + AfterDecode + System.getProperty("line.separator"))
// println("> " + AfterDecode )
var xindex3 = 1
var zindex3 = 0
var boucle3 = 0
while (boucle3 != -1) {
AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex3) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex3 + 1)))
fw.write("-> " + AfterDecode2 + System.getProperty("line.separator"))
// println("-> " + AfterDecode2)
xindex3 = AfterDecode.indexOf(" 0x", xindex3) + 1
boucle3 = AfterDecode.indexOf(" 0x", xindex3)
zindex3 = AfterDecode.indexOf(")", zindex3) + 1
}
}
}
if ((isMultiPacketEx != -1 || isMultiPacketExSlot != -1) && isSlotted != 0) {
fw.write(AfterDecode + System.getProperty("line.separator"))
// println( AfterDecode )
var xindex = 1
var zindex = 0
var boucle = 0
while (boucle != -1) {
AfterDecode2 = Fdecode(AfterDecode.drop(AfterDecode.indexOf(" 0x", xindex) + 3).dropRight(AfterDecode.length - AfterDecode.indexOf(")", zindex + 1)))
fw.write("> " + AfterDecode2 + System.getProperty("line.separator"))
// println("> " + AfterDecode2)
xindex = AfterDecode.indexOf(" 0x", xindex) + 1
boucle = AfterDecode.indexOf(" 0x", xindex)
zindex = AfterDecode.indexOf(")", zindex) + 1
}
}
} else {
i += 1
}
}
}
catch {
case e: Throwable =>
println(s"File ${file.getName} threw an exception")
e.printStackTrace()
}
finally {
fw.close()
moveFile(FileToWrite, FileToMoveTo)
}
}
}
// TODO : end
}
import java.nio.file.{Files, Paths, StandardCopyOption}
def moveFile(sourcePath: String, targetPath: String): Boolean = {
var flag = true
try
Files.move(Paths.get(sourcePath), Paths.get(targetPath), StandardCopyOption.REPLACE_EXISTING)
catch {
case e: Exception =>
flag = false
e.printStackTrace()
}
flag
}
def Fdecode(toto: String): String = {
val ADecode = PacketCoding.DecodePacket(ByteVector.fromValidHex(toto)).toString;
return ADecode
}
def handlePkt(pkt : Attempt[PlanetSidePacket]) : Unit = pkt match {
case ctrl : PlanetSideControlPacket =>
println(ctrl)
// handleControlPkt(ctrl)
case game : PlanetSideGamePacket =>
println(game)
// handleGamePkt(game)
case default => println(s"Invalid packet class received: $default")
}
def handlePktContainer(pkt : PlanetSidePacketContainer) : Unit = pkt match {
case ctrl @ ControlPacket(opcode, ctrlPkt) =>
// println(pkt)
println(ctrlPkt)
// handleControlPkt(ctrlPkt)
case game @ GamePacket(opcode, seq, gamePkt) =>
// println(pkt)
println(gamePkt)
// handleGamePkt(gamePkt)
case default => println(s"Invalid packet container class received: $default")
}
// def handleControlPkt(pkt : PlanetSideControlPacket) = {
// // println(pkt)
// pkt match {
// case SlottedMetaPacket(slot, subslot, innerPacket) =>
//// sendResponse(PacketCoding.CreateControlPacket(SlottedMetaAck(slot, subslot)))
//
// PacketCoding.DecodePacket(innerPacket) match {
// case Failure(e) =>
// println(innerPacket.toString)
// println(s"Failed to decode inner packet of SlottedMetaPacket: $e")
// case Successful(v) =>
// handlePkt(v)
// }
// case sync @ ControlSync(diff, unk, f1, f2, f3, f4, fa, fb) =>
// println(s"SYNC: ${sync}")
// val serverTick = Math.abs(System.nanoTime().toInt) // limit the size to prevent encoding error
//// sendResponse(PacketCoding.CreateControlPacket(ControlSyncResp(diff, serverTick, fa, fb, fb, fa)))
// case MultiPacket(packets) =>
// packets.foreach { pkt =>
// PacketCoding.DecodePacket(pkt) match {
// case Failure(e) =>
// println(pkt.toString)
// println(s"Failed to decode inner packet of MultiPacket: $e")
// case Successful(v) =>
// handlePkt(v)
// }
// }
// case MultiPacketEx(packets) =>
// packets.foreach { pkt =>
// PacketCoding.DecodePacket(pkt) match {
// case Failure(e) =>
// println(pkt.toString)
// println(s"Failed to decode inner packet of MultiPacketEx: $e")
// case Successful(v) =>
// handlePkt(v)
// }
// }
// case default =>
// println(s"Unhandled ControlPacket $default")
// }
// }
}