Tribes2/t2-stat-parser
1
0
Fork 0
mirror of https://github.com/amineo/t2-stat-parser.git synced 2026-07-12 23:04:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Restructure for go-modules

Browse source
This commit is contained in:
Anthony Mineo 2020-03-07 09:47:28 -05:00
parent 397415fbd8
commit d17efed959
4 changed files with 324 additions and 271 deletions
Download patch file Download diff file Expand all files Collapse all files

12
app/t2-stats/go.mod Normal file
View file

@ -0,0 +1,12 @@
module github.com/amineo/t2-stat-parser
go 1.13
require (
github.com/jackc/pgx v3.6.2+incompatible
github.com/jmoiron/sqlx v1.2.0
github.com/pkg/errors v0.9.1
github.com/rs/xid v1.2.1
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect
golang.org/x/text v0.3.2 // indirect
)

21
app/t2-stats/go.sum Normal file
View file

@ -0,0 +1,21 @@
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o=
github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

273
app/t2-stats/main.go
View file

@ -12,278 +12,9 @@ Parser for DarkTiger's T2 Server Stats
package main package main
import ( import (
"bufio" _ "github.com/amineo/t2-stat-parser"
"database/sql"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
_ "github.com/jackc/pgx/stdlib"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"github.com/rs/xid"
) )
var (
connectionString = flag.String("conn", getenvWithDefault("DATABASE_URL", ""), "PostgreSQL connection string")
db *sqlx.DB
maxStatOverwrite int = 100
debugLevel int = 1 // 0 off,min | 1 Basic output checks | 2 Output all the things
)
func getenvWithDefault(name, defaultValue string) string {
val := os.Getenv(name)
if val == "" {
val = defaultValue
}
return val
}
func genXid() string {
id := xid.New()
return id.String()
}
// Game is...
type Game struct {
playerGUID int `db.players:"player_guid"`
playerName string `db.players:"player_name"`
totalGames int `db.players:"total_games"`
dbStatOverWrite int `db.players:"stat_overwrite"`
statOverWrite int
gameMap string `db.games:"map"`
gameType string `db.games:"gametype"`
dateStamp string `db.games:"datestamp"`
stats string `db.games:"stats"`
uuid string `db.games:"uuid"`
}
func main() { func main() {
start := time.Now() init()
flag.Parse()
var err error
// postgres connection
if *connectionString == "" {
log.Fatalln("Please pass the connection string using the -conn option")
}
db, err = sqlx.Connect("pgx", *connectionString)
if err != nil {
log.Fatalf("Unable to establish connection: %v\n", err)
}
// ----
var statFiles []string
statFolder := "serverStats/CTFGame" // GameType is CTFGame
fileErr := filepath.Walk(statFolder, func(path string, info os.FileInfo, fileErr error) error {
r, err := regexp.MatchString("g.cs", path)
if err == nil && r {
statFiles = append(statFiles, path)
}
return nil
})
if fileErr != nil {
panic(fileErr)
}
for _, file := range statFiles {
fmt.Println(file)
g := Game{}
// The file name is the player GUID so we need to store that in the object since its not part of the stat line
// Add playerGUID, gameType as part of the stat line
g.gameType = strings.Split(file, "/")[1]
g.playerGUID, err = strconv.Atoi(regexp.MustCompile("[0-9]+").FindAllString(file, -1)[0])
if err != nil {
fmt.Println("Couldn't convert playerGUID to an int", err)
}
file, err := os.Open(file)
//file, err := os.Open("serverStats/CTFGame/1g.cs")
if err != nil {
log.Fatal(err)
}
defer file.Close()
// Raw map for the stat lines in the file
mStatLine := make(map[string][]string)
scanner := bufio.NewScanner(file)
for scanner.Scan() {
statScanLine := scanner.Text()
// Split the lines on %t
lineParts := strings.Split(statScanLine, "%t")
// The first token of the part is the key
statHeading := lineParts[0]
// Loop over the parts from the string.
for i := range lineParts {
if i > 0 {
mStatLine[statHeading] = append(mStatLine[statHeading], lineParts[i])
}
}
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
g.playerName = mStatLine["playerName"][0]
g.totalGames, err = strconv.Atoi(mStatLine["totalGames"][0])
if err != nil {
fmt.Println("Couldn't convert totalGames to an int", err)
}
g.statOverWrite, err = strconv.Atoi(mStatLine["statsOverWrite"][0])
if err != nil {
fmt.Println("Couldn't convert statOverWrite to an int", err)
}
if debugLevel >= 1 {
// These dont have a position offset
fmt.Println("playerGUID", g.playerGUID)
fmt.Println("GameType", g.gameType)
fmt.Println("Player Name", g.playerName)
fmt.Println("Stat Overwrite", g.statOverWrite)
fmt.Println("maxStatOverwrite", maxStatOverwrite)
}
checkPlayer(g)
g.dbStatOverWrite = getDBStatOverWrite(g.playerGUID)
statCron := 0
if g.statOverWrite < g.dbStatOverWrite {
statCron = (maxStatOverwrite - g.statOverWrite) + g.dbStatOverWrite
} else {
statCron = g.statOverWrite - g.dbStatOverWrite
}
if debugLevel >= 1 {
fmt.Println("statCron", statCron)
}
for i := 1; i <= statCron; i++ {
arrPosition := i - 1
fmt.Println(arrPosition)
parseStatOverWriteLine(g, mStatLine, arrPosition)
}
fmt.Println("---")
}
// Close connection to DB
//defer conn.Close(context.Background())
fmt.Println("Total Execution time: ", time.Since(start))
}
func parseStatOverWriteLine(g Game, mStatLine map[string][]string, arrPosition int) {
if debugLevel == 1 {
fmt.Println("Running fn parseStatOverWriteLine")
fmt.Println("playerGUID", g.playerGUID, "arrPosition", arrPosition)
}
cleanStatLine := make(map[string][]string)
for index, element := range mStatLine {
if len(element) > 1 {
cleanStatLine[index] = append(cleanStatLine[index], element[arrPosition])
}
}
jsonStats, err := json.Marshal(cleanStatLine)
if err != nil {
log.Fatal("Error parsing jsonStats", err)
}
g.stats = string(jsonStats)
g.dateStamp = mStatLine["dateStamp"][arrPosition]
g.uuid = genXid()
g.gameMap = cleanStatLine["map"][0]
if debugLevel >= 2 {
// log the game struct
fmt.Println(g)
}
// insert game stat
addPlayerGameStat(g)
}
func rowExists(query string, args ...interface{}) bool {
var exists bool
query = fmt.Sprintf("SELECT exists (%s)", query)
err := db.QueryRow(query, args...).Scan(&exists)
if err != nil && err != sql.ErrNoRows {
errors.Wrap(err, "error checking if row exists")
}
return exists
}
func checkPlayer(g Game) {
check := rowExists("select player_guid from players where player_guid = $1", g.playerGUID)
if !check {
createPlayer(genXid(), g)
} else {
fmt.Println("Player already exists", g.playerName)
}
}
func createPlayer(uuid string, g Game) {
fmt.Println("Creating new player", g.playerName)
_, err := db.Exec("insert into players(uuid, player_guid, player_name) values($1,$2,$3)", uuid, g.playerGUID, g.playerName)
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to add player's game stat: %v\n", err)
os.Exit(1)
}
}
func getDBStatOverWrite(playerGUID int) int {
var dbStatOverWrite, dbPlayerGUID int
// Used to compare ingested statOverWrite to known dbStatOverWrite
err := db.QueryRow("select player_guid, stat_overwrite from players where player_guid = $1", playerGUID).Scan(&dbPlayerGUID, &dbStatOverWrite)
if err != nil {
fmt.Fprintf(os.Stderr, "QueryRow failed: %v\n", err)
os.Exit(1)
}
return dbStatOverWrite
}
func addPlayerGameStat(g Game) {
if debugLevel == 1 {
fmt.Println("g.dbStatOverWrite", g.dbStatOverWrite, "g.statOverWrite", g.statOverWrite)
}
if g.dbStatOverWrite != g.statOverWrite {
// Insert new stat line
fmt.Println("New stat line!", g.playerName)
sqlInsert := `insert into games(player_guid, player_name, stat_overwrite, map, stats, datestamp, uuid, gametype) values($1,$2,$3,$4,$5,$6,$7,$8)`
_, err := db.Exec(sqlInsert, g.playerGUID, g.playerName, g.statOverWrite, g.gameMap, g.stats, g.dateStamp, g.uuid, g.gameType)
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to add player's game stat: %v\n", err)
os.Exit(1)
}
// Need to update a players statOverWrite after comparison
sqlUpdate := `UPDATE players SET stat_overwrite = $2, total_games = $3 WHERE player_guid = $1;`
_, err = db.Exec(sqlUpdate, g.playerGUID, g.statOverWrite, g.totalGames)
if err != nil {
panic(err)
}
} else {
fmt.Println("This stat line already exists")
}
} }

289
app/t2-stats/parser.go Normal file
View file

@ -0,0 +1,289 @@
/*
Parser for DarkTiger's T2 Server Stats
[TODO]
- Read Additional GameTypes on the fly
- Use go modules
- Update to v4 SQL driver
- Ability to download stat files from remote server via FTP
*/
package parser
import (
"bufio"
"database/sql"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
_ "github.com/jackc/pgx/stdlib"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"github.com/rs/xid"
)
var (
connectionString = flag.String("conn", getenvWithDefault("DATABASE_URL", ""), "PostgreSQL connection string")
db *sqlx.DB
maxStatOverwrite int = 100
debugLevel int = 1 // 0 off,min | 1 Basic output checks | 2 Output all the things
)
func getenvWithDefault(name, defaultValue string) string {
val := os.Getenv(name)
if val == "" {
val = defaultValue
}
return val
}
func genXid() string {
id := xid.New()
return id.String()
}
// Game is...
type Game struct {
playerGUID int `db.players:"player_guid"`
playerName string `db.players:"player_name"`
totalGames int `db.players:"total_games"`
dbStatOverWrite int `db.players:"stat_overwrite"`
statOverWrite int
gameMap string `db.games:"map"`
gameType string `db.games:"gametype"`
dateStamp string `db.games:"datestamp"`
stats string `db.games:"stats"`
uuid string `db.games:"uuid"`
}
func init() {
start := time.Now()
flag.Parse()
var err error
// postgres connection
if *connectionString == "" {
log.Fatalln("Please pass the connection string using the -conn option")
}
db, err = sqlx.Connect("pgx", *connectionString)
if err != nil {
log.Fatalf("Unable to establish connection: %v\n", err)
}
// ----
var statFiles []string
statFolder := "serverStats/CTFGame" // GameType is CTFGame
fileErr := filepath.Walk(statFolder, func(path string, info os.FileInfo, fileErr error) error {
r, err := regexp.MatchString("g.cs", path)
if err == nil && r {
statFiles = append(statFiles, path)
}
return nil
})
if fileErr != nil {
panic(fileErr)
}
for _, file := range statFiles {
fmt.Println(file)
g := Game{}
// The file name is the player GUID so we need to store that in the object since its not part of the stat line
// Add playerGUID, gameType as part of the stat line
g.gameType = strings.Split(file, "/")[1]
g.playerGUID, err = strconv.Atoi(regexp.MustCompile("[0-9]+").FindAllString(file, -1)[0])
if err != nil {
fmt.Println("Couldn't convert playerGUID to an int", err)
}
file, err := os.Open(file)
//file, err := os.Open("serverStats/CTFGame/1g.cs")
if err != nil {
log.Fatal(err)
}
defer file.Close()
// Raw map for the stat lines in the file
mStatLine := make(map[string][]string)
scanner := bufio.NewScanner(file)
for scanner.Scan() {
statScanLine := scanner.Text()
// Split the lines on %t
lineParts := strings.Split(statScanLine, "%t")
// The first token of the part is the key
statHeading := lineParts[0]
// Loop over the parts from the string.
for i := range lineParts {
if i > 0 {
mStatLine[statHeading] = append(mStatLine[statHeading], lineParts[i])
}
}
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
g.playerName = mStatLine["playerName"][0]
g.totalGames, err = strconv.Atoi(mStatLine["totalGames"][0])
if err != nil {
fmt.Println("Couldn't convert totalGames to an int", err)
}
g.statOverWrite, err = strconv.Atoi(mStatLine["statsOverWrite"][0])
if err != nil {
fmt.Println("Couldn't convert statOverWrite to an int", err)
}
if debugLevel >= 1 {
// These dont have a position offset
fmt.Println("playerGUID", g.playerGUID)
fmt.Println("GameType", g.gameType)
fmt.Println("Player Name", g.playerName)
fmt.Println("Stat Overwrite", g.statOverWrite)
fmt.Println("maxStatOverwrite", maxStatOverwrite)
}
checkPlayer(g)
g.dbStatOverWrite = getDBStatOverWrite(g.playerGUID)
statCron := 0
if g.statOverWrite < g.dbStatOverWrite {
statCron = (maxStatOverwrite - g.statOverWrite) + g.dbStatOverWrite
} else {
statCron = g.statOverWrite - g.dbStatOverWrite
}
if debugLevel >= 1 {
fmt.Println("statCron", statCron)
}
for i := 1; i <= statCron; i++ {
arrPosition := i - 1
fmt.Println(arrPosition)
parseStatOverWriteLine(g, mStatLine, arrPosition)
}
fmt.Println("---")
}
// Close connection to DB
//defer conn.Close(context.Background())
fmt.Println("Total Execution time: ", time.Since(start))
}
func parseStatOverWriteLine(g Game, mStatLine map[string][]string, arrPosition int) {
if debugLevel == 1 {
fmt.Println("Running fn parseStatOverWriteLine")
fmt.Println("playerGUID", g.playerGUID, "arrPosition", arrPosition)
}
cleanStatLine := make(map[string][]string)
for index, element := range mStatLine {
if len(element) > 1 {
cleanStatLine[index] = append(cleanStatLine[index], element[arrPosition])
}
}
jsonStats, err := json.Marshal(cleanStatLine)
if err != nil {
log.Fatal("Error parsing jsonStats", err)
}
g.stats = string(jsonStats)
g.dateStamp = mStatLine["dateStamp"][arrPosition]
g.uuid = genXid()
g.gameMap = cleanStatLine["map"][0]
if debugLevel >= 2 {
// log the game struct
fmt.Println(g)
}
// insert game stat
addPlayerGameStat(g)
}
func rowExists(query string, args ...interface{}) bool {
var exists bool
query = fmt.Sprintf("SELECT exists (%s)", query)
err := db.QueryRow(query, args...).Scan(&exists)
if err != nil && err != sql.ErrNoRows {
errors.Wrap(err, "error checking if row exists")
}
return exists
}
func checkPlayer(g Game) {
check := rowExists("select player_guid from players where player_guid = $1", g.playerGUID)
if !check {
createPlayer(genXid(), g)
} else {
fmt.Println("Player already exists", g.playerName)
}
}
func createPlayer(uuid string, g Game) {
fmt.Println("Creating new player", g.playerName)
_, err := db.Exec("insert into players(uuid, player_guid, player_name) values($1,$2,$3)", uuid, g.playerGUID, g.playerName)
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to add player's game stat: %v\n", err)
os.Exit(1)
}
}
func getDBStatOverWrite(playerGUID int) int {
var dbStatOverWrite, dbPlayerGUID int
// Used to compare ingested statOverWrite to known dbStatOverWrite
err := db.QueryRow("select player_guid, stat_overwrite from players where player_guid = $1", playerGUID).Scan(&dbPlayerGUID, &dbStatOverWrite)
if err != nil {
fmt.Fprintf(os.Stderr, "QueryRow failed: %v\n", err)
os.Exit(1)
}
return dbStatOverWrite
}
func addPlayerGameStat(g Game) {
if debugLevel == 1 {
fmt.Println("g.dbStatOverWrite", g.dbStatOverWrite, "g.statOverWrite", g.statOverWrite)
}
if g.dbStatOverWrite != g.statOverWrite {
// Insert new stat line
fmt.Println("New stat line!", g.playerName)
sqlInsert := `insert into games(player_guid, player_name, stat_overwrite, map, stats, datestamp, uuid, gametype) values($1,$2,$3,$4,$5,$6,$7,$8)`
_, err := db.Exec(sqlInsert, g.playerGUID, g.playerName, g.statOverWrite, g.gameMap, g.stats, g.dateStamp, g.uuid, g.gameType)
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to add player's game stat: %v\n", err)
os.Exit(1)
}
// Need to update a players statOverWrite after comparison
sqlUpdate := `UPDATE players SET stat_overwrite = $2, total_games = $3 WHERE player_guid = $1;`
_, err = db.Exec(sqlUpdate, g.playerGUID, g.statOverWrite, g.totalGames)
if err != nil {
panic(err)
}
} else {
fmt.Println("This stat line already exists")
}
}