mirror of
https://github.com/Ragora/TSScraper.git
synced 2026-01-19 20:24:57 +00:00
Functional analytics; stub definitions for various Db types
This commit is contained in:
Robert MacGregor
parent
e164a5a201
commit
5cccabb0a3
507
scriptScrape.py
507
scriptScrape.py
|
|
@ -23,32 +23,440 @@ class Function(object):
|
|||
filepath = None
|
||||
line = None
|
||||
|
||||
aliases = None
|
||||
comments = None
|
||||
|
||||
def __init__(self, name, type, parameters, filepath, line):
|
||||
self.name = name
|
||||
self.parameters = parameters
|
||||
self.filepath = filepath
|
||||
self.line = line
|
||||
self.aliases = [ ]
|
||||
self.type = type
|
||||
|
||||
class Global(object):
|
||||
name = None
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __repr__(self):
|
||||
return "$%s" % self.name
|
||||
|
||||
class Datablock(object):
|
||||
name = None
|
||||
type = None
|
||||
derived = None
|
||||
line = None
|
||||
|
||||
def __init__(self, name, type, derived):
|
||||
aliases = None
|
||||
properties = None
|
||||
filepath = None
|
||||
comments = None
|
||||
|
||||
def __init__(self, name, type, properties, filepath, line, derived):
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.derived = derived
|
||||
self.line = line
|
||||
self.aliases = [ ]
|
||||
self.properties = properties
|
||||
self.filepath = filepath
|
||||
|
||||
class Application(object):
|
||||
bound_function_pattern = re.compile("function +(([A-z]|_)+::)([A-z]|_)+\( *(%[A-z]+( *, *%[A-z]+)*)* *\)")
|
||||
function_pattern = re.compile("function +([A-z]|_)+\( *(%[A-z]+( *, *%[A-z]+)*)* *\)")
|
||||
datablock_pattern = re.compile("datablock +[A-z]+ *( *[A-z]+ *)( *: *[A-z]+)?")
|
||||
bound_function_pattern = re.compile("(?<!.)\s*function\s+(([A-z]|_)+::)([A-z]|_)+\(\s*(%[A-z]+(\s*,\s*%[A-z]+)*)*\s*\)")
|
||||
function_pattern = re.compile("(?<!.)\s*function\s+([A-z]|_)+\(\s*(%[A-z]+(\w*,\s*%[A-z]+)*)*\s*\)")
|
||||
datablock_pattern = re.compile("(?<!.)\s*datablock\s+[A-z]+\s*\(\s*\S+\s*\)\s*(:\s*[A-z]+)?\s*(//.*)?\s*\{(\s|\S)*?\s*(?<!.)\};")
|
||||
|
||||
key_value_pattern = re.compile("(?<!.)\s*[A-z]+\s*=\s*(\S+);")
|
||||
|
||||
#block_iterator = re.compile("function\s+[A-z]+\s*\(\s*(%[A-z]+(\w*,\s*%[A-z]+)*)*\s*\)\{\S*\}")
|
||||
|
||||
# (?<!{)\s*\$[A-z]+(::([A-z]+))*?\s*(?!})
|
||||
global_usages = re.compile("\{.*\$[A-z]+(::([A-z]+))*?.*\}")
|
||||
global_pattern = re.compile("(?<!.)\s*$[A-z]+(::([A-z]+))*?")
|
||||
|
||||
parameter_split = re.compile("\s*,\s*")
|
||||
assignment_split = re.compile("\s*=\s*")
|
||||
|
||||
def print_usage(self):
|
||||
print("Usage: '%s <target directory> <exporter>'" % sys.argv[0])
|
||||
print("Or: '%s exporters' for a list of known exporters." % sys.argv[0])
|
||||
|
||||
# Tables for checking datablock data
|
||||
datablock_reference_table = {
|
||||
"tracerprojectiledata": {
|
||||
"references": ["splash", "explosion", "sound"],
|
||||
"declared": [ ],
|
||||
"checks": {
|
||||
"fizzletimems": (lambda x: x >= 0, "Cannot use negative fizzle time!")
|
||||
}
|
||||
},
|
||||
|
||||
"shapebaseimagedata": {
|
||||
"references": ["item", "projectile"],
|
||||
"declared": ["projectiletype"],
|
||||
"checks": {
|
||||
}
|
||||
},
|
||||
|
||||
"itemdata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { }
|
||||
},
|
||||
|
||||
"audioprofile": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"simdatablock": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"jeteffectdata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"hovervehicledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"stationfxpersonaldata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"cameradata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"triggerdata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"wheeledvehicledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"tsshapeconstructor": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"bombprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"stationfxvehicledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"staticshapedata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"decaldata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"repairprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"explosiondata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"linearprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"elfprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"linearflareprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"sensordata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"forcefieldbaredata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"particledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"particleemitterdata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"playerdata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"turretdata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"turretimagedata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"shockwavedata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"seekerprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"debrisdata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"grenadeprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"sniperprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"sniperprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"flyingvehicledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"splashdata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"energyprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"flareprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"targetprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
|
||||
"shocklanceprojectiledata": {
|
||||
"references": [ ],
|
||||
"declared": [ ],
|
||||
"checks": { },
|
||||
},
|
||||
}
|
||||
|
||||
"""
|
||||
TracerProjectileData:
|
||||
splash
|
||||
explosion
|
||||
sound
|
||||
|
||||
ShapeBaseImageData:
|
||||
item
|
||||
projectile
|
||||
projectileType == projectile.type
|
||||
"""
|
||||
def check_datablock_references(self, data, known_datablocks):
|
||||
|
||||
# For each file entry
|
||||
for file in data:
|
||||
# For each datablock
|
||||
for datablock in file.datablocks:
|
||||
if (datablock.type in self.datablock_reference_table):
|
||||
# Flip through each reference in the table
|
||||
for reference in self.datablock_reference_table[datablock.type]["references"]:
|
||||
if (reference not in datablock.properties):
|
||||
print("Reference Warning: %s datablock '%s' has no '%s' declaration! (Declaration in %s, line %u)" % (datablock.type, datablock.name, reference, datablock.filepath, datablock.line))
|
||||
else:
|
||||
if (datablock.properties[reference] not in known_datablocks.keys()):
|
||||
print("Reference Warning: %s Datablock '%s' references '%s' in property '%s', which does not exist! (Declaration in %s, line %u)" % (datablock.type, datablock.name, datablock.properties[reference], reference, datablock.filepath, datablock.line))
|
||||
|
||||
# Check each declaration
|
||||
for declaration in self.datablock_reference_table[datablock.type]["declared"]:
|
||||
if (declaration not in datablock.properties):
|
||||
print("Declaration Warning: %s Datablock '%s' required property '%s' not declared! (Declaration in %s, line %u)" % (datablock.type, datablock.name, declaration, datablock.filepath, datablock.line))
|
||||
|
||||
# Run custom checks
|
||||
for check in self.datablock_reference_table[datablock.type]["checks"].keys():
|
||||
# Is it declared?
|
||||
if (check not in datablock.properties):
|
||||
print("Property Warning: %s Datablock %s '%s' property not declared! (Declaration in %s, line %u)" % (datablock.type, datablock.name, check, datablock.filepath, datablock.line))
|
||||
else:
|
||||
method, message = self.datablock_reference_table[datablock.type]["checks"][check]
|
||||
|
||||
if (not method(datablock.properties[check])):
|
||||
print("Property Warning (Datablock '%s', type %s. Declaration in %s, line %u): %s" % (datablock.name, datablock.type, datablock.filepath, datablock.line, message))
|
||||
else:
|
||||
print("Program Error: Unknown datablock type '%s'! This means the software does not know how to check this datablock. (Declaration in %s, line %u)" % (datablock.type, datablock.filepath, datablock.line))
|
||||
|
||||
def resolve_datablock_parents(self, data, known_datablocks):
|
||||
# For each file entry
|
||||
for file in data:
|
||||
# For each datablock
|
||||
for datablock in file.datablocks:
|
||||
if (datablock.derived is not None and datablock.derived not in known_datablocks.keys()):
|
||||
print("Warning: Datablock '%s' derives from non-existent parent '%s'! (Declaration in %s, line %u)" % (datablock.name, datablock.derived,datablock.filepath, datablock.line))
|
||||
elif (datablock.derived is not None):
|
||||
datablock.derived = known_datablocks[datablock.derived]
|
||||
|
||||
def process_data(self, data):
|
||||
# Entries we've already processed
|
||||
processed_entries = { }
|
||||
|
||||
# For each file entry
|
||||
for file in data:
|
||||
# For each global function
|
||||
for global_function in file.global_functions:
|
||||
processed_entries.setdefault(global_function.name, global_function)
|
||||
|
||||
# Check for declarations
|
||||
if (processed_entries[global_function.name] is not global_function):
|
||||
known_entry = processed_entries[global_function.name]
|
||||
|
||||
# Redeclaration with different param count
|
||||
if (len(known_entry.parameters) != len(global_function.parameters)):
|
||||
global_function.aliases.append(known_entry)
|
||||
known_entry.aliases.append(global_function)
|
||||
print("Warning: Global function '%s' redeclared with %u parameters in %s, line %u! (Original declaration in %s, line %u with %u parameters)" % (known_entry.name, len(global_function.parameters), global_function.filepath, global_function.line, known_entry.filepath, known_entry.line, len(known_entry.parameters)))
|
||||
# Regular Redeclaration
|
||||
else:
|
||||
global_function.aliases.append(known_entry)
|
||||
known_entry.aliases.append(global_function)
|
||||
print("Warning: Global function '%s' redeclared in %s, line %u! (Original declaration in %s, line %u)" % (known_entry.name, global_function.filepath, global_function.line, known_entry.filepath, known_entry.line))
|
||||
|
||||
processed_entries = { }
|
||||
|
||||
# For each bound function
|
||||
for bound_type in file.bound_functions.keys():
|
||||
for bound_function in file.bound_functions[bound_type]:
|
||||
processed_entries.setdefault(bound_function.type, {})
|
||||
processed_entries[bound_function.type].setdefault(bound_function.name, bound_function)
|
||||
|
||||
# Check for declarations
|
||||
if (processed_entries[bound_function.type][bound_function.name] is not bound_function):
|
||||
known_entry = processed_entries[bound_function.type][bound_function.name]
|
||||
|
||||
# Redeclaration with different param count
|
||||
if (len(known_entry.parameters) != len(bound_function.parameters)):
|
||||
bound_function.aliases.append(known_entry)
|
||||
known_entry.aliases.append(bound_function)
|
||||
print("Warning: Bound function '%s::%s' redeclared with %u parameters in %s, line %u! (Original declaration in %s, line %u with %u parameters)" % (known_entry.type, known_entry.name, len(bound_function.parameters), bound_function.filepath, bound_function.line, known_entry.filepath, known_entry.line, len(known_entry.parameters)))
|
||||
# Regular Redeclaration
|
||||
else:
|
||||
bound_function.aliases.append(known_entry)
|
||||
known_entry.aliases.append(bound_function)
|
||||
print("Warning: Bound function '%s::%s' redeclared in %s, line %u! (Original declaration in %s, line %u)" % (known_entry.type, known_entry.name, bound_function.filepath, bound_function.line, known_entry.filepath, known_entry.line))
|
||||
|
||||
processed_entries = { }
|
||||
|
||||
# For each datablock
|
||||
known_datablocks = { }
|
||||
for datablock in file.datablocks:
|
||||
processed_entries.setdefault(datablock.name, datablock)
|
||||
known_datablocks.setdefault(datablock.name, [])
|
||||
known_datablocks[datablock.name].append(datablock)
|
||||
|
||||
# Check for declarations
|
||||
if (processed_entries[datablock.name] is not datablock):
|
||||
known_entry = processed_entries[datablock.name]
|
||||
|
||||
# Redeclaration with different parent
|
||||
if (known_entry.derived != datablock.derived):
|
||||
known_entry.aliases.append(datablock)
|
||||
datablock.aliases.append(known_entry)
|
||||
print("Warning: Datablock '%s' redeclared in %s, line %u with parent '%s'! (Original declaration in %s, line %u with parent '%s')" % (datablock.name, datablock.filepath, datablock.line, datablock.derived, known_entry.filepath, known_entry.line, known_entry.derived))
|
||||
# Regular Redeclaration
|
||||
else:
|
||||
known_entry.aliases.append(datablock)
|
||||
datablock.aliases.append(known_entry)
|
||||
print("Warning: Datablock '%s' redeclared in %s, line %u! (Original declaration in %s, line %u" % (datablock.name, datablock.filepath, datablock.line, known_entry.filepath, known_entry.line))
|
||||
|
||||
return known_datablocks
|
||||
|
||||
def main(self):
|
||||
# Load exporters
|
||||
exporters = { }
|
||||
|
|
@ -96,6 +504,11 @@ class Application(object):
|
|||
|
||||
if (not os.path.isfile(filepath)):
|
||||
continue
|
||||
|
||||
# Only check TS files
|
||||
name, extension = os.path.splitext(filepath)
|
||||
if (extension != ".cs"):
|
||||
continue
|
||||
|
||||
with open(filepath, "r") as handle:
|
||||
file_entry = FileEntry(filepath)
|
||||
|
|
@ -105,10 +518,10 @@ class Application(object):
|
|||
# Grab Global function definitions
|
||||
for match in re.finditer(self.function_pattern, file_data):
|
||||
line = file_data[0:match.start()].count("\n") + 1
|
||||
match_split = match.group(0).lstrip("function ").split("(")
|
||||
name = match_split[0]
|
||||
match_split = match.group(0).lstrip().rstrip().lstrip("function ").split("(")
|
||||
name = match_split[0].lower()
|
||||
|
||||
match_split = match_split[1].replace(")", "").split(",")
|
||||
match_split = re.split(self.parameter_split, match_split[1].replace(")", ""))
|
||||
|
||||
parameters = [ ]
|
||||
for parameter in match_split:
|
||||
|
|
@ -119,26 +532,15 @@ class Application(object):
|
|||
|
||||
file_entry.global_functions.append(Function(name, None, parameters, filepath, line))
|
||||
|
||||
tracked_name = name.lower()
|
||||
global_aliases.setdefault(tracked_name, (0, filepath, line))
|
||||
|
||||
occurrence_count, old_filepath, old_line = global_aliases[tracked_name]
|
||||
occurrence_count = occurrence_count + 1
|
||||
global_aliases[tracked_name] = (occurrence_count, old_filepath, old_line)
|
||||
|
||||
if (occurrence_count != 1):
|
||||
print("Warning: Found a multiple declaration of global function '%s' in %s, line %u! (Original detection: %s, line %u)" % (tracked_name, filepath, line, old_filepath, old_line))
|
||||
|
||||
|
||||
# Grab bound function definitions
|
||||
for match in re.finditer(self.bound_function_pattern, file_data):
|
||||
line = file_data[0:match.start()].count("\n") + 1
|
||||
|
||||
match_split = match.group(0).lstrip("function ").split("::")
|
||||
type = match_split[0]
|
||||
match_split = match.group(0).lstrip().rstrip().lstrip("function ").split("::")
|
||||
type = match_split[0].lower()
|
||||
|
||||
match_split = match_split[1].split("(")
|
||||
name = match_split[0]
|
||||
name = match_split[0].lower()
|
||||
match_split = match_split[1].replace(")", "").split(",")
|
||||
|
||||
parameters = [ ]
|
||||
|
|
@ -149,28 +551,57 @@ class Application(object):
|
|||
|
||||
file_entry.bound_functions.setdefault(type, [])
|
||||
file_entry.bound_functions[type].append(Function(name, type, parameters, filepath, line))
|
||||
|
||||
tracked_name = name.lower()
|
||||
tracked_type = type.lower()
|
||||
typed_aliases.setdefault(tracked_type, {})
|
||||
typed_aliases[tracked_type].setdefault(tracked_name, (0, filepath, line))
|
||||
|
||||
occurrence_count, old_filepath, old_line = typed_aliases[tracked_type][tracked_name]
|
||||
occurrence_count = occurrence_count + 1
|
||||
typed_aliases[tracked_type][tracked_name] = (occurrence_count, old_filepath, old_line)
|
||||
|
||||
if (occurrence_count != 1):
|
||||
print("Warning: Found a multiple declaration of bound function '%s::%s' in %s, line %u! (Original detection: %s, line %u)" % (tracked_type, tracked_name, filepath, line, old_filepath, old_line))
|
||||
|
||||
# Grab DB definitions
|
||||
|
||||
# Grab non-inherited DB definitions
|
||||
for match in re.finditer(self.datablock_pattern, file_data):
|
||||
match_text = match.group(0).lstrip("datablock ")
|
||||
line = file_data[0:match.start()].count("\n") + 1
|
||||
match_text = match.group(0).lstrip().rstrip()
|
||||
|
||||
#print(match_text)
|
||||
header = match_text[0:match_text.find("{")]
|
||||
type = header[len("datablock") + 1:header.find("(")].lstrip().rstrip().lower()
|
||||
name = header[header.find("(") + 1:header.find(")")].lstrip().rstrip().lower()
|
||||
|
||||
# Inherited?
|
||||
inherited = None
|
||||
inheritor = header.find(":")
|
||||
if (inheritor != -1):
|
||||
inherited = header[inheritor + 1:].lstrip().rstrip().lower()
|
||||
|
||||
# Blow through key, values
|
||||
properties = { }
|
||||
for property_match in re.finditer(self.key_value_pattern, match_text):
|
||||
property_text = property_match.group(0)
|
||||
|
||||
key, value = re.split(self.assignment_split, property_text, 1)
|
||||
key = key.lstrip().lower()
|
||||
|
||||
value = value.rstrip().rstrip(";")
|
||||
|
||||
# Global reference
|
||||
if (value[0] == "$"):
|
||||
value = Global(value[1:])
|
||||
# String
|
||||
elif (value[0] == "\""):
|
||||
value = value[1:value.rfind("\"")]
|
||||
# Numerics
|
||||
else:
|
||||
try:
|
||||
value = float(value)
|
||||
except ValueError as e:
|
||||
# If this was raised, treat it as a string
|
||||
pass
|
||||
|
||||
properties[key] = value
|
||||
|
||||
file_entry.datablocks.append(Datablock(name, type, properties, filepath, line, inherited))
|
||||
|
||||
# Stick in results
|
||||
results.append(file_entry)
|
||||
|
||||
|
||||
known_datablocks = self.process_data(results)
|
||||
self.resolve_datablock_parents(results, known_datablocks)
|
||||
self.check_datablock_references(results, known_datablocks)
|
||||
|
||||
# Init the DokuOutput
|
||||
output = exporter.Exporter(results)
|
||||
output.write()
|
||||
|
|
|
|||
Loading…
Reference in a new issue