Tribes2/TSScraper
1
0
Fork 0
mirror of https://github.com/Ragora/TSScraper.git synced 2026-01-19 20:24:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Initial commit.

Browse source
This commit is contained in:
Robert MacGregor 2015-07-15 02:40:12 -04:00
commit c149d6bad8
6 changed files with 329 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
data/filetempl.html Normal file
View file

@ -0,0 +1,15 @@
<HTML>
<HEAD>
<TITLE>TS Scraper | {{ file }}</TITLE>
</HEAD>
<BODY>
<CENTER>
Globals: </br>
{% for global in globals %}
{{ global.name }}</br>
{% endfor %}
</CENTER>
</BODY>
</HTML>

24
data/indextempl.html Normal file
View file

@ -0,0 +1,24 @@
<HTML>
<HEAD>
<TITLE>TS Scraper | Index</TITLE>
</HEAD>
<BODY>
<CENTER>
<FONT SIZE="12">
TS Scraper
</FONT>
<HR></br>
<FONT SIZE="6"><U>File Index</U></FONT></br></br>
{% for file in files %}
<a href="{{ file.path }}">{{ file.path }}</a></br>
{% endfor %}
</br>
This file was auto-generated by TS Scraper.</br>
TS Scraper is written in Python 2 by Robert MacGregor.
</CENTER>
</BODY>
</HTML>

1
exporters/__init__.py Normal file
View file

@ -0,0 +1 @@

64
exporters/doku.py Normal file
View file

@ -0,0 +1,64 @@
import re
import os
import sys
import importlib
import os.path
class Exporter(object):
data = None
def __init__(self, data):
self.data = data
def write(self):
with open("Out.txt", "w") as handle:
# Write the header
handle.write("====== Test ======\n\n")
# For each file entry...
for file in self.data:
if (len(file.global_functions) == 0 and len(file.bound_functions.keys()) == 0 and len(file.datablocks) == 0):
continue
# Calculate the total entry count
entry_count = len(file.global_functions) + len(file.datablocks)
for type in file.bound_functions.keys():
entry_count = entry_count + len(file.bound_functions[type])
handle.write("===== Entries in %s (%u total) =====\n\n" % (file.path, entry_count))
handle.write("===== Global Functions (%u total) =====\n\n" % len(file.global_functions))
# For each global function...
for function in file.global_functions:
handle.write("==== %s ====\n" % function.name)
handle.write("File (line %u): %s\n\n" % (function.line, file.path))
if (len(function.parameters) != 0):
handle.write("Parameters (in order):\n")
for parameter in function.parameters:
handle.write(" * %s\n" % parameter)
else:
handle.write("Parameters: None\n")
handle.write("\n")
# For each known type...
for type in file.bound_functions.keys():
handle.write("===== Bound Functions on %s (%u total) =====\n\n" % (type, len(file.bound_functions[type])))
# For each function for this type...
for function in file.bound_functions[type]:
handle.write("==== %s::%s ====\n" % (function.type, function.name))
handle.write("File (line %u): %s\n\n" % (function.line, file.path))
if (len(function.parameters) != 0):
handle.write("Parameters (in order):\n")
for parameter in function.parameters:
handle.write(" * %s\n" % parameter)
else:
handle.write("Parameters: None\n")
handle.write("\n")
print("Done processing.")

46
exporters/html.py Normal file
View file

@ -0,0 +1,46 @@
import re
import os
import sys
import importlib
import os.path
class Exporter(object):
data = None
def __init__(self, data):
self.data = data
def write(self):
import jinja2
# Read the template files first
file_template = None
with open("data/filetempl.html", "r") as handle:
file_template = handle.read()
index_template = None
with open("data/indextempl.html", "r") as handle:
index_template = handle.read()
html_filenames = [ ]
# For each file entry...
for file in self.data:
if (len(file.global_functions) == 0 and len(file.bound_functions.keys()) == 0 and len(file.datablocks) == 0):
continue
html_filename = file.path.lstrip("./").replace("/", "-")
html_filename, oldextension = os.path.splitext(html_filename)
html_filename = "%s.html" % html_filename
html_filenames.append(html_filename)
with open(html_filename, "w") as handle:
template = jinja2.Template(file_template)
handle.write(template.render(file=file.path, globals=file.global_functions))
# Dump the index file
with open("index.html", "w") as handle:
template = jinja2.Template(index_template)
handle.write(template.render(files=self.data))
print("Done processing.")

179
scriptScrape.py Normal file
View file

@ -0,0 +1,179 @@
import re
import os
import sys
import importlib
import os.path
class FileEntry(object):
path = None
global_functions = None
bound_functions = None
datablocks = None
def __init__(self, path):
self.path = path
self.global_functions = [ ]
self.bound_functions = { }
self.datablocks = [ ]
class Function(object):
name = None
parameters = None
type = None
filepath = None
line = None
def __init__(self, name, type, parameters, filepath, line):
self.name = name
self.parameters = parameters
self.filepath = filepath
self.line = line
self.type = type
class Datablock(object):
name = None
type = None
derived = None
def __init__(self, name, type, derived):
self.name = name
self.type = type
self.derived = derived
class Application(object):
bound_function_pattern = re.compile("function +(([A-z]|_)+::)([A-z]|_)+\( *(%[A-z]+( *, *%[A-z]+)*)* *\)")
function_pattern = re.compile("function +([A-z]|_)+\( *(%[A-z]+( *, *%[A-z]+)*)* *\)")
datablock_pattern = re.compile("datablock +[A-z]+ *( *[A-z]+ *)( *: *[A-z]+)?")
def print_usage(self):
print("Usage: '%s <target directory> <exporter>'" % sys.argv[0])
print("Or: '%s exporters' for a list of known exporters." % sys.argv[0])
def main(self):
# Load exporters
exporters = { }
for root, dirs, files in os.walk("exporters"):
for filename in files:
module_name, extension = os.path.splitext(filename)
if (module_name == "__init__"):
continue
try:
module = importlib.import_module('exporters.%s' % (module_name))
exporters[module_name] = module
except ImportError as e:
print(e)
if (len(sys.argv) < 2):
self.print_usage()
return
if (sys.argv[1] == "exporters"):
print("Available Exporters: ")
for exporter in exporters.keys():
print("\t- %s" % exporter)
return
elif(len(sys.argv) != 3):
self.print_usage()
return
exporter = None
try:
exporter = exporters[sys.argv[2]]
except KeyError as e:
print("Error: No such exporter '%s'." % sys.argv[2])
self.print_usage()
return
results = [ ]
global_aliases = { }
typed_aliases = { }
for root, dirs, files in os.walk(sys.argv[1]):
for filename in files:
filepath = os.path.join(root, filename)
if (not os.path.isfile(filepath)):
continue
with open(filepath, "r") as handle:
file_entry = FileEntry(filepath)
file_data = handle.read()
# Grab Global function definitions
for match in re.finditer(self.function_pattern, file_data):
line = file_data[0:match.start()].count("\n") + 1
match_split = match.group(0).lstrip("function ").split("(")
name = match_split[0]
match_split = match_split[1].replace(")", "").split(",")
parameters = [ ]
for parameter in match_split:
if (parameter == ""):
continue
parameters.append(parameter.lstrip().rstrip())
file_entry.global_functions.append(Function(name, None, parameters, filepath, line))
tracked_name = name.lower()
global_aliases.setdefault(tracked_name, (0, filepath, line))
occurrence_count, old_filepath, old_line = global_aliases[tracked_name]
occurrence_count = occurrence_count + 1
global_aliases[tracked_name] = (occurrence_count, old_filepath, old_line)
if (occurrence_count != 1):
print("Warning: Found a multiple declaration of global function '%s' in %s, line %u! (Original detection: %s, line %u)" % (tracked_name, filepath, line, old_filepath, old_line))
# Grab bound function definitions
for match in re.finditer(self.bound_function_pattern, file_data):
line = file_data[0:match.start()].count("\n") + 1
match_split = match.group(0).lstrip("function ").split("::")
type = match_split[0]
match_split = match_split[1].split("(")
name = match_split[0]
match_split = match_split[1].replace(")", "").split(",")
parameters = [ ]
for parameter in match_split:
if (parameter == ""):
continue
parameters.append(parameter.lstrip().rstrip())
file_entry.bound_functions.setdefault(type, [])
file_entry.bound_functions[type].append(Function(name, type, parameters, filepath, line))
tracked_name = name.lower()
tracked_type = type.lower()
typed_aliases.setdefault(tracked_type, {})
typed_aliases[tracked_type].setdefault(tracked_name, (0, filepath, line))
occurrence_count, old_filepath, old_line = typed_aliases[tracked_type][tracked_name]
occurrence_count = occurrence_count + 1
typed_aliases[tracked_type][tracked_name] = (occurrence_count, old_filepath, old_line)
if (occurrence_count != 1):
print("Warning: Found a multiple declaration of bound function '%s::%s' in %s, line %u! (Original detection: %s, line %u)" % (tracked_type, tracked_name, filepath, line, old_filepath, old_line))
# Grab DB definitions
for match in re.finditer(self.datablock_pattern, file_data):
match_text = match.group(0).lstrip("datablock ")
#print(match_text)
# Stick in results
results.append(file_entry)
# Init the DokuOutput
output = exporter.Exporter(results)
output.write()
if __name__ == "__main__":
Application().main()