* Adjustment: Initial CMake reworking.

Engine/lib/assimp/test/regression/run.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+# -*- Coding: UTF-8 -*-
+
+# ---------------------------------------------------------------------------
+# Open Asset Import Library (ASSIMP)
+# ---------------------------------------------------------------------------
+#
+# Copyright (c) 2006-2020, ASSIMP Development Team
+#
+# All rights reserved.
+#
+# Redistribution and use of this software in source and binary forms,
+# with or without modification, are permitted provided that the following
+# conditions are met:
+#
+# * Redistributions of source code must retain the above
+#   copyright notice, this list of conditions and the
+#   following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above
+#   copyright notice, this list of conditions and the
+#   following disclaimer in the documentation and/or other
+#   materials provided with the distribution.
+#
+# * Neither the name of the ASSIMP team, nor the names of its
+#   contributors may be used to endorse or promote products
+#   derived from this software without specific prior
+#   written permission of the ASSIMP Development Team.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# ---------------------------------------------------------------------------
+
+"""
+Run the regression test suite using settings from settings.py.
+
+The assimp_cmd (or assimp) binary to use is specified by the first
+command line argument and defaults to ``assimp``.
+
+To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating
+configs for an IDE, make sure to build the assimp_cmd project.
+
+On Windows, use ``py run.py <path to assimp>`` to make sure the command
+line parameter is forwarded to the script.
+"""
+
+import sys
+import os
+import subprocess
+import zipfile
+import collections
+import multiprocessing
+
+import settings
+import utils
+
+# -------------------------------------------------------------------------------
+EXPECTED_FAILURE_NOT_MET, DATABASE_LENGTH_MISMATCH, \
+DATABASE_VALUE_MISMATCH, IMPORT_FAILURE, \
+FILE_NOT_READABLE, COMPARE_SUCCESS, EXPECTED_FAILURE = range(7)
+
+messages = collections.defaultdict(lambda: "<unknown", {
+        EXPECTED_FAILURE_NOT_MET:
+"""Unexpected success during import\n\
+\tReturn code was 0""",
+
+        DATABASE_LENGTH_MISMATCH:
+"""Database mismatch: lengths don't match\n\
+\tExpected: {0} Actual: {1}""",
+
+        DATABASE_VALUE_MISMATCH:
+"""Database mismatch: """,
+
+        IMPORT_FAILURE:
+"""Unexpected failure during import\n\
+\tReturn code was {0}""",
+
+        FILE_NOT_READABLE:
+"""Unexpected failure reading file""",
+
+        COMPARE_SUCCESS:
+"""Results match archived reference dump in database\n\
+\tNumber of bytes compared: {0}""",
+
+        EXPECTED_FAILURE:
+"""Expected failure was met.""",
+})
+
+outfilename_output = "run_regression_suite_output.txt"
+outfilename_failur = "run_regression_suite_failures.csv"
+Environment        = {}
+
+# -------------------------------------------------------------------------------
+class results:
+
+    """ Handle formatting of results"""
+
+    def __init__(self, zipin):
+        """Init, given a ZIPed database """
+        self.failures = []
+        self.success = []
+        self.zipin = zipin
+
+
+    def fail(self, failfile, filename_expect, pp, msg, *args):
+        """
+        Report failure of a sub-test
+
+        File f failed a test for pp config pp, failure notice is msg,
+        *args is format()ting args for msg
+
+        """
+        print("[FAILURE] " + messages[msg].format(*args))
+        self.failures.append((failfile, filename_expect, pp))
+
+
+    def ok(self, f, pp, msg, *args):
+        """
+        Report success of a sub-test
+
+        File f passed the test, msg is a happy success note,
+        *args is format()ing args for msg.
+
+        """
+        print("[SUCCESS] " + messages[msg].format(*args))
+        self.success.append(f)
+
+
+    def report_results(self):
+        """Write results to ../results/run_regression_suite_failures.txt"""
+
+        count_success = len(self.success)
+        count_fail = len(self.failures)
+        percent_good = float(count_success) / (count_success + count_fail)
+        print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\nFAILURE: {1}\nPercentage good: {2}".format(
+            count_success, count_fail, percent_good) +
+              "\n" + ('='*60) + "\n")
+
+        with open(os.path.join('..', 'results',outfilename_failur), "wt") as f:
+            f.write("ORIGINAL FILE;EXPECTED DUMP\n")
+            f.writelines(map(
+                lambda x: x[0] + ' ' + x[2] + ";" + x[1] + "\n", self.failures))
+
+        if self.failures:
+           print("\nSee " + settings.results + "\\" + outfilename_failur
+                 + " for more details\n\n")
+
+    def hasFailures( self ):
+        """ Return True, if any failures there. """
+        return 0 != len( self.failures )
+
+# -------------------------------------------------------------------------------
+def setEnvVar( var, value ):
+    print ( "set var " + var +" to" + value)
+    Environment[ var ] = value
+
+# -------------------------------------------------------------------------------
+def getEnvVar( var ):
+    if var in Environment:
+        return Environment[ var ]
+    else:
+        print ( "Error: cannot find " + var )
+    return ""
+    
+# -------------------------------------------------------------------------------
+def prepare_output_dir(fullpath, myhash, app):
+    outfile = os.path.join(settings.results, "tmp", os.path.split(fullpath)[1] + "_" + myhash)
+    try:
+        os.mkdir(outfile)
+    except OSError:
+        pass
+
+    outfile = os.path.join(outfile, app)
+    return outfile
+
+# -------------------------------------------------------------------------------
+def process_dir(d, outfile_results, zipin, result ):
+    shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False}
+
+    print("Processing directory " + d)
+    all = ""
+    for f in sorted(os.listdir(d)):
+        fullpath = os.path.join(d, f)
+        if os.path.isdir(fullpath) and not f[:1] == '.':
+            process_dir(fullpath, outfile_results, zipin, result)
+            continue
+                
+        if f in settings.files_to_ignore or os.path.splitext(f)[1] in settings.exclude_extensions:
+            print("Ignoring " + f)
+            return
+
+        for pppreset in settings.pp_configs_to_test:
+            filehash = utils.hashing(fullpath, pppreset)
+            failure = False
+
+            try:
+                input_expected = zipin.open(filehash, "r").read()
+                # empty dump files indicate 'expected import failure'
+                if not len(input_expected):
+                   failure = True
+            except KeyError:
+                # TODO(acgessler): Keep track of this and report as error in the end.
+                print("Didn't find "+fullpath+" (Hash is "+filehash+") in database. Outdated "+\
+                    "regression database? Use gen_db.zip to re-generate.")
+                continue
+
+            print("-"*60 + "\n  " + os.path.realpath(fullpath) + " pp: " + pppreset)
+
+            outfile_actual = prepare_output_dir(fullpath, filehash, "ACTUAL")
+            outfile_expect = prepare_output_dir(fullpath, filehash, "EXPECT")
+            outfile_results.write("assimp dump    "+"-"*80+"\n")
+            outfile_results.flush()
+            assimp_bin_path = getEnvVar("assimp_path")
+            command = [assimp_bin_path,
+                "dump",
+                fullpath, outfile_actual, "-b", "-s", "-l" ] +\
+                pppreset.split()
+            print( "command = " + str( command ) )
+            r = subprocess.call(command, **shellparams)
+            outfile_results.flush()
+
+            if r and not failure:
+                result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r)
+                outfile_results.write("Failed to import\n")
+                continue
+            elif failure and not r:
+                result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET)
+                outfile_results.write("Expected import to fail\n")
+                continue
+            elif failure and r:
+                result.ok(fullpath, pppreset, EXPECTED_FAILURE)
+                outfile_results.write("Failed as expected, skipping.\n")
+                continue
+
+            with open(outfile_expect, "wb") as s:
+                s.write(input_expected)
+
+            try:
+                with open(outfile_actual, "rb") as s:
+                    input_actual = s.read()
+            except IOError:
+                continue
+
+            outfile_results.write("Expected data length: {0}\n".format(len(input_expected)))
+            outfile_results.write("Actual data length: {0}\n".format(len(input_actual)))
+            failed = False
+            if len(input_expected) != len(input_actual):
+                result.fail(fullpath, outfile_expect, pppreset, DATABASE_LENGTH_MISMATCH,
+                        len(input_expected), len(input_actual))
+                # Still compare the dumps to see what the difference is
+                failed = True
+
+            outfile_results.write("assimp cmpdump "+"-"*80+"\n")
+            outfile_results.flush()
+            command = [ assimp_bin_path, 'cmpdump', outfile_actual, outfile_expect ]
+            if subprocess.call(command, **shellparams) != 0:
+                if not failed:
+                    result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH)
+                continue
+
+            result.ok(fullpath, pppreset, COMPARE_SUCCESS, len(input_expected))
+
+# -------------------------------------------------------------------------------
+def del_folder_with_contents(folder):
+    for root, dirs, files in os.walk(folder, topdown=False):
+        for name in files:
+            os.remove(os.path.join(root, name))
+        for name in dirs:
+            os.rmdir(os.path.join(root, name))
+
+
+# -------------------------------------------------------------------------------
+def run_test():
+    tmp_target_path = os.path.join(settings.results, "tmp")
+    try:
+        print( "try to make " + tmp_target_path )
+        os.mkdir(tmp_target_path)
+    except OSError as oerr:
+        # clear contents if tmp folder exists already
+       del_folder_with_contents(tmp_target_path)
+
+    try:
+        zipin = zipfile.ZipFile(settings.database_name + ".zip",
+            "r", zipfile.ZIP_STORED)
+    except IOError:
+        print("Regression database ", settings.database_name,
+              ".zip was not found")
+        return
+
+    res = results(zipin)
+    with open(os.path.join(settings.results, outfilename_output), "wt") as outfile:
+        for tp in settings.model_directories:
+            process_dir(tp, outfile, zipin, res)
+
+    res.report_results()
+    if res.hasFailures():
+        return 1
+
+    return 0
+
+# -------------------------------------------------------------------------------
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        assimp_bin_path = sys.argv[1]
+    else:
+        assimp_bin_path = 'assimp'
+    setEnvVar("assimp_path", assimp_bin_path)
+    print('Using assimp binary: ' + assimp_bin_path)
+    sys.exit( run_test() )
+
+# vim: ai ts=4 sts=4 et sw=4