working update

cs101courseware/homework1.py

+
+"""
+Question 1. Write a function reverse_list which accepts a list, and returns a new list
+with the same elements but in opposite order.
+"""
+def reverse_list(mylist):
+    # TODO: Your solution here
+    # (What follows is my solution; obviously it would not be part of the code handed out to the students)
+    res = []
+    for k in mylist:
+        res = [k] + l
+    return res
+
+if __name__ == "__main__":
+    print("We will now test the reverse list function")
+    l = [1, 2, 3, 4]
+    print("List was:", l, "reversed version", reverse_list(l))

unitgrade/__init__.py

+import os
+import compress_pickle
+
+def cache_write(object, file_name, verbose=True):
+    dn = os.path.dirname(file_name)
+    if not os.path.exists(dn):
+        os.mkdir(dn)
+    if verbose: print("Writing cache...", file_name)
+    with open(file_name, 'wb', ) as f:
+        compress_pickle.dump(object, f, compression="lzma")
+    if verbose: print("Done!")
+
+
+def cache_exists(file_name):
+    # file_name = cn_(file_name) if cache_prefix else file_name
+    return os.path.exists(file_name)
+
+
+def cache_read(file_name):
+    # file_name = cn_(file_name) if cache_prefix else file_name
+    if os.path.exists(file_name):
+        with open(file_name, 'rb') as f:
+            return compress_pickle.load(f, compression="lzma")
+            # return pickle.load(f)
+    else:
+        return None
+

unitgrade/example/report0.py

+from unitgrade.autograde import QuestionGroup, QItem, Report, QPrintItem
+from unitgrade.autograde_helpers import evaluate_report_student
+
+class AlgebraQuestionGroup(QuestionGroup):
+    title = "Graph search"
+    class AdditionQuestion(QItem):
+        title = "Testing addition"
+        def compute_answer(self):
+            return 4 + 6
+
+    class MultiplicationQuestion(QItem):
+        title = "Testing multiplication"
+        def compute_answer(self):
+            return 2*12
+
+    class ComplicatedQuestion(QPrintItem):
+        def compute_answer_print(self):
+            print("The answer is 43.5 and 23434 and -23.4 asdf")
+
+        def process_output(self, res, txt, numbers):
+            return txt
+
+class Week1QuestionGroup(QuestionGroup):
+    title = "Week 1: Simple DP"
+    class ChessTournamentQuestion(QPrintItem):
+        tol = 0.05
+        testfun = QPrintItem.assertL2Relative
+        def compute_answer_print(self):
+            from irlc.ex01.chess import main
+            main()
+        def process_output(self, res, txt, numbers):
+            return numbers
+
+    class GraphTraversalQuestion(QPrintItem):
+        def compute_answer_print(self):
+            from irlc.ex01 import graph_traversal
+            graph_traversal.main()
+
+class Report0(Report):
+    title = "Example report script"
+    questions = [(AlgebraQuestionGroup, 1), (Week1QuestionGroup, 3)]
+
+if __name__ == "__main__":
+    from unitgrade.hidden.autograde_setup_hidden import setup_answers
+    setup_answers(Report0())
+    evaluate_report_student(Report0())

unitgrade/hidden/unitgrade_setup_hidden.py

+from ..autograde_helpers import evaluate_report
+from ..example.report0 import Report0
+from .. import cache_read, cache_write
+import jinja2
+import pickle
+from tabulate import tabulate
+from datetime import datetime
+import bz2
+import inspect
+import json
+import os
+
+data = """
+{{head}}
+
+report1_source = {{source}}
+report1_payload = {{payload}}
+name="{{Report1}}"
+report = source_instantiate(name, report1_source, report1_payload)
+
+# campusnet_token_out = {{token_out}}
+report = source_instantiate(name, report1_source, report1_payload)
+gather_upload_to_campusnet(report)
+"""
+
+def setup_answers(report):
+    """
+    Obtain student answers by executing the test in the report and then same them to the disk.
+    """
+    payloads = {}
+    for q, _ in report.questions:
+        payloads[q.name] = {}
+        for item, _ in q.items:
+            answer = item.compute_answer()
+            payloads[q.name][item.name] = {'payload': answer}
+
+    cache_write(payloads, report.computed_answers_file, verbose=False)
+
+def bzwrite(json_str, token): # to get around obfuscation issues
+    with getattr(bz2, 'open')(token, "wt") as f:
+        f.write(json_str)
+
+def strip_main(report1_source):
+    dx = report1_source.find("__main__")
+    report1_source = report1_source[:dx]
+    report1_source = report1_source[:report1_source.rfind("\n")]
+    return report1_source
+
+def pack_report_for_sudents(Report1, obfuscate=False, minify=False, bzip=True, nonlatin=False):
+    # pack report into a binary blob thingy the students can run on their own.
+    report = Report1()
+    fn = inspect.getfile(Report1)
+    with open(fn, 'r') as f:
+        report1_source = f.read()
+    report1_source = strip_main(report1_source)
+
+    payload = cache_read(report.computed_answers_file)
+    picklestring = pickle.dumps(payload)
+
+    with open("autograde_helpers.py", 'r') as f:
+        shelp = f.read()
+
+    s = jinja2.Environment().from_string(data).render({'Report1':  Report1.__name__,
+                        'source': repr(report1_source),
+                         'payload': repr(picklestring),
+                         'token_out': repr(fn[:-3]+"_handin"),
+                         'head': shelp} )
+
+    output = fn[:-3]+"_grade.py"
+    with open(output, 'w') as f:
+        f.write(s)
+
+    if minify: #obfuscate:
+        obs = '-O ' if obfuscate else ""
+        # output_obfuscated = output[:-3]+"_obfuscated.py"
+        extra = [#"--nonlatin",
+                 # '--bzip2',
+                 ]
+        if bzip: extra.append("--bzip2")
+        os.system(f'pyminifier {obs} {" ".join(extra)} --replacement-length=20 -o {output} {output}')
+
+        import time
+
+        time.sleep(0.2)
+        with open(output, 'r') as f:
+            sauce = f.read().splitlines()
+        wa = """WARNING: Modifying, decompiling or otherwise tampering with this script, it's data or the resulting .token file will be investigated as a cheating attempt. 
+        Note we perform manual and static analysis of the uploaded results.
+        """
+        sauce = ["'''"+wa+"'''"] + sauce[:-1]
+        sauce = "\n".join(sauce)
+        with open(output, 'w') as f:
+            f.write(sauce)
+
+
+def setup_grade_file_report(ReportClass, execute=True):
+    # from irlc.autograde.autograde import setup_answers
+    import time
+    print("Seeting up answers...")
+    setup_answers(ReportClass())
+    time.sleep(0.1)
+    print("Packing student files...")
+    pack_report_for_sudents(ReportClass, minify=True, bzip=True, obfuscate=True)
+    if execute:
+        time.sleep(0.1)
+        print("Testing packed files...")
+        fn = inspect.getfile(ReportClass)
+        s = os.path.basename(fn)[:-3] + "_grade"
+        exec("import " + s)
+
+
+
+def gather_upload_to_campusnet(report, payload_out_base=None):
+    n = 80
+    results, table_data = evaluate_report(report)
+    print(" ")
+    print("="*n)
+    print("Final evaluation")
+    print(tabulate(table_data))
+    # also load the source code of missing files...
+    results['sources'] = {}
+    print("Gathering files...")
+    for m in report.pack_imports:
+        with open(m.__file__, 'r') as f:
+            results['sources'][m.__name__] = f.read()
+        print(f"*** {m.__name__}")
+
+    results['sources'] = {}
+    json_str = json.dumps(results, indent=4)
+    now = datetime.now()
+    dname = os.path.dirname(inspect.getfile(report.__class__))
+    payload_out_base = report.__class__.__name__ + "_handin"
+
+    token = payload_out_base + ".token"
+    token = os.path.join(dname, token)
+    bzwrite(json_str, token)
+
+    print(" ")
+    # print("=======================================================================")
+    print("To get credit for your results, please upload the file: ")
+    # rout =
+    print(">", token)
+    print("To campusnet without any modifications.")
+
+
+if __name__ == "__main__":
+    setup_grade_file_report(Report0, execute=False)
+    from irlc.reports.report1example import Report1Example
+    setup_grade_file_report(Report1Example, execute=False)
+
+
+

unitgrade/unitgrade.py

+from . import cache_read
+import inspect
+import unittest
+import numpy as np
+import pathlib
+import os
+from io import StringIO
+import sys
+import collections
+
+myround = lambda x: int(x) # Seems required for obfuscation
+
+def setup_dir_by_class(C,base_dir):
+    name = C.__class__.__name__
+    base_dir = os.path.join(base_dir, name)
+    if not os.path.isdir(base_dir):
+        os.makedirs(base_dir)
+    return base_dir, name
+
+class Capturing(list):
+    def __enter__(self, capture_errors=True):
+        self._stdout = sys.stdout
+        sys.stdout = self._stringio = StringIO()
+        if capture_errors:
+            self._sterr = sys.stderr
+            sys.sterr = StringIO() # memory hole it
+        self.capture_errors = capture_errors
+        return self
+
+    def __exit__(self, *args):
+        self.extend(self._stringio.getvalue().splitlines())
+        del self._stringio    # free up some memory
+        sys.stdout = self._stdout
+        if self.capture_errors:
+            sys.sterr = self._sterr
+
+
+class QItem(unittest.TestCase):
+    title = None
+    testfun = unittest.TestCase.assertEqual
+    tol = 0
+
+    def __init__(self, working_directory=None, correct_answer_payload=None, *args, **kwargs):
+        self.name = self.__class__.__name__
+        self._correct_answer_payload = correct_answer_payload
+        super().__init__(*args, **kwargs)
+        if self.title is None:
+            self.title = self.name
+
+    def assertL2(self, computed, expected, tol=None):
+        if tol == None:
+            tol = self.tol
+        diff = np.abs( (np.asarray(computed) - np.asarray(expected)) )
+        if np.max(diff) > tol:
+            print("Not equal within tolerance {tol}")
+            print(f"Element-wise differences {diff.tolist()}")
+            self.assertEqual(computed, expected, msg=f"Not equal within tolerance {tol}")
+
+    def assertL2Relative(self, computed, expected, tol=None):
+        if tol == None:
+            tol = self.tol
+        diff = np.abs( (np.asarray(computed) - np.asarray(expected)) )
+        diff = diff / (1e-8 + np.abs( (np.asarray(computed) + np.asarray(expected)) ) )
+        if np.sum(diff > tol) > 0:
+            print(f"Not equal within tolerance {tol}")
+            print(f"Element-wise differences {diff.tolist()}")
+            self.assertEqual(computed, expected, msg=f"Not equal within tolerance {tol}")
+
+    def compute_answer(self):
+        raise NotImplementedError("test code here")
+
+    def test(self, computed, expected):
+        self.testfun(computed, expected)
+
+    def get_points(self):
+        possible = 1
+        try:
+            computed = self.compute_answer()
+        except Exception as e:
+            print("\n=================================================================================")
+            print(f"When trying to run test class '{self.name}' your code threw an error:")
+            print(e)
+            print("=================================================================================")
+            import traceback
+            traceback.print_exc()
+            return (0, possible)
+        try:
+            correct = self._correct_answer_payload
+            self.test(computed=computed, expected=correct)
+        except Exception as e:
+            print("=================================================================================")
+            print(f"Test output from test class '{self.name}' does not match expected result. Test error:")
+            print(e)
+            print("-------------------------Your output:--------------------------------------------")
+            print(computed)
+            print("-------------------------Expected output:----------------------------------------")
+            print(correct)
+            print("=================================================================================")
+            return (0, possible)
+        return (1, possible)
+
+    def score(self):
+        try:
+            self.test()
+        except Exception as e:
+            return 0
+        return 1
+
+class QPrintItem(QItem):
+    def compute_answer_print(self):
+        raise Exception("Generate output here")
+
+    def process_output(self, res, txt, numbers):
+        return txt
+
+    def compute_answer(self):
+        with Capturing() as output:
+            res = self.compute_answer_print()
+        s = "\n".join(output)
+        numbers = extract_numbers(s)
+        return self.process_output(res, s, numbers)
+
+class OrderedClassMembers(type):
+    @classmethod
+    def __prepare__(self, name, bases):
+        return collections.OrderedDict()
+    def __new__(self, name, bases, classdict):
+        classdict['__ordered__'] = [key for key in classdict.keys() if key not in ('__module__', '__qualname__')]
+        return type.__new__(self, name, bases, classdict)
+
+class QuestionGroup(metaclass=OrderedClassMembers):
+    title = "Graph search"
+    items = None
+    def __init__(self, *args, **kwargs):
+        # self.wdir, self.name = setup_dir_by_class(self, working_directory)
+        self.name = self.__class__.__name__
+        if self.items is None:
+            self.items = []
+            members = [gt for gt in [getattr(self, gt) for gt in self.__ordered__] if inspect.isclass(gt) and issubclass(gt, QItem)]
+            for gt in members:
+                self.items.append( (gt, 1) )
+        self.items = [(I(), w) for I, w in self.items]
+
+class Report():
+    title = "report title"
+    questions = []
+
+    def __init__(self):
+        working_directory = os.path.join(pathlib.Path(__file__).parent.absolute(), "answers/")
+        self.wdir, self.name = setup_dir_by_class(self, working_directory)
+        self.computed_answers_file = self.wdir + "/" + self.name + ".report"
+        self.questions = [(Q(working_directory=self.wdir),w) for Q,w in self.questions]
+        if os.path.isfile(self.computed_answers_file):
+            self.set_payload(cache_read(self.computed_answers_file))
+
+    def set_payload(self, payloads):
+        for q, _ in self.questions:
+            for item, _ in q.items:
+                if q.name not in payloads or item.name not in payloads[q.name]:
+                    continue
+                item._correct_answer_payload = payloads[q.name][item.name]['payload']
+
+def extract_numbers(txt):
+    import re
+    numeric_const_pattern = '[-+]? (?: (?: \d* \. \d+ ) | (?: \d+ \.? ) )(?: [Ee] [+-]? \d+ ) ?'
+    rx = re.compile(numeric_const_pattern, re.VERBOSE)
+    all = rx.findall(txt)
+    all = [float(a) if '.' in a else int(a) for a in all]
+    return all

unitgrade/unitgrade_helpers.py

+import numpy as np
+from tabulate import tabulate
+from datetime import datetime
+import pickle
+
+def source_instantiate(name, report1_source, payload):
+    eval("exec")(report1_source, globals())
+    report = eval(name)()
+    pl = pickle.loads(payload)
+    report.set_payload(pl)
+    return report
+
+def evaluate_report_student(report, question=None, qitem=None):
+    results, table_data = evaluate_report(report, question=question, qitem=qitem)
+    print("Provisional evaluation")
+    tabulate(table_data)
+    table = table_data
+    print(tabulate(table))
+    print(" ")
+    print("Note your results have not yet been registered. \nTo register your grade, please follow instructions on campusnet")
+    return results
+
+def evaluate_report(report, question=None, qitem=None):
+    now = datetime.now()
+    dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
+
+    print("Starting on " + dt_string)
+    print("Evaluating " + report.title)
+    print("")
+
+    report.obtained = 0
+    report.possible = 0
+    table_data = []
+    nL = 80
+    for n, (q, w) in enumerate(report.questions):
+        if question is not None and n != question:
+            continue
+
+        print(f"Question {n+1}: {q.title}")
+        print("="*nL)
+        q.possible = 0
+        q.obtained = 0
+        for j, (item, iw) in enumerate(q.items):
+            if question is not None and item is not None and j != qitem:
+                continue
+
+            ss = f"*** q{n+1}.{j+1}) {item.title}"
+            el = nL-4
+            if len(ss) < el:
+                ss += '.'*(el-len(ss))
+            print(ss, end="")
+            (current, possible) = item.get_points()
+            q.possible += possible * iw
+            q.obtained += current * iw
+            if current == possible:
+                print(f"PASS")
+            else:
+                print(f"*** FAILED")
+        from .autograde import myround
+        q.obtained = myround(np.floor((w * q.obtained) / q.possible)) if q.possible > 0 else 0
+        report.obtained += q.obtained
+        report.possible += w
+
+        s1 = f"*** Question q{n+1}"
+        s2 = f" {q.obtained}/{w}"
+        print(s1 + ("."* (nL-len(s1)-len(s2) )) + s2 )
+        print(" ")
+        table_data.append([f"Question q{n+1}", f"{q.obtained}/{w}"])
+
+    now = datetime.now()
+    dt_string = now.strftime("%H:%M:%S")
+    print(f"Finished at "+ dt_string)
+    table_data.append(["Total", ""+str(report.obtained)+"/"+str(report.possible) ])
+    results = {'total': (report.obtained, report.possible)}
+    return results, table_data
+
+
+
+