diff --git a/README.md b/README.md
index 6ac3f12b1e89ab3772e48267b661ec712f63092d..5c4e241d3d24d8deb07c041b01645515f651738a 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,65 @@
-# unitgrade_private
+# Unitgrade-private
+**Do not distribute this repository, or files from this repository, to students**
+This repository contains the secret parts of the unitgrade framework.
+
+
+
+## What it looks like to a student
+Homework is broken down into **reports**. A report is a collection of questions which are individually scored, and each question may in turn involve multiple tests. Each report is therefore given an overall score based on a weighted average of how many tests are passed.
+
+In practice, a report consist of an ordinary python file which they simply run. It looks like this:
+
+```
+python cs101report1.py
+```
+The file `cs101report1.py` is just an ordinary, non-obfuscated file which they can navigate and debug using a debugger. The file may contain the homework, or it may call functions the students have written. Running the file creates console output which tells the students their current score for each test:
+
+```
+Starting on 02/12/2020 14:57:06
+Evaluating CS 101 Report 1
+
+Question 1: Reversal of list
+================================================================================
+*** q1.1) ListReversalItem..................................................PASS
+*** q1.2) ListReversalWordsItem.............................................PASS
+*** Question q1............................................................. 5/5
+
+Question 2: Linear regression and Boston dataset
+================================================================================
+*** q2.1) CoefficientsItem..................................................PASS
+*** q2.2) RMSEItem..........................................................PASS
+*** Question q2........................................................... 13/13
+
+Finished at 14:57:06
+Provisional evaluation
+----------- -----
+Question q1 5/5
+Question q2 13/13
+Total 18/18
+----------- -----
+
+Note your results have not yet been registered.
+To register your results, please run the file:
+>>> cs101report1_grade.py
+In the same manner as you ran this file.
+```
+Once students are happy with the result, they run an alternative, not-easy-to-tamper-with script called `cs101report1_grade.py`:
+
+```
+python report1_grade.py
+```
+This runs the same tests, and generates a file `cs101report1.token` which they upload to campusnet. This file contains the results of the report evaluation, the script output, and so on.
+
+
+## How to develop tests
+The framework is build around the build-in `unittest` framework in python. Using the framework therefore also familiarizes students with the use of automatic testing.
+
+A unittest normally consist of three things:
+ - The result of the users code,
+ - The expected result,
+ - A comparison operation of the two which may either fail or succeed.
+
+The comparisons are build on top of pythons `unittest` framework to obtain a wide variety of well-documented comparisons, and it is easy to write your own.
+
+To get the expected result, one option (which is certainly possible) is to specify it yourself, however the recommended (and much easier option) is to maintain a working branch of the code with all the funtionality the students must implement and then **automatically** compute the expected output. In other words, we save the step of specifying the expected result, which also has the benefit we don't need to update expected output when the scripts change. The video contains a full working example of how a test is developed.
diff --git a/cs101courseware/Report0_handin_18_of_18.token b/cs101courseware/Report0_handin_18_of_18.token
new file mode 100644
index 0000000000000000000000000000000000000000..6cd543c842071373e70b9d1396817cb69ccd159e
Binary files /dev/null and b/cs101courseware/Report0_handin_18_of_18.token differ
diff --git a/cs101courseware/Report0_resources_do_not_hand_in.dat b/cs101courseware/Report0_resources_do_not_hand_in.dat
new file mode 100644
index 0000000000000000000000000000000000000000..4f47053cc9a973b0c4505598c4779e85885c930e
Binary files /dev/null and b/cs101courseware/Report0_resources_do_not_hand_in.dat differ
diff --git a/cs101courseware/__init__.py b/cs101courseware/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/cs101courseware/__pycache__/__init__.cpython-36.pyc b/cs101courseware/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f0120104402414327be5b9e01a5f44b6705be45c
Binary files /dev/null and b/cs101courseware/__pycache__/__init__.cpython-36.pyc differ
diff --git a/cs101courseware/__pycache__/cs101report1.cpython-36.pyc b/cs101courseware/__pycache__/cs101report1.cpython-36.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..10e0fe726c06a4bb64737430fb2635d6af031acd
Binary files /dev/null and b/cs101courseware/__pycache__/cs101report1.cpython-36.pyc differ
diff --git a/cs101courseware/__pycache__/cs101report1_grade.cpython-36.pyc b/cs101courseware/__pycache__/cs101report1_grade.cpython-36.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..09c08be151e15128351f96a97897e615ecfa3360
Binary files /dev/null and b/cs101courseware/__pycache__/cs101report1_grade.cpython-36.pyc differ
diff --git a/cs101courseware/cs101report1.py b/cs101courseware/cs101report1.py
new file mode 100644
index 0000000000000000000000000000000000000000..9945a79168348faf1057e98d76e21a8d3f486f05
--- /dev/null
+++ b/cs101courseware/cs101report1.py
@@ -0,0 +1,43 @@
+from unitgrade.unitgrade import QuestionGroup, Report, QPrintItem
+from unitgrade.unitgrade_helpers import evaluate_report_student
+from unitgrade_private.hidden_create_files import setup_answers, setup_grade_file_report
+
+class ListReversalQuestion(QuestionGroup):
+ title = "Reversal of list"
+
+ class ListReversalItem(QPrintItem):
+ l = [1, 3, 5, 1, 610]
+ def compute_answer_print(self):
+ from cs101courseware_example.homework1 import reverse_list
+ return reverse_list(self.l)
+
+ class ListReversalWordsItem(ListReversalItem):
+ l = ["hello", "world", "summer", "dog"]
+
+class LinearRegressionQuestion(QuestionGroup):
+ title = "Linear regression and Boston dataset"
+ class CoefficientsItem(QPrintItem):
+ testfun = QPrintItem.assertL2
+ tol = 0.03
+
+ def compute_answer_print(self):
+ from cs101courseware_example.homework1 import boston_linear
+ boston_linear()
+
+ def process_output(self, res, txt, numbers):
+ return numbers[:-1]
+
+ class RMSEItem(CoefficientsItem):
+ def process_output(self, res, txt, numbers):
+ return numbers[-1]
+
+from cs101courseware_example import homework1
+class Report0(Report):
+ title = "CS 101 Report 1"
+ questions = [(ListReversalQuestion, 5), (LinearRegressionQuestion, 13)]
+ pack_imports = [homework1] # Include this file in .token file
+
+if __name__ == "__main__":
+ # setup_answers(Report0()) # hidden for students
+ # setup_grade_file_report(Report0,minify=True, bzip=True, obfuscate=True)
+ evaluate_report_student(Report0())
diff --git a/cs101courseware/cs101report1_grade.py b/cs101courseware/cs101report1_grade.py
new file mode 100644
index 0000000000000000000000000000000000000000..766113ffdae50779fbb1b9aae5757ae3cfafaeef
--- /dev/null
+++ b/cs101courseware/cs101report1_grade.py
@@ -0,0 +1,5 @@
+'''WARNING: Modifying, decompiling or otherwise tampering with this script, it's data or the resulting .token file will be investigated as a cheating attempt.
+ Note we perform manual and static analysis of the uploaded results.
+ '''
+import bz2, base64
+exec(bz2.decompress(base64.b64decode('QlpoOTFBWSZTWWIfYn8ABUHfgERQev9/9/v//+7/////YAZcNlvrc3TmVWgFAoENCUPSan6o9TymgNPImj1MepA0D1HqNPUaAB6I0MjQaFPU01PJqeknpkp4jUeiPJkyBNMnqGgDEyMRmUGHGTBNDIZGRk0NAGgyMIBoNGmQxDQAYSCJPVP1NNTT9SempkaaaaAAAMjQaAGm0mmQHGTBNDIZGRk0NAGgyMIBoNGmQxDQASKCNAmgjU0Y0MSYU8k0MmmmhoGmNTT0gaaMkiSM1yP9ZCRt3GNNAzCwQz488b4lnfz4Kw6XfLjtxYyCAKkkaYaNb1tnoHIqxbG4iLzwUVyV61UBbW2NiHICyZ4fsHwmTcYZCBzUKRbBmjNCoviZ0bPxWugnO9VCbDF0+ernyWYxcL61vmxYwqTEIW5NCQggjXMbtZHbHnCVWCTy4w6hj6Kt54eYNhoG1ncxy/cxWeC56ioqCoIJ6sJQMGaBfAGlgcpxG+izDA8dpK3kfwD1JQ1T3A/MxwgqLxiqikiCZEZSiUnZWVD33fm3Ox/ENcBodEb0s9nODFBlTGrKbUjExCyDKjDPzdIvartrGmnoNmjJvR1Im4zXxo/XLTrmMHWUdv9zAjf1UrqcgANENbwIs40CbaySAY0WEYVkpihFcxZoZgq6z0MkxFV9OS2XoKpgcqV/BHLzPDtu+7JFbXVQF52xLGLB9ImJhGUrKTmVV56IzMdICCtVzZZDKQGGjfqzCNxCIV0EBuY8uLYwzs0LtNrYTbxbHdLXkr25GrGnhsGRWI88iIATCFeIiIAJzLojwE6Bcm/Q7V3N4OO2X/Z1kZLBPWY7xg7QBoFUSFAAEBWl2i2221McQ76JIAZBzAIGgoLrkoERDpikgPh3VKt3t4Af10nrSQHYkgJmQ+lJAcWiLCSAMvu1Lyt3jJIAxHHvAKaS2r9Qt7xiG0dnBgYebFZ5bem3RYh6V65WHXIO0udhHJnmRQbDY10h+m6IZlIanIPItduelmsdiCQ9lPO2bNp+Oyfga1zPWEs+f//C2Fm4gVi00xctaL8OBHdWhl5u6yGTBbVX36OhC9D1xHG+7LYIbJI/T1sILsn09ASURkr4LzjKhk167TwEkWT1+T1LNpyJICkc0J2JzOSUS4RHJhWwPQEQqVvz+HKqEbgXcV4IucAXNquQGdlnEUpU5JIuX229YHyZt1fUAZCfVVrp8m26sTntiVadSKiVclWe7U2x8LsLVGAb27w1U33CANA0q1gt0yA1DSDFT5XJEDBRexPrEs1N06Sw2VkVx1ykEiEg0CieBybJlqqg1xlCESp0KrRShxqVAchjaOGXiaDfRLTzBSUgQm5GTkk22kk5BKQOEINSjKUjIjOGIlhzjwXAsMGwZo4SytbCLOVIv3NKsIYRIhk1CIeLepAmoEln0szIIklmDiWw40d7TA3s6L798TKbujqgF1Vqqu6FYsAfkEAMaN3mNR1uLfceAHVTPsqRcwxBppIDDR09dVXNBQo2W9sfoGWhy3KrvyAmomdg0wtGTzdnP59TT110IJa7gGmFqJ5KygDlXvIOdUUcDRbvRkDzjNT6zqVGBrqA51S5fJsY0NnEjSYshon44qy8LsQsYpgc2r4xKBgWNrooErJKYAhyFAEv8SROcg5aRbibYccUGxAyAE2J00jNdts3VAWqahcg0zCKqTVrBwGrgh2G7sF9zQg4uWREMwzR+at0Uml7mHAqCNob5pzzaIMUD4QDX12AfE5Ua0CyyqH79pdituRGKy6C7b0QvOd8iO1JAfZsOnDacS4ldZtIiTSZ3MqmRN0TKC7fVmg3J8mSMW28tOKzv4wx45c1S1pICITPKIoihJaeA0bgMrB+NMs1r3hthci9ZInCI+9USCaqQg0RPLp6xNuQLZFiRsYooFVIZDN5mEpfGo7GgWQuZsRGOgkohj0EuTgWWgF42GZIiQSMq7uMfH8agx3GYsqatYEN55o1lRGJ5hkkPqSQEkEIzCGc3MtRmtGr2qDDwiXigqpoIMsg4XFFKYh7FNchSa03C72PO71dXejA22Soiz2oPZ8qncD49ma8Gnuq0hAUE1rbAYq2YhuNIg1vyVaiHDY84CRFMEFph7EkBLcplL/NHWe33eU5fwyfV6XF14/4u5IpwoSDEPsT+A==')))
\ No newline at end of file
diff --git a/cs101courseware/deploy_cs101.py b/cs101courseware/deploy_cs101.py
new file mode 100644
index 0000000000000000000000000000000000000000..9416cad5602eaf1258639f856d03a3a4efa96112
--- /dev/null
+++ b/cs101courseware/deploy_cs101.py
@@ -0,0 +1,20 @@
+import inspect, os
+import shutil
+import unitgrade as ug
+from cs101courseware.cs101report1 import Report0
+from unitgrade_private.hidden_create_files import setup_answers, setup_grade_file_report
+from unitgrade.unitgrade_helpers import evaluate_report_student
+
+if __name__ == "__main__":
+ report = Report0()
+ setup_answers(Report0()) # hidden for students; re-computes the answers file.
+ setup_grade_file_report(Report0, minify=True, bzip=True, obfuscate=True)
+ evaluate_report_student(Report0())
+
+ # now copy scripts to public directory
+ dest = os.path.join( os.path.dirname(inspect.getfile(ug)), "../cs101courseware_example")
+ source = os.path.dirname(__file__)
+ for f in os.listdir(source):
+ if f == "__pycache__" or f.endswith(".token") or "deploy" in f:
+ continue
+ shutil.copy(f, dest)
diff --git a/cs101courseware/homework1.py b/cs101courseware/homework1.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ed701c0f7870469469d51c0ae354474d8a880a6
--- /dev/null
+++ b/cs101courseware/homework1.py
@@ -0,0 +1,40 @@
+import numpy as np
+from sklearn.datasets import load_boston
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error, r2_score
+
+####################
+# Question 1. Write a function reverse_list which accepts a list, and returns a new list
+# with the same elements but in opposite order.
+####################
+def reverse_list(mylist):
+ # TODO: Your solution here
+ result = []
+ for k in mylist:
+ result = [k] + result
+ return result
+
+def simple_list_question():
+ print("The reverse list function can reverse a list")
+ l = [1, 2, 3, 4]
+ print("List was:", l, "reversed version", reverse_list(l))
+
+
+####################
+# Question 2: Write a function which performs linear regression on the Boston housing dataset using scipy
+####################
+def boston_linear():
+ X,y = load_boston(return_X_y=True) # Load the dataset here
+ y += np.random.randn(y.size ) * 0.01
+ # TODO: Fit a linear regression model and print the coefficients
+ lin_model = LinearRegression()
+ lin_model.fit(X, y)
+ print("Coefficients are", lin_model.coef_)
+ # TODO: Compute the RMSE here (on the training set) and print it
+ y_predict = lin_model.predict(X)
+ rmse = (np.sqrt(mean_squared_error(y, y_predict)))
+ print("RMSE is", rmse)
+
+if __name__ == "__main__":
+ simple_list_question()
+ boston_linear()
diff --git a/unitgrade_demo.mp4 b/unitgrade_demo.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..3826fec3bb8c9fc33acf056d26479a42be0630be
Binary files /dev/null and b/unitgrade_demo.mp4 differ
diff --git a/unitgrade_private/__init__.py b/unitgrade_private/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3164fd600024d6e68669c78454d997897d638063
--- /dev/null
+++ b/unitgrade_private/__init__.py
@@ -0,0 +1,27 @@
+import os
+import compress_pickle
+
+def cache_write(object, file_name, verbose=True):
+ dn = os.path.dirname(file_name)
+ if not os.path.exists(dn):
+ os.mkdir(dn)
+ if verbose: print("Writing cache...", file_name)
+ with open(file_name, 'wb', ) as f:
+ compress_pickle.dump(object, f, compression="lzma")
+ if verbose: print("Done!")
+
+
+def cache_exists(file_name):
+ # file_name = cn_(file_name) if cache_prefix else file_name
+ return os.path.exists(file_name)
+
+
+def cache_read(file_name):
+ # file_name = cn_(file_name) if cache_prefix else file_name
+ if os.path.exists(file_name):
+ with open(file_name, 'rb') as f:
+ return compress_pickle.load(f, compression="lzma")
+ # return pickle.load(f)
+ else:
+ return None
+
diff --git a/unitgrade_private/__pycache__/__init__.cpython-36.pyc b/unitgrade_private/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..55b1e08fb7a2e153288f98c9683097b615a31d04
Binary files /dev/null and b/unitgrade_private/__pycache__/__init__.cpython-36.pyc differ
diff --git a/unitgrade_private/__pycache__/hidden_create_files.cpython-36.pyc b/unitgrade_private/__pycache__/hidden_create_files.cpython-36.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fb1668b582af516f0125e1835de23764cd48a63e
Binary files /dev/null and b/unitgrade_private/__pycache__/hidden_create_files.cpython-36.pyc differ
diff --git a/unitgrade_private/__pycache__/hidden_gather_upload.cpython-36.pyc b/unitgrade_private/__pycache__/hidden_gather_upload.cpython-36.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d1b8579b18a393e2fdb792be88d9ac27a13a0e6b
Binary files /dev/null and b/unitgrade_private/__pycache__/hidden_gather_upload.cpython-36.pyc differ
diff --git a/unitgrade_private/example/report0.py b/unitgrade_private/example/report0.py
new file mode 100644
index 0000000000000000000000000000000000000000..de59c7dde606fbda9b6f88c964980b8affdfe5b9
--- /dev/null
+++ b/unitgrade_private/example/report0.py
@@ -0,0 +1,46 @@
+from unitgrade.unitgrade import QuestionGroup, QItem, Report, QPrintItem
+from unitgrade.unitgrade_helpers import evaluate_report_student
+
+class AlgebraQuestionGroup(QuestionGroup):
+ title = "Graph search"
+ class AdditionQuestion(QItem):
+ title = "Testing addition"
+ def compute_answer(self):
+ return 4 + 6
+
+ class MultiplicationQuestion(QItem):
+ title = "Testing multiplication"
+ def compute_answer(self):
+ return 2*12
+
+ class ComplicatedQuestion(QPrintItem):
+ def compute_answer_print(self):
+ print("The answer is 43.5 and 23434 and -23.4 asdf")
+
+ def process_output(self, res, txt, numbers):
+ return txt
+
+class Week1QuestionGroup(QuestionGroup):
+ title = "Week 1: Simple DP"
+ class ChessTournamentQuestion(QPrintItem):
+ tol = 0.05
+ testfun = QPrintItem.assertL2Relative
+ def compute_answer_print(self):
+ from irlc.ex01.chess import main
+ main()
+ def process_output(self, res, txt, numbers):
+ return numbers
+
+ class GraphTraversalQuestion(QPrintItem):
+ def compute_answer_print(self):
+ from irlc.ex01 import graph_traversal
+ graph_traversal.main()
+
+class Report0(Report):
+ title = "Example report script"
+ questions = [(AlgebraQuestionGroup, 1), (Week1QuestionGroup, 3)]
+
+if __name__ == "__main__":
+ from unitgrade.hidden_create_files import setup_answers
+ setup_answers(Report0())
+ evaluate_report_student(Report0())
diff --git a/unitgrade_private/hidden_create_files.py b/unitgrade_private/hidden_create_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5fbd218db5156c954990d24a0f41071f8a4e77e
--- /dev/null
+++ b/unitgrade_private/hidden_create_files.py
@@ -0,0 +1,113 @@
+from unitgrade import cache_read, cache_write
+import jinja2
+import pickle
+from tabulate import tabulate
+from datetime import datetime
+import bz2
+import inspect
+import json
+import os
+
+data = """
+{{head}}
+
+report1_source = {{source}}
+report1_payload = {{payload}}
+name="{{Report1}}"
+from unitgrade.unitgrade_helpers import source_instantiate
+report = source_instantiate(name, report1_source, report1_payload)
+output_dir = os.path.dirname(__file__)
+gather_upload_to_campusnet(report, output_dir)
+"""
+
+def setup_answers(report):
+ """
+ Obtain student answers by executing the test in the report and then same them to the disk.
+ """
+ payloads = {}
+ for q, _ in report.questions:
+ payloads[q.name] = {}
+ for item, _ in q.items:
+ answer = item.compute_answer()
+ payloads[q.name][item.name] = {'payload': answer}
+
+ cache_write(payloads, report.computed_answers_file, verbose=False)
+
+
+def strip_main(report1_source):
+ dx = report1_source.find("__main__")
+ report1_source = report1_source[:dx]
+ report1_source = report1_source[:report1_source.rfind("\n")]
+ return report1_source
+
+# def pack_report_for_students(Report1, obfuscate=False, minify=False, bzip=True, nonlatin=False):
+
+def setup_grade_file_report(ReportClass, execute=True, obfuscate=False, minify=False, bzip=True, nonlatin=False):
+ # from irlc.autograde.autograde import setup_answers
+ import time
+ print("Seeting up answers...")
+ setup_answers(ReportClass())
+ time.sleep(0.1)
+ print("Packing student files...")
+ # pack report into a binary blob thingy the students can run on their own.
+ report = ReportClass()
+ fn = inspect.getfile(ReportClass)
+ with open(fn, 'r') as f:
+ report1_source = f.read()
+ report1_source = strip_main(report1_source)
+
+ payload = cache_read(report.computed_answers_file)
+ picklestring = pickle.dumps(payload)
+
+ from unitgrade_private import hidden_gather_upload
+ # from unitgrade_private import unitgrade_helpers
+
+ pyhead = ""
+ for fname in [hidden_gather_upload.__file__]: #, unitgrade_helpers.__file__]:
+ with open(fname, 'r') as f:
+ pyhead += f.read() + "\n" + "\n"
+
+ s = jinja2.Environment().from_string(data).render({'Report1': ReportClass.__name__,
+ 'source': repr(report1_source),
+ 'payload': repr(picklestring),
+ 'token_out': repr(fn[:-3] + "_handin"),
+ 'head': pyhead})
+
+ output = fn[:-3] + "_grade.py"
+ with open(output, 'w') as f:
+ f.write(s)
+
+ if minify: # obfuscate:
+ obs = '-O ' if obfuscate else ""
+ # output_obfuscated = output[:-3]+"_obfuscated.py"
+ extra = [ # "--nonlatin",
+ # '--bzip2',
+ ]
+ if bzip: extra.append("--bzip2")
+ os.system(f'pyminifier {obs} {" ".join(extra)} --replacement-length=20 -o {output} {output}')
+ import time
+ time.sleep(0.2)
+ with open(output, 'r') as f:
+ sauce = f.read().splitlines()
+ wa = """WARNING: Modifying, decompiling or otherwise tampering with this script, it's data or the resulting .token file will be investigated as a cheating attempt.
+ Note we perform manual and static analysis of the uploaded results.
+ """
+ sauce = ["'''" + wa + "'''"] + sauce[:-1]
+ sauce = "\n".join(sauce)
+ with open(output, 'w') as f:
+ f.write(sauce)
+
+ if execute:
+ time.sleep(0.1)
+ print("Testing packed files...")
+ fn = inspect.getfile(ReportClass)
+ s = os.path.basename(fn)[:-3] + "_grade"
+ exec("import " + s)
+
+
+if __name__ == "__main__":
+ from unitgrade.example.report0 import Report0
+ setup_grade_file_report(Report0, execute=False)
+
+
+
diff --git a/unitgrade_private/hidden_gather_upload.py b/unitgrade_private/hidden_gather_upload.py
new file mode 100644
index 0000000000000000000000000000000000000000..48ea72f3a6f08763a717a65a3bcb772966581f55
--- /dev/null
+++ b/unitgrade_private/hidden_gather_upload.py
@@ -0,0 +1,50 @@
+from unitgrade.unitgrade_helpers import evaluate_report
+from tabulate import tabulate
+from datetime import datetime
+import inspect
+import json
+import os
+import bz2
+
+def bzwrite(json_str, token): # to get around obfuscation issues
+ with getattr(bz2, 'open')(token, "wt") as f:
+ f.write(json_str)
+
+def gather_upload_to_campusnet(report, output_dir=None):
+ n = 80
+ results, table_data = evaluate_report(report)
+ print(" ")
+ print("="*n)
+ print("Final evaluation")
+ print(tabulate(table_data))
+ # also load the source code of missing files...
+ results['sources'] = {}
+ print("Gathering files...")
+ for m in report.pack_imports:
+ with open(m.__file__, 'r') as f:
+ results['sources'][m.__name__] = f.read()
+ print(f"*** {m.__name__}")
+
+ results['sources'] = {}
+ json_str = json.dumps(results, indent=4)
+ now = datetime.now()
+ # dname = os.path.dirname(inspect.getfile(report.__class__))
+ # dname = os.getcwd()
+ if output_dir is None:
+ output_dir = os.getcwd()
+
+ # raise Exception( dname )
+ #
+ # print(dname)
+
+ payload_out_base = report.__class__.__name__ + "_handin"
+
+ obtain, possible = results['total']
+ token = "%s_%i_of_%i.token"%(payload_out_base, obtain, possible) # + str(obtained) +"_" + ".token"
+ token = os.path.join(output_dir, token)
+ bzwrite(json_str, token)
+
+ print(" ")
+ print("To get credit for your results, please upload the single file: ")
+ print(">", token)
+ print("To campusnet without any modifications.")