From a9bc2f95dffc76dae13608ffd4028a96f9f19eb5 Mon Sep 17 00:00:00 2001
From: Tue Herlau <tuhe@win.dtu.dk>
Date: Sat, 12 Dec 2020 19:01:08 +0100
Subject: [PATCH] Readme and framework
---
README.md | 79 ++++++++++++++----
cs101courseware/cs101report1.py | 5 +-
.../hidden_create_files.cpython-36.pyc | Bin 3165 -> 2880 bytes
unitgrade_private/hidden_create_files.py | 22 ++---
unitgrade_private/hidden_gather_upload.py | 6 +-
5 files changed, 72 insertions(+), 40 deletions(-)
diff --git a/README.md b/README.md
index 5c4e241..6b5b4d3 100644
--- a/README.md
+++ b/README.md
@@ -1,20 +1,15 @@
# Unitgrade-private
**Do not distribute this repository, or files from this repository, to students**
-
This repository contains the secret parts of the unitgrade framework.
-
-
-## What it looks like to a student
+## At a glance
Homework is broken down into **reports**. A report is a collection of questions which are individually scored, and each question may in turn involve multiple tests. Each report is therefore given an overall score based on a weighted average of how many tests are passed.
-In practice, a report consist of an ordinary python file which they simply run. It looks like this:
-
+In practice, a report consist of an ordinary python file which they simply run, and which executes a sequence of tests. An example:
```
python cs101report1.py
```
-The file `cs101report1.py` is just an ordinary, non-obfuscated file which they can navigate and debug using a debugger. The file may contain the homework, or it may call functions the students have written. Running the file creates console output which tells the students their current score for each test:
-
+The file `cs101report1.py` is a non-obfuscated file which they can navigate and debug using a debugger. The file may contain the homework, or it may call functions the students have written. Running the file creates console output which tells the students their current score for each test:
```
Starting on 02/12/2020 14:57:06
Evaluating CS 101 Report 1
@@ -49,17 +44,67 @@ Once students are happy with the result, they run an alternative, not-easy-to-ta
```
python report1_grade.py
```
-This runs the same tests, and generates a file `cs101report1.token` which they upload to campusnet. This file contains the results of the report evaluation, the script output, and so on.
-
+This runs the same tests, and generates a file `cs101report1.token` which they upload to campusnet. This file contains the results of the report evaluation, the script output, and optionally local files from the users system for plagiarism checking. The `.token` file itself is in a binary but easily readable format.
## How to develop tests
-The framework is build around the build-in `unittest` framework in python. Using the framework therefore also familiarizes students with the use of automatic testing.
+To develop a new test, all you need is a working version of the students homework and the api will automatically created the expected output. This saves about half the work of creating tests and ensures tests are always in sync with the code.
-A unittest normally consist of three things:
- - The result of the users code,
- - The expected result,
- - A comparison operation of the two which may either fail or succeed.
+As an examle, suppose the students write the code for the function `reverse_list` in the `cs101courseware_example/homework1.py` file. Our test script `cs101report1.py`, which was the same file the students ran before, contains code such as (we omitted one question for brevity):
+```
+class ListReversalQuestion(QuestionGroup):
+ title = "Reversal of list"
+
+ class ListReversalItem(QPrintItem):
+ l = [1, 3, 5, 1, 610]
+ def compute_answer_print(self):
+ from cs101courseware_example.homework1 import reverse_list
+ return reverse_list(self.l)
+
+ class ListReversalWordsItem(ListReversalItem):
+ l = ["hello", "world", "summer", "dog"]
-The comparisons are build on top of pythons `unittest` framework to obtain a wide variety of well-documented comparisons, and it is easy to write your own.
+class Report0(Report):
+ title = "CS 101 Report 1"
+ questions = [(ListReversalQuestion, 5), ] # In this case only a single question
+ pack_imports = [homework1] # Include this file in .token file
+
+if __name__ == "__main__":
+ evaluate_report_student(Report0())
+```
+This code instantiates a group of questions and run two tests on the students code: one in which a short list of integers is reversed, and another where a list of strings has to be reversed. The API contains quite a lot of flexibility when creating tests such as easy parsing of printed output.
+
+All that remains is to prepare the files which are distributed to the students. This can be done in a single line:
+```
+if __name__ == "__main__":
+ setup_grade_file_report(Report0)
+```
+This code will create two files:
+ - `cs101report1_grade.py`: This file contains all tests/expected output rolled into a binary tamper-resistant binary blob. Students run this file to generate their token files.
+ - `Report0_resource_do_not_hand_in.dat`: This contains the expected output of the tests (i.e., the students output is compared against what is in this file)
+
+Both of these files, plus the actual tests `cs101report1.py`, are distributed to the students as is.
+
+### Why is there a seperate `.dat` file and seperate test/grading scripts?
+Ideally, tests should be a help in creating correct code, and therefore we want the publically-facing test code to be as easy to work with as possible.
+For this reason the user test script, `homework1.py`, is completely transparent and does not include any `exec(...)` magic or similar. It simply
+
+ - Instantiates the Report class and loads the expected output from the `.dat` file (a simple, pickled dictionary with output)
+ - Execute the tests one by one in a for loop
+
+Therefore, it will integrate well with a debugger and the homework could in principle be stated within the test script itself.
+Transparency flies in the face of security, which is why there is an obfuscated `homework1_grade.py` file which bundles the expected output with the test code.
+
+## Security/cheating
+Any system that relies on local code execution is vulnerable to tampering. In this case, tampering will involve either changing the tests, or the `.token`file.
+Both attempts involve figuring out what the main `_grade.py` does. The content of this file is a single line which executes a binary blob:
+```
+'''WARNING: Modifying, decompiling or otherwise tampering with this script, it's data or the resulting .token file will be investigated as a cheating attempt.
+ Note we perform manual and static analysis of the uploaded results.
+ '''
+import bz2, base64
+exec(bz2.decompress(base64.b64decode('QlpoOTFBWSZTWWIfYn8ABUHfgERQev9/9/v//+7/////YAZc...etc.etc.')))
+```
+If this blob is decompressed, it will contain badly obfuscated source code bundled with the result of the tests, and figuring this out involves some work and there are no possibilities for accidential tampering.
-To get the expected result, one option (which is certainly possible) is to specify it yourself, however the recommended (and much easier option) is to maintain a working branch of the code with all the funtionality the students must implement and then **automatically** compute the expected output. In other words, we save the step of specifying the expected result, which also has the benefit we don't need to update expected output when the scripts change. The video contains a full working example of how a test is developed.
+If the script is decompiled successfully, and the user manage to generate a tampered `.token` file, the `.token` file will still contain the source code. Since the source is collected at runtime, it will be possible to prove definitely that cheating occurred. This also adds an extra layer of security for code-copying (or simply copying someone elses .token file).
+In light of this I think the most realistic form of cheating is simply copying someone elses code. This option is available for all homework, but a benefit of this system is we are guaranteed to have the code included in an easy-to-read format.
diff --git a/cs101courseware/cs101report1.py b/cs101courseware/cs101report1.py
index 9945a79..1ee2d8c 100644
--- a/cs101courseware/cs101report1.py
+++ b/cs101courseware/cs101report1.py
@@ -1,6 +1,6 @@
from unitgrade.unitgrade import QuestionGroup, Report, QPrintItem
from unitgrade.unitgrade_helpers import evaluate_report_student
-from unitgrade_private.hidden_create_files import setup_answers, setup_grade_file_report
+from cs101courseware_example import homework1
class ListReversalQuestion(QuestionGroup):
title = "Reversal of list"
@@ -31,13 +31,10 @@ class LinearRegressionQuestion(QuestionGroup):
def process_output(self, res, txt, numbers):
return numbers[-1]
-from cs101courseware_example import homework1
class Report0(Report):
title = "CS 101 Report 1"
questions = [(ListReversalQuestion, 5), (LinearRegressionQuestion, 13)]
pack_imports = [homework1] # Include this file in .token file
if __name__ == "__main__":
- # setup_answers(Report0()) # hidden for students
- # setup_grade_file_report(Report0,minify=True, bzip=True, obfuscate=True)
evaluate_report_student(Report0())
diff --git a/unitgrade_private/__pycache__/hidden_create_files.cpython-36.pyc b/unitgrade_private/__pycache__/hidden_create_files.cpython-36.pyc
index fb1668b582af516f0125e1835de23764cd48a63e..5bbc47a3f98795e19bc84bc75aea42370b40e855 100644
GIT binary patch
delta 509
zcmcaBaX?Jjn3tF99K)6PY<>oY#|%h-m4Si5fq{XcIB24>S`bqXYZPk=Lkd$4TNGOg
za};|DOA2cXLlj2}TMBy%LlkEUPYPEGcMD?_R|-=wgC_4wkWrdUx44oMlQUA|i&7I)
zZgE4{<wcn#seT)OFJNL6oIHoQj!}5BG)q0VAOizK5ttB~yp%<eQEKyXmV-=;qLVw>
zRT)_(uV>F-oIP2BWA<c0PAhp81_p+g|NsC0Uj$-lD%@fz$;?f?#ad95nO9OIJUNR~
zlUoX8AQOn-nmmtFjZtRuQBG6FTa$lt7WqYS<|n0<7AGf`q(-sjX69w4RYtKSRb>`L
zapdLa<s_D5<`t=dv~n{rFx+BF%e%#xRwMxuU}0cjh~g>8OwP_pEiNg_%u6qlnY^1z
zRym3}KdJZ@YidPFQDPKpabjt5>Mf@7M2MH5{`$?8qs+*{D8<Od$ipbY#Ky?Q$ic|P
zD8X3d&cMK+$r|G37UHJKJ$WIwSeyaKHeC>*2_h^&gb|1k1`%M-fixkR53vN~U=Ewy
i{FKt1R69mc3<@wXFbFVmFmkYPFmdp(Fmdp4Z~y>X_INx1
delta 790
zcmX>gc2`2#n3tEUsq1q5QGN!7#|%h-m4Si5fq{Xcc-BN^wR+|p)?C&oRz`*th7^_@
zwp_L-HZYqhhdqitg&~D0ha-vuOmjwYf@!WOt`ycN?i98Z_7;XHo)nH0&K8C!-W0(U
zo)q2|#wfm2{uI7!rlM&n3@Q9M0=a@wf{YBQe5ryd+$jRROpFXEOu-DALN7s1(wx{J
zEuqPHi=!klsWd0CBo)L?Ni0b%$;?gNIC}vTqwr)K<~l~v$-9~BxrG@R7>dAz$YfU*
zMMjy;=`07C7{w=Rva2$(P4;KcVEj3G2m9<e1ug|7;Fn*Ls!*P)P>@=bmS2>skeirS
znwX=In3tkZT#{IlnXHhQmzYyooLQ`ppQccfk*ZKykdvR7lA5AWlv-SxQ&OylV&3Fn
z4l8*z1_p+g|NsC0Uj$-lD%@fLIr$cAK~ZL2Ns;K}H5{7UG9dRdfe4<-4>;5q<tDRp
znleUBw&g7HyTzHGlvY}toLG{2i!C=ZFEg$37E4l9W<eB3UVdIqVo7FRkp@UBF9QR^
zEvB@*Ta0N%QXl~~1_p*Go`THe?3~o%lA_GK^dh;*e>r89Z!za572je_ttcr<jAAWL
zEKN?m#Z;aM2{CA}*>dI7GqW&CF>*2TFp4m-F>*0-Fmf?UFc$fM%obx{V9;c|#U7Me
zkY7|{pvhPSN-3JGA#QFVZkj^3I7{;~OVW!HQ&Nk-{$d7cG6E6$Ac6%%fYR43=A<g4
zB5jb<S+a`r^NK7%GA1BGv>Zf&g9dD+DabpLaC7xiD-v@Ha#Hn*AWkUq1t|x+9zlqJ
v4Cb)O%}*)KNws4HC1yDW1_l8}4n__}9!4H8&B4gQ!@|VD$HBp64-y3c!Rf!U
diff --git a/unitgrade_private/hidden_create_files.py b/unitgrade_private/hidden_create_files.py
index e5fbd21..b1dcda2 100644
--- a/unitgrade_private/hidden_create_files.py
+++ b/unitgrade_private/hidden_create_files.py
@@ -1,11 +1,11 @@
from unitgrade import cache_read, cache_write
import jinja2
import pickle
-from tabulate import tabulate
-from datetime import datetime
-import bz2
+# from tabulate import tabulate
+# from datetime import datetime
+# import bz2
import inspect
-import json
+# import json
import os
data = """
@@ -89,9 +89,7 @@ def setup_grade_file_report(ReportClass, execute=True, obfuscate=False, minify=F
time.sleep(0.2)
with open(output, 'r') as f:
sauce = f.read().splitlines()
- wa = """WARNING: Modifying, decompiling or otherwise tampering with this script, it's data or the resulting .token file will be investigated as a cheating attempt.
- Note we perform manual and static analysis of the uploaded results.
- """
+ wa = """WARNING: Modifying, decompiling or otherwise tampering with this script, it's data or the resulting .token file will be investigated as a cheating attempt."""
sauce = ["'''" + wa + "'''"] + sauce[:-1]
sauce = "\n".join(sauce)
with open(output, 'w') as f:
@@ -104,10 +102,6 @@ def setup_grade_file_report(ReportClass, execute=True, obfuscate=False, minify=F
s = os.path.basename(fn)[:-3] + "_grade"
exec("import " + s)
-
-if __name__ == "__main__":
- from unitgrade.example.report0 import Report0
- setup_grade_file_report(Report0, execute=False)
-
-
-
+# if __name__ == "__main__":
+# from unitgrade.example.report0 import Report0
+# setup_grade_file_report(Report0, execute=False)
diff --git a/unitgrade_private/hidden_gather_upload.py b/unitgrade_private/hidden_gather_upload.py
index 48ea72f..21a02d6 100644
--- a/unitgrade_private/hidden_gather_upload.py
+++ b/unitgrade_private/hidden_gather_upload.py
@@ -27,16 +27,12 @@ def gather_upload_to_campusnet(report, output_dir=None):
results['sources'] = {}
json_str = json.dumps(results, indent=4)
- now = datetime.now()
+ # now = datetime.now()
# dname = os.path.dirname(inspect.getfile(report.__class__))
# dname = os.getcwd()
if output_dir is None:
output_dir = os.getcwd()
- # raise Exception( dname )
- #
- # print(dname)
-
payload_out_base = report.__class__.__name__ + "_handin"
obtain, possible = results['total']
--
GitLab