error modes for bad configuration

.idea/workspace.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ChangeListManager">
-    <list default="true" id="5c346737-53d8-4e87-88c5-7be2c8e7baeb" name="Default" comment="" />
+    <list default="true" id="5c346737-53d8-4e87-88c5-7be2c8e7baeb" name="Default" comment="">
+      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/cs101courseware_example/Report1_resources_do_not_hand_in.dat" beforeDir="false" afterPath="$PROJECT_DIR$/cs101courseware_example/Report1_resources_do_not_hand_in.dat" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/cs101courseware_example/Report2_resources_do_not_hand_in.dat" beforeDir="false" afterPath="$PROJECT_DIR$/cs101courseware_example/Report2_resources_do_not_hand_in.dat" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/cs101courseware_example/cs101report1_grade.py" beforeDir="false" afterPath="$PROJECT_DIR$/cs101courseware_example/cs101report1_grade.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/cs101courseware_example/cs101report2_grade.py" beforeDir="false" afterPath="$PROJECT_DIR$/cs101courseware_example/cs101report2_grade.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/unitgrade/unitgrade.py" beforeDir="false" afterPath="$PROJECT_DIR$/unitgrade/unitgrade.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/unitgrade/unitgrade_helpers.py" beforeDir="false" afterPath="$PROJECT_DIR$/unitgrade/unitgrade_helpers.py" afterDir="false" />
+    </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
@@ -134,7 +143,7 @@
       <recent name="C:\Users\tuhe\Documents\unitgrade_private" />
     </key>
   </component>
-  <component name="RunManager" selected="Python.cs101report1">
+  <component name="RunManager" selected="Python.deploy_cs101">
     <configuration name="cs101report1" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
       <module name="unitgrade" />
       <option name="INTERPRETER_OPTIONS" value="" />
@@ -156,7 +165,7 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
-    <configuration name="cs101report2" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+    <configuration name="cs101report2_grade" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
       <module name="unitgrade" />
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
@@ -168,7 +177,7 @@
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/../unitgrade_private/cs101courseware/cs101report2.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/../unitgrade_private/cs101courseware/cs101report2_grade.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
@@ -177,7 +186,7 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
-    <configuration name="cs101report2_grade" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+    <configuration name="deploy_cs101" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
       <module name="unitgrade" />
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
@@ -189,7 +198,7 @@
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/../unitgrade_private/cs101courseware/cs101report2_grade.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/../unitgrade_private/cs101courseware/deploy_cs101.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
@@ -263,19 +272,19 @@
     </configuration>
     <list>
       <item itemvalue="Python.cs101report1" />
-      <item itemvalue="Python.cs101report2" />
       <item itemvalue="Python.cs101report2_grade" />
       <item itemvalue="Python.submission_autograder" />
       <item itemvalue="Python.shopSmart" />
       <item itemvalue="Python.fruit_project" />
+      <item itemvalue="Python.deploy_cs101" />
     </list>
     <recent_temporary>
       <list>
+        <item itemvalue="Python.deploy_cs101" />
         <item itemvalue="Python.fruit_project" />
         <item itemvalue="Python.shopSmart" />
         <item itemvalue="Python.submission_autograder" />
         <item itemvalue="Python.cs101report2_grade" />
-        <item itemvalue="Python.cs101report2" />
       </list>
     </recent_temporary>
   </component>
@@ -349,11 +358,6 @@
           <line>145</line>
           <option name="timeStamp" value="85" />
         </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/unitgrade/unitgrade.py</url>
-          <line>123</line>
-          <option name="timeStamp" value="87" />
-        </line-breakpoint>
         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
           <url>file://$PROJECT_DIR$/unitgrade/unitgrade.py</url>
           <line>54</line>

README.md

@@ -15,7 +15,11 @@ Unitgrade can be installed through pip using
 ```
 pip install git+ssh://git@gitlab.compute.dtu.dk/tuhe/unitgrade.git
 ```
-This will install unitgrade in your site-packages directory. If you are using anaconda+virtual environment you can install it as
+This will install unitgrade in your site-packages directory. If you want to upgrade an old installation of unitgrade:
+```
+pip install git+ssh://git@gitlab.compute.dtu.dk/tuhe/unitgrade.git --upgrade
+```
+If you are using anaconda+virtual environment you can install it as
 ```
 source activate myenv
 conda install git pip
@@ -89,7 +93,51 @@ The reason why we use a standard test script, and one with the `_grade.py` exten
 
 ## FAQ
  - **My non-grade script and the `_grade.py` script gives different number of points**
-Since the two scripts should contain the same code, the reason is nearly certainly that you have made an (accidental) change to the test scripts. Please ensure both scripts are up-to-date and if the problem persists try to get support.
+Since the two scripts should contain the same code, the reason is nearly certainly that you have made an (accidental) change to the test scripts. Please ensure both scripts are up-to-date and if the problem persists, try to get support.
    
  - **Why is there a `*_resources_do_not_hand_in.dat` file? Should I also upload it?**
-No. The file contains the pre-computed test results your code is compared against.
+No. The file contains the pre-computed test results your code is compared against. If you want to load this file manually, the unitgrade package contains helpful functions for doing so.
+   
+ - **I am worried you might think I cheated because I opened the '_grade.py' script/token file**
+This should not be a concern. Both files are in a binary format (i.e., if you open them in a text editor they look like garbage), which means that if you make an accidential change, they will with all probability simply fail to work. 
+   
+ - **I think I might have edited the `report1.py` file. Is this a problem since one of the tests have now been altered?**
+Feel free to edit/break this file as much as you like if it helps you work out the correct solution. In fact, I recommend you just run `report1.py` from your IDE and use the debugger to work out the current state of your program. However, since the `report1_grade.py` script contains a seperate version of the tests, please ensure your `report1.py` file is up to date.
+   
+### Debugging your code/making the tests pass
+The course material should contain information about the intended function of the scripts used in the tests, and the file `report1.py` should mainly be used to check which of your code is being run. In other words, first make sure your code solves the exercises, and only later run the test script which is less easy/nice to read. 
+However, obivously you might get to a situation where your code seems to work, but a test fails. In that case, it is worth looking into the code in `report1.py` to work out what is going on. 
+
+ - **I am 99% sure my code is correct, but the test still fails. Why is that?**
+The testing framework offers a great deal of flexibility in terms of what is compared. This is either: (i) The value a function returns, (ii) what the code print to the console (iii) something derived from these.
+   Since the test *might* compare the console output, i.e. what you generate using `print("...")`-statements, innnocent changes to the script, like an extra print statement, can cause the test to fail, which is counter-intuitive. For this reason, please look at the error message carefully (or the code in `report1.py`) to understand what is being compared. 
+   
+One possibility that might trick some is that if the test compares a value computed by your code, the datatype of that value is important. For instance, a `list` is not the same as a python `ndarray`, and a `tuple` is different from a `list`. This is the correct behavior of a test: These things are not alike and correct code should not confuse them. 
+
+ - **The `report1.py` class is really confusing. I can see the code it runs on my computer, but not the expected output. Why is it like this?**
+To make sure the desired output of the tests is always up to date, the tests are computed from a working version of the code and loaded from the disk rather than being hard-coded.
+
+ - **How do I see the output of my programs in the tests? Or the intended output?**
+There are a number of console options available to help you figure out what your program should output and what it currently outputs. They can be found using:
+ ```python report1.py --help```
+Note these are disabled for the `report1_grade.py` script to avoid confusion. It is not recommended you use the grade script to debug your code.  
+
+ - **How do I see the output generated by my scripts in the IDE?**
+The file `unitgrade/unitgrade.py` contains all relevant information. Look at the `QItem` class and the function `get_points`, which is the function that strings together all the tests. 
+
+ - **Since I cannot read the `.token` file, can I trust it contains the same number of points internally as the file name indicate?**
+Yes. 
+
+### Privacy/security   
+ - **I managed to reverse engineer the `report1_grade.py`/`*.token` files in about 30 minutes. If the safety measures are so easily broken, how do you ensure people do not cheat?**
+That the script `report1_grade.py` is difficult to read is not the principle safety measure. Instead, it ensures there is no accidential tampering. If you muck around with these files and upload the result, we will very likely know.     
+
+- **I have private data on my computer. Will this be read or uploaded?**
+No. The code will look for and upload your solutions, but it will not read/look at other directories in your computer. In the example provided with this code, this means you should expect unitgrade to read/run all files in the `cs101courseware_example`-directory, but **no** other files on your computer (unless some code in this directory load other files). So as long as you keep your private files out of the base courseware directory, you should be fine. 
+
+- **Does this code install any spyware/etc.? Does it communicate with a website/online service?**
+No. Unitgrade makes no changes outside the courseware directory and it does not do anything tricky. It reads/runs code and write the `.token` file.
+  
+- **I still have concerns about running code on my computer I cannot easily read**
+Please contact me and we can discuss your specific concerns.
+  

unitgrade/unitgrade.py

@@ -218,21 +218,34 @@ class Report():
     questions = []
     pack_imports = []
 
-    def __init__(self):
+    def __init__(self, strict=False,payload=None):
         working_directory = os.path.abspath(os.path.dirname(inspect.getfile(type(self))))
         # working_directory = os.path.join(pathlib.Path(__file__).parent.absolute(), "payloads/")
         self.wdir, self.name = setup_dir_by_class(self, working_directory)
         self.computed_answers_file = os.path.join(self.wdir, self.name + "_resources_do_not_hand_in.dat")
-        # print(self.computed_answers_file)
         self.questions = [(Q(working_directory=self.wdir),w) for Q,w in self.questions]
+        # self.strict = strict
+        if payload is not None:
+            self.set_payload(payload, strict=strict)
+        else:
             if os.path.isfile(self.computed_answers_file):
-            self.set_payload(cache_read(self.computed_answers_file))
+                self.set_payload(cache_read(self.computed_answers_file), strict=strict)
+            else:
+                s = f"> Warning: The pre-computed answer file, {os.path.abspath(self.computed_answers_file)} is missing. The framework will NOT work as intended. Reasons may be a broken local installation."
+                if strict:
+                    raise Exception(s)
+                else:
+                    print(s)
 
-    def set_payload(self, payloads):
+    def set_payload(self, payloads, strict=False):
         for q, _ in self.questions:
             for item, _ in q.items:
                 if q.name not in payloads or item.name not in payloads[q.name]:
-                    continue
+                    s = f"> Broken resource dictionary submitted to unitgrade for question {q.name} and subquestion {item.name}. Framework will not work."
+                    if strict:
+                        raise Exception(s)
+                    else:
+                        print(s)
                 item._correct_answer_payload = payloads[q.name][item.name]['payload']
 
 def extract_numbers(txt):

unitgrade/unitgrade_helpers.py

@@ -94,7 +94,7 @@ def evaluate_report(report, question=None, qitem=None, passall=False, verbose=Fa
     dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
     print("Started: " + dt_string)
     print("Evaluating " + report.title, "(use --help for options)")
-    print("")
+    print(f"Loaded answers from: ", report.computed_answers_file, "\n")
     table_data = []
     nL = 80
 
@@ -134,7 +134,7 @@ def evaluate_report(report, question=None, qitem=None, passall=False, verbose=Fa
         ws, possible, obtained = upack(q_)
         possible = int(ws @ possible)
         obtained = int(ws @ obtained)
-        obtained = myround(int((w * obtained) / possible )) if possible > 0 else 0
+        obtained = int(myround(int((w * obtained) / possible ))) if possible > 0 else 0
         score[n] = {'w': w, 'possible': w, 'obtained': obtained, 'ítems': q_, 'hidden': q_hidden}
 
         q.obtained = obtained