diff --git a/src/unitgrade_devel.egg-info/PKG-INFO b/src/unitgrade_devel.egg-info/PKG-INFO index 06857de4aefbbafcb5c43c2bbc4ee94113b5c700..c24c14555aa8fdd79cede48d7bd49653ffaca96d 100644 --- a/src/unitgrade_devel.egg-info/PKG-INFO +++ b/src/unitgrade_devel.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: unitgrade-devel -Version: 0.1.27 +Version: 0.1.28 Summary: A set of tools to develop unitgrade tests and reports and later evaluate them Home-page: https://lab.compute.dtu.dk/tuhe/unitgrade_private Author: Tue Herlau @@ -41,7 +41,14 @@ pip install unitgrade-devel This will install `unitgrade-devel` (this package) and all dependencies to get you started. ### Videos -For videos see the `/videos` directory : https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/videos +Videos where I try to talk and code my way through the examples can be found on youtube: + + - First test: https://youtu.be/jC9AzZA5FcQ + - Framework and hints: https://youtu.be/xyY9Qan1b1Q + - MOSS plagiarism check: https://youtu.be/Cp4PvOnYozo + - Hidden tests and Docker: https://youtu.be/vP6ZqeDwC5U + - Jupyter notebooks: https://youtu.be/B6nzVuFTEsA + - Autolab: https://youtu.be/h5mqR8iNMwM # Instructions and examples of use The examples can be found in the `/examples` directory: https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/examples @@ -197,7 +204,7 @@ One of the main advantages of `unitgrade` over web-based autograders it that tes ```python # example_framework/instructor/cs102/report2.py -from unitgrade import UTestCase +from unitgrade import UTestCase, cache class Week1(UTestCase): def test_add(self): @@ -220,7 +227,9 @@ class Week1Titles(UTestCase): def test_add(self): """ Test the addition method add(a,b) """ self.assertEqualC(add(2,2)) + print("output generated by test") self.assertEqualC(add(-100, 5)) + # self.assertEqual(2,3, msg="This test automatically fails.") def test_reverse(self): ls = [1, 2, 3] @@ -236,9 +245,9 @@ When this is run, the titles are shown as follows: | | | |_ __ _| |_| | \/_ __ __ _ __| | ___ | | | | '_ \| | __| | __| '__/ _` |/ _` |/ _ \ | |_| | | | | | |_| |_\ \ | | (_| | (_| | __/ - \___/|_| |_|_|\__|\____/_| \__,_|\__,_|\___| v0.1.5, started: 16/09/2021 17:42:18 + \___/|_| |_|_|\__|\____/_| \__,_|\__,_|\___| v0.1.17, started: 21/09/2021 11:56:22 -CS 101 Report 2 (use --help for options) +CS 102 Report 2 Question 1: Week1 * q1.1) test_add...................................................................................................PASS * q1.2) test_reverse...............................................................................................PASS @@ -248,9 +257,17 @@ Question 1: Week1 Question 2: The same problem as before with nicer titles * q2.1) Test the addition method add(a,b)..........................................................................PASS * q2.2) Checking if reverse_list([1, 2, 3]) = [3, 2, 1]............................................................PASS - * q2) Total...................................................................................................... 8/8 + * q2) Total...................................................................................................... 6/6 + +Total points at 11:56:22 (0 minutes, 0 seconds)....................................................................16/16 + +Including files in upload... + * cs102 +> Testing token file integrity... +Done! -Total points at 17:42:18 (0 minutes, 0 seconds)....................................................................18/18 +To get credit for your results, please upload the single unmodified file: +> C:\Users\tuhe\Documents\unitgrade_private\examples\example_framework\instructor\cs102\Report2_handin_16_of_16.token ``` What happens behind the scenes when we set `self.title` is that the result is pre-computed on the instructors machine and cached. This means the last test will display the correct result regardless of how `reverse_list` has been implemented by the student. The titles are also shown correctly when the method is run as a unittest. @@ -287,16 +304,14 @@ Finally, notice how one of the tests has a return value. This will be automatica ## Example 3: Hidden and secure tests To use `unitgrade` as part of automatic grading, it is recommended you check the students output locally and use hidden tests. Fortunately, this is very easy. -Let's start with the hidden tests. As usual we write a complete report script (`report3_complete.py`), but this time we use the `@hide`-decorator to mark tests as hidden: - +Let's start with the hidden tests. As usual we write a complete report script (`report3_complete.py`), but this time we use the `@hide`-decorator to mark tests as hidden: ```python # example_docker/instructor/cs103/report3_complete.py -from unitgrade import UTestCase, Report +from unitgrade import UTestCase, Report from unitgrade.utils import hide from unitgrade import evaluate_report_student import cs103 - class AutomaticPass(UTestCase): def test_automatic_pass(self): self.assertEqual(2, 2) # For simplicity, this test will always pass @@ -305,7 +320,6 @@ class AutomaticPass(UTestCase): def test_hidden_fail(self): self.assertEqual(2, 3) # For simplicity, this test will always fail. - class Report3(Report): title = "CS 101 Report 3" questions = [(AutomaticPass, 10)] # Include a single question for 10 credits. @@ -324,18 +338,16 @@ if __name__ == "__main__": setup_grade_file_report(Report) # Create report3_grade.py for the students student_directory = "../../students/cs103" - snip_dir("./", student_directory, exclude=['__pycache__', '*.token', 'deploy.py', 'report3_complete*.py', '.*']) + snip_dir("./", student_directory, exclude=['*.token', 'deploy.py', 'report3_complete*.py', '.*']) ``` Just to check, let's have a quick look at the students report script `report3.py`: - ```python # example_docker/instructor/cs103/report3.py -from unitgrade import UTestCase, Report +from unitgrade import UTestCase, Report from unitgrade.utils import hide from unitgrade import evaluate_report_student import cs103 - class AutomaticPass(UTestCase): def test_automatic_pass(self): self.assertEqual(2, 2) # For simplicity, this test will always pass @@ -354,15 +366,16 @@ The grade script works as normal, and just to make the example self-contained, l ``` ### Setting up and using Docker We are going to run the students tests in a Docker virtual machine so that we avoid any underhanded stuff, and also because it makes sure we get the same result every time (i.e., we can pass the task on to TAs). -To do that, you first have to install Docker (easy), and then build a Docker image. We are going to use one of the pre-baked images from https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/docker_images, which simply consists of a lean Linux distribution with python 3.8 and whatever packages are found in `requirements.txt`. If you need more, it is very easy to add. -To build the Docker image simply run: +To do that, you first have to install Docker (easy), and then build a Docker image. We are going to use one of the pre-baked images from https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/docker_images, which simply consists of a lean Linux distribution with python 3.8 and whatever packages are found in `requirements.txt`. If you need more it is very easy to add. +To download and build the Docker image simply run: ```python # example_docker/instructor/cs103/deploy.py # Step 3: Compile the Docker image (obviously you should only do this once). - Dockerfile = os.path.dirname(__file__) + "/../../../../docker_images/unitgrade_v1-docker/Dockerfile" - os.system(f"cd {os.path.dirname(Dockerfile)} && docker build --tag unitgrade_v1-docker .") + download_docker_images(destination="../docker") # Download an up-to-date docker image from gitlab. + Dockerfile = "../docker/unitgrade-docker/Dockerfile" # Location of just downloaded docker file + compile_docker_image(Dockerfile, tag="unitgrade-docker") ``` -This takes about 3 minutes but only needs to be done once. If you are keeping track we have the following: +This takes about 2 minutes but only needs to be done once. If you are keeping track we have the following: - A grade script with all tests, `report3_complete_grade.py`, which we build when the file was deployed - A (student) `.token` file we simulated, but in general would have downloaded from DTU Learn - A Docker image with the right packages @@ -374,7 +387,8 @@ Next we feed this into unitgrade: token = docker_run_token_file(Dockerfile_location=Dockerfile, host_tmp_dir=os.path.dirname(Dockerfile) + "/home", student_token_file=student_token_file, - instructor_grade_script="report3_complete_grade.py") + instructor_grade_script="report3_complete_grade.py", + tag="unitgrade-docker") ``` Behind the scenes, this code does the following: - Load the docker image @@ -386,10 +400,9 @@ Just to show it works we will load both `.token`-files and print the results: ```python # example_docker/instructor/cs103/deploy.py # Load the two token files and compare their scores - with open(token, 'rb') as f: - checked_token = pickle.load(f) - with open(student_token_file, 'rb') as f: - results = pickle.load(f) + checked_token, _ = load_token(token) + results, _ = load_token(student_token_file) + print("Student's score was:", results['total']) print("My independent evaluation of the students score was", checked_token['total']) ``` @@ -478,12 +491,75 @@ What happens behind the scenes is that a code-coverage tool is run on the instru to determine which methods are actually used in solving a problem, and then the hint-texts of those methods are collected and displayed. This feature requires no external configuration; simply write `Hints:` in the source code. +# CMU Autolab support (Experimental) +CMU Autolab is a mature, free and opensource web-based autograder developed at Carnegie Mellon University and used across the world. You can find more information here: https://autolabproject.com/. It offers all features you expect from an online autograder + - Web-based submission of homework + - Class-management + - Build in TA feedback mechanism + - Class monitoring/statistics + - Automatic integration with enrollment data (Autolab supports LDAP and Shibboleth) means Autolab can be `plugged in` to existing IT infrastructure (including DTUs) + - CLI Tools + +An important design choice behind CMU Autolab is the grading is entirely based on Makefiles and Docker VMs. I.e., if you can make your autograding scheme work as Makefile that runs code on a Docker image you specify it will work on Autolab. This makes it very easy to let third-party platforms work with an **unmodified** version of Autolab. The following contains all steps needed to compile a Unitgrade test to Autolab + +### Step 1: Set up Autolab +Simply follow the guide here: https://docs.autolabproject.com/installation/overview/ to set up Autolab. I used the 'manual' installation, but it should also work with the Docker-compose installation. + +### Step 2: Compile a unitgrade test to Autolab lab-assignment format +Autolab calls handins for `lab assignments`, and allow you to import them as `.tar`-files (see the Autolab documentation for more information). We can build these automatically in a few lines as this example demonstrates. +The code for the example can be found in `examples/autolab_example`. It consists of two steps. The first is that you need to build the Docker image for Autolab/Tango used for grading. This is exactly like our earlier example using Docker for Unitgrade, except the image contains a few additional autolab-specific things. You can find the image here: + - https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/docker_images/docker_tango_python + +Concretely, the following code will download and build the image (note this code must be run on the same machine that you have installed Autolab on) +```python +# autolab_example/deploy_autolab.py + # Step 1: Download and compile docker grading image. You only need to do this once. + download_docker_images("./docker") # Download docker images from gitlab (only do this once. + dockerfile = f"./docker/docker_tango_python/Dockerfile" + autograde_image = 'tango_python_tue' + compile_docker_image(Dockerfile=dockerfile, tag=autograde_image) # Compile docker image. +``` +Next, simply call the framework to compile any `_grade.py`-file into an Autolab-compatible `.tar` file that can be imported from the web interface. The script requires you to specify +both the instructor-directory and the directory with the files the student have been handed out (i.e., the same file-system format we have seen earlier). +```python +# autolab_example/deploy_autolab.py + # Step 2: Create the cs102.tar file from the grade scripts. + instructor_base = f"../example_framework/instructor" + student_base = f"../example_framework/students" + output_tar = deploy_assignment("cs102", # Autolab name of assignment (and name of .tar file) + INSTRUCTOR_BASE=instructor_base, + INSTRUCTOR_GRADE_FILE=f"{instructor_base}/cs102/report2_grade.py", + STUDENT_BASE=student_base, + STUDENT_GRADE_FILE=f"{student_base}/cs102/report2_grade.py", + autograde_image_tag=autograde_image) +``` +This will produce a file `cs102.tar`. Whereas you needed to build the Docker image on the machine where you are running Autolab, you can build the lab assignments on any computer. +### Step 3: Upload the `.tar` lab-assignment file +To install the `cs102.tar`-file, simply open your course in Autolab and click the `INSTALL ASSESSMENT` button. Click `Browse` and upload the `cs102.tar` file: + + +You will immediately see the page for the assignment where you can begin to upload solutions! +The solutions are (of course!) `.token` files, and they will be automatically unpacked and run on Autolab. + +To test it, press the big upload square and select the `.token` file for the second assignment found in `examples/example_framework/instructor/cs102/Report2_handin_18_of_18.token`. +The file will now be automatically evaluated and the score registered as any other Autolab assignment: + + + +The students can choose to view both the console output or a nicer formatted overview of the individual problems: + + + +and TAs can choose to annotate the students code directly in Autolab -- we are here making use of the fact the code is automatically included in the top of the `.token`-file. + + + # Citing ```bibtex @online{unitgrade_devel, - title={Unitgrade-devel (0.1.7): \texttt{pip install unitgrade-devel}}, + title={Unitgrade-devel (0.1.27): \texttt{pip install unitgrade-devel}}, url={https://lab.compute.dtu.dk/tuhe/unitgrade_private}, - urldate = {2021-09-16}, + urldate = {2021-09-21}, month={9}, publisher={Technical University of Denmark (DTU)}, author={Tue Herlau}, diff --git a/src/unitgrade_private/version.py b/src/unitgrade_private/version.py index 62631cb2ee426821e404c1394ae51b3ad057b25b..32eedaa05654cc78c0ec385c0529a2a4a668d89d 100644 --- a/src/unitgrade_private/version.py +++ b/src/unitgrade_private/version.py @@ -1 +1 @@ -version = "0.1.27" +version = "0.1.28"