Skip to content
Snippets Groups Projects
Commit e656d950 authored by tuhe's avatar tuhe
Browse files

Updated docs to include description of Autolab

parent e97a5b81
No related branches found
No related tags found
No related merge requests found
Showing with 141 additions and 40 deletions
...@@ -180,7 +180,7 @@ One of the main advantages of `unitgrade` over web-based autograders it that tes ...@@ -180,7 +180,7 @@ One of the main advantages of `unitgrade` over web-based autograders it that tes
```python ```python
# example_framework/instructor/cs102/report2.py # example_framework/instructor/cs102/report2.py
from unitgrade import UTestCase from unitgrade import UTestCase, cache
class Week1(UTestCase): class Week1(UTestCase):
def test_add(self): def test_add(self):
...@@ -203,7 +203,9 @@ class Week1Titles(UTestCase): ...@@ -203,7 +203,9 @@ class Week1Titles(UTestCase):
def test_add(self): def test_add(self):
""" Test the addition method add(a,b) """ """ Test the addition method add(a,b) """
self.assertEqualC(add(2,2)) self.assertEqualC(add(2,2))
print("output generated by test")
self.assertEqualC(add(-100, 5)) self.assertEqualC(add(-100, 5))
# self.assertEqual(2,3, msg="This test automatically fails.")
def test_reverse(self): def test_reverse(self):
ls = [1, 2, 3] ls = [1, 2, 3]
...@@ -271,7 +273,6 @@ Finally, notice how one of the tests has a return value. This will be automatica ...@@ -271,7 +273,6 @@ Finally, notice how one of the tests has a return value. This will be automatica
To use `unitgrade` as part of automatic grading, it is recommended you check the students output locally and use hidden tests. Fortunately, this is very easy. To use `unitgrade` as part of automatic grading, it is recommended you check the students output locally and use hidden tests. Fortunately, this is very easy.
Let's start with the hidden tests. As usual we write a complete report script (`report3_complete.py`), but this time we use the `@hide`-decorator to mark tests as hidden: Let's start with the hidden tests. As usual we write a complete report script (`report3_complete.py`), but this time we use the `@hide`-decorator to mark tests as hidden:
```python ```python
# example_docker/instructor/cs103/report3_complete.py # example_docker/instructor/cs103/report3_complete.py
from unitgrade import UTestCase, Report from unitgrade import UTestCase, Report
...@@ -279,7 +280,6 @@ from unitgrade.utils import hide ...@@ -279,7 +280,6 @@ from unitgrade.utils import hide
from unitgrade import evaluate_report_student from unitgrade import evaluate_report_student
import cs103 import cs103
class AutomaticPass(UTestCase): class AutomaticPass(UTestCase):
def test_automatic_pass(self): def test_automatic_pass(self):
self.assertEqual(2, 2) # For simplicity, this test will always pass self.assertEqual(2, 2) # For simplicity, this test will always pass
...@@ -288,7 +288,6 @@ class AutomaticPass(UTestCase): ...@@ -288,7 +288,6 @@ class AutomaticPass(UTestCase):
def test_hidden_fail(self): def test_hidden_fail(self):
self.assertEqual(2, 3) # For simplicity, this test will always fail. self.assertEqual(2, 3) # For simplicity, this test will always fail.
class Report3(Report): class Report3(Report):
title = "CS 101 Report 3" title = "CS 101 Report 3"
questions = [(AutomaticPass, 10)] # Include a single question for 10 credits. questions = [(AutomaticPass, 10)] # Include a single question for 10 credits.
...@@ -307,10 +306,9 @@ if __name__ == "__main__": ...@@ -307,10 +306,9 @@ if __name__ == "__main__":
setup_grade_file_report(Report) # Create report3_grade.py for the students setup_grade_file_report(Report) # Create report3_grade.py for the students
student_directory = "../../students/cs103" student_directory = "../../students/cs103"
snip_dir("./", student_directory, exclude=['__pycache__', '*.token', 'deploy.py', 'report3_complete*.py', '.*']) snip_dir("./", student_directory, exclude=['*.token', 'deploy.py', 'report3_complete*.py', '.*'])
``` ```
Just to check, let's have a quick look at the students report script `report3.py`: Just to check, let's have a quick look at the students report script `report3.py`:
```python ```python
# example_docker/instructor/cs103/report3.py # example_docker/instructor/cs103/report3.py
from unitgrade import UTestCase, Report from unitgrade import UTestCase, Report
...@@ -318,7 +316,6 @@ from unitgrade.utils import hide ...@@ -318,7 +316,6 @@ from unitgrade.utils import hide
from unitgrade import evaluate_report_student from unitgrade import evaluate_report_student
import cs103 import cs103
class AutomaticPass(UTestCase): class AutomaticPass(UTestCase):
def test_automatic_pass(self): def test_automatic_pass(self):
self.assertEqual(2, 2) # For simplicity, this test will always pass self.assertEqual(2, 2) # For simplicity, this test will always pass
...@@ -337,15 +334,16 @@ The grade script works as normal, and just to make the example self-contained, l ...@@ -337,15 +334,16 @@ The grade script works as normal, and just to make the example self-contained, l
``` ```
### Setting up and using Docker ### Setting up and using Docker
We are going to run the students tests in a Docker virtual machine so that we avoid any underhanded stuff, and also because it makes sure we get the same result every time (i.e., we can pass the task on to TAs). We are going to run the students tests in a Docker virtual machine so that we avoid any underhanded stuff, and also because it makes sure we get the same result every time (i.e., we can pass the task on to TAs).
To do that, you first have to install Docker (easy), and then build a Docker image. We are going to use one of the pre-baked images from https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/docker_images, which simply consists of a lean Linux distribution with python 3.8 and whatever packages are found in `requirements.txt`. If you need more, it is very easy to add. To do that, you first have to install Docker (easy), and then build a Docker image. We are going to use one of the pre-baked images from https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/docker_images, which simply consists of a lean Linux distribution with python 3.8 and whatever packages are found in `requirements.txt`. If you need more it is very easy to add.
To build the Docker image simply run: To download and build the Docker image simply run:
```python ```python
# example_docker/instructor/cs103/deploy.py # example_docker/instructor/cs103/deploy.py
# Step 3: Compile the Docker image (obviously you should only do this once). # Step 3: Compile the Docker image (obviously you should only do this once).
Dockerfile = os.path.dirname(__file__) + "/../../../../docker_images/unitgrade_v1-docker/Dockerfile" download_docker_images(destination="../docker") # Download an up-to-date docker image from gitlab.
os.system(f"cd {os.path.dirname(Dockerfile)} && docker build --tag unitgrade_v1-docker .") Dockerfile = "../docker/unitgrade-docker/Dockerfile" # Location of just downloaded docker file
compile_docker_image(Dockerfile, tag="unitgrade-docker")
``` ```
This takes about 3 minutes but only needs to be done once. If you are keeping track we have the following: This takes about 2 minutes but only needs to be done once. If you are keeping track we have the following:
- A grade script with all tests, `report3_complete_grade.py`, which we build when the file was deployed - A grade script with all tests, `report3_complete_grade.py`, which we build when the file was deployed
- A (student) `.token` file we simulated, but in general would have downloaded from DTU Learn - A (student) `.token` file we simulated, but in general would have downloaded from DTU Learn
- A Docker image with the right packages - A Docker image with the right packages
...@@ -357,7 +355,8 @@ Next we feed this into unitgrade: ...@@ -357,7 +355,8 @@ Next we feed this into unitgrade:
token = docker_run_token_file(Dockerfile_location=Dockerfile, token = docker_run_token_file(Dockerfile_location=Dockerfile,
host_tmp_dir=os.path.dirname(Dockerfile) + "/home", host_tmp_dir=os.path.dirname(Dockerfile) + "/home",
student_token_file=student_token_file, student_token_file=student_token_file,
instructor_grade_script="report3_complete_grade.py") instructor_grade_script="report3_complete_grade.py",
tag="unitgrade-docker")
``` ```
Behind the scenes, this code does the following: Behind the scenes, this code does the following:
- Load the docker image - Load the docker image
...@@ -369,10 +368,9 @@ Just to show it works we will load both `.token`-files and print the results: ...@@ -369,10 +368,9 @@ Just to show it works we will load both `.token`-files and print the results:
```python ```python
# example_docker/instructor/cs103/deploy.py # example_docker/instructor/cs103/deploy.py
# Load the two token files and compare their scores # Load the two token files and compare their scores
with open(token, 'rb') as f: checked_token, _ = load_token(token)
checked_token = pickle.load(f) results, _ = load_token(student_token_file)
with open(student_token_file, 'rb') as f:
results = pickle.load(f)
print("Student's score was:", results['total']) print("Student's score was:", results['total'])
print("My independent evaluation of the students score was", checked_token['total']) print("My independent evaluation of the students score was", checked_token['total'])
``` ```
...@@ -461,12 +459,75 @@ What happens behind the scenes is that a code-coverage tool is run on the instru ...@@ -461,12 +459,75 @@ What happens behind the scenes is that a code-coverage tool is run on the instru
to determine which methods are actually used in solving a problem, and then the hint-texts of those methods are to determine which methods are actually used in solving a problem, and then the hint-texts of those methods are
collected and displayed. This feature requires no external configuration; simply write `Hints:` in the source code. collected and displayed. This feature requires no external configuration; simply write `Hints:` in the source code.
# CMU Autolab support (Experimental)
CMU Autolab is a mature, free and opensource web-based autograder developed at Carnegie Mellon University and used across the world. You can find more information here: https://autolabproject.com/. It offers all features you expect from an online autograder
- Web-based submission of homework
- Class-management
- Build in TA feedback mechanism
- Class monitoring/statistics
- Automatic integration with enrollment data (Autolab supports LDAP and Shibboleth) means Autolab can be `plugged in` to existing IT infrastructure (including DTUs)
- CLI Tools
An important design choice behind CMU Autolab is the grading is entirely based on Makefiles and Docker VMs. I.e., if you can make your autograding scheme work as Makefile that runs code on a Docker image you specify it will work on Autolab. This makes it very easy to let third-party platforms work with an **unmodified** version of Autolab. The following contains all steps needed to compile a Unitgrade test to Autolab
### Step 1: Set up Autolab
Simply follow the guide here: https://docs.autolabproject.com/installation/overview/ to set up Autolab. I used the 'manual' installation, but it should also work with the Docker-compose installation.
### Step 2: Compile a unitgrade test to Autolab lab-assignment format
Autolab calls handins for `lab assignments`, and allow you to import them as `.tar`-files (see the Autolab documentation for more information). We can build these automatically in a few lines as this example demonstrates.
The code for the example can be found in `examples/autolab_example`. It consists of two steps. The first is that you need to build the Docker image for Autolab/Tango used for grading. This is exactly like our earlier example using Docker for Unitgrade, except the image contains a few additional autolab-specific things. You can find the image here:
- https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/docker_images/docker_tango_python
Concretely, the following code will download and build the image (note this code must be run on the same machine that you have installed Autolab on)
```python
# autolab_example/deploy_autolab.py
# Step 1: Download and compile docker grading image. You only need to do this once.
download_docker_images("./docker") # Download docker images from gitlab (only do this once.
dockerfile = f"./docker/docker_tango_python/Dockerfile"
autograde_image = 'tango_python_tue'
compile_docker_image(Dockerfile=dockerfile, tag=autograde_image) # Compile docker image.
```
Next, simply call the framework to compile any `_grade.py`-file into an Autolab-compatible `.tar` file that can be imported from the web interface. The script requires you to specify
both the instructor-directory and the directory with the files the student have been handed out (i.e., the same file-system format we have seen earlier).
```python
# autolab_example/deploy_autolab.py
# Step 2: Create the cs102.tar file from the grade scripts.
instructor_base = f"../example_framework/instructor"
student_base = f"../example_framework/students"
output_tar = deploy_assignment("cs102", # Autolab name of assignment (and name of .tar file)
INSTRUCTOR_BASE=instructor_base,
INSTRUCTOR_GRADE_FILE=f"{instructor_base}/cs102/report2_grade.py",
STUDENT_BASE=student_base,
STUDENT_GRADE_FILE=f"{student_base}/cs102/report2_grade.py",
autograde_image_tag=autograde_image)
```
This will produce a file `cs102.tar`. Whereas you needed to build the Docker image on the machine where you are running Autolab, you can build the lab assignments on any computer.
### Step 3: Upload the `.tar` lab-assignment file
To install the `cs102.tar`-file, simply open your course in Autolab and click the `INSTALL ASSESSMENT` button. Click `Browse` and upload the `cs102.tar` file:
![alt text|small](https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/raw/master/docs/images/autolab1.png)
You will immediately see the page for the assignment where you can begin to upload solutions!
The solutions are (of course!) `.token` files, and they will be automatically unpacked and run on Autolab.
To test it, press the big upload square and select the `.token` file for the second assignment found in `examples/example_framework/instructor/cs102/Report2_handin_18_of_18.token`.
The file will now be automatically evaluated and the score registered as any other Autolab assignment:
![alt text|small](https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/raw/master/docs/images/autolab2.png)
The students can choose to view both the console output or a nicer formatted overview of the individual problems:
![alt text|small](https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/raw/master/docs/images/autolab4.png)
and TAs can choose to annotate the students code directly in Autolab -- we are here making use of the fact the code is automatically included in the top of the `.token`-file.
![alt text|small](https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/raw/master/docs/images/autolab3.png)
# Citing # Citing
```bibtex ```bibtex
@online{unitgrade_devel, @online{unitgrade_devel,
title={Unitgrade-devel (0.1.7): \texttt{pip install unitgrade-devel}}, title={Unitgrade-devel (0.1.27): \texttt{pip install unitgrade-devel}},
url={https://lab.compute.dtu.dk/tuhe/unitgrade_private}, url={https://lab.compute.dtu.dk/tuhe/unitgrade_private},
urldate = {2021-09-16}, urldate = {2021-09-20},
month={9}, month={9},
publisher={Technical University of Denmark (DTU)}, publisher={Technical University of Denmark (DTU)},
author={Tue Herlau}, author={Tue Herlau},
......
...@@ -286,16 +286,15 @@ The solutions are (of course!) `.token` files, and they will be automatically un ...@@ -286,16 +286,15 @@ The solutions are (of course!) `.token` files, and they will be automatically un
To test it, press the big upload square and select the `.token` file for the second assignment found in `examples/example_framework/instructor/cs102/Report2_handin_18_of_18.token`. To test it, press the big upload square and select the `.token` file for the second assignment found in `examples/example_framework/instructor/cs102/Report2_handin_18_of_18.token`.
The file will now be automatically evaluated and the score registered as any other Autolab assignment: The file will now be automatically evaluated and the score registered as any other Autolab assignment:
(overview here) ![alt text|small]({{resources}}/docs/images/autolab2.png)
The students can choose to view both the console output or a nicer formatted overview of the individual problems: The students can choose to view both the console output or a nicer formatted overview of the individual problems:
(scores) ![alt text|small]({{resources}}/docs/images/autolab4.png)
and TAs can choose to annotate the students code directly in Autolab -- we are here making use of the fact the code is automatically included in the top of the `.token`-file. and TAs can choose to annotate the students code directly in Autolab -- we are here making use of the fact the code is automatically included in the top of the `.token`-file.
(code overview). ![alt text|small]({{resources}}/docs/images/autolab3.png)
# Citing # Citing
```bibtex ```bibtex
......
docs/images/autolab3.png

176 KiB | W: | H:

docs/images/autolab3.png

209 KiB | W: | H:

docs/images/autolab3.png
docs/images/autolab3.png
docs/images/autolab3.png
docs/images/autolab3.png
  • 2-up
  • Swipe
  • Onion skin
docs/images/autolab4.png

141 KiB

# example_moss/tmp/submissions/s1003/0_homework1.py
def reverse_list(mylist): #!f
"""
Given a list 'mylist' returns a list consisting of the same elements in reverse order. E.g.
reverse_list([1,2,3]) should return [3,2,1] (as a list).
"""
ls = []
for l in mylist:
ls = [l] + ls
return ls
# return list(reversed(mylist))
def add(a,b): #!f
""" Given two numbers `a` and `b` this function should simply return their sum:
> add(a,b) = a+b """
sum2 = a + b
return sum2
if __name__ == "__main__":
# Example usage:
print(f"Your result of 2 + 2 = {add(2,2)}")
print(f"Reversing a small list", reverse_list([2,3,5,7]))
\ No newline at end of file
# autolab_example/deploy_autolab.py
# Step 1: Download and compile docker grading image. You only need to do this once.
download_docker_images("./docker") # Download docker images from gitlab (only do this once.
dockerfile = f"./docker/docker_tango_python/Dockerfile"
autograde_image = 'tango_python_tue'
compile_docker_image(Dockerfile=dockerfile, tag=autograde_image) # Compile docker image.
\ No newline at end of file
# autolab_example/deploy_autolab.py
# Step 2: Create the cs102.tar file from the grade scripts.
instructor_base = f"../example_framework/instructor"
student_base = f"../example_framework/students"
output_tar = deploy_assignment("cs102", # Autolab name of assignment (and name of .tar file)
INSTRUCTOR_BASE=instructor_base,
INSTRUCTOR_GRADE_FILE=f"{instructor_base}/cs102/report2_grade.py",
STUDENT_BASE=student_base,
STUDENT_GRADE_FILE=f"{student_base}/cs102/report2_grade.py",
autograde_image_tag=autograde_image)
\ No newline at end of file
...@@ -7,4 +7,4 @@ if __name__ == "__main__": ...@@ -7,4 +7,4 @@ if __name__ == "__main__":
setup_grade_file_report(Report) # Create report3_grade.py for the students setup_grade_file_report(Report) # Create report3_grade.py for the students
student_directory = "../../students/cs103" student_directory = "../../students/cs103"
snip_dir("./", student_directory, exclude=['__pycache__', '*.token', 'deploy.py', 'report3_complete*.py', '.*']) snip_dir("./", student_directory, exclude=['*.token', 'deploy.py', 'report3_complete*.py', '.*'])
\ No newline at end of file \ No newline at end of file
# example_docker/instructor/cs103/deploy.py # example_docker/instructor/cs103/deploy.py
# Step 3: Compile the Docker image (obviously you should only do this once). # Step 3: Compile the Docker image (obviously you should only do this once).
Dockerfile = os.path.dirname(__file__) + "/../../../../docker_images/unitgrade_v1-docker/Dockerfile" download_docker_images(destination="../docker") # Download an up-to-date docker image from gitlab.
os.system(f"cd {os.path.dirname(Dockerfile)} && docker build --tag unitgrade_v1-docker .") Dockerfile = "../docker/unitgrade-docker/Dockerfile" # Location of just downloaded docker file
\ No newline at end of file compile_docker_image(Dockerfile, tag="unitgrade-docker")
\ No newline at end of file
...@@ -3,4 +3,5 @@ ...@@ -3,4 +3,5 @@
token = docker_run_token_file(Dockerfile_location=Dockerfile, token = docker_run_token_file(Dockerfile_location=Dockerfile,
host_tmp_dir=os.path.dirname(Dockerfile) + "/home", host_tmp_dir=os.path.dirname(Dockerfile) + "/home",
student_token_file=student_token_file, student_token_file=student_token_file,
instructor_grade_script="report3_complete_grade.py") instructor_grade_script="report3_complete_grade.py",
\ No newline at end of file tag="unitgrade-docker")
\ No newline at end of file
# example_docker/instructor/cs103/deploy.py # example_docker/instructor/cs103/deploy.py
# Load the two token files and compare their scores # Load the two token files and compare their scores
with open(token, 'rb') as f: checked_token, _ = load_token(token)
checked_token = pickle.load(f) results, _ = load_token(student_token_file)
with open(student_token_file, 'rb') as f:
results = pickle.load(f)
print("Student's score was:", results['total']) print("Student's score was:", results['total'])
print("My independent evaluation of the students score was", checked_token['total']) print("My independent evaluation of the students score was", checked_token['total'])
\ No newline at end of file
# example_framework/instructor/cs102/report2.py # example_framework/instructor/cs102/report2.py
from unitgrade import UTestCase from unitgrade import UTestCase, cache
class Week1(UTestCase): class Week1(UTestCase):
def test_add(self): def test_add(self):
......
...@@ -4,7 +4,9 @@ class Week1Titles(UTestCase): ...@@ -4,7 +4,9 @@ class Week1Titles(UTestCase):
def test_add(self): def test_add(self):
""" Test the addition method add(a,b) """ """ Test the addition method add(a,b) """
self.assertEqualC(add(2,2)) self.assertEqualC(add(2,2))
print("output generated by test")
self.assertEqualC(add(-100, 5)) self.assertEqualC(add(-100, 5))
# self.assertEqual(2,3, msg="This test automatically fails.")
def test_reverse(self): def test_reverse(self):
ls = [1, 2, 3] ls = [1, 2, 3]
......
@online{unitgrade_devel, @online{unitgrade_devel,
title={Unitgrade-devel (0.1.7): \texttt{pip install unitgrade-devel}}, title={Unitgrade-devel (0.1.27): \texttt{pip install unitgrade-devel}},
url={https://lab.compute.dtu.dk/tuhe/unitgrade_private}, url={https://lab.compute.dtu.dk/tuhe/unitgrade_private},
urldate = {2021-09-16}, urldate = {2021-09-20},
month={9}, month={9},
publisher={Technical University of Denmark (DTU)}, publisher={Technical University of Denmark (DTU)},
author={Tue Herlau}, author={Tue Herlau},
......
Metadata-Version: 2.1 Metadata-Version: 2.1
Name: unitgrade-devel Name: unitgrade-devel
Version: 0.1.26 Version: 0.1.27
Summary: A set of tools to develop unitgrade tests and reports and later evaluate them Summary: A set of tools to develop unitgrade tests and reports and later evaluate them
Home-page: https://lab.compute.dtu.dk/tuhe/unitgrade_private Home-page: https://lab.compute.dtu.dk/tuhe/unitgrade_private
Author: Tue Herlau Author: Tue Herlau
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment