Skip to content
Snippets Groups Projects
Commit 2111008c authored by tuhe's avatar tuhe
Browse files

updates to readme.md

parent 178c3726
No related branches found
No related tags found
No related merge requests found
...@@ -5,8 +5,8 @@ Unitgrade is an automatic report and exam evaluation framework that enables inst ...@@ -5,8 +5,8 @@ Unitgrade is an automatic report and exam evaluation framework that enables inst
Unitgrade is build on pythons `unittest` framework so that the tests can be specified in a familiar syntax and will integrate with any modern IDE. What it offers beyond `unittest` is the ability to collect tests in reports (for automatic evaluation) and an easy and 100% safe mechanism for verifying the students results and creating additional, hidden tests. A powerful cache system allows instructors to automatically create test-answers based on a working solution. Unitgrade is build on pythons `unittest` framework so that the tests can be specified in a familiar syntax and will integrate with any modern IDE. What it offers beyond `unittest` is the ability to collect tests in reports (for automatic evaluation) and an easy and 100% safe mechanism for verifying the students results and creating additional, hidden tests. A powerful cache system allows instructors to automatically create test-answers based on a working solution.
- 100% Python `unittest` compatible - 100% Python `unittest` compatible
- No external configuration files: Just write a `unittest` - No configuration files
- No unnatural limitations: Use any package or framework. If you can `unittest` it, it works. - No limitations: If you can `unittest` it, it works
- Tests are quick to run and will integrate with your IDE - Tests are quick to run and will integrate with your IDE
- Cache and hint-system makes tests easy to develop - Cache and hint-system makes tests easy to develop
- Granular security model: - Granular security model:
...@@ -39,7 +39,7 @@ instructor/cs101/deploy.py # A private file to deploy the tests ...@@ -39,7 +39,7 @@ instructor/cs101/deploy.py # A private file to deploy the tests
### The homework ### The homework
The homework is just any old python code you would give to the students. For instance: The homework is just any old python code you would give to the students. For instance:
```python ```python
def reverse_list(mylist): #!f def reverse_list(mylist): #!f #!s
""" """
Given a list 'mylist' returns a list consisting of the same elements in reverse order. E.g. Given a list 'mylist' returns a list consisting of the same elements in reverse order. E.g.
reverse_list([1,2,3]) should return [3,2,1] (as a list). reverse_list([1,2,3]) should return [3,2,1] (as a list).
...@@ -54,16 +54,15 @@ def add(a,b): #!f ...@@ -54,16 +54,15 @@ def add(a,b): #!f
if __name__ == "__main__": if __name__ == "__main__":
# Example usage: # Example usage:
print(f"Your result of 2 + 2 = {add(2,2)}") print(f"Your result of 2 + 2 = {add(2,2)}")
print(f"Reversing a small list", reverse_list([2,3,5,7])) print(f"Reversing a small list", reverse_list([2,3,5,7])) #!s
``` ```
### The test: ### The test:
The test consists of individual problems and a report-class. The tests themselves are just regular Unittest (we will see a slightly smarter idea in a moment). For instance: The test consists of individual problems and a report-class. The tests themselves are just regular Unittest (we will see a slightly smarter idea in a moment). For instance:
```python ```python
from cs101 import reverse_list, add # example_simplest/instructor/cs101/report1.py
import unittest from cs101.homework1 import reverse_list, add
class Week1(unittest.TestCase): class Week1(unittest.TestCase):
def test_add(self): def test_add(self):
...@@ -72,15 +71,14 @@ class Week1(unittest.TestCase): ...@@ -72,15 +71,14 @@ class Week1(unittest.TestCase):
def test_reverse(self): def test_reverse(self):
self.assertEqual(reverse_list([1,2,3]), [3,2,1]) self.assertEqual(reverse_list([1,2,3]), [3,2,1])
``` ```
A number of tests can be collected into a `Report`, which will allow us to assign points to the tests and use the more advanced features of the framework later. A complete, minimal example: A number of tests can be collected into a `Report`, which will allow us to assign points to the tests and use the more advanced features of the framework later. A complete, minimal example:
```python ```python
# example_simplest/instructor/cs101/report1.py
import unittest import unittest
from unitgrade2.unitgrade2 import Report from unitgrade2 import Report, evaluate_report_student
from unitgrade2.unitgrade_helpers2 import evaluate_report_student
from cs101.homework1 import reverse_list, add
import cs101 import cs101
from cs101.homework1 import reverse_list, add #!s
class Week1(unittest.TestCase): class Week1(unittest.TestCase):
def test_add(self): def test_add(self):
...@@ -88,7 +86,7 @@ class Week1(unittest.TestCase): ...@@ -88,7 +86,7 @@ class Week1(unittest.TestCase):
self.assertEqual(add(-100, 5), -95) self.assertEqual(add(-100, 5), -95)
def test_reverse(self): def test_reverse(self):
self.assertEqual(reverse_list([1,2,3]), [3,2,1]) self.assertEqual(reverse_list([1,2,3]), [3,2,1]) #!s
class Report1(Report): class Report1(Report):
title = "CS 101 Report 1" title = "CS 101 Report 1"
...@@ -96,35 +94,23 @@ class Report1(Report): ...@@ -96,35 +94,23 @@ class Report1(Report):
pack_imports = [cs101] # Include all .py files in this folder pack_imports = [cs101] # Include all .py files in this folder
if __name__ == "__main__": if __name__ == "__main__":
# unittest.main(verbosity=2) # Uncomment to run everything as a regular unittest
evaluate_report_student(Report1()) evaluate_report_student(Report1())
``` ```
### Deployment ### Deployment
The above is all you need if you simply want to use the framework as a self-check: Students can run the code and see how well they did. The above is all you need if you simply want to use the framework as a self-check: Students can run the code and see how well they did.
In order to begin using the framework for evaluation we need to create a bit more structure. We do that by deploying the report class as follows: In order to begin using the framework for evaluation we need to create a bit more structure. We do that by deploying the report class as follows:
```python ```python
from cs101.report1 import Report1 #!s # example_simplest/instructor/cs101/deploy.py
from cs101.report1 import Report1
from unitgrade_private2.hidden_create_files import setup_grade_file_report from unitgrade_private2.hidden_create_files import setup_grade_file_report
import shutil, os
from snipper import snip_dir from snipper import snip_dir
# wd = os.path.dirname(__file__)
if __name__ == "__main__": if __name__ == "__main__":
setup_grade_file_report(Report1) setup_grade_file_report(Report1) # Make the report1_grade.py report file
# Deploy the files using snipper: https://gitlab.compute.dtu.dk/tuhe/snipper # Deploy the files using snipper: https://gitlab.compute.dtu.dk/tuhe/snipper
snip_dir.snip_dir("./", "../../students/cs101", clean_destination_dir=True, exclude=['__pycache__', '*.token', 'deploy.py']) #!s snip_dir("./", "../../students/cs101", exclude=['__pycache__', '*.token', 'deploy.py'])
# For my own sake, copy the homework file to the other examples.
for f in ['../../../example_framework/instructor/cs102/homework1.py',
'../../../example_docker/instructor/cs103/homework1.py',
'../../../example_flat/instructor/cs101flat/homework1.py']:
shutil.copy('homework1.py', f)
``` ```
- The first line creates the `report1_grade.py` script and any additional data files needed by the tests (none in this case) - The first line creates the `report1_grade.py` script and any additional data files needed by the tests (none in this case)
- The second line set up the students directory (remember, we have included the solutions!) and remove the students solutions. You can check the results in the students folder. - The second line set up the students directory (remember, we have included the solutions!) and remove the students solutions. You can check the results in the students folder.
...@@ -323,8 +309,9 @@ submissions/ # Where you dump student submissions. ...@@ -323,8 +309,9 @@ submissions/ # Where you dump student submissions.
``` ```
The whitelist directory is optional, and the submissions directory contains student submissions (one folder per student): The whitelist directory is optional, and the submissions directory contains student submissions (one folder per student):
```terminal ```terminal
/submissions/<student-id-1>/.. /submissions/<student-id-1>/Report1_74_of_144.token
/submissions/<student-id-2>/.. /submissions/<student-id-2>/Report1_130_of_144.token
...
``` ```
The files in the whitelist/student directory can be either `.token` files (which are unpacked) or python files, and they may contain subdirectories: Everything will be unpacked and flattened. The simplest way to set it up is simply to download all files from DTU learn as a zip-file and unzip it somewhere. The files in the whitelist/student directory can be either `.token` files (which are unpacked) or python files, and they may contain subdirectories: Everything will be unpacked and flattened. The simplest way to set it up is simply to download all files from DTU learn as a zip-file and unzip it somewhere.
When done just call moss as follows: When done just call moss as follows:
...@@ -339,7 +326,7 @@ if __name__ == "__main__": ...@@ -339,7 +326,7 @@ if __name__ == "__main__":
moss_it(whitelist_dir="whitelist", submissions_dir="student_submissions", moss_id=id) moss_it(whitelist_dir="whitelist", submissions_dir="student_submissions", moss_id=id)
``` ```
This will generate a report. You can view the example including the report here: https://lab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/examples/example_moss This will generate a report. You can see the example including the report here: https://lab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/examples/example_moss
# Smart hinting # Smart hinting
To help students get started, unitgrade will collect hints to solve failed tests from across the codebase and display them. Consider the following homework where two problems depends on each other and the To help students get started, unitgrade will collect hints to solve failed tests from across the codebase and display them. Consider the following homework where two problems depends on each other and the
...@@ -397,7 +384,7 @@ When students run this homework it will fail and display the hints from the two ...@@ -397,7 +384,7 @@ When students run this homework it will fail and display the hints from the two
![alt text|small](https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/raw/master/docs/hints.png) ![alt text|small](https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/raw/master/docs/hints.png)
What happens behind the scenes is that a code-coverage tool is run on the instructors computer What happens behind the scenes is that a code-coverage tool is run on the instructors computer
to determine which methods are actually used in solving a problem, and then the hint-texts of those methods (and none other) are to determine which methods are actually used in solving a problem, and then the hint-texts of those methods are
collected and displayed. This feature requires no external configuration; simply write `Hints:` in the source code. collected and displayed. This feature requires no external configuration; simply write `Hints:` in the source code.
# Citing # Citing
......
...@@ -45,29 +45,18 @@ The homework is just any old python code you would give to the students. For ins ...@@ -45,29 +45,18 @@ The homework is just any old python code you would give to the students. For ins
The test consists of individual problems and a report-class. The tests themselves are just regular Unittest (we will see a slightly smarter idea in a moment). For instance: The test consists of individual problems and a report-class. The tests themselves are just regular Unittest (we will see a slightly smarter idea in a moment). For instance:
```python ```python
from cs101 import reverse_list, add {{report1_py}}
import unittest
class Week1(unittest.TestCase):
def test_add(self):
self.assertEqual(add(2, 2), 4)
self.assertEqual(add(-100, 5), -95)
def test_reverse(self):
self.assertEqual(reverse_list([1, 2, 3]), [3, 2, 1])
``` ```
A number of tests can be collected into a `Report`, which will allow us to assign points to the tests and use the more advanced features of the framework later. A complete, minimal example: A number of tests can be collected into a `Report`, which will allow us to assign points to the tests and use the more advanced features of the framework later. A complete, minimal example:
```python ```python
{{example_simplest_instructor_cs101_report1_py}} {{report1_all_py}}
``` ```
### Deployment ### Deployment
The above is all you need if you simply want to use the framework as a self-check: Students can run the code and see how well they did. The above is all you need if you simply want to use the framework as a self-check: Students can run the code and see how well they did.
In order to begin using the framework for evaluation we need to create a bit more structure. We do that by deploying the report class as follows: In order to begin using the framework for evaluation we need to create a bit more structure. We do that by deploying the report class as follows:
```python ```python
{{example_simplest_instructor_cs101_deploy_py}} {{deploy_py}}
``` ```
- The first line creates the `report1_grade.py` script and any additional data files needed by the tests (none in this case) - The first line creates the `report1_grade.py` script and any additional data files needed by the tests (none in this case)
- The second line set up the students directory (remember, we have included the solutions!) and remove the students solutions. You can check the results in the students folder. - The second line set up the students directory (remember, we have included the solutions!) and remove the students solutions. You can check the results in the students folder.
...@@ -266,15 +255,16 @@ submissions/ # Where you dump student submissions. ...@@ -266,15 +255,16 @@ submissions/ # Where you dump student submissions.
``` ```
The whitelist directory is optional, and the submissions directory contains student submissions (one folder per student): The whitelist directory is optional, and the submissions directory contains student submissions (one folder per student):
```terminal ```terminal
/submissions/<student-id-1>/.. /submissions/<student-id-1>/Report1_74_of_144.token
/submissions/<student-id-2>/.. /submissions/<student-id-2>/Report1_130_of_144.token
...
``` ```
The files in the whitelist/student directory can be either `.token` files (which are unpacked) or python files, and they may contain subdirectories: Everything will be unpacked and flattened. The simplest way to set it up is simply to download all files from DTU learn as a zip-file and unzip it somewhere. The files in the whitelist/student directory can be either `.token` files (which are unpacked) or python files, and they may contain subdirectories: Everything will be unpacked and flattened. The simplest way to set it up is simply to download all files from DTU learn as a zip-file and unzip it somewhere.
When done just call moss as follows: When done just call moss as follows:
```python ```python
{{example_moss_moss_example_py}} {{example_moss_moss_example_py}}
``` ```
This will generate a report. You can view the example including the report here: https://lab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/examples/example_moss This will generate a report. You can see the example including the report here: https://lab.compute.dtu.dk/tuhe/unitgrade_private/-/tree/master/examples/example_moss
# Smart hinting # Smart hinting
To help students get started, unitgrade will collect hints to solve failed tests from across the codebase and display them. Consider the following homework where two problems depends on each other and the To help students get started, unitgrade will collect hints to solve failed tests from across the codebase and display them. Consider the following homework where two problems depends on each other and the
...@@ -291,7 +281,7 @@ When students run this homework it will fail and display the hints from the two ...@@ -291,7 +281,7 @@ When students run this homework it will fail and display the hints from the two
![alt text|small]({{resources}}/docs/hints.png) ![alt text|small]({{resources}}/docs/hints.png)
What happens behind the scenes is that a code-coverage tool is run on the instructors computer What happens behind the scenes is that a code-coverage tool is run on the instructors computer
to determine which methods are actually used in solving a problem, and then the hint-texts of those methods (and none other) are to determine which methods are actually used in solving a problem, and then the hint-texts of those methods are
collected and displayed. This feature requires no external configuration; simply write `Hints:` in the source code. collected and displayed. This feature requires no external configuration; simply write `Hints:` in the source code.
# Citing # Citing
......
if __name__ == "__main__": def dump_data(base):
from jinjafy.bibliography_maker import make_bibliography data = {}
bib = make_bibliography("../setup.py", "./") fls = [f for f in glob.glob(base +"/**/*.*", recursive=True) if f.split(".")[-1] in ["py", "txt", "shell"]]
import jinja2
data = {'bibtex': bib}
import glob
fls = [f for f in glob.glob("../examples/**/*.*", recursive=True) if f.split(".")[-1] in ["py", "txt", "shell"] ]
import os import os
# fls = [(f, ) for f in fls] # fls = [(f, ) for f in fls]
for file in fls: for file in fls:
with open(file, 'r') as f: with open(file, 'r') as f:
k = os.path.relpath(file, "../examples").replace(os.sep, "_").replace(".", "_") k = os.path.relpath(file, base).replace(os.sep, "_").replace(".", "_")
data[k] = f.read() data[k] = f.read()
return data
if __name__ == "__main__":
# Snip the examples directory.
from snipper import snip_dir
import glob
import jinja2
snip_dir("../examples", output_dir="./snips")
data = dump_data("./snips")
from jinjafy.bibliography_maker import make_bibliography
bib = make_bibliography("../setup.py", "./")
data['bibtex'] = bib
data = {**data, **dump_data("../examples")}
# import glob
# fls = [f for f in glob.glob("../examples/**/*.*", recursive=True) if f.split(".")[-1] in ["py", "txt", "shell"] ]
# import os
# # fls = [(f, ) for f in fls]
# for file in fls:
# with open(file, 'r') as f:
# k = os.path.relpath(file, "../examples").replace(os.sep, "_").replace(".", "_")
# data[k] = f.read()
data['resources'] = "https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/raw/master" data['resources'] = "https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/raw/master"
......
# example_simplest/instructor/cs101/deploy.py
from cs101.report1 import Report1
from unitgrade_private2.hidden_create_files import setup_grade_file_report
from snipper import snip_dir
if __name__ == "__main__":
setup_grade_file_report(Report1) # Make the report1_grade.py report file
# Deploy the files using snipper: https://gitlab.compute.dtu.dk/tuhe/snipper
snip_dir("./", "../../students/cs101", exclude=['__pycache__', '*.token', 'deploy.py'])
\ No newline at end of file
# example_simplest/instructor/cs101/homework1.py
def reverse_list(mylist): #!f
"""
Given a list 'mylist' returns a list consisting of the same elements in reverse order. E.g.
reverse_list([1,2,3]) should return [3,2,1] (as a list).
"""
return list(reversed(mylist))
def add(a,b): #!f
""" Given two numbers `a` and `b` this function should simply return their sum:
> add(a,b) = a+b """
return a+b
if __name__ == "__main__":
# Example usage:
print(f"Your result of 2 + 2 = {add(2,2)}")
print(f"Reversing a small list", reverse_list([2,3,5,7]))
\ No newline at end of file
# example_simplest/instructor/cs101/report1.py
from cs101.homework1 import reverse_list, add
class Week1(unittest.TestCase):
def test_add(self):
self.assertEqual(add(2,2), 4)
self.assertEqual(add(-100, 5), -95)
def test_reverse(self):
self.assertEqual(reverse_list([1,2,3]), [3,2,1])
\ No newline at end of file
# example_simplest/instructor/cs101/report1.py
import unittest
from unitgrade2 import Report, evaluate_report_student
import cs101
from cs101.homework1 import reverse_list, add #!s
class Week1(unittest.TestCase):
def test_add(self):
self.assertEqual(add(2,2), 4)
self.assertEqual(add(-100, 5), -95)
def test_reverse(self):
self.assertEqual(reverse_list([1,2,3]), [3,2,1]) #!s
class Report1(Report):
title = "CS 101 Report 1"
questions = [(Week1, 10)] # Include a single question for a total of 10 credits.
pack_imports = [cs101] # Include all .py files in this folder
if __name__ == "__main__":
evaluate_report_student(Report1())
\ No newline at end of file
import shutil
from cs101.report1 import Report1 #!s from cs101.report1 import Report1 #!s
from unitgrade_private2.hidden_create_files import setup_grade_file_report from unitgrade_private2.hidden_create_files import setup_grade_file_report
import shutil, os
from snipper import snip_dir from snipper import snip_dir
# wd = os.path.dirname(__file__)
if __name__ == "__main__": if __name__ == "__main__":
setup_grade_file_report(Report1) setup_grade_file_report(Report1) # Make the report1_grade.py report file
# Deploy the files using snipper: https://gitlab.compute.dtu.dk/tuhe/snipper # Deploy the files using snipper: https://gitlab.compute.dtu.dk/tuhe/snipper
snip_dir.snip_dir("./", "../../students/cs101", clean_destination_dir=True, exclude=['__pycache__', '*.token', 'deploy.py']) #!s snip_dir("./", "../../students/cs101", exclude=['__pycache__', '*.token', 'deploy.py']) #!s
# For my own sake, copy the homework file to the other examples. # For my own sake, copy the homework file to the other examples.
for f in ['../../../example_framework/instructor/cs102/homework1.py', for f in ['../../../example_framework/instructor/cs102/homework1.py',
......
def reverse_list(mylist): #!f def reverse_list(mylist): #!f #!s
""" """
Given a list 'mylist' returns a list consisting of the same elements in reverse order. E.g. Given a list 'mylist' returns a list consisting of the same elements in reverse order. E.g.
reverse_list([1,2,3]) should return [3,2,1] (as a list). reverse_list([1,2,3]) should return [3,2,1] (as a list).
...@@ -13,4 +13,4 @@ def add(a,b): #!f ...@@ -13,4 +13,4 @@ def add(a,b): #!f
if __name__ == "__main__": if __name__ == "__main__":
# Example usage: # Example usage:
print(f"Your result of 2 + 2 = {add(2,2)}") print(f"Your result of 2 + 2 = {add(2,2)}")
print(f"Reversing a small list", reverse_list([2,3,5,7])) print(f"Reversing a small list", reverse_list([2,3,5,7])) #!s
import unittest import unittest #!s=all
from unitgrade2.unitgrade2 import Report from unitgrade2 import Report, evaluate_report_student
from unitgrade2.unitgrade_helpers2 import evaluate_report_student
from cs101.homework1 import reverse_list, add
import cs101 import cs101
from cs101.homework1 import reverse_list, add #!s
class Week1(unittest.TestCase): class Week1(unittest.TestCase):
def test_add(self): def test_add(self):
...@@ -10,7 +9,7 @@ class Week1(unittest.TestCase): ...@@ -10,7 +9,7 @@ class Week1(unittest.TestCase):
self.assertEqual(add(-100, 5), -95) self.assertEqual(add(-100, 5), -95)
def test_reverse(self): def test_reverse(self):
self.assertEqual(reverse_list([1,2,3]), [3,2,1]) self.assertEqual(reverse_list([1,2,3]), [3,2,1]) #!s
class Report1(Report): class Report1(Report):
title = "CS 101 Report 1" title = "CS 101 Report 1"
...@@ -18,5 +17,5 @@ class Report1(Report): ...@@ -18,5 +17,5 @@ class Report1(Report):
pack_imports = [cs101] # Include all .py files in this folder pack_imports = [cs101] # Include all .py files in this folder
if __name__ == "__main__": if __name__ == "__main__":
evaluate_report_student(Report1()) #!s=all
# unittest.main(verbosity=2) # Uncomment to run everything as a regular unittest # unittest.main(verbosity=2) # Uncomment to run everything as a regular unittest
evaluate_report_student(Report1())
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment