Skip to content
Snippets Groups Projects
Commit edfbb0c2 authored by tuhe's avatar tuhe
Browse files

Autolab score computation working

parent bceb1c3f
No related branches found
No related tags found
No related merge requests found
Showing
with 633 additions and 6 deletions
...@@ -4,4 +4,5 @@ jinja2 ...@@ -4,4 +4,5 @@ jinja2
tabulate tabulate
compress_pickle compress_pickle
pyfiglet pyfiglet
colorama colorama
\ No newline at end of file unitgrade-devel>=0.1.23
\ No newline at end of file
import os import os
from unitgrade_private.autolab.autolab import format_autolab_json, docker_build_image
from unitgrade_private.autolab.autolab import deploy_assignment from unitgrade_private.autolab.autolab import deploy_assignment
"""
Semantic score formatting. See.
https://docs.autolabproject.com/lab/https://docs.autolabproject.com/features/formatted-feedback/
{
"_presentation": "semantic",
"stages": ["Build","Test","Timing"],
"Test": {
"Add Things": {
"passed": true
},
"Return Values": {
"passed": false,
"hint": "You need to return 1"
}
},
"Build": {
"compile": {
"passed": true
},
"link": {
"passed": true
}
},
"Timing": {
"Stage 1 (ms)": 10,
"Stage 2 (ms)": 20
}
}
{"scores": {"Correctness": 20, "TA/Design/Readability": 40}}
"""
if __name__ == "__main__": if __name__ == "__main__":
wdir = os.getcwd() wdir = os.getcwd()
args = [('example_simplest', 'programs', 'report1_grade.py', 'report1_grade.py'), args = [('example_simplest', 'cs101', 'report1_grade.py', 'report1_grade.py'),
('example_framework', 'cs102', 'report2_grade.py', 'report2_grade.py'), ('example_framework', 'cs102', 'report2_grade.py', 'report2_grade.py'),
('example_docker', 'cs103', 'report3_complete_grade.py', 'report3_grade.py'), ('example_docker', 'cs103', 'report3_complete_grade.py', 'report3_grade.py'),
] ]
from unitgrade_private import load_token
# data, _ = load_token("../example_framework/instructor/cs102/Report2_handin_18_of_18.token")
data, _ = load_token("../example_framework/students/cs102/Report2_handin_3_of_16.token")
format_autolab_json(data, indent=2)
# import sys
# sys.exit()
for bdir, name, instructor, student in args: for bdir, name, instructor, student in args:
instructor_base = f"{wdir}/../{bdir}/instructor" instructor_base = f"{wdir}/../{bdir}/instructor"
student_base = f"{wdir}/../{bdir}/students" student_base = f"{wdir}/../{bdir}/students"
output_tar = deploy_assignment(name, INSTRUCTOR_BASE=instructor_base, INSTRUCTOR_GRADE_FILE=f"{instructor_base}/{name}/{instructor}", output_tar = deploy_assignment(name, INSTRUCTOR_BASE=instructor_base, INSTRUCTOR_GRADE_FILE=f"{instructor_base}/{name}/{instructor}",
STUDENT_BASE=student_base, STUDENT_GRADE_FILE=f"{student_base}/{name}/{student}", STUDENT_BASE=student_base, STUDENT_GRADE_FILE=f"{student_base}/{name}/{student}",
output_tar=None) output_tar=None,
autograde_image='tango_python_tue')
print(output_tar) print(output_tar)
docker_build_image() # Make sure docker grading image is up-to-date.
docker_build_image() # Make sure docker grading image is up-to-date.
\ No newline at end of file
No preview for this file type
No preview for this file type
No preview for this file type
## Startup (development notes)
## Startup (development) To start `Tango` follow this guide:
- https://docs.autolabproject.com/installation/tango/
```terminal ```terminal
cd ~/Documents/Tango cd ~/Documents/Tango
source bin/activate source bin/activate
python restful_tango/server.py 3000 redis-server &
python restful_tango/server.py 3000 &
python jobManager.py
```
To start `Autolab`
```terminal
cd ~/Autolab && bundle exec rails s -p 8000 --binding=0.0.0.0
``` ```
Access Autolab on https://autolab:8000
It is important to use this address because Tango expects requests from an `autolab` hostname.
#
# Makefile to manage the example Hello Lab
#
# Get the name of the lab directory
# LAB = $(notdir $(PWD)) # Fail on windows for some reason...
all: handout handout-tarfile
handout:
# Rebuild the handout directory that students download
(rm -rf cs101-handout; mkdir cs101-handout)
cp -p src/Makefile-handout cs101-handout/Makefile
cp -p src/README-handout cs101-handout/README
cp -p src/driver_python.py cs101-handout
cp -p src/student_sources.zip cs101-handout
cp -p src/Report1_handin.token cs101-handout
cp -p src/docker_helpers.py cs101-handout
cp -p src/report1_grade.py cs101-handout
handout-tarfile: handout
# Build *-handout.tar and autograde.tar
tar cvf cs101-handout.tar cs101-handout
cp -p cs101-handout.tar autograde.tar
clean:
# Clean the entire lab directory tree. Note that you can run
# "make clean; make" at any time while the lab is live with no
# adverse effects.
rm -f *~ *.tar
(cd src; make clean)
(cd test-autograder; make clean)
rm -rf cs101-handout
rm -f autograde.tar
#
# CAREFULL!!! This will delete all student records in the logfile and
# in the handin directory. Don't run this once the lab has started.
# Use it to clean the directory when you are starting a new version
# of the lab from scratch, or when you are debugging the lab prior
# to releasing it to the students.
#
cleanallfiles:
# Reset the lab from scratch.
make clean
rm -f log.txt
rm -rf handin/*
all:
tar xf autograde.tar
cp Report1_handin.token cs101-handout
(cd cs101-handout; python3 driver_python.py)
clean:
rm -rf *~ hello3-handout
\ No newline at end of file
File added
File added
# Makefile for the Hello Lab
all:
echo "Makefile called... it is empty so far. "
#gcc hello3.c -o hello3
clean:
rm -rf *~ hello3
This directory contains all of the code files for the Hello Lab,
including the files that are handed out to students.
Files:
# Autograder and solution files
Makefile Makefile and ...
README ... README for this directory
driver.sh* Autograder
hello.c Solution hello.c file
# Files that are handed out to students
Makefile-handout Makefile and ...
README-handout ... README handed out to students
hello.c-handout Blank hello.c file handed out to students
This diff is collapsed.
# from cs202courseware.ug2report1 import Report1
import pickle
import os
import glob
import shutil
import time
import zipfile
import io
import inspect
import subprocess
def compile_docker_image(Dockerfile, tag=None):
assert os.path.isfile(Dockerfile)
base = os.path.dirname(Dockerfile)
if tag == None:
tag = os.path.basename(base)
os.system(f"cd {base} && docker build --tag {tag} .")
return tag
def student_token_file_runner(host_tmp_dir, student_token_file, instructor_grade_script, grade_file_relative_destination):
"""
Use by autolab code.
:param Dockerfile_location:
:param host_tmp_dir:
:param student_token_file:
:param ReportClass:
:param instructor_grade_script:
:return:
"""
assert os.path.exists(student_token_file)
assert os.path.exists(instructor_grade_script)
start = time.time()
from unitgrade_private import load_token
results, _ = load_token(student_token_file)
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
sources = results['sources'][0]
with io.BytesIO(sources['zipfile']) as zb:
with zipfile.ZipFile(zb) as zip:
zip.extractall(host_tmp_dir)
# Done extracting the zip file! Now time to move the (good) report test class into the location.
gscript = instructor_grade_script
print(f"{sources['report_relative_location']=}")
print(f"{sources['name']=}")
print("Now in docker_helpers.py")
print(f'{gscript=}')
print(f'{instructor_grade_script=}')
gscript_destination = host_tmp_dir + "/" + grade_file_relative_destination
print(f'{gscript_destination=}')
shutil.copy(gscript, gscript_destination)
# Now everything appears very close to being set up and ready to roll!.
d = os.path.normpath(grade_file_relative_destination).split(os.sep)
d = d[:-1] + [os.path.basename(instructor_grade_script)[:-3]]
pycom = ".".join(d)
"""
docker run -v c:/Users/tuhe/Documents/2021/python-docker/tmp:/app python-docker python3 -m cs202courseware.ug2report1_grade
"""
pycom = "python3 -m " + pycom # pycom[:-3]
print(f"{pycom=}")
token_location = host_tmp_dir + "/" + os.path.dirname( grade_file_relative_destination ) + "/*.token"
elapsed = time.time() - start
# print("Elapsed time is", elapsed)
return pycom, token_location
def docker_run_token_file(Dockerfile_location, host_tmp_dir, student_token_file, tag=None, instructor_grade_script=None, fix_user=True):
"""
This thingy works:
To build the image, run:
docker build --tag python-docker .
To run the app run:
docker run -v c:/Users/tuhe/Documents/2021/python-docker/tmp:/app python-docker > output.log
"""
# A bunch of tests. This is going to be great!
assert os.path.exists(Dockerfile_location)
start = time.time()
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
from unitgrade_private import load_token
results, _ = load_token(student_token_file)
sources = results['sources'][0]
if os.path.exists(host_tmp_dir):
shutil.rmtree(host_tmp_dir)
with io.BytesIO(sources['zipfile']) as zb:
with zipfile.ZipFile(zb) as zip:
zip.extractall(host_tmp_dir)
# Done extracting the zip file! Now time to move the (good) report test class into the location.
gscript = instructor_grade_script
student_grade_script = host_tmp_dir + "/" + sources['report_relative_location']
instructor_grade_script = os.path.dirname(student_grade_script) + "/"+os.path.basename(gscript)
shutil.copy(gscript, instructor_grade_script)
"""
docker run -v c:/Users/tuhe/Documents/2021/python-docker/tmp:/home python-docker python3 -m cs202courseware.ug2report1_grade
"""
if tag is None:
dockname = os.path.basename( os.path.dirname(Dockerfile_location) )
else:
dockname = tag
tmp_grade_file = sources['name'] + "/" + sources['report_relative_location']
pycom = ".".join( sources['report_module_specification'][:-1] + [os.path.basename(gscript)[:-3],] )
pycom = "python3 -m " + pycom
if fix_user:
user_cmd = ' --user "$(id -u):$(id -g)" '
else:
user_cmd = ''
tmp_path = os.path.abspath(host_tmp_dir).replace("\\", "/")
dcom = f"docker run {user_cmd} -v {tmp_path}:/home {dockname} {pycom}"
cdcom = f"cd {os.path.dirname(Dockerfile_location)}"
fcom = f"{cdcom} && {dcom}"
print("> Running docker command")
print(fcom)
init = time.time() - start
# thtools.execute_command(fcom.split())
subprocess.check_output(fcom, shell=True).decode("utf-8")
host_tmp_dir +"/" + os.path.dirname(tmp_grade_file) + "/"
tokens = glob.glob( os.path.dirname(instructor_grade_script) + "/*.token" )
for t in tokens:
print("Source image produced token", t)
elapsed = time.time() - start
print("Elapsed time is", elapsed, f"({init=} seconds)")
return tokens[0]
import os
import glob
import sys
import pickle
# import io
import subprocess
from unitgrade_private.docker_helpers import student_token_file_runner
from unitgrade_private import load_token
# import docker_helpers
import time
verbose = False
tag = "[driver_python.py]"
if not verbose:
print("="*10)
print(tag, "Starting unitgrade evaluation...")
sys.stderr = sys.stdout
wdir = os.getcwd()
def pfiles():
print("> Files in dir:")
for f in glob.glob(wdir + "/*"):
print(f)
print("---")
student_token_file = 'Report1_handin.token'
instructor_grade_script = 'report1_grade.py'
grade_file_relative_destination = "cs101/report1_grade.py"
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
host_tmp_dir = wdir + "/tmp"
if not verbose:
pfiles()
print(f"{host_tmp_dir=}")
print(f"{student_token_file=}")
print(f"{instructor_grade_script=}")
command, token = student_token_file_runner(host_tmp_dir, student_token_file, instructor_grade_script, grade_file_relative_destination)
command = f"cd tmp && {command} --noprogress --autolab"
def rcom(cm):
# print(f"running... ", cm)
# start = time.time()
rs = subprocess.run(cm, capture_output=True, text=True, shell=True)
print(rs.stdout)
if len(rs.stderr) > 0:
print(tag, "There were errors in executing the file:")
print(rs.stderr)
# print(rs)
# print("result of running command was", rs.stdout, "err", rs.stderr, "time", time.time() - start)
# results, _ = load_token(student_token_file)
# sources = results['sources'][0]
start = time.time()
rcom(command)
# pfiles()
# for f in glob.glob(host_tmp_dir + "/programs/*"):
# print("programs/", f)
# print("---")
ls = glob.glob(token)
# print(ls)
f = ls[0]
# with open(f, 'rb') as f:
# results = pickle.load(f)
results, _ = load_token(ls[0])
# print("results")
# print(results.keys())
if verbose:
print(f"{token=}")
print(results['total'])
# if os.path.exists(host_tmp_dir):
# shutil.rmtree(host_tmp_dir)
# with io.BytesIO(sources['zipfile']) as zb:
# with zipfile.ZipFile(zb) as zip:
# zip.extractall(host_tmp_dir
# print("="*10)
# print('{"scores": {"Correctness": 100, "Problem 1": 4}}')
## Format the scores here.
# sc = [('Total', results['total'][0])] + [(q['title'], q['obtained']) for k, q in results['details'].items()]
# ss = ", ".join([f'"{t}": {s}' for t, s in sc])
# scores = '{"scores": {' + ss + '}}'
# print('{"_presentation": "semantic"}')
# print(scores)
from unitgrade_private.autolab.autolab import format_autolab_json
format_autolab_json(results)
\ No newline at end of file
This diff is collapsed.
File added
require "AssessmentBase.rb"
module Cs101
include AssessmentBase
def assessmentInitialize(course)
super("cs101",course)
@problems = []
end
end
\ No newline at end of file
---
general:
name: cs101
description: ''
display_name: CS 101 Report 1
handin_filename: Report1_handin.token
handin_directory: handin
max_grace_days: 0
handout: cs101-handout.tar
writeup: writeup/cs101.html
max_submissions: -1
disable_handins: false
max_size: 2
has_svn: false
category_name: Lab
problems:
- name: Unitgrade score
description: ''
max_score: 10
optional: false
autograder:
autograde_timeout: 180
autograde_image: tango_python_tue
release_score: true
# problems:
# - name: Correctness
# description: ''
# max_score: 100.0
# optional: false
\ No newline at end of file
# Makefile for the Hello Lab
all:
echo "Makefile called... it is empty so far. "
#gcc hello3.c -o hello3
clean:
rm -rf *~ hello3
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment