Skip to content
Snippets Groups Projects
Commit 47c407cb authored by tuhe's avatar tuhe
Browse files

Docker build and download updates

parent edfbb0c2
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 429 additions and 1000 deletions
examples/autolab_example/tmp/cs101/src/Report1_handin.token
+ 167
− 166
View file @ 47c407cb
This diff is collapsed.
examples/autolab_example/tmp/cs101/src/docker_helpers.py
+ 30
− 4
View file @ 47c407cb
# from cs202courseware.ug2report1 import Report1
import pickle
import os
import glob
import shutil
import time
import zipfile
import io
import inspect
import subprocess
import urllib.request
def download_docker_images(destination=None):
if destination is None:
destination = os.getcwd()
if not os.path.exists(destination):
os.makedirs(destination)
print('Beginning file download with urllib2...')
url = 'https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/archive/master/unitgrade_private-master.zip?path=docker_images'
result, headers = urllib.request.urlretrieve(url)
ex = result +"_extract"
zf = zipfile.ZipFile(result)
zf.extractall(path=ex)
dockers = [f for f in zf.namelist() if f[-1] == "/" and len( [s for s in f if s == "/"] ) == 3]
for f in dockers: # zf.namelist():
tmp_dir = ex + "/" + f
if os.path.isdir(tmp_dir):
dest = destination +"/"+os.path.basename(tmp_dir[:-1])
if os.path.isdir(dest):
print("> Destination for docker image", dest, "exists. Skipping download.")
else:
print("> Extracting docker image", os.path.basename(f[:-1]), "to", dest)
shutil.copytree(tmp_dir, dest)
# zf.extract(f, path=destination)
a = 234
def compile_docker_image(Dockerfile, tag=None):
assert os.path.isfile(Dockerfile)
......
examples/autolab_example/tmp/cs101/src/driver_python.py
+ 5
− 31
View file @ 47c407cb
import os
import glob
import sys
import pickle
# import io
import subprocess
from unitgrade_private.autolab.autolab import format_autolab_json
from unitgrade_private.docker_helpers import student_token_file_runner
from unitgrade_private import load_token
# import docker_helpers
import time
verbose = False
......@@ -28,10 +25,7 @@ def pfiles():
student_token_file = 'Report1_handin.token'
instructor_grade_script = 'report1_grade.py'
grade_file_relative_destination = "cs101/report1_grade.py"
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
grade_file_relative_destination = "cs101\report1_grade.py"
host_tmp_dir = wdir + "/tmp"
if not verbose:
......@@ -44,40 +38,24 @@ command, token = student_token_file_runner(host_tmp_dir, student_token_file, ins
command = f"cd tmp && {command} --noprogress --autolab"
def rcom(cm):
# print(f"running... ", cm)
# start = time.time()
rs = subprocess.run(cm, capture_output=True, text=True, shell=True)
print(rs.stdout)
if len(rs.stderr) > 0:
print(tag, "There were errors in executing the file:")
print(rs.stderr)
# print(rs)
# print("result of running command was", rs.stdout, "err", rs.stderr, "time", time.time() - start)
# results, _ = load_token(student_token_file)
# sources = results['sources'][0]
start = time.time()
rcom(command)
# pfiles()
# for f in glob.glob(host_tmp_dir + "/programs/*"):
# print("programs/", f)
# print("---")
ls = glob.glob(token)
# print(ls)
f = ls[0]
# with open(f, 'rb') as f:
# results = pickle.load(f)
results, _ = load_token(ls[0])
# print("results")
# print(results.keys())
if verbose:
print(f"{token=}")
print(results['total'])
format_autolab_json(results)
# if os.path.exists(host_tmp_dir):
# shutil.rmtree(host_tmp_dir)
# with io.BytesIO(sources['zipfile']) as zb:
......@@ -87,12 +65,8 @@ if verbose:
# print('{"scores": {"Correctness": 100, "Problem 1": 4}}')
## Format the scores here.
# sc = [('Total', results['total'][0])] + [(q['title'], q['obtained']) for k, q in results['details'].items()]
# ss = ", ".join([f'"{t}": {s}' for t, s in sc])
# scores = '{"scores": {' + ss + '}}'
# print('{"_presentation": "semantic"}')
# print(scores)
from unitgrade_private.autolab.autolab import format_autolab_json
format_autolab_json(results)
\ No newline at end of file
examples/autolab_example/tmp/cs101/src/driver_python.py-handout
+ 5
− 31
View file @ 47c407cb
import os
import glob
import sys
import pickle
# import io
import subprocess
from unitgrade_private.autolab.autolab import format_autolab_json
from unitgrade_private.docker_helpers import student_token_file_runner
from unitgrade_private import load_token
# import docker_helpers
import time
verbose = False
......@@ -28,10 +25,7 @@ def pfiles():
student_token_file = 'Report1_handin.token'
instructor_grade_script = 'report1_grade.py'
grade_file_relative_destination = "cs101/report1_grade.py"
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
grade_file_relative_destination = "cs101\report1_grade.py"
host_tmp_dir = wdir + "/tmp"
if not verbose:
......@@ -44,40 +38,24 @@ command, token = student_token_file_runner(host_tmp_dir, student_token_file, ins
command = f"cd tmp && {command} --noprogress --autolab"
def rcom(cm):
# print(f"running... ", cm)
# start = time.time()
rs = subprocess.run(cm, capture_output=True, text=True, shell=True)
print(rs.stdout)
if len(rs.stderr) > 0:
print(tag, "There were errors in executing the file:")
print(rs.stderr)
# print(rs)
# print("result of running command was", rs.stdout, "err", rs.stderr, "time", time.time() - start)
# results, _ = load_token(student_token_file)
# sources = results['sources'][0]
start = time.time()
rcom(command)
# pfiles()
# for f in glob.glob(host_tmp_dir + "/programs/*"):
# print("programs/", f)
# print("---")
ls = glob.glob(token)
# print(ls)
f = ls[0]
# with open(f, 'rb') as f:
# results = pickle.load(f)
results, _ = load_token(ls[0])
# print("results")
# print(results.keys())
if verbose:
print(f"{token=}")
print(results['total'])
format_autolab_json(results)
# if os.path.exists(host_tmp_dir):
# shutil.rmtree(host_tmp_dir)
# with io.BytesIO(sources['zipfile']) as zb:
......@@ -87,12 +65,8 @@ if verbose:
# print('{"scores": {"Correctness": 100, "Problem 1": 4}}')
## Format the scores here.
# sc = [('Total', results['total'][0])] + [(q['title'], q['obtained']) for k, q in results['details'].items()]
# ss = ", ".join([f'"{t}": {s}' for t, s in sc])
# scores = '{"scores": {' + ss + '}}'
# print('{"_presentation": "semantic"}')
# print(scores)
from unitgrade_private.autolab.autolab import format_autolab_json
format_autolab_json(results)
\ No newline at end of file
examples/autolab_example/tmp/cs101/src/student_sources.zip
+ 0
− 0
View file @ 47c407cb
No preview for this file type
examples/autolab_example/tmp/cs102/autograde.tar deleted 100644 → 0
+ 0
− 0
View file @ edfbb0c2
File deleted
examples/autolab_example/tmp/cs102/cs102-handout.tar deleted 100644 → 0
+ 0
− 0
View file @ edfbb0c2
File deleted
examples/autolab_example/tmp/cs102/cs102-handout/Makefile deleted 100644 → 0
+ 0
− 7
View file @ edfbb0c2
# Makefile for the Hello Lab
all:
echo "Makefile called... it is empty so far. "
#gcc hello3.c -o hello3
clean:
rm -rf *~ hello3
examples/autolab_example/tmp/cs102/cs102-handout/README deleted 100644 → 0
+ 0
− 15
View file @ edfbb0c2
This directory contains all of the code files for the Hello Lab,
including the files that are handed out to students.
Files:
# Autograder and solution files
Makefile Makefile and ...
README ... README for this directory
driver.sh* Autograder
hello.c Solution hello.c file
# Files that are handed out to students
Makefile-handout Makefile and ...
README-handout ... README handed out to students
hello.c-handout Blank hello.c file handed out to students
examples/autolab_example/tmp/cs102/cs102-handout/docker_helpers.py deleted 100644 → 0
+ 0
− 146
View file @ edfbb0c2
# from cs202courseware.ug2report1 import Report1
import pickle
import os
import glob
import shutil
import time
import zipfile
import io
import inspect
import subprocess
def compile_docker_image(Dockerfile, tag=None):
assert os.path.isfile(Dockerfile)
base = os.path.dirname(Dockerfile)
if tag == None:
tag = os.path.basename(base)
os.system(f"cd {base} && docker build --tag {tag} .")
return tag
def student_token_file_runner(host_tmp_dir, student_token_file, instructor_grade_script, grade_file_relative_destination):
"""
Use by autolab code.
:param Dockerfile_location:
:param host_tmp_dir:
:param student_token_file:
:param ReportClass:
:param instructor_grade_script:
:return:
"""
assert os.path.exists(student_token_file)
assert os.path.exists(instructor_grade_script)
start = time.time()
from unitgrade_private import load_token
results, _ = load_token(student_token_file)
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
sources = results['sources'][0]
with io.BytesIO(sources['zipfile']) as zb:
with zipfile.ZipFile(zb) as zip:
zip.extractall(host_tmp_dir)
# Done extracting the zip file! Now time to move the (good) report test class into the location.
gscript = instructor_grade_script
print(f"{sources['report_relative_location']=}")
print(f"{sources['name']=}")
print("Now in docker_helpers.py")
print(f'{gscript=}')
print(f'{instructor_grade_script=}')
gscript_destination = host_tmp_dir + "/" + grade_file_relative_destination
print(f'{gscript_destination=}')
shutil.copy(gscript, gscript_destination)
# Now everything appears very close to being set up and ready to roll!.
d = os.path.normpath(grade_file_relative_destination).split(os.sep)
d = d[:-1] + [os.path.basename(instructor_grade_script)[:-3]]
pycom = ".".join(d)
"""
docker run -v c:/Users/tuhe/Documents/2021/python-docker/tmp:/app python-docker python3 -m cs202courseware.ug2report1_grade
"""
pycom = "python3 -m " + pycom # pycom[:-3]
print(f"{pycom=}")
token_location = host_tmp_dir + "/" + os.path.dirname( grade_file_relative_destination ) + "/*.token"
elapsed = time.time() - start
# print("Elapsed time is", elapsed)
return pycom, token_location
def docker_run_token_file(Dockerfile_location, host_tmp_dir, student_token_file, tag=None, instructor_grade_script=None, fix_user=True):
"""
This thingy works:
To build the image, run:
docker build --tag python-docker .
To run the app run:
docker run -v c:/Users/tuhe/Documents/2021/python-docker/tmp:/app python-docker > output.log
"""
# A bunch of tests. This is going to be great!
assert os.path.exists(Dockerfile_location)
start = time.time()
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
from unitgrade_private import load_token
results, _ = load_token(student_token_file)
sources = results['sources'][0]
if os.path.exists(host_tmp_dir):
shutil.rmtree(host_tmp_dir)
with io.BytesIO(sources['zipfile']) as zb:
with zipfile.ZipFile(zb) as zip:
zip.extractall(host_tmp_dir)
# Done extracting the zip file! Now time to move the (good) report test class into the location.
gscript = instructor_grade_script
student_grade_script = host_tmp_dir + "/" + sources['report_relative_location']
instructor_grade_script = os.path.dirname(student_grade_script) + "/"+os.path.basename(gscript)
shutil.copy(gscript, instructor_grade_script)
"""
docker run -v c:/Users/tuhe/Documents/2021/python-docker/tmp:/home python-docker python3 -m cs202courseware.ug2report1_grade
"""
if tag is None:
dockname = os.path.basename( os.path.dirname(Dockerfile_location) )
else:
dockname = tag
tmp_grade_file = sources['name'] + "/" + sources['report_relative_location']
pycom = ".".join( sources['report_module_specification'][:-1] + [os.path.basename(gscript)[:-3],] )
pycom = "python3 -m " + pycom
if fix_user:
user_cmd = ' --user "$(id -u):$(id -g)" '
else:
user_cmd = ''
tmp_path = os.path.abspath(host_tmp_dir).replace("\\", "/")
dcom = f"docker run {user_cmd} -v {tmp_path}:/home {dockname} {pycom}"
cdcom = f"cd {os.path.dirname(Dockerfile_location)}"
fcom = f"{cdcom} && {dcom}"
print("> Running docker command")
print(fcom)
init = time.time() - start
# thtools.execute_command(fcom.split())
subprocess.check_output(fcom, shell=True).decode("utf-8")
host_tmp_dir +"/" + os.path.dirname(tmp_grade_file) + "/"
tokens = glob.glob( os.path.dirname(instructor_grade_script) + "/*.token" )
for t in tokens:
print("Source image produced token", t)
elapsed = time.time() - start
print("Elapsed time is", elapsed, f"({init=} seconds)")
return tokens[0]
examples/autolab_example/tmp/cs102/cs102-handout/driver_python.py deleted 100644 → 0
+ 0
− 98
View file @ edfbb0c2
import os
import glob
import sys
import pickle
# import io
import subprocess
from unitgrade_private.docker_helpers import student_token_file_runner
from unitgrade_private import load_token
# import docker_helpers
import time
verbose = False
tag = "[driver_python.py]"
if not verbose:
print("="*10)
print(tag, "Starting unitgrade evaluation...")
sys.stderr = sys.stdout
wdir = os.getcwd()
def pfiles():
print("> Files in dir:")
for f in glob.glob(wdir + "/*"):
print(f)
print("---")
student_token_file = 'Report2_handin.token'
instructor_grade_script = 'report2_grade.py'
grade_file_relative_destination = "cs102/report2_grade.py"
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
host_tmp_dir = wdir + "/tmp"
if not verbose:
pfiles()
print(f"{host_tmp_dir=}")
print(f"{student_token_file=}")
print(f"{instructor_grade_script=}")
command, token = student_token_file_runner(host_tmp_dir, student_token_file, instructor_grade_script, grade_file_relative_destination)
command = f"cd tmp && {command} --noprogress --autolab"
def rcom(cm):
# print(f"running... ", cm)
# start = time.time()
rs = subprocess.run(cm, capture_output=True, text=True, shell=True)
print(rs.stdout)
if len(rs.stderr) > 0:
print(tag, "There were errors in executing the file:")
print(rs.stderr)
# print(rs)
# print("result of running command was", rs.stdout, "err", rs.stderr, "time", time.time() - start)
# results, _ = load_token(student_token_file)
# sources = results['sources'][0]
start = time.time()
rcom(command)
# pfiles()
# for f in glob.glob(host_tmp_dir + "/programs/*"):
# print("programs/", f)
# print("---")
ls = glob.glob(token)
# print(ls)
f = ls[0]
# with open(f, 'rb') as f:
# results = pickle.load(f)
results, _ = load_token(ls[0])
# print("results")
# print(results.keys())
if verbose:
print(f"{token=}")
print(results['total'])
# if os.path.exists(host_tmp_dir):
# shutil.rmtree(host_tmp_dir)
# with io.BytesIO(sources['zipfile']) as zb:
# with zipfile.ZipFile(zb) as zip:
# zip.extractall(host_tmp_dir
# print("="*10)
# print('{"scores": {"Correctness": 100, "Problem 1": 4}}')
## Format the scores here.
# sc = [('Total', results['total'][0])] + [(q['title'], q['obtained']) for k, q in results['details'].items()]
# ss = ", ".join([f'"{t}": {s}' for t, s in sc])
# scores = '{"scores": {' + ss + '}}'
# print('{"_presentation": "semantic"}')
# print(scores)
from unitgrade_private.autolab.autolab import format_autolab_json
format_autolab_json(results)
\ No newline at end of file
examples/autolab_example/tmp/cs102/cs102-handout/student_sources.zip deleted 100644 → 0
+ 0
− 0
View file @ edfbb0c2
File deleted
examples/autolab_example/tmp/cs102/src/Report2_handin.token
+ 182
− 181
View file @ 47c407cb
This diff is collapsed.
examples/autolab_example/tmp/cs102/src/docker_helpers.py
+ 30
− 4
View file @ 47c407cb
# from cs202courseware.ug2report1 import Report1
import pickle
import os
import glob
import shutil
import time
import zipfile
import io
import inspect
import subprocess
import urllib.request
def download_docker_images(destination=None):
if destination is None:
destination = os.getcwd()
if not os.path.exists(destination):
os.makedirs(destination)
print('Beginning file download with urllib2...')
url = 'https://gitlab.compute.dtu.dk/tuhe/unitgrade_private/-/archive/master/unitgrade_private-master.zip?path=docker_images'
result, headers = urllib.request.urlretrieve(url)
ex = result +"_extract"
zf = zipfile.ZipFile(result)
zf.extractall(path=ex)
dockers = [f for f in zf.namelist() if f[-1] == "/" and len( [s for s in f if s == "/"] ) == 3]
for f in dockers: # zf.namelist():
tmp_dir = ex + "/" + f
if os.path.isdir(tmp_dir):
dest = destination +"/"+os.path.basename(tmp_dir[:-1])
if os.path.isdir(dest):
print("> Destination for docker image", dest, "exists. Skipping download.")
else:
print("> Extracting docker image", os.path.basename(f[:-1]), "to", dest)
shutil.copytree(tmp_dir, dest)
# zf.extract(f, path=destination)
a = 234
def compile_docker_image(Dockerfile, tag=None):
assert os.path.isfile(Dockerfile)
......
examples/autolab_example/tmp/cs102/src/driver_python.py
+ 5
− 31
View file @ 47c407cb
import os
import glob
import sys
import pickle
# import io
import subprocess
from unitgrade_private.autolab.autolab import format_autolab_json
from unitgrade_private.docker_helpers import student_token_file_runner
from unitgrade_private import load_token
# import docker_helpers
import time
verbose = False
......@@ -28,10 +25,7 @@ def pfiles():
student_token_file = 'Report2_handin.token'
instructor_grade_script = 'report2_grade.py'
grade_file_relative_destination = "cs102/report2_grade.py"
# with open(student_token_file, 'rb') as f:
# results = pickle.load(f)
grade_file_relative_destination = "cs102\report2_grade.py"
host_tmp_dir = wdir + "/tmp"
if not verbose:
......@@ -44,40 +38,24 @@ command, token = student_token_file_runner(host_tmp_dir, student_token_file, ins
command = f"cd tmp && {command} --noprogress --autolab"
def rcom(cm):
# print(f"running... ", cm)
# start = time.time()
rs = subprocess.run(cm, capture_output=True, text=True, shell=True)
print(rs.stdout)
if len(rs.stderr) > 0:
print(tag, "There were errors in executing the file:")
print(rs.stderr)
# print(rs)
# print("result of running command was", rs.stdout, "err", rs.stderr, "time", time.time() - start)
# results, _ = load_token(student_token_file)
# sources = results['sources'][0]
start = time.time()
rcom(command)
# pfiles()
# for f in glob.glob(host_tmp_dir + "/programs/*"):
# print("programs/", f)
# print("---")
ls = glob.glob(token)
# print(ls)
f = ls[0]
# with open(f, 'rb') as f:
# results = pickle.load(f)
results, _ = load_token(ls[0])
# print("results")
# print(results.keys())
if verbose:
print(f"{token=}")
print(results['total'])
format_autolab_json(results)
# if os.path.exists(host_tmp_dir):
# shutil.rmtree(host_tmp_dir)
# with io.BytesIO(sources['zipfile']) as zb:
......@@ -87,12 +65,8 @@ if verbose:
# print('{"scores": {"Correctness": 100, "Problem 1": 4}}')
## Format the scores here.
# sc = [('Total', results['total'][0])] + [(q['title'], q['obtained']) for k, q in results['details'].items()]
# ss = ", ".join([f'"{t}": {s}' for t, s in sc])
# scores = '{"scores": {' + ss + '}}'
# print('{"_presentation": "semantic"}')
# print(scores)
from unitgrade_private.autolab.autolab import format_autolab_json
format_autolab_json(results)
\ No newline at end of file
examples/autolab_example/tmp/cs102/src/student_sources.zip
+ 0
− 0
View file @ 47c407cb
No preview for this file type
examples/autolab_example/tmp/cs103/autograde.tar deleted 100644 → 0
+ 0
− 0
View file @ edfbb0c2
File deleted
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment