Skip to content
Snippets Groups Projects
Commit 678051eb authored by tuhe's avatar tuhe
Browse files

Changes to evaluation code

parent 87e2197e
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
src/unitgrade_private/deployment.py
+ 47
− 2
View file @ 678051eb
...@@ -4,6 +4,38 @@ import os ...@@ -4,6 +4,38 @@ import os
import importlib import importlib
import snipper import snipper
# import re, inspect
#
# FUNC_BODY = re.compile('^(?P<tabs>[\t ]+)?def (?P<name>[a-zA-Z0-9_]+)([^\n]+)\n(?P<body>(^([\t ]+)?([^\n]+)\n)+)', re.M)
#
# class Source(object):
# @staticmethod
# def investigate(focus: object, strfocus: str) -> str:
# with open(inspect.getsourcefile(focus), 'r') as f:
# for m in FUNC_BODY.finditer(f.read()):
# if m.group('name') == strfocus:
# tabs = m.group('tabs') if not m.group('tabs') is None else ''
# return f"{tabs}'''\n{m.group('body')}{tabs}'''"
#
#
# def decorator(func):
# def inner():
# print("I'm decorated")
# func()
#
# return inner
#
#
# @decorator
# def test():
# a = 5
# b = 6
# return a + b
#
#
# print(Source.investigate(test, 'test'))
def remove_hidden_methods(ReportClass, outfile=None): def remove_hidden_methods(ReportClass, outfile=None):
# Given a ReportClass, clean out all @hidden tests from the imports of that class. # Given a ReportClass, clean out all @hidden tests from the imports of that class.
file = ReportClass()._file() file = ReportClass()._file()
...@@ -12,14 +44,24 @@ def remove_hidden_methods(ReportClass, outfile=None): ...@@ -12,14 +44,24 @@ def remove_hidden_methods(ReportClass, outfile=None):
lines_to_rem = [] lines_to_rem = []
for l in source:
if l.strip().startswith("@hide"):
print(l)
for Q,_ in ReportClass.questions: for Q,_ in ReportClass.questions:
ls = list(methodsWithDecorator(Q, hide)) ls = list(methodsWithDecorator(Q, hide))
# print("hide decorateed is", ls) # print("hide decorateed is", ls)
for f in ls: for f in ls:
assert inspect.getsourcefile(f) == file, "You must apply the @hide decorator as the inner-most decorator, i.e., just above the function you wish to remove."
s, start = inspect.getsourcelines(f) s, start = inspect.getsourcelines(f)
end = len(s) + start end = len(s) + start
lines_to_rem += list(range(start-1, end-1)) lines_to_rem += list(range(start-1, end-1))
print("All hidden funcs")
print(ls)
source = list([l for k, l in enumerate(source) if k not in lines_to_rem]) source = list([l for k, l in enumerate(source) if k not in lines_to_rem])
source = "\n".join(source) source = "\n".join(source)
...@@ -31,9 +73,12 @@ def remove_hidden_methods(ReportClass, outfile=None): ...@@ -31,9 +73,12 @@ def remove_hidden_methods(ReportClass, outfile=None):
# Allows us to use the !b;silent tags in the code. This is a bit hacky, but allows timeouts, etc. to make certain tests more robust # Allows us to use the !b;silent tags in the code. This is a bit hacky, but allows timeouts, etc. to make certain tests more robust
from snipper.fix_bf import fix_b from snipper.fix_bf import fix_b
lines, _, _ = fix_b(source.splitlines()) from snipper.snipper_main import fix_tags
source = "\n".join(lines) from snipper.snip_dir import snip_dir
lines, _, _ = fix_b(fix_tags(source.rstrip().splitlines()))
source = "\n".join(lines)
# print(source)
with open(os.path.dirname(file) + "/" + outfile, 'w') as f: with open(os.path.dirname(file) + "/" + outfile, 'w') as f:
f.write(source) f.write(source)
......
src/unitgrade_private/pipelines/dtulearn.py
+ 67
− 26
View file @ 678051eb
...@@ -324,6 +324,9 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -324,6 +324,9 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
# configuration # configuration
""" Unpack token or prep python files. """ """ Unpack token or prep python files. """
for fid in glob.glob(stage2_dir + "/*"): for fid in glob.glob(stage2_dir + "/*"):
if "s234792" in fid:
print(fid)
# print(fid) # print(fid)
id, type = os.path.basename(fid).split("-") id, type = os.path.basename(fid).split("-")
s3dir = f"{stage3_dir}/{os.path.basename(fid)}" s3dir = f"{stage3_dir}/{os.path.basename(fid)}"
...@@ -361,9 +364,10 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -361,9 +364,10 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
if len(fn) == 0: if len(fn) == 0:
print("I was unable to locate", g) print("I was unable to locate", g)
print("Bad?") print("Bad?")
os.path.relpath(fn[0], student_handout_folder) # os.path.relpath(fn[0], student_handout_folder)
dst = os.path.relpath(g, fid) # Take it relative to the currnet directory.
dst = s3dir + "/"+os.path.dirname(grade_script_relative) + "/"+ os.path.basename(g) else:
# dst = s3dir + "/"+os.path.dirname(grade_script_relative) + "/"+ os.path.basename(g)
dst = s3dir + "/" + os.path.relpath(fn[0], student_handout_folder) dst = s3dir + "/" + os.path.relpath(fn[0], student_handout_folder)
if os.path.isfile(dst): if os.path.isfile(dst):
...@@ -417,6 +421,8 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -417,6 +421,8 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
conf = configuration.get('stage3', {}) conf = configuration.get('stage3', {})
for fid in glob.glob(stage3_dir + "/*"): for fid in glob.glob(stage3_dir + "/*"):
# if "s234792" in fid:
# print(fid)
s4dir = f"{stage4_dir}/{os.path.basename(fid)}" s4dir = f"{stage4_dir}/{os.path.basename(fid)}"
grade_script_relative = get_grade_script_location(instructor_grade_script) grade_script_relative = get_grade_script_location(instructor_grade_script)
grade_script_destination = os.path.dirname(fid + "/" + grade_script_relative) + "/" + os.path.basename(instructor_grade_script) grade_script_destination = os.path.dirname(fid + "/" + grade_script_relative) + "/" + os.path.basename(instructor_grade_script)
...@@ -441,6 +447,7 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -441,6 +447,7 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
elif len(products) == 1: elif len(products) == 1:
rc = load_token(products[0])[0] rc = load_token(products[0])[0]
if len(products) == 0: # No .token file has actually been generated. So obviously we have to re-generate it. if len(products) == 0: # No .token file has actually been generated. So obviously we have to re-generate it.
RERUN_TOKEN = True RERUN_TOKEN = True
elif len(student_token_file) > 0 and id not in configuration.get('stage2', {}).get('skip_students', []): elif len(student_token_file) > 0 and id not in configuration.get('stage2', {}).get('skip_students', []):
...@@ -459,6 +466,11 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -459,6 +466,11 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
print("no sources") print("no sources")
ptoken = load_token(products[0])[0] ptoken = load_token(products[0])[0]
rename_map = conf.get('rename_items', {}) # Why give them a single test when I can sit on my ass and give them incompatible tests, WCGW?
for q in stoken['details']:
stoken['details'][q]['items'] = {rename_map.get(k, k): v for k, v in stoken['details'][q]['items'].items()}
if ".".join(stoken['sources'][0]['report_module_specification']).lower().replace(" ", "") == ".".join(ptoken['sources'][0]['report_module_specification']).replace("_tests_complete", "").lower(): # if ".".join(stoken['sources'][0]['report_module_specification']).lower().replace(" ", "") == ".".join(ptoken['sources'][0]['report_module_specification']).replace("_tests_complete", "").lower(): #
s_better_than_i, _ = determine_token_difference(stoken, rc) s_better_than_i, _ = determine_token_difference(stoken, rc)
acceptable_broken = False acceptable_broken = False
...@@ -469,16 +481,26 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -469,16 +481,26 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
else: else:
print(".".join(stoken['sources'][0]['report_module_specification']).lower()) print(".".join(stoken['sources'][0]['report_module_specification']).lower())
print(".".join(rc['sources'][0]['report_module_specification']).replace("_tests_complete", "").lower()) print(".".join(rc['sources'][0]['report_module_specification']).replace("_tests_complete", "").lower())
messages['stage3'].append(f"{id}> Bad student token. Add id incompatible token names ['stage3']['accept_incompatible_token_names']. This likely occured because the student renamed the grade script. " + str(student_token_file))
raise Exception("Bad student token. Add id incompatible token names " + str(student_token_file) ) RERUN_TOKEN = True # Not hat it really helps.
pass acceptable_broken = True
if len(s_better_than_i) > 0: if len(s_better_than_i) > 0:
for q in s_better_than_i: for q in s_better_than_i:
for item in s_better_than_i[q]['items']: for item in s_better_than_i[q]['items']:
if item == ('Week06SentimentAnalysis', 'test_sentiment_analysis'):
print("Yes we were better but it had to do with idiotic sentiment analysis thanks a fuck...")
continue
messages['stage3'].append(f"{id}> ERROR: Student strictly better than instructor. q{q}. item: {item}") messages['stage3'].append(f"{id}> ERROR: Student strictly better than instructor. q{q}. item: {item}")
RERUN_TOKEN = True RERUN_TOKEN = True
# for q in stoken['details']:
# print(stoken['details'][q]['name'], ptoken['details'][q]['name'] )
#
# print(stoken['details'][5] )
# print( ptoken['details'][5] )
rch = token_gather_hidden(rc) rch = token_gather_hidden(rc)
for q in stoken['details']: for q in stoken['details']:
...@@ -494,9 +516,12 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -494,9 +516,12 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
if item not in rch['details'][q]['items']: if item not in rch['details'][q]['items']:
print( rch['details'][q]['items'].keys() ) print( rch['details'][q]['items'].keys() )
# print(rch['details'][q]['items'].keys())
iitems = rch['details'][q]['items'][item] iitems = rch['details'][q]['items'][item]
if sitem['status'] == 'pass' and not all([i['status'] == 'pass' for i in iitems]) and id not in conf.get('verified_problematic_items', {}).get(item, []): if sitem['status'] == 'pass' and not all([i['status'] == 'pass' for i in iitems]) and id not in conf.get('verified_problematic_items', {}).get(item, []) and not conf.get("accept_public_ok_hidden_failed", False):
# print('disagreement found.') # print('disagreement found.')
iitems = rch['details'][q]['items'][item] iitems = rch['details'][q]['items'][item]
fails = [i['nice_title'] for i in iitems if i['status'] != 'pass'] fails = [i['nice_title'] for i in iitems if i['status'] != 'pass']
...@@ -530,9 +555,9 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -530,9 +555,9 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
# a '234 # a '234
import filecmp import filecmp
if not filecmp.cmp(instructor_grade_script, grade_script_destination, shallow=False): if not filecmp.cmp(instructor_grade_script, grade_script_destination, shallow=False) and not conf.get("forgive_changed_grade_script", False):
print("grade script has been updated subsequently. Rerunning the tests...") print("grade script has been updated subsequently. Rerunning the tests...")
messages['stage3'].append(f"Rerunning token bc. of new grade script {grade_script_destination}") messages['stage3'].append(f"{id}> Rerunning token bc. of new grade script {grade_script_destination}")
RERUN_TOKEN = True RERUN_TOKEN = True
else: else:
continue continue
...@@ -571,6 +596,8 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -571,6 +596,8 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
a = 234 a = 234
from unitgrade.utils import Capturing2, Capturing, Logger from unitgrade.utils import Capturing2, Capturing, Logger
# from spb.defaults import * # spb / defaults.py
if unmute: # This is a pretty big mess. if unmute: # This is a pretty big mess.
from unitgrade_private.run import run from unitgrade_private.run import run
out = run(fcom, print_output=True, log_output=False, check=False) out = run(fcom, print_output=True, log_output=False, check=False)
...@@ -650,12 +677,14 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -650,12 +677,14 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
if len(p_best) > 0: if len(p_best) > 0:
for q in p_best.values(): for q in p_best.values():
for item in q['items']: for item in q['items']:
messages['report'].append(f"{id}> Evaluation of student code was better than the token file evaluation. " + str(item) ) # + " student stderr: \n" + str(q['items'][item]['a']['stderr']) + "\n instructor stderr: \n" + str(q['items'][item]['b']['stderr'])) if not configuration.get("stage_report", {}).get("accept_student_code_better_than_token", False):
messages['report'].append(f"{id}> Evaluation of student code (i.e. .py handins) was better than the token file evaluation. " + str(item) ) # + " student stderr: \n" + str(q['items'][item]['a']['stderr']) + "\n instructor stderr: \n" + str(q['items'][item]['b']['stderr']))
elif 'token' in found_students[id] and 'python' not in found_students[id]: elif 'token' in found_students[id] and 'python' not in found_students[id]:
pass pass
elif 'token' not in found_students[id] and 'python' in found_students[id]: elif 'token' not in found_students[id] and 'python' in found_students[id]:
if id not in configuration.get('stage_report', {}).get("python_handin_checked", []): if id not in configuration.get('stage_report', {}).get("python_handin_checked", []):
if not configuration.get("stage_report", {}).get("accept_only_py_no_token", False):
print("=" * 50) print("=" * 50)
s = f"{id}> only handed in the .py files and not the .token files. " +str(found_students[id]['python'] + " to skip this mesage, alter the stage_report['python_handin_checked'] field. ") s = f"{id}> only handed in the .py files and not the .token files. " +str(found_students[id]['python'] + " to skip this mesage, alter the stage_report['python_handin_checked'] field. ")
messages['report'].append(s) messages['report'].append(s)
...@@ -682,17 +711,29 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad ...@@ -682,17 +711,29 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
return rs return rs
rs = _stage_report() rs = _stage_report()
all_msgs = []
if len(messages) > 0: if len(messages) > 0:
print("=" * 50) print("=" * 50)
print("Oy veh, there are messages") print("Oy veh, there are messages")
for stage in messages: for stage in messages:
print("Messages from", stage) print("Messages from", stage)
for s in messages[stage]: for s in messages[stage]:
print(">> ", s) print(m_ := ">> "+ s)
all_msgs.append(m_)
print("-" * 50) print("-" * 50)
if accept_problems:
if not accept_problems:
assert False, "No messages allowed!" assert False, "No messages allowed!"
with open(base_directory +"/log.txt", "w") as f:
f.write("\n".join(all_msgs))
# rs['messages'] = messages
# with open()
if plagiarism_check and True: if plagiarism_check and True:
from unitgrade_private.plagiarism.mossit import moss_it2023 from unitgrade_private.plagiarism.mossit import moss_it2023
moss_it2023(submissions_base_dir=stage4_dir, submissions_pattern="*-token", instructor_grade_script=instructor_grade_script) moss_it2023(submissions_base_dir=stage4_dir, submissions_pattern="*-token", instructor_grade_script=instructor_grade_script)
...@@ -759,7 +800,7 @@ def docker_verify_projects(learn_zip_file_path, Dockerfile=None, instructor_grad ...@@ -759,7 +800,7 @@ def docker_verify_projects(learn_zip_file_path, Dockerfile=None, instructor_grad
Dockerfile = images['unitgrade-docker'] Dockerfile = images['unitgrade-docker']
# info = class_information() # info = class_information()
# Dockerfile = paths['02450instructors'] + "/docker/Dockerfile" # Dockerfile = paths['02450instructors'] + "/docker/Dockerfile"
tag = compile_docker_image(Dockerfile) tag = compile_docker_image(Dockerfile, verbose=True)
print("Docker verify project image tag:", tag) print("Docker verify project image tag:", tag)
if not os.path.isdir(dzip + "/verified_tokens"): if not os.path.isdir(dzip + "/verified_tokens"):
......
src/unitgrade_private/plagiarism/mossit.py
+ 2
− 1
View file @ 678051eb
...@@ -93,7 +93,8 @@ def moss_it2023(submissions_base_dir=None, submissions_pattern="*-token", whitel ...@@ -93,7 +93,8 @@ def moss_it2023(submissions_base_dir=None, submissions_pattern="*-token", whitel
for q in cov_files: for q in cov_files:
for i in cov_files[q].values(): for i in cov_files[q].values():
for g in i: for g in i:
shutil.copy(f"{tmpdirname}/{g}", f"{sdir}/{os.path.basename(g)}") if os.path.isfile(student_file := f"{tmpdirname}/{g}"):
shutil.copy(student_file, f"{sdir}/{os.path.basename(g)}")
# Now submit it to moss. # Now submit it to moss.
import mosspy import mosspy
......
src/unitgrade_private/version.py
+ 0
− 4
View file @ 678051eb
<<<<<<< HEAD
__version__ = "0.1.62" __version__ = "0.1.62"
=======
__version__ = "0.1.61"
>>>>>>> 05f66ebff680f2f18cf59f9adf46bc6c8ecea3c0
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment