Worked on timeout function

src/unitgrade_devel.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: unitgrade-devel
-Version: 0.1.55
+Version: 0.1.60
 Summary: A set of tools to develop unitgrade tests and reports and later evaluate them
 Home-page: https://lab.compute.dtu.dk/tuhe/unitgrade_private
 Author: Tue Herlau
@@ -13,6 +13,16 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: unitgrade
+Requires-Dist: numpy
+Requires-Dist: codesnipper
+Requires-Dist: tabulate
+Requires-Dist: tqdm
+Requires-Dist: pyfiglet
+Requires-Dist: jinja2
+Requires-Dist: colorama
+Requires-Dist: coverage
+Requires-Dist: mosspy
 
 # Unitgrade-devel
 **Note: This is the development version of unitgrade. If you are a student, please see http://gitlab.compute.dtu.dk/tuhe/unitgrade.**

src/unitgrade_private/deployment.py

@@ -2,7 +2,7 @@ import inspect
 from unitgrade.utils import hide, methodsWithDecorator
 import os
 import importlib
-
+import snipper
 
 def remove_hidden_methods(ReportClass, outfile=None):
     # Given a ReportClass, clean out all @hidden tests from the imports of that class.
@@ -14,7 +14,7 @@ def remove_hidden_methods(ReportClass, outfile=None):
 
     for Q,_ in ReportClass.questions:
         ls = list(methodsWithDecorator(Q, hide))
-        print("hide decorateed is", ls)
+        # print("hide decorateed is", ls)
         for f in ls:
             s, start = inspect.getsourcelines(f)
             end = len(s) + start
@@ -28,6 +28,12 @@ def remove_hidden_methods(ReportClass, outfile=None):
 
     if os.path.exists(outfile) and os.path.samefile(file, outfile):
         raise Exception("Similar file paths identified!")
+
+    # Allows us to use the !b;silent tags in the code. This is a bit hacky, but allows timeouts, etc. to make certain tests more robust
+    from snipper.fix_bf import fix_b
+    lines, _, _ = fix_b(source.splitlines())
+    source = "\n".join(lines)
+
     with open(os.path.dirname(file) + "/" + outfile, 'w') as f:
         f.write(source)
 

src/unitgrade_private/hidden_create_files.py

@@ -52,7 +52,7 @@ def setup_grade_file_report(ReportClass, execute=False, obfuscate=False, minify=
                 raise Exception("Must spacify name to give new report file")
 
         fout, Report = remove_hidden_methods(ReportClass, outfile=name_without_hidden)  # Create report3.py without @hide-methods
-        return setup_grade_file_report(Report, remove_hidden=False) # Create report3_grade.py for the students
+        return setup_grade_file_report(Report, remove_hidden=False, bzip=bzip) # Create report3_grade.py for the students
 
     print("Setting up answers...")
     url = ReportClass.url
@@ -164,14 +164,11 @@ def setup_grade_file_report(ReportClass, execute=False, obfuscate=False, minify=
                             unitgrade.evaluate.__file__, hidden_gather_upload.__file__,
                             version.__file__], excl) + "\n" + report1_source
 
-    # print(sys.getsizeof(picklestring))
-    # print(len(picklestring))
     s = jinja2.Environment().from_string(data).render({'Report1': ReportClass.__name__,
                                                        'source': repr(report1_source),
                                                        'payload': picklestring.hex(), #repr(picklestring),
                                                        'token_out': repr(fn[:-3] + "_handin"),
                                                        'head': pyhead})
-
     # if fn[:-3].endswith("_test.py"):
     #     output = fn[:-8] + "_grade.py"
     # elif fn.endswith("_tests.py"):

src/unitgrade_private/hidden_gather_upload.py

 import sys
-from unitgrade.evaluate import evaluate_report, python_code_str_id, _print_header
+from unitgrade.evaluate import evaluate_report, _print_header, python_code_binary_id
 import textwrap
 import bz2
 import pickle
@@ -36,13 +36,23 @@ def gather_imports(imp):
         for root, dirs, files in os.walk(top_package):
             for file in files:
                 if file.endswith(".py"):
+                    # print("file is", file)
                     fpath = os.path.join(root, file)
                     v = os.path.relpath(fpath, os.path.dirname(top_package) if not module_import else top_package)
                     zip.write(fpath, v)
                     if not fpath.endswith("_grade.py"): # Exclude grade files.
-                        with open(fpath, 'r') as f:
+
+                        # print("I am reading the file", fpath)
+                        try:
+                            with open(fpath, 'rb') as f:
                                 s = f.read()
-                        found_hashes[v] = python_code_str_id(s)
+                        except UnicodeDecodeError as e:
+                            print("Unitgrade> Oh no, I tried to read the file", f)
+                            print("But I got an error since the file contains unusual content (non-unicode characters). Please go over the file and check that it looks okay. Things that can cause this problem are unusual characters (Japanese, Arabic, etc.) or Emojis.")
+                            print("This problem sometimes also occurs when virus scanners or viruses write unusual files on your computer. In that case, try to disable or remove the program.")
+                            raise e
+                        # print("Read")
+                        found_hashes[v] = python_code_binary_id(s)
                         resources['pycode'][v] = s
 
     resources['zipfile'] = zip_buffer.getvalue()
@@ -105,11 +115,13 @@ def gather_upload_to_campusnet(report, output_dir=None, token_include_plaintext_
 
     if report.remote_url is not None and args.evaluation:
         import datetime
+        # print("Performing a remote evaluation...")
         _print_header(now=datetime.datetime.now(), big_header=True)
 
         # get the remote evaluation.
         # os.path.dirname(report._artifact_file())
         mf = report._manifest_file()
+        # print("looking for manifest file", mf)
         if os.path.isfile(mf):
             with open(mf, 'r') as f:
                 s = f.read()
@@ -118,7 +130,7 @@ def gather_upload_to_campusnet(report, output_dir=None, token_include_plaintext_
             results = checkout_remote_results(report.remote_url, s.strip())
             if results['html'] is None:
                 print("""Oy! I failed to download the verified results. 
-There can be three reasons why this this command failed:
+There are four likely reasons why this this command failed:
                 
     * You have not yet uploaded a .token file (i.e., you have not yet handed in)
     * You did not upload a .token file, but rather a file in some other format. 
@@ -131,8 +143,7 @@ for more information about what happened to your evaluation. Please don't contac
                 sys.exit()
             print("Your verified results are:")
             import tabulate
-            # import sys
-            print( tabulate.tabulate(results['df'], showindex=False) )
+            print( tabulate.tabulate(results['dict'], headers="keys" ) )
             print("These scores are based on our internal tests.")
             print("To see all your results, visit:")
             print(">", results['url'])

src/unitgrade_private/pipelines/dtulearn.py

@@ -114,7 +114,19 @@ def write_summary_report_xlsx_file(write_html=True, open_browser=True):
     return dd
 
 
-
+def f2date(f):
+    date = os.path.basename(f).split("-")[-1].strip()
+
+    import datetime
+    # date = " 31 August, 2023 3:44 PM"
+    # date = "31 August, 2023"
+    # date = "3:44 PM"
+    # datetime_obj = datetime.datetime.strptime(date.strip(), "%I:%M %p")
+    if ":" in date:
+        datetime_obj = datetime.datetime.strptime(date.strip(), "%d %B, %Y  %I:%M %p")
+    else:
+        datetime_obj = datetime.datetime.strptime(date.strip(), "%d %B, %Y  %I%M %p")
+    return datetime_obj
 
 
 def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grade_script=None,
@@ -126,6 +138,7 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
                                 configuration=None,
                                 unmute_docker = True,
                                 plagiarism_check=False,
+                                accept_problems=False, # No!
                                 ):
     """
     This is the main verification scripts. It is the main entry point for project verifications as downloaded from DTU Learn.
@@ -189,23 +202,54 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
         # stage0_excluded_files = ["*.pdf"]
         stage0_excluded_files = configuration['stage0']['excluded_files']
         found = []
+
+        # ids_and_directories = []
+
+        relevant_directories = {}
+
         # Set up stage 1:
         for z in glob.glob(f"{stage0_dir}/*.*"):
             if not z.endswith(".zip"):
                 raise Exception("The downloaded files must be .zip files from DTU Learn")
 
             unpack_zip_file_recursively(z[:-4] + ".zip", z[:-4] + "/raw", remove_zipfiles=True)
+
             for f in glob.glob(z[:-4] + "/raw/*"):
                 if os.path.basename(f) == "index.html":
                     continue
                 elif os.path.isdir(f):
                     id = fname2id(os.path.basename(f), info)
-                    if id in found:
-                        assert False
+
+                    # now get the directory.
+
+                    if id not in relevant_directories:
+                        relevant_directories[id] = f
+                    else:
+                        dold = f2date(relevant_directories[id])
+                        dnew = f2date(f)
+                        if dnew == dold:
+                            pass
+                            raise Exception("User has two handins with the same date. Not possible. \n" + f + "\n " + relevant_directories[id])
+
+                        if dnew > dold:
+                            relevant_directories[id] = f
+                else:
+                    assert student_handout_folder is not None
+                    raise Exception(
+                        "The .zip files can only contain directories with names such as: '67914-43587 - s214598, Andreas Rahbek-Palm - 09 February, 2023 441 PM', got " + student_handout_folder)
+
+            for id, f in relevant_directories.items():
                 found.append(id)
                 dest = stage1_dir +"/" + id
+
                 if not os.path.isdir(dest):
                     shutil.copytree(f, dest )
+                else:
+                    # merge the files...
+                    for new_file in glob.glob(f +"/**/*", recursive=True):
+                        # print(os.path.relpath(new_file, f))
+                        shutil.copy(new_file, dest + "/"+os.path.relpath(new_file, f))
+
                 # Now remove blacklisted files to simplify it.
                 for g in glob.glob(dest +"/**/*", recursive=True):
                     import fnmatch
@@ -214,15 +258,13 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
                     if os.path.basename(g) in configuration['stage0']['rename']:
                         dst_name = configuration['stage0']['rename'][os.path.basename(g)]
                         dst_name = os.path.dirname(g) + "/" + dst_name
-                                assert not os.path.isfile(dst_name)
+                        if not os.path.isfile(dst_name):
                             shutil.move(g, dst_name)
 
                     if len([ex for ex in stage0_excluded_files if fnmatch.fnmatch(g, ex)]) > 0:
                         os.remove(g)
 
-                else:
-                    assert student_handout_folder is not None
-                    raise Exception("The .zip files can only contain directories with names such as: '67914-43587 - s214598, Andreas Rahbek-Palm - 09 February, 2023 441 PM'")
+
     _stage0()
 
     def _stage1():
@@ -279,6 +321,7 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
         return os.path.dirname(gs) + "/"+os.path.basename(instructor_grade_script)
 
     def _stage2(fix_user=True, xvfb=True):
+        # configuration
         """ Unpack token or prep python files. """
         for fid in glob.glob(stage2_dir + "/*"):
             # print(fid)
@@ -289,26 +332,76 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
             grade_script_relative = get_grade_script_location(instructor_grade_script)
             if type == "token":
                 tokens = glob.glob(fid + "/**/*.token", recursive=True)
-                assert len(tokens) == 1
+                assert len(tokens) == 1, f"{id} has too many tokens: The tokens found are {tokens}"
+                try:
                     unpack_sources_from_token(tokens[0], s3dir)
+                except Exception as e:
+                    print("-" * 100)
+                    print("Not a valid token file", tokens[0], "investigate and potentially blacklist", id)
+
+                    if id in configuration.get('stage2', {}).get('skip_students', []):
+                        pass
+                    else:
+                        raise e
+
                 # This will copy in resource files etc. that may not be contained in the .token file.
                 for g in glob.glob(student_handout_folder + "/**/*.*", recursive=True):
                     rg = os.path.relpath(g, student_handout_folder)
                     if not os.path.isfile(s3dir + "/"+rg) and not rg.endswith(".py"):
                         if not os.path.isdir(os.path.dirname(s3dir + "/"+rg)): os.makedirs(os.path.dirname(s3dir + "/"+rg))
+                        if os.path.isfile(g):
                             shutil.copy(g, s3dir + "/"+rg)
+                        else:
+                            shutil.copytree(g, s3dir + "/" + g)
             else:
                 shutil.copytree(student_handout_folder, s3dir)
                 for g in glob.glob(fid+"/**/*.*", recursive=True):
+                    # Find location in student handout folder.
+                    fn = glob.glob(student_handout_folder + "/**/" + os.path.basename(g), recursive=True)
+                    if len(fn) == 0:
+                        print("I was unable to locate", g)
+                        print("Bad?")
+                    os.path.relpath(fn[0], student_handout_folder)
+
                     dst = s3dir + "/"+os.path.dirname(grade_script_relative) + "/"+ os.path.basename(g)
+                    dst = s3dir + "/" + os.path.relpath(fn[0], student_handout_folder)
+
                     if os.path.isfile(dst):
                         shutil.copy(g, dst)
                     else:
                         shutil.move(g, dst)
                         print("> Stage two: Created", dst)
+
             ### Copy in the instructor grade script. We are now ready for deployment.
             shutil.copy(instructor_grade_script, os.path.dirname(s3dir + "/" + grade_script_relative) + "/" + os.path.basename(instructor_grade_script))
 
+            ## Check files are readable...
+        for fid in glob.glob(stage2_dir + "/*"):
+            # print(fid)
+            id, type = os.path.basename(fid).split("-")
+            s3dir = f"{stage3_dir}/{os.path.basename(fid)}"
+            # if os.path.isdir(s3dir):
+            #     continue
+            for f in glob.glob(s3dir +"/**/*.py", recursive=True):
+                if os.path.isdir(f):
+                    continue
+                try:
+                    with open(f, 'r') as ff:
+                        ff.read()
+                except UnicodeDecodeError as e:
+                    print("""Student file not readable. add to stage2 kill list as in { configurations['projects']['project1']['stage3']['exclude_if_bad_encoding'] += ['*/~BROMIUM/*.py'] }""", f)
+                    for p in configuration['stage2'].get('exclude_if_bad_encoding', []):
+                        if fnmatch.fnmatch(f, p):
+                            print("Skipping file with shit encoding", f)
+                            os.remove(f)
+                            break
+                    if os.path.isfile(f):
+                        raise e
+
+
+
+
+
     _stage2()
 
     def _stage3(Dockerfile, fix_user=True, xvfb=True, unmute=False, verbose=False):
@@ -350,9 +443,35 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
 
                 if len(products) == 0: # No .token file has actually been generated. So obviously we have to re-generate it.
                     RERUN_TOKEN = True
-                elif len(student_token_file) > 0:
+                elif len(student_token_file) > 0 and id not in configuration.get('stage2', {}).get('skip_students', []):
+                    # We check if the student id is marked as skipped. This is reserved for cases where student uploads a token file, but it is fundamentally broken (as determined by manual inspection).
+                    if len(student_token_file) == 0:
+                        print(f"Strange error in stage 3: this student did not have a token file {id}")
+                    try:
                         stoken, _ = load_token(student_token_file[0])
+                    except Exception as e:
+                        print(f"did not load token file for student {id}: {student_token_file}")
+                        raise e
+                    # if os.path.basename(student_token_file[0]).split("_")[0] != os.path.basename(products[0]).split("_")[0]:
+                    #     print("bad")
+                    # We check if the student ran the actual token file they were supposed to run. If not, it may still have the right sources...
+                    if "sources" not in rc:
+                        print("no sources")
+
+                    ptoken = load_token(products[0])[0]
+                    if ".".join(stoken['sources'][0]['report_module_specification']).lower().replace(" ", "") == ".".join(ptoken['sources'][0]['report_module_specification']).replace("_tests_complete", "").lower(): #
                         s_better_than_i, _ = determine_token_difference(stoken, rc)
+                        acceptable_broken = False
+                    elif id in configuration.get('stage3', {}).get('accept_incompatible_token_names', []):
+                        print("Incompatible token names accepted...")
+                        s_better_than_i = []
+                        acceptable_broken = True
+                    else:
+                        print(".".join(stoken['sources'][0]['report_module_specification']).lower())
+                        print(".".join(rc['sources'][0]['report_module_specification']).replace("_tests_complete", "").lower())
+
+                        raise Exception("Bad student token. Add id incompatible token names " + str(student_token_file) )
+                        pass
 
                     if len(s_better_than_i) > 0:
                         for q in s_better_than_i:
@@ -363,9 +482,20 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
                     rch = token_gather_hidden(rc)
 
                     for q in stoken['details']:
+                        if acceptable_broken:
+                            continue
                         for item in stoken['details'][q]['items']:
+                            if item ==  ('Week06SentimentAnalysis', 'test_sentiment_analysis'):
+                                continue
                             sitem = stoken['details'][q]['items'][item]
+                            if item == ("Week06SpellCheck", "test_SpellCheck"):
+                                item = ("Week06SpellCheck", "test_spell_check")
+
+                            if item not in rch['details'][q]['items']:
+
+                                print( rch['details'][q]['items'].keys() )
                             iitems = rch['details'][q]['items'][item]
+
                             if sitem['status'] == 'pass' and not all([i['status'] == 'pass' for i in iitems]) and id not in conf.get('verified_problematic_items', {}).get(item, []):
                                 # print('disagreement found.')
                                 iitems = rch['details'][q]['items'][item]
@@ -392,6 +522,8 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
 
                                 RERUN_TOKEN = True
                                 nn += 1
+            else:
+                print("No token rerunning", s4dir)
 
             if not RERUN_TOKEN:
                 # Check if the instructor script is identical to the current one.
@@ -442,7 +574,7 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
             if unmute: # This is a pretty big mess.
                 from unitgrade_private.run import run
                 out = run(fcom, print_output=True, log_output=False, check=False)
-                stdout = out.stderr.getvalue()
+                stdout = out.stdout.getvalue()
                 stderr = out.stderr.getvalue()
 
                 if not os.path.isdir(s4dir):
@@ -466,7 +598,14 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
 
             for f in glob.glob(s4dir + "/*.token"):
                 os.remove(f)
+            try:
                 shutil.move(tokens[0], s4dir + "/" + os.path.basename(tokens[0]))
+            except Exception as e:
+                print("-"*50)
+                print("Got a problem wit hthis student")
+                print("dir", s4dir)
+                print("tokens", tokens)
+                raise e
 
     _stage3(Dockerfile, unmute=unmute_docker)
 
@@ -481,6 +620,9 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
             for tf in glob.glob(fid +"/**/*.token", recursive=True):
                 rs[id]['token_downloaded'] = tf
 
+                if id in configuration.get('stage2', {}).get('skip_students', []):
+                    blake_hash = "BAD TOKEN: STUDENT IS MARKED AS SKIPPED."
+                else:
                     tdata, _ = load_token(tf)
                     blake_hash = tdata['metadata']['file_reported_hash']
 
@@ -515,7 +657,7 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
             elif 'token' not in found_students[id] and 'python' in found_students[id]:
                 if id not in configuration.get('stage_report', {}).get("python_handin_checked", []):
                     print("="*50)
-                    s = f"{id}> only handed in the .py files and not the .token files. " +str(found_students[id]['python'])
+                    s = f"{id}> only handed in the .py files and not the .token files. " +str(found_students[id]['python'] + " to skip this mesage, alter the stage_report['python_handin_checked'] field. ")
                     messages['report'].append(s)
                     stoken =token_gather_hidden(load_token(found_students[id]['python'])[0])
                     print(s)
@@ -548,6 +690,8 @@ def docker_stagewise_evaluation(base_directory, Dockerfile=None, instructor_grad
             for s in messages[stage]:
                 print(">> ", s)
         print("-" * 50)
+        if accept_problems:
+            assert False, "No messages allowed!"
 
     if plagiarism_check and True:
         from unitgrade_private.plagiarism.mossit import moss_it2023
@@ -841,7 +985,10 @@ def fname2id(fname, info=None):
         id = id_cand
         if info is not None:
             if not id[1:].isdigit():
-                real_id = [sid for sid in info['students'] if info['students'][sid]['initials']  == id].pop()
+                possible = [sid for sid in info['students'] if info['students'][sid]['initials']  == id]
+                if len(possible) == 0:
+                    raise Exception(f"Tried to find student id {id} but was not found. You need to download the CSV file from inside and put it in the main config excel sheet.")
+                real_id = possible.pop()
                 if real_id != id:
                     print("Changing id from", id, "to", real_id)
                     id = real_id

src/unitgrade_private/token_loader.py

@@ -36,13 +36,16 @@ def get_coverage_files(token_file, instructor_grade_script_dir):
 
     for q in stoken['details']:
         cov_files[q] = {}
-
-        with open(f"{instructor_grade_script_dir}/unitgrade_data/{stoken['details'][q]['name']}.pkl", 'rb') as f:
+        try:
+            with open(pkl_file := f"{instructor_grade_script_dir}/unitgrade_data/{stoken['details'][q]['name']}.pkl", 'rb') as f:
                 pk = pickle.load(f)
             for item in stoken['details'][q]['items']:
                 key = (item, 'coverage')
                 if key in pk:
                     cov_files[q][key] = list( pk[(item, 'coverage')].keys() )
+        except Exception as e:
+            print("Unitgrade> Failed to load a coverage file. This may indicate that files have been removed from the unitgrade_data directory. Skipping and possibly returning a too small list.", pkl_file)
+
     return cov_files
 
 
@@ -117,8 +120,12 @@ def determine_token_difference(student_token_rs, instructor_token_rs):
                         a_better_than_b[q] = {'items': {}}
                     a_better_than_b[q]['items'][item] = {'a': a[q]['items'][item], 'b': b[q]['items'].get(item,None)}
         return a_better_than_b
+    try:
         a_better_than_b = where_a_better_b(rsa['details'], rsb['details'])
         b_better_than_a = where_a_better_b(rsb['details'], rsa['details'])
+    except Exception as e:
+        print("Oh no", student_token_rs, instructor_token_rs)
+        raise e
     return a_better_than_b, b_better_than_a
 
 
@@ -176,7 +183,7 @@ def combine_token_results(token_a_rs, token_b_rs):
         n_obt += nc
 
 
-    rs = dict(total=(n_obt, n_tot), details=rsd, sources=None)
+    rs = dict(total=(n_obt, n_tot), details=rsd, sources=None, metadata=None)
     return rs
 
 

src/unitgrade_private/utils.py

+"""
+Tue: This may become part of unitgrade proper at some point. It will allow automatic timeout of tests, but right now it is used
+to timeout hidden tests which ends up in an infinite loop.
+
+"""
+import sys
+import threading
+from time import sleep
+# try:
+#     import thread
+# except ImportError:
+import _thread as thread
+
+def quit_function(fn_name):
+    # print to stderr, unbuffered in Python 2.
+    # print('{0} took too long'.format(fn_name), file=sys.stderr)
+    # sys.stderr.flush() # Python 3 stderr is likely buffered.
+    thread.interrupt_main() # raises KeyboardInterrupt
+
+def exit_after(s):
+    '''
+    use as decorator to exit process if
+    function takes longer than s seconds
+    '''
+    def outer(fn):
+        def inner(*args, **kwargs):
+            timer = threading.Timer(s, quit_function, args=[fn.__name__])
+            timer.start()
+            try:
+                result = fn(*args, **kwargs)
+            finally:
+                timer.cancel()
+            return result
+        return inner
+    return outer
+

src/unitgrade_private/version.py

-__version__ = "0.1.56"
-
+__version__ = "0.1.60"