Caching configuration to remove dependency on information

setup.py

@@ -15,7 +15,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 # beamer-slider
 setuptools.setup(
     name="coursebox",
-    version="0.1.17.11",
+    version="0.1.18.0",
     author="Tue Herlau",
     author_email="tuhe@dtu.dk",
     description="A course management system currently used at DTU",

src/coursebox.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: coursebox
-Version: 0.1.17.11
+Version: 0.1.17.12
 Summary: A course management system currently used at DTU
 Home-page: https://lab.compute.dtu.dk/tuhe/coursebox
 Author: Tue Herlau

src/coursebox/core/info.py

+import glob
+import os
+import copy
+import re
+import pickle
 from datetime import timedelta
 from datetime import datetime
 import coursebox
-# import thtools
 from coursebox.thtools_base import list_dict2dict_list
-# import jinjafy
 import openpyxl
 from coursebox.core.projects_info import populate_student_report_results
 from coursebox.core.info_paths import get_paths, semester_id, semester, year, today
 from coursebox.core.info_paths import core_conf
-# import six
-# import pybtex.database.input.bibtex
-# import pybtex.plugin
-# import io
-# from line_profiler_pycharm import profile
 import time
 
 # @profile
-def xlsx_to_dicts(xlsx_file,sheet=None, as_dict_list=False):
+def xlsx_to_dicts(xlsx_file,sheet=None, as_dict_list=False, columns=None):
     # print("Loading...", xlsx_file, sheet, as_dict_list)
     t0 = time.time()
     wb = openpyxl.load_workbook(xlsx_file, data_only=True, read_only=True)
@@ -28,46 +26,19 @@ def xlsx_to_dicts(xlsx_file,sheet=None, as_dict_list=False):
             return None
         else:
             ws = ws.pop()
-    # print(time.time()-t0)
-    # dd = []
-    # key_cols = [j for j in range(ws.max_column) if ws.cell(row=1, column=j + 1).value is not None]
-    # print(time.time()-t0, ws.max_row)
-    # np.array([[i.value for i in j[1:5]] for j in ws.rows])
 
     import numpy as np
     A = np.array([[i.value for i in j] for j in ws.rows])
-    # print(time.time() - t0, ws.max_row, len(key_cols))
 
+    if columns is not None:
+        I = [a in columns for a in A[0,:] ]
 
-    # for j in range(A.shape[1]):
-
-
-
-
-    a = 234
-
-    # for i in range(1, ws.max_row):
-    #     rdict = {}
-    #     if not any( [ws.cell(row=i+1, column=j+1).value is not None for j in key_cols] ):
-    #         continue
-    #     for j in key_cols:
-    #         key = ws.cell(row=1, column=j+1).value
-    #         if key is not None:
-    #             key = key.strip() if isinstance(key,str) else key
-    #             value = ws.cell(row=i + 1, column=j + 1).value
-    #             value = value.strip() if isinstance(value,str) else value
-    #             if isinstance(value, str):
-    #                 if value == 'True':
-    #                     value = True
-    #                 if value == 'False':
-    #                     value = False
-    #             rdict[key] = value
-    #     dd.append(rdict)
-
-    # print(time.time()-t0)
-
-    A = A[:, A[0] != None]
-    A = A[(A != None).sum(axis=1) > 0, :]
+        a = A[:, I]
+        a = a[(a != None).all(axis=1),:]
+        A = a
+    else:
+        A = A[:, A[0] != None]
+        A = A[(A != None).sum(axis=1) > 0, :]
 
     dd2 = []
     for i in range(1, A.shape[0]):
@@ -77,16 +48,7 @@ def xlsx_to_dicts(xlsx_file,sheet=None, as_dict_list=False):
         d = dict(zip(A[0, :].tolist(), [a.strip() if isinstance(a,str) else a for a in A[i, :].tolist() ]))
         dd2.append(d)
 
-    # print(time.time() - t0)
     dd = dd2
-    # if dd != dd2:
-    #     for k in range(len(dd)):
-    #         if dd[k] != dd2[k]:
-    #             print(k)
-    #             print(dd)
-    #             print(dd2)
-    #     assert False
-    #     print("BAd!")
     if as_dict_list:
         dl = list_dict2dict_list(dd)
         for k in dl.keys():
@@ -95,7 +57,6 @@ def xlsx_to_dicts(xlsx_file,sheet=None, as_dict_list=False):
             dl[k] = x
         dd = dl
     wb.close()
-    # print("xlsx2dicts", time.time()-t0)
     return dd
 
 def get_enrolled_students():
@@ -306,10 +267,25 @@ def get_forum(paths):
     return d2
 
 # @profile
-def class_information(verbose=False):
+def class_information(verbose=False,
+                      update_with_core_conf=False, # Whether to include (module) level core-conf items. Nearly always yes, but core is likely to include classes that are not easily pickled. so when 0xxxxprivate is excluded, this should be False.
+                      ):
+    paths = get_paths()
+    if not os.path.isfile(paths['information.xlsx']):
+        print("Tried loading", paths['information.xlsx'])
+        cf = _info_cache_file()
+        print("Information configuration file not found. Loading from cache:", cf)
+        if not os.path.isfile(cf):
+            raise Exception("No configuration found. Please set up configuration file at: " + paths['information.xlsx'])
+        else:
+            with open(cf, 'rb') as f:
+                print("Loaded cached configuration from", cf)
+                info = pickle.load(f)
+                info = _update_with_core_conf(info)
+            return info
+
     course_number = core_conf['course_number']
     piazza = 'https://piazza.com/dtu.dk/%s%s/%s' % (semester().lower(), year(), course_number)
-    paths = get_paths()
     teachers = xlsx_to_dicts(paths['information.xlsx'], sheet='teachers')
     students, all_groups = populate_student_report_results( get_enrolled_students(), verbose=verbose)
     continuing_education_mode = core_conf['continuing_education_mode']
@@ -318,9 +294,8 @@ def class_information(verbose=False):
     d = {'year': year(),
          'piazza': piazza, # deprecated.
          'course_number': course_number,
+         'exam': list_dict2dict_list(xlsx_to_dicts(paths['information.xlsx'], sheet='exam')),
          'semester': semester(),
-         # 'reports_handout': [1,6], # Set in excel conf.
-         # 'reports_handin': [6, 11], # set in excel conf.
          'semester_id': semester_id(),
          'today': today(),
          'instructors': get_instructors(),
@@ -338,16 +313,17 @@ def class_information(verbose=False):
 
     d['written_exam'] = written_exam
 
+    kv = xlsx_to_dicts(paths['information.xlsx'], sheet='general_information', as_dict_list=False, columns=("key", "value") )
+    kvs = {}
+    for k in kv:
+        kvs[k['key']] = k['value']
+
     gi = xlsx_to_dicts(paths['information.xlsx'], sheet='general_information', as_dict_list=True)
-    for (k, v) in zip(gi['key'], gi['value']):
-        if v == 'True':
-            v = True
-        if v == 'False':
-            v= False
-        gi[k] = v
     del gi['key']
     del gi['value']
 
+    gi = {**gi, **kvs}
+
     from snipper.load_citations import get_bibtex, get_aux
     if "pensum_bib" in gi:
         bibtex = get_bibtex(paths['02450public'] + "/" + gi['pensum_bib'])
@@ -384,9 +360,6 @@ def class_information(verbose=False):
         freeze = freeze[0] if isinstance(freeze, list) else freeze
         gi[k] = freeze
 
-    for k,v in core_conf.items():
-        d[k] = v
-
     d['CE2'] = gi.get("days", 5) == 2 if continuing_education_mode else False
     d['CE5'] = gi.get("days", 5) == 5 if continuing_education_mode else False
 
@@ -399,10 +372,6 @@ def class_information(verbose=False):
     d['teams'] = xlsx_to_dicts(paths['information.xlsx'], sheet='teams')
     fix_instructor_comma(d['teams'], d['instructors'])
 
-    if 'reports_delta' in d:
-        print(234)
-
-
 
     if 'handin_day_delta' in d:
         d['reports_info'] = {}
@@ -413,20 +382,8 @@ def class_information(verbose=False):
             nd = d['lectures'][r-1]['date'] + timedelta(days=int(d['handin_day_delta']))
             ri['date'] = nd
             ri['html'] = f"{nd.day} {nd.strftime('%b')}"
-            # ab = 'st'
-            # if nd.day == 2:
-            #     ab = "nd"
-            # elif nd.day == 3:
-            #     ab = 'rd'
-            # elif nd.day >= 4:
-            #     ab = 'th'
-            # latex_short, latex_long = date2format(nd)
-
-            # ri['latex_long'] = latex_long # f"{nd.strftime('%A')} {nd.day}{ab} {nd.strftime('%B')}, {nd.year}"
-            # ri['latex_short'] = latex_short # f"{nd.strftime('%B')} {nd.day}{ab}, {nd.year}"
-            ri = {**ri, **date2format(nd)}
-
 
+            ri = {**ri, **date2format(nd)}
 
             d['reports_info'][k] = ri
 
@@ -449,15 +406,7 @@ def class_information(verbose=False):
 
         ri['date'] = nd
         ri['html'] = f"{nd.day} {nd.strftime('%b')}"
-        # ab = 'st'
-        # if nd.day == 2:
-        #     ab = "nd"
-        # elif nd.day == 3:
-        #     ab = 'rd'
-        # elif nd.day >= 4:
-        #     ab = 'th'
-        # ri['latex_long'] = f"{nd.strftime('%A')} {nd.day}{ab} {nd.strftime('%B')}, {nd.year}"
-        # ri['latex_short'] = f"{nd.strftime('%B')} {nd.day}{ab}, {nd.year}"
+
         ri = {**ri, **date2format(nd)}
         # d['reports_info'][k] = ri
 
@@ -474,10 +423,69 @@ def class_information(verbose=False):
                 r['handout'] = get_lecture_date(r['handout'], delta_days=0)
                 r['exercises'] = [e.strip() for e in r['exercises'].split(",") if len(e.strip()) > 0]
 
-
     ice = xlsx_to_dicts(paths['information.xlsx'], sheet='ce', as_dict_list=True)
+
+    d['release_rules'] = {}
+
+    for l in d['lectures']:
+        n = l['number']
+        date = l['date']
+
+        dd = timedelta(days=l['show_solutions_after'])
+        d['release_rules'][str(n)] = dict(start=date+dd, end=date+timedelta(days=2000))
+
+    if update_with_core_conf:
+        d = _update_with_core_conf(d)
+
+    return d
+
+def _update_with_core_conf(d):
+    for k,v in core_conf.items():
+        d[k] = v
     return d
 
+
+
+def _info_cache_file():
+    paths = get_paths()
+    f = glob.glob(paths['02450public'] + "/src/*_box").pop()
+    return f + f"/cache/{semester_id()}.pkl"
+
+
+def _save_info_cache():
+    """ Save a cached version of info.
+    """
+    paths = get_paths()
+    if not os.path.isfile(paths['information.xlsx']):
+        print("Tried saving cache file from installation without information.xlsx file. Exiting without saving...")
+        return
+
+    d = class_information(update_with_core_conf=False)
+    cdir = _info_cache_file()
+    if not os.path.isdir(os.path.dirname(cdir)):
+        os.makedirs(os.path.dirname(cdir))
+    known = {}
+
+    def _remove_ids(d):
+        if isinstance(d, str):
+            # v = "asdf s123456 safdasfd"
+            o = re.findall(r'(s\d{6})', d)
+            for id in o:
+                if id not in known:
+                    known[id] = f"s{len(known):6d}".replace(" ", "0")
+                d = d.replace(id, known[id])
+        elif isinstance(d, dict):
+            for k, v in d.items():
+                d[_remove_ids(k)] = _remove_ids(v)
+        elif isinstance(d, list):
+            d = [_remove_ids(k) for k in d]
+        elif isinstance(d, tuple):
+            d = tuple(_remove_ids(k) for k in d)
+        return d
+    dd = _remove_ids(copy.deepcopy(d))
+    with open(cdir, 'wb') as f:
+        pickle.dump(dd, f)
+
 def fix_instructor_comma(dd, instructors):
     for r in dd:
         ri_shortnames = [i.strip().lower() for i in r['instructors'].split(",")]

src/coursebox/core/info_paths.py

@@ -22,13 +22,13 @@ def get_paths():
     root_02450public = root_02450public.replace("\\", "/")
     root_02450private = root_02450private.replace("\\", "/")
 
-    if not os.path.isdir(root_02450private):
-        root_02450private = f'{root_02450public}/{num}private'
-        warn('Private repository not found at the expected location.')
-        warn('Using mock info from resources folder at:')
-        warn(root_02450private)
-        # Tue: always overwrite semester path.
-        # semester_path = root_02450private +"/resources/mock_semesters/" + semester_id()
+    # if not os.path.isdir(root_02450private):
+    #     root_02450private = f'{root_02450public}/{num}private'
+    #     warn('Private repository not found at the expected location.')
+    #     warn('Using mock info from resources folder at:')
+    #     warn(root_02450private)
+    # Tue: always overwrite semester path.
+    # semester_path = root_02450private +"/resources/mock_semesters/" + semester_id()
     # else:
     semester_path = root_02450private + "/semesters/" + semester_id()
 
@@ -36,10 +36,16 @@ def get_paths():
         os.makedirs(semester_path)
 
     main_conf = semester_path + "/" + semester_id() + ".xlsx"
+
     if not os.path.exists(main_conf):
         main_conf = f"{semester_path}/{course_number}_{semester_id()}.xlsx"
     if not os.path.exists(main_conf):
-        raise Exception("Main config file not found " + main_conf)
+        # cf = _info_cache_file()
+        # print("Information configuration file not found. Loading from cache:", cf)
+        # if not os.path.isfile(cf):
+        #     raise Exception("No configuration found. Please set up configuration file at: " + paths['information.xlsx'])
+        pass
+        # raise Exception("Main config file not found " + main_conf)
 
     _files = []
     sCE = "CE" if core_conf['continuing_education_mode'] else ""
@@ -48,27 +54,27 @@ def get_paths():
         # 'docs':
         # 'docs':
         '02450private': root_02450private,
-            '02450public': root_02450public,
-            '02450instructors': root_02450instructors,
-            '02450students': root_02450students,
-            'shared': root_02450public+"/shared",
-            'exams': root_02450private+"/Exam",
-            'course_number': course_number,
-            'semester': semester_path,
-            'information.xlsx': main_conf,
-            'homepage_template': "%s/WEB/index_partial.html"%root_02450public,
-            'homepage_out': "%s/WEB/%sindex.html"%(root_02450public, sCE),
-            'pdf_out': "%s/%spdf_out"%(root_02450public, sCE),
-            'instructor': root_02450public + "/Exercises",
-            'shared_latex_compilation_dir': root_02450public + "/Exercises/LatexCompilationDir/Latex",
-            'book': root_02450public + "/MLBOOK/Latex",
-            'lectures': root_02450public + "/Lectures",
-            'instructor_project_evaluations': "%s/project_evaluations_%s" % (root_02450instructors, semester_id()),
-            'project_evaluations_template.xlsx': root_02450private +"/ReportEvaluation/%s_project_template.xlsx"%num,
-            'collected_project_evaluations.xlsx': semester_path + "/"+course_number+"_project_" + semester_id() + ".xlsx",
-            'electronic_exam_handin_dir': semester_path + "/exam/electronic_handin",
-            'exam_results_template.xlsx': root_02450private +"/Exam/%s_results_TEMPLATE.xlsx"%num,
-            'exam_instructions': root_02450public + "/ExamInstructions",
+        '02450public': root_02450public,
+        '02450instructors': root_02450instructors,
+        '02450students': root_02450students,
+        'shared': root_02450public+"/shared",
+        'exams': root_02450private+"/Exam",
+        'course_number': course_number,
+        'semester': semester_path,
+        'information.xlsx': main_conf,
+        'homepage_template': "%s/WEB/index_partial.html"%root_02450public,
+        'homepage_out': "%s/WEB/%sindex.html"%(root_02450public, sCE),
+        'pdf_out': "%s/%spdf_out"%(root_02450public, sCE),
+        'instructor': root_02450public + "/Exercises",
+        'shared_latex_compilation_dir': root_02450public + "/Exercises/LatexCompilationDir/Latex",
+        'book': root_02450public + "/MLBOOK/Latex",
+        'lectures': root_02450public + "/Lectures",
+        'instructor_project_evaluations': "%s/project_evaluations_%s" % (root_02450instructors, semester_id()),
+        'project_evaluations_template.xlsx': root_02450private +"/ReportEvaluation/%s_project_template.xlsx"%num,
+        'collected_project_evaluations.xlsx': semester_path + "/"+course_number+"_project_" + semester_id() + ".xlsx",
+        'electronic_exam_handin_dir': semester_path + "/exam/electronic_handin",
+        'exam_results_template.xlsx': root_02450private +"/Exam/%s_results_TEMPLATE.xlsx"%num,
+        'exam_instructions': root_02450public + "/ExamInstructions",
     }
     if os.path.exists(os.path.dirname(paths['instructor_project_evaluations'])):
         if not os.path.isdir(paths['instructor_project_evaluations']):

src/coursebox/material/homepage_lectures_exercises.py

@@ -291,10 +291,8 @@ def fix_shared(paths, output_dir, pdf2png=False,dosvg=True,verbose=False, compil
     # def get_cache_from_dir(shared_base):
     # print("Beginning file cache..")
 
-
     source = get_hash_from_base(shared_base)
     target = get_hash_from_base(output_dir)
-
     # update_source_cache = False
     source_extra = {}
     for rel in source:
@@ -338,6 +336,7 @@ def fix_shared(paths, output_dir, pdf2png=False,dosvg=True,verbose=False, compil
         pickle.dump(target, f)
 
 
+
 def jinjafy_shared_templates_dir(paths, info):
     tpd = paths['shared'] + "/templates"
     for f in glob.glob(tpd + "/*.*"):