diff --git a/setup.py b/setup.py index 6c26e5368975471a983736973cc710277f696718..f50928f6e0d96278e9e082e0e66d183317949f1a 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setuptools.setup( name="coursebox", - version="0.1.18.20", + version="0.1.18.21", author="Tue Herlau", author_email="tuhe@dtu.dk", description="A course management system currently used at DTU", diff --git a/src/coursebox.egg-info/PKG-INFO b/src/coursebox.egg-info/PKG-INFO index ceb4f447391d8c28f7a18fc1dfbf5214576e4337..ce04bbe641e62174bd412a9c77e68d264fa86860 100644 --- a/src/coursebox.egg-info/PKG-INFO +++ b/src/coursebox.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: coursebox -Version: 0.1.18.19 +Version: 0.1.18.21 Summary: A course management system currently used at DTU Home-page: https://lab.compute.dtu.dk/tuhe/coursebox Author: Tue Herlau @@ -13,6 +13,16 @@ Classifier: Operating System :: OS Independent Requires-Python: >=3.8 Description-Content-Type: text/markdown License-File: LICENSE +Requires-Dist: numpy +Requires-Dist: pycode_similar +Requires-Dist: tika +Requires-Dist: openpyxl +Requires-Dist: xlwings +Requires-Dist: matplotlib +Requires-Dist: langdetect +Requires-Dist: beamer-slider +Requires-Dist: tinydb +Requires-Dist: python-gitlab # Coursebox DTU DTU course management software. diff --git a/src/coursebox/__init__.py b/src/coursebox/__init__.py index 790ab999fff754e7ff4531055b8dd9e72c3dc88e..97ca307a4affcbd499196d9e2f4514e3bbcd3898 100644 --- a/src/coursebox/__init__.py +++ b/src/coursebox/__init__.py @@ -8,3 +8,10 @@ from coursebox.core.info import class_information from coursebox.admin.gitlab import sync_tas_with_git # from coursebox.core import info_paths + + +def setup_student_files(*args, **kwargs): + from coursebox.setup_coursebox import funcs + + funcs['setup_student_files'](*args, **kwargs) + funcs['fix_all_shared_files'](*args, **kwargs) diff --git a/src/coursebox/core/info_paths.py b/src/coursebox/core/info_paths.py index cc07d6e417b99b899a7998665e1e8fe4594881e4..2a19f50b1f178cc34b0bf3b8ef771704516c1392 100644 --- a/src/coursebox/core/info_paths.py +++ b/src/coursebox/core/info_paths.py @@ -11,6 +11,7 @@ def get_paths(): cd = core_conf['working_dir'] cd = os.path.basename( os.path.dirname( os.path.dirname(cd) ) ) num = cd[:-6] # course number + # num = cd[:cd.find("public")] CDIR = core_conf['working_dir'] course_number = core_conf['course_number'] diff --git a/src/coursebox/material/documentation.py b/src/coursebox/material/documentation.py new file mode 100644 index 0000000000000000000000000000000000000000..b95974583d4327531aa3ad166ce92786c3520de2 --- /dev/null +++ b/src/coursebox/material/documentation.py @@ -0,0 +1,368 @@ +import time +import PIL.Image +import os +import shutil +from slider.legacy_importer import slide_to_image +import glob +import re +import pickle +import datetime +import subprocess +from unitgrade_private.run import run + +def build_sphinx_documentation(cut_files=False, open_browser=True, build_and_copy_censored=True, CE=False, languages=('en', 'da'), show_all_solutions=False, + tolerate_problems=False, # If False, WARNINGS and ERRORS in the spinx build process will lead to a compile error. Set to True during local development. + sphinx_cache=False, # If False, disable the Sphinx cache, i.e. include the -a (slightly longer rebuilds but all errors are caught & more reliable; recommended on stackexchange). + update_translations=False, + CE_public=False, + ): + + # print("This functionality has been moved to coursebox.") + + + from coursebox.core.info_paths import get_paths + paths = get_paths() + if CE: + languages = ("en",) + + if CE: + SPHINX_TAG = " -t ce" + + if CE_public: + SPHINX_TAG += " -t ce_public" + PUBLIC_BUILD_DEST = f"{paths['02450public']}/public/ce_public" + else: + PUBLIC_BUILD_DEST = f"{paths['02450public']}/public/ce" + + else: + SPHINX_TAG = "" + PUBLIC_BUILD_DEST = f"{paths['02450public']}/public" + + # This will build the student documentation (temporary). + # The functionality here should match the gitlab ci file closely. + from cp_box.material.student_files import fix_all_shared_files + from cp_box.material.student_files import setup_student_files + + + + if os.path.isfile(d_ := f"{paths['book']}/{paths['course_number']}_Notes.pdf"): + book_frontpage_png = paths['shared']+"/figures/book.png" + slide_to_image(d_, book_frontpage_png, page_to_take=1) + image = PIL.Image.open(book_frontpage_png) + im = _makeShadow(image, iterations=100, offset=(25,)*2, shadowColour="#aaaaaa") + im.save(book_frontpage_png) + + fix_all_shared_files(dosvg=True) + + """ Return extra information required for building the documentation. + """ + # from coursebox.core.info_paths import get_paths + from coursebox.core.info import class_information + from coursebox.core import info_paths + # paths = get_paths() + info = class_information() + # {{ (date1|to_datetime - date2|to_datetime).days < lecture['show_slides_after'] }} + # (info['lectures'][2]['date'].now() - info['lectures'][2]['date']).days < lecture['show_slides_after'] }} + source = _get_source(paths) + PACKAGE = info.get('package', 'cp') + + x = {} + for f in glob.glob(f"{source}/projects/project*.rst"): + k = int(re.findall(r'\d+', f)[-1]) + x[k] = {} + + exfiles = [] + for g in glob.glob(f"{paths['02450students']}/{PACKAGE}/project{k}/*.py"): + with open(g, 'r') as ff: + if "TODO" in ff.read(): + exfiles.append(g) + files = [os.path.relpath(ff, paths['02450students']) for ff in exfiles] + x[k]['files'] = files + # print(">>> k class is: ") + # print(info_paths.core_conf['projects_all'][k]) + f = info_paths.core_conf['projects_all'][k]['class'].mfile() + with open(f.split("_grade.py")[0], 'r') as ff: + l = [l for l in ff.read().splitlines() if "(Report)" in l].pop().split("(")[0].split(" ")[-1] + + token = f"{os.path.relpath(os.path.dirname(f), paths['02450public'] + '/src')}/{l}_handin_k_of_n.token" + x[k]['token'] = token + f = os.path.relpath(f, paths['02450public'] + "/src") + if f.endswith("_complete.py"): + f = f.split("_complete.py")[0] + f = "_".join(f.split("_")[:-1]) + f = f + "_grade.py" + else: + f = f.split(".py")[0] + "_grade.py" + x[k]['grade_file'] = f + x[k]['grade_module'] = f[:-3].replace("/", ".") + + """ TH: What happens here is that we cache the report information so we can later load it (above) when rst source is build. + The reason we split the code like this is because we don't have access to the report classes during the Sphinx build process, + and that means some of the variables are not easily set. This is a bit of a dirty fix, but it works. """ + with open(os.path.dirname(os.path.abspath(__file__)) + "/_extra_info.pkl", 'wb') as f: + pickle.dump(x, f) + + for f in glob.glob(f"{paths['02450public']}/src/docs/templates/*.rst"): + if f.endswith("blurb.rst") or f.endswith("base.rst"): + continue # We dealt with these; nb. very hacky stuff. + + PROJECTS = [int(os.path.basename(f)[len("project"):]) for f in glob.glob(f"{paths['02450public']}/src/cp/project*")] + WEEKS = [int(os.path.basename(f)[len("ex"):]) for f in glob.glob(f"{paths['02450public']}/src/cp/ex*") if not os.path.basename(f) == 'exam'] + + pdfs = [] + for g in glob.glob(paths['pdf_out'] +"/handout/*.pdf"): + dst = paths['02450public'] + "/src/docs/assets/"+os.path.basename(g)[:-4] + "-handout.pdf" + shutil.copy(g, dst) + pdfs.append(dst) + for g in [paths['pdf_out'] + "/" + paths['course_number'] + "_Notes.pdf"]: + dst = paths['02450public'] + "/src/docs/assets/" + os.path.basename(g) + shutil.copy(g, dst) + pdfs.append(dst) + + # Copy shared templates. + if not os.path.isdir(paths['02450public'] + "/src/docs/source/templates_generated"): + os.mkdir(paths['02450public'] + "/src/docs/source/templates_generated") + for g in glob.glob(paths['shared'] + "/templates/*.rst"): + if '_partial.rst' in g: + continue + shutil.copy(g, f"{paths['02450public']}/src/docs/source/templates_generated/{os.path.basename(g)}") + ## Update the translation files. + + if update_translations: + print("build_documentation> updating ze translations.") + pr1 = run(f"cd {paths['docs']} && make gettext", print_output=False, log_output=True, check=True) + pr2 = run(f"cd {paths['docs']} && sphinx-intl update -p build/gettext", print_output=False, log_output=True, check=True) + assert pr1.returncode == 0 and pr2.returncode ==0, "you done goofed in the translation building." + print("build_documentation> I am done updating ze translation!") + + # from cp_box.checks.checks import deploy_student_repos + BAD_MODULES_DIR = False + deploy_students_complete() # I guess this will build public and private. + students_complete = paths['02450students'] + "_complete" + # PUBLIC_BUILD_DEST + fns = [] + + # Blow public modules. This is because renamed files will otherwise stay around and when you later build the version + # without solutions, and verify the without/with solutions version has the same files, the old files will cause problems. + if os.path.isdir(d_ := f"{PUBLIC_BUILD_DEST}/_modules"): + shutil.rmtree(d_) + + for l in languages: + if l=='en': + lang = "" + TAG_EXTRA = '' + else: + lang = " -D language='da' " + TAG_EXTRA = '-t da' + if not sphinx_cache: + TAG_EXTRA += ' -a' + # " sphinx-build -b html source build -D language='da' -t da " + + _FINAL_BUILD_DEST = f"{PUBLIC_BUILD_DEST}{'/da' if l=='da' else ''}" + + + SET_PATH = f"""PYTHONPATH="{paths['02450students'] + '_complete'}" """ + if os.name == 'nt': + SET_PATH = "set "+SET_PATH +" && " + + cmd_ = f"""cd "{students_complete}/docs" && {SET_PATH} sphinx-build -b html source "{os.path.relpath(_FINAL_BUILD_DEST, students_complete+'/docs')}" {TAG_EXTRA} {SPHINX_TAG} {lang}""" + cmd = f"""cd "{students_complete}/docs" && sphinx-build -b html source "{os.path.relpath(_FINAL_BUILD_DEST, students_complete + '/docs')}" {TAG_EXTRA} {SPHINX_TAG} {lang}""" + + # if os.name == "nt": + # cmd = cmd.replace("&&", ";") + # cd "C:\Users\tuhe\Documents\02002students_complete/docs" ; $env:PYTHONPATH="C:\Users\tuhe\Documents\02002students_complete" ; sphinx-build -b html source ..\..\02002public\public -a + + # p = subprocess.run(cmd, shell=True, capture_output=True, encoding="utf-8") + # print(">>>>>>>>>> inside build_documentation.py. The python binary is", sys.executable) + cmd = cmd.replace("\\", "/") + print(">>> Sphinx main build command is\n", cmd_) + print(" ") + + # subprocess.run(cmd, shell=True) + # subprocess.run('cd "C:/Users/tuhe/Documents/02002students_complete/docs" && set PYTHONPATH="C:/Users/tuhe/Documents/02002students_complete" && sphinx-build -b html source "../../02002public/public" -a ', shell=True) + + problems = [] + if os.name == "nt": + + # do win specific stuff here. + # >> > result = subprocess.run(['ls', '-l'], stdout=subprocess.PIPE) + # >> > result.stdout + pass + + my_env = os.environ.copy() + my_env['PYTHONPATH'] = (paths['02450students'] + '_complete').replace("\\", "/") + process = run(cmd, print_output=True, log_output=True, check=False, env=my_env) + + + if os.name == 'nt': + # time.sleep(10) + process = run(cmd, print_output=True, log_output=True, check=False, env=my_env) + print("TH 2023 Juli: Running sphinx compilation job twice bc of path error on windows. This should be easy to fix but I don't know enough about windows to do so.") + print(process.stderr.getvalue()) + + errors = process.stderr.getvalue() + + file = f"{os.path.normpath(PUBLIC_BUILD_DEST)}/log_{l}.txt" + fns.append(file) + + if not os.path.isdir(d_ := os.path.dirname(file)): + os.makedirs(d_) + with open(file,'w') as f: + f.write("\n".join(["stdout","="*100,"", process.stdout.getvalue(), "stderr","="*100,"", errors, " ","build at", datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") ])) + + problems = [l for l in errors.splitlines() if ("WARNING:" in l ) or "ERROR" in l] + + if len(problems) > 0 and not tolerate_problems: + print("=" * 50) + print("""Sphinx compilation encountered errors and/or warnings. Although the documentation can build, we are running build_documentation(..., tolerate_problems=False), which is the +default on gitlab, as we don't want a pileup of small(ish) build errors. So carefully read through the output above to identify errors and fix them. +Remember you can also use the tolerate_problems argument locally to fix problems in that way. + +Below is a summary of the problems we found: """) + for p in problems: + print(">", p) + raise Exception("There were compilation problems when compiling documentation using sphinx. Please read output above carefully for details. ") + + # Slightly silly code that copies the language thumbnails. I guess I could find a way to include them and do it automatically but oh well. + if not os.path.isdir(fbd := f"{_FINAL_BUILD_DEST}/_images"): + os.makedirs(fbd) + + for im in ["gb.png", "dk.png"]: + shutil.copy(f"{paths['shared']}/figures/{im}", f"{_FINAL_BUILD_DEST}/_images/{im}") + + + + + if build_and_copy_censored: + verbose = False + setup_student_files(run_files=False, cut_files=False, censor_files=True, setup_lectures=cut_files, + week=WEEKS, projects=PROJECTS, + fix_shared_files=True, verbose=verbose, include_docs=True) + + cmd_with_pythonpath = f"cd {paths['02450students']} && PYTHONPATH={paths['02450students']} sphinx-build -b html docs/source ./public {SPHINX_TAG}" + cmd = f"cd {paths['02450students']} && sphinx-build -b html docs/source ./public {SPHINX_TAG}" + + # try: + # print("Running>", cmd) + my_env = os.environ.copy() + my_env['PYTHONPATH'] = paths['02450students'].replace("\\","/") + print("> Building documentation based on .py-files that do not contain solutions based on command:\n", cmd_with_pythonpath) + out = subprocess.run(cmd, shell=True, check=True, env=my_env, capture_output=False) + + # glob.glob(f"{PUBLIC_BUILD_DEST}/_modules/**/*.html") + + known_modules = set([os.path.relpath(f, PUBLIC_BUILD_DEST) for f in glob.glob(f"{PUBLIC_BUILD_DEST}/_modules/**/*.html", recursive=True) ] ) + build_modules = set([os.path.relpath(f, f"{paths['02450students']}/public") for f in glob.glob(f"{paths['02450students']}/public/_modules/**/*.html", recursive=True) ] ) + + # known_modules == build_modules + # set.difference() + for f in known_modules.difference(build_modules): + print(f) + for f in known_modules.difference(build_modules): + print("> Documentation error. View source function did not build correctly since the (censored) files did not contain the html file: ", f) + print("> The likely cause of this problem is that you got a top-level #!b tag in the corresponding python file, meaning the documentation cannot be build for this file. ") + print("> To fix the problem, use the #!b;noerror command to suppress Exceptions.") + raise Exception(f"View source file not found for {f}. Please see terminal output above.") + + shutil.rmtree(paths['02450students'] + "/docs") + if os.path.isdir(f"{PUBLIC_BUILD_DEST}/_modules"): + shutil.rmtree(f"{PUBLIC_BUILD_DEST}/_modules") + + if os.path.isdir(f"{paths['02450students']}/public/_modules"): + shutil.copytree(f"{paths['02450students']}/public/_modules", f"{PUBLIC_BUILD_DEST}/_modules") + else: + BAD_MODULES_DIR = True + + if os.path.isdir(f"{paths['02450students']}/docs"): + shutil.rmtree(f"{paths['02450students']}/docs") + + if os.path.isdir(f"{paths['02450students']}/public"): + shutil.rmtree(f"{paths['02450students']}/public") + + # copy images into the _image folder. + + + if BAD_MODULES_DIR: + print("WARNING!: Student _modules dir not generated. Probably script crash. This is a bad situation. Documentation view source links not up to date. ") + + if open_browser: + import webbrowser + try: + if os.name == "nt": + chrome_path = "C:/Program Files/Google/Chrome/Application/chrome.exe" + webbrowser.register('chrome', None, webbrowser.BackgroundBrowser(chrome_path)) + webbrowser.get("chrome").open(f"{PUBLIC_BUILD_DEST}/index.html") + else: + webbrowser.get("chromium").open(f"{PUBLIC_BUILD_DEST}/index.html") + except Exception as e: + print("URL to local host website:",f"{PUBLIC_BUILD_DEST}/index.html") + webbrowser.get("firefox").open(f"{PUBLIC_BUILD_DEST}/index.html") + pass + for f in fns: + print("> See log file", f, "at", f"https://cp.pages.compute.dtu.dk/02002public/{os.path.relpath(f, PUBLIC_BUILD_DEST)}") + + +def _makeShadow(image, iterations, border=8, offset=(3,3), backgroundColour="#ffffff", shadowColour="#444444"): + # backgroundColour = () + from PIL import Image, ImageFilter + # from PIL import + + # image: base image to give a drop shadow + # iterations: number of times to apply the blur filter to the shadow + # border: border to give the image to leave space for the shadow + # offset: offset of the shadow as [x,y] + # backgroundCOlour: colour of the background + # shadowColour: colour of the drop shadow + + # Calculate the size of the shadow's image + fullWidth = image.size[0] + abs(offset[0]) + 2 * border + fullHeight = image.size[1] + abs(offset[1]) + 2 * border + + # Create the shadow's image. Match the parent image's mode. + shadow = Image.new(image.mode, (fullWidth, fullHeight), backgroundColour) + + # Place the shadow, with the required offset + shadowLeft = border + max(offset[0], 0) # if <0, push the rest of the image right + shadowTop = border + max(offset[1], 0) # if <0, push the rest of the image down + # Paste in the constant colour + shadow.paste(shadowColour, + [shadowLeft, shadowTop, + shadowLeft + image.size[0], + shadowTop + image.size[1]]) + + # Apply the BLUR filter repeatedly + for i in range(iterations): + shadow = shadow.filter(ImageFilter.BLUR) + + # Paste the original image on top of the shadow + imgLeft = border - min(offset[0], 0) # if the shadow offset was <0, push right + imgTop = border - min(offset[1], 0) # if the shadow offset was <0, push down + shadow.paste(image, (imgLeft, imgTop)) + + return shadow + + +def _get_source(paths): + return paths['02450public'] + "/src/docs/source" + # return source + + + +def deploy_students_complete(verbose=False): + from coursebox.core.info_paths import get_paths + paths = get_paths() + from cp_box.material.student_files import setup_student_files + studens_complete = paths['02450students'] + '_complete' + if os.path.isdir(studens_complete): + shutil.rmtree(paths['02450students'] + '_complete') + PROJECTS = [int(os.path.basename(f)[len("project"):]) for f in + glob.glob(f"{paths['02450public']}/src/cp/project*")] + WEEKS = [int(os.path.basename(f)[len("ex"):]) for f in glob.glob(f"{paths['02450public']}/src/cp/ex*") if + not os.path.basename(f) == 'exam'] + # shutil.rmtree(paths['02450students'] + '_complete') + cut_files=False + # verbose=False + # cut_files = False # This deploy to students_complete. + setup_student_files(run_files=False, cut_files=cut_files, censor_files=False, setup_lectures=cut_files, + week=WEEKS, projects=PROJECTS, + fix_shared_files=cut_files, verbose=verbose, include_docs=True) diff --git a/src/coursebox/setup_coursebox.py b/src/coursebox/setup_coursebox.py index f3d790743e3213e777cd52a7ec9d925a1a74e7d7..3b46bc81b1c0717f6eed1fb0ac69d77ae0af7ddf 100644 --- a/src/coursebox/setup_coursebox.py +++ b/src/coursebox/setup_coursebox.py @@ -1,11 +1,23 @@ from coursebox.core import info_paths +def _no_such_function(*args, **kwargs): + raise NotImplementedError("The function does nto exist. You muast pass it to coursebox setup_coursebox(..) for this to work") + +funcs = {'setup_student_files': _no_such_function, + 'fix_all_shared_files' : _no_such_function + } + def setup_coursebox(working_dir, course_number="02450", semester='spring', year=2019, slides_showsolutions=True, slides_includelabels=False, continuing_education_mode = False, slides_shownotes=False, - continuing_education_month = "March", post_process_info=None, **kwargs): + continuing_education_month = "March", post_process_info=None, + setup_student_files=None, + fix_all_shared_files=None, + **kwargs): + funcs['setup_student_files'] = setup_student_files + funcs['fix_all_shared_files'] = fix_all_shared_files info_paths.core_conf['working_dir'] = working_dir info_paths.core_conf['course_number'] = course_number diff --git a/src/coursebox/student_files/student_files.py b/src/coursebox/student_files/student_files.py index 20ea4a0482275bbf4bef92cb377de88d5875013e..329fa3ea08283ce6f9b86369fd80f43659da3c4f 100644 --- a/src/coursebox/student_files/student_files.py +++ b/src/coursebox/student_files/student_files.py @@ -74,6 +74,7 @@ def setup_student_files(run_files=True, print(m) if extra_dirs is None: + extra_dirs = ['utils', 'tests', 'exam/exam2023spring'] # 'assignments', for m in midterms: if m == 0: @@ -93,19 +94,19 @@ def setup_student_files(run_files=True, # 'workshop' ] - import coursebox.core.info + import coursebox.core.info - for id in projects: - # if setup_projects: - if 'projects_all' in info_paths.core_conf: - p = info_paths.core_conf['projects_all'][id]['class'] - extra_dirs += [os.path.basename(os.path.dirname(p.mfile()))] - else: + for id in projects: + # if setup_projects: + if 'projects_all' in info_paths.core_conf: + p = info_paths.core_conf['projects_all'][id]['class'] + extra_dirs += [os.path.basename(os.path.dirname(p.mfile()))] + else: - if id <= 3: - extra_dirs += [f'project{id}'] # , 'project1', 'project2', 'project3'] - else: - extra_dirs.append(f'project3i') + if id <= 3: + extra_dirs += [f'project{id}'] # , 'project1', 'project2', 'project3'] + else: + extra_dirs.append(f'project3i') if week is not None: for w in week: @@ -143,6 +144,8 @@ def setup_student_files(run_files=True, else: info = None for hw in hws: + if "ex08" in hw['out']: + print("ex08") n = fix_hw(paths=paths, info=info, hw=hw, out= hw['out'], output_dir=paths['shared'] +"/output", run_files=run_files, cut_files=cut_files, package_base_dir=os.path.dirname(students_irlc_tools), censor_files=censor_files, strict=strict, include_solutions=hw.get('include_solutions', False),