diff --git a/setup.py b/setup.py index a7fc926195803d749a51d95f64d5cd39bb7cc54c..5d605ec95e52575a3760b247f24386c66ac21229 100644 --- a/setup.py +++ b/setup.py @@ -5,16 +5,16 @@ import setuptools import pkg_resources -# with open("src/unitgrade2/version.py", "r", encoding="utf-8") as fh: -# __version__ = fh.read().split(" = ")[1].strip()[1:-1] -# long_description = fh.read() +with open("src/snipper/version.py", "r", encoding="utf-8") as fh: + __version__ = fh.read().split(" = ")[1].strip()[1:-1] + with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() setuptools.setup( name="codesnipper", - version="0.1.1", + version=__version__, author="Tue Herlau", author_email="tuhe@dtu.dk", description="A lightweight framework for censoring student solutions files and extracting code + output", diff --git a/src/codesnipper.egg-info/PKG-INFO b/src/codesnipper.egg-info/PKG-INFO index 51473de1e497fa3563c8b96b5e5441be9d3d7e2b..f95b07cf7df87424af270c862bf798e7b9b9673d 100644 --- a/src/codesnipper.egg-info/PKG-INFO +++ b/src/codesnipper.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: codesnipper -Version: 0.1.0 +Version: 0.1.7 Summary: A lightweight framework for censoring student solutions files and extracting code + output Home-page: https://lab.compute.dtu.dk/tuhe/snipper Author: Tue Herlau @@ -24,12 +24,14 @@ pip install codesnipper ## What it does This project address the following three challenges for administering a python-based course - - You need to maintain a (working) version for debugging as well as a version handed out to students (with code missing) - - You ideally want to make references in source code to course material *"(see equation 2.1 in exercise 5)"* but these tend to go out of date - - You want to include code snippets and code output in lectures notes/exercises/beamer slides - - You want to automatically create student solutions + - Maintain a (working) version for debugging as well as a version handed out to students (with code missing) + - Use LaTeX references in source code to link to course material (i.e. `\ref{mylabel}` -> *"(see equation 2.1 in exercise 5)"*) + - Including code snippets and console output in lectures notes/exercises/beamer slides + - Automatically create student solutions -This framework address these problems and allow you to maintain a **single**, working project repository. +This framework address these problems and allow you to maintain a **single**, working project repository. Below is an example of the snippets produced and included using simple `\inputminted{python}{...}` commands (see the `examples/` directory): + + The project is currently used in **02465** at DTU. An example of student code can be found at: - https://gitlab.gbar.dtu.dk/02465material/02465students/blob/master/irlc/ex02/dp.py @@ -45,9 +47,9 @@ examples/cs101_students # This directory contains the (processed) student fil examples/cs101_output # This contains automatically generated contents (snippets, etc.). ``` The basic functionality is you insert special comment tags in your source, such as `#!b` or `#!s` and the script then process -the sources based on the tags. The following will show most basic usages: +the sources based on the tags. The following will show most common usages: -## The #f!-tag +## The #!f-tag Let's start with the simplest example, blocking out a function (see `examples/cs101_instructor/f_tag.py`; actually it will work for any scope) You insert a comment like: `#!f <exception message>` like so: ```python @@ -71,7 +73,7 @@ def myfun(a,b): return sm ``` -## The #b!-tag +## The #!b-tag The #!b-tag allows you more control over what is cut out. The instructor file: ```python def primes_sieve(limit): @@ -104,7 +106,7 @@ This allows you to cut out text across scopes, but still allows you to insert ex -## The #s!-tag +## The #!s-tag The #!s-tag is useful for making examples to include in exercises and lecture notes. The #!s (snip) tag cuts out the text between tags and places it in files found in the output-directory. As an example, here is the instructor file: ```python @@ -161,7 +163,7 @@ and finally: I recommend using `\inputminted{filename}` to insert the cutouts in LaTeX. -## The #o!-tag +## The #!o-tag The #!o-tag allows you to capture output from the code, which can be useful when showing students the expected behavior of their scripts. Like the #!s-tag, the #!o-tags can be named. @@ -193,7 +195,7 @@ Area of square of width 2 and height 4 is: and that is a fact! ``` -## The #i!-tag +## The #!i-tag The #!i-tag allows you to create interactive python shell-snippets that can be imported using the minted `pycon` environment (`\inputminted{python}{input.shell}`). As an example, consider the instructor file @@ -227,8 +229,8 @@ Note that apparently there is no library for converting python code to shell sessions so I had to write it myself, which means it can properly get confused with multi-line statements (lists, etc.). On the plus-side, it will automatically insert newlines after the end of scopes. My parse is also known to be a bit confused if your code outputs `...` since it has to manually parse the interactive python session and this normally indicates a new line. -## References and citations (`\ref` and `\cite`) -One of the most annoying parts of maintaining student code is to constantly write "see equation on slide 41 bottom" only to have the reference go stale because slide 23 got removed. Well now anymore, now you can direcly refence anything with a bibtex or aux file! +# References and citations (`\ref` and `\cite`) +One of the most annoying parts of maintaining student code is to constantly write *"see equation on slide 41 bottom"* only to have the reference go stale because slide 23 got removed. Well not anymore, now you can direcly refence anything with a bibtex or aux file! Let's consider the following example of a simple document with a couple of references: (see `examples/latex/index.pdf`):  @@ -314,7 +316,7 @@ Since the aux/bibtex databases are just dictionaries it is easy to join them tog I have written reference tags to lecture and exercise material as well as my notes and it makes reference management very easy. -## Advanced block processing +# Partial solutions The default behavior for code removal (#!b and #!f-tags) is to simply remove the code and insert the number of missing lines. We can easily create more interesting behavior. The code for the following example can be found in `examples/b_example.py` and will deal with the following problem: ```python @@ -331,10 +333,10 @@ def primes_sieve(limit): return primes #!b # Example use: print(primes_sieve(42)) ``` -The example shows how we can easily define custom functions for processing the code which is to be removed. -All such a function need is to take a list of lines (to be obfuscated) and return a new list of lines (the obfuscated code). -A couple of short examples: -### Permute lines +The examples below shows how we can easily define custom functions for processing the code which is to be removed; I have not included the functions here for brevity, +but they are all just a few lines long and all they do is take a list of lines (to be obfuscated) and return a new list of lines (the obfuscated code). + +### Example 1: Permute lines This example simple randomly permutes the line and prefix them with a comment tag to ensure the code still compiles ```python import numpy as np @@ -352,7 +354,7 @@ def primes_sieve(limit): # Example use: print(primes_sieve(42)) raise NotImplementedError('Complete the above program') ``` -### Partial replacement +### Example 2: Partial replacement This example replaces non-keyword, non-special-symbol parts of the lines: ```python import numpy as np @@ -371,8 +373,8 @@ def primes_sieve(limit): raise NotImplementedError('Complete the above program') ``` -### Half of the solution -The final solution display half of the proposed solution: +### Example 3: Half of the solution +The final example displays half of the proposed solution: ```python import numpy as np # Implement a sieve here. @@ -390,3 +392,17 @@ def primes_sieve(limit): raise NotImplementedError('Complete the above program') ``` + +# Citing +```bibtex +@online{codesnipper, + title={Codesnipper (0.1.0): \texttt{pip install codesnipper}}, + url={https://lab.compute.dtu.dk/tuhe/snipper}, + urldate = {2021-09-07}, + month={9}, + publisher={Technical University of Denmark (DTU)}, + author={Tue Herlau}, + year={2021}, +} +``` + diff --git a/src/codesnipper.egg-info/SOURCES.txt b/src/codesnipper.egg-info/SOURCES.txt index aa653c49760598429cda3986784fafc79595a575..31883456cf9554c60cbb0d0d6e049b6a3b7965a6 100644 --- a/src/codesnipper.egg-info/SOURCES.txt +++ b/src/codesnipper.egg-info/SOURCES.txt @@ -14,8 +14,10 @@ src/snipper/fix_bf.py src/snipper/fix_cite.py src/snipper/fix_i.py src/snipper/fix_o.py +src/snipper/fix_r.py src/snipper/fix_s.py src/snipper/legacy.py src/snipper/load_citations.py src/snipper/snip_dir.py -src/snipper/snipper_main.py \ No newline at end of file +src/snipper/snipper_main.py +src/snipper/version.py \ No newline at end of file diff --git a/src/snipper/__init__.py b/src/snipper/__init__.py index 4e820bd71b5043c9aa88372b27090362b28a3900..32d269e53dfb2d71d10aaf454ec7df6393d79010 100644 --- a/src/snipper/__init__.py +++ b/src/snipper/__init__.py @@ -1,3 +1,4 @@ -__version__ = "0.0.1" +from snipper.version import __version__ from snipper.snip_dir import snip_dir + diff --git a/src/snipper/__pycache__/__init__.cpython-38.pyc b/src/snipper/__pycache__/__init__.cpython-38.pyc index eae9fb89b94899b9aab163c73358f43efa116212..b69da3bc3256d708cd9d94a26ff0de84cbe7c47d 100644 Binary files a/src/snipper/__pycache__/__init__.cpython-38.pyc and b/src/snipper/__pycache__/__init__.cpython-38.pyc differ diff --git a/src/snipper/__pycache__/block_parsing.cpython-38.pyc b/src/snipper/__pycache__/block_parsing.cpython-38.pyc index f863b8116bd2e1ce5093ddf2a92c11e93ae21392..cb029cce281298d5b7f761b09068fdbd689ab7af 100644 Binary files a/src/snipper/__pycache__/block_parsing.cpython-38.pyc and b/src/snipper/__pycache__/block_parsing.cpython-38.pyc differ diff --git a/src/snipper/__pycache__/fix_bf.cpython-38.pyc b/src/snipper/__pycache__/fix_bf.cpython-38.pyc index 08110df11cf8ba047f40b6a36f84bec56cdd98a5..d658b0d20470302c45daa0e3245fa0823a229ac8 100644 Binary files a/src/snipper/__pycache__/fix_bf.cpython-38.pyc and b/src/snipper/__pycache__/fix_bf.cpython-38.pyc differ diff --git a/src/snipper/__pycache__/legacy.cpython-38.pyc b/src/snipper/__pycache__/legacy.cpython-38.pyc index de81fc57ca76b392afb215f36ef762b6c4553601..5b42f09e6c1319e32d100c7978700b585efb237c 100644 Binary files a/src/snipper/__pycache__/legacy.cpython-38.pyc and b/src/snipper/__pycache__/legacy.cpython-38.pyc differ diff --git a/src/snipper/__pycache__/snip_dir.cpython-38.pyc b/src/snipper/__pycache__/snip_dir.cpython-38.pyc index 4ce3da3636250429478c87ecb3bafc063b86f0dc..f19c4b3b4d60e3eb2edd09eecf507974dbf6878e 100644 Binary files a/src/snipper/__pycache__/snip_dir.cpython-38.pyc and b/src/snipper/__pycache__/snip_dir.cpython-38.pyc differ diff --git a/src/snipper/__pycache__/snipper_main.cpython-38.pyc b/src/snipper/__pycache__/snipper_main.cpython-38.pyc index 44240208f1eec24853d05c9805203b70e188365e..bf692bfec5b10c0f396eee21b21d79be8ddc833f 100644 Binary files a/src/snipper/__pycache__/snipper_main.cpython-38.pyc and b/src/snipper/__pycache__/snipper_main.cpython-38.pyc differ diff --git a/src/snipper/block_parsing.py b/src/snipper/block_parsing.py index 23d4271adbc88234cab2988ae60c4568e2472126..27ca1b91d2f8b21173ffa5186bf863c83cf73672 100644 --- a/src/snipper/block_parsing.py +++ b/src/snipper/block_parsing.py @@ -31,8 +31,6 @@ def block_join(contents): def block_split(lines, tag): stag = tag[:2] # Start of any next tag. - - contents = {} i, j = f2(lines, tag) @@ -80,7 +78,7 @@ def block_split(lines, tag): def argpost(line, j): nx_tag = line.find(stag, j+1) - arg1 = line[j+len(tag):nx_tag] + arg1 = line[j+len(tag):nx_tag] if nx_tag >= 0 else line[j+len(tag):] if nx_tag >= 0: post = line[nx_tag:] else: diff --git a/src/snipper/fix_bf.py b/src/snipper/fix_bf.py index c4d5254adb5e889060e50488e132291397c82700..d06de2f5444ac39777bcd65bae9662c39e068e45 100644 --- a/src/snipper/fix_bf.py +++ b/src/snipper/fix_bf.py @@ -4,7 +4,7 @@ from snipper.block_parsing import indent from snipper.block_parsing import block_split, block_join -def fix_f(lines, debug): +def fix_f(lines, debug, keep=False): lines2 = [] i = 0 while i < len(lines): @@ -20,31 +20,54 @@ def fix_f(lines, debug): if j+1 == len(lines) or ( jid < len(id) and len(lines[j].strip() ) > 0): break - if len(lines[j-1].strip()) == 0: + if len(lines[j-1].strip()) == 0: # Visual aid? j = j - 1 funbody = "\n".join( lines[i+1:j] ) if i == j: raise Exception("Empty function body") i = j + # print(len(funbody)) + # print("fun body") + # print(funbody) comments, funrem = gcoms(funbody) - comments = [id + c for c in comments] - if len(comments) > 0: - lines2 += comments[0].split("\n") + # print(len(comments.splitlines()) + len(funrem)) + # comments = [id + c for c in comments] + for c in comments: + lines2 += c.split("\n") + # print(funrem) f = [id + l.strip() for l in funrem.splitlines()] f[0] = f[0] + "#!b" errm = l_head if len(l_head) > 0 else "Implement function body" - f[-1] = f[-1] + f' #!b {errm}' + + """ If function body is more than 1 line long and ends with a return we keep the return. """ + if keep or (len( funrem.strip().splitlines() ) == 1 or not f[-1].strip().startswith("return ")): + f[-1] = f[-1] + f' #!b {errm}' + else: + f[-2] = f[-2] + f' #!b {errm}' lines2 += f + i = len(lines2) else: lines2.append(l) i += 1 + + if len(lines2) != len(lines) and keep: + print("Very bad. The line length is changing.") + print(len(lines2), len(lines)) + for k in range(len(lines2)): + l2 = (lines2[k] +" "*1000)[:40] + l1 = (lines[k] + " " * 1000)[:40] + + print(l1 + " || " + l2) + assert False + return lines2 # stats = {'n': 0} def _block_fun(lines, start_extra, end_extra, keep=False, silent=False): id = indent(lines[0]) - lines = lines[1:] if len(lines[0].strip()) == 0 else lines - lines = lines[:-1] if len(lines[-1].strip()) == 0 else lines + if not keep: + lines = lines[1:] if len(lines[0].strip()) == 0 else lines + lines = lines[:-1] if len(lines[-1].strip()) == 0 else lines cc = len(lines) ee = end_extra.strip() if len(ee) >= 2 and ee[0] == '"': @@ -59,13 +82,9 @@ def _block_fun(lines, start_extra, end_extra, keep=False, silent=False): else: l2 = ([id + start_extra] if len(start_extra) > 0 else []) + [id + f"# TODO: {cc} lines missing.", id + f'raise NotImplementedError("{ee}")'] - - # stats['n'] += cc return l2, cc - - -def fix_b2(lines, keep=False): +def fix_b(lines, keep=False): cutout = [] n = 0 while True: diff --git a/src/snipper/legacy.py b/src/snipper/legacy.py index d52e5ab2de07ed264b85fca7a9e6f3b6dc0bf5e0..6d3f4ece9a92f6b6b7c659589c0cc1c3e9502e5d 100644 --- a/src/snipper/legacy.py +++ b/src/snipper/legacy.py @@ -2,6 +2,38 @@ COMMENT = '"""' def gcoms(s): + lines = s.splitlines() + coms = [] + rem = [] + in_cm = False + for l in lines: + i = l.find(COMMENT) + if i >= 0: + if not in_cm: + in_cm = True + coms.append( [l]) + if l.find(COMMENT, i+len(COMMENT)) > 0: + in_cm = False + else: + coms[-1].append(l) + in_cm = False + else: + if in_cm: + coms[-1].append(l) + else: + rem.append(l) + if sum( map(len, coms) ) + len(rem) != len(lines): + print("Very bad. Comment-lengths change. This function MUST preserve length") + import sys + sys.exit() + + coms = ["\n".join(c) for c in coms] + rem = "\n".join(rem) + return coms, rem + + + + coms = [] while True: i = s.find(COMMENT) diff --git a/src/snipper/snip_dir.py b/src/snipper/snip_dir.py index f264c59b152f3e228b20f8462ef604c881e69f3d..40b084fddc7e4e3b00b9632baf4259ee3a747dda 100644 --- a/src/snipper/snip_dir.py +++ b/src/snipper/snip_dir.py @@ -5,6 +5,7 @@ import time import fnmatch import tempfile + def snip_dir(source_dir, # Sources dest_dir=None, # Will write to this directory output_dir=None, # Where snippets are going to be stored @@ -15,6 +16,7 @@ def snip_dir(source_dir, # Sources license_head=None, ): + if dest_dir == None: dest_dir = tempfile.mkdtemp() print("[snipper]", "no destination dir was specified so using nonsense destination:", dest_dir) @@ -52,46 +54,22 @@ def snip_dir(source_dir, # Sources shutil.copytree(source_dir, dest_dir) time.sleep(0.1) - ls = list(Path(dest_dir).glob('**/*.*')) + ls = list(Path(dest_dir).glob('**/*')) acceptable = [] for l in ls: split = os.path.normpath(os.path.relpath(l, dest_dir)) m = [fnmatch.fnmatch(split, ex) for ex in exclude] acceptable.append( (l, not any(m) )) - # for f,ac in acceptable: - # if not ac: - # print(f) - - # print(acceptable) - # now we have acceptable files in list. - # run_out_dirs = ["./output"] n = 0 - # edirs = [os.path.join(out, f_) for f_ in hw['exclusion']] # Exclude directories on exclude list (speed) - # edirs = {os.path.normpath(os.path.dirname(f_) if not os.path.isdir(f_) else f_) for f_ in edirs} - # edirs.remove(os.path.normpath(out)) for f, accept in acceptable: - if os.path.isdir(f) or not str(f).endswith(".py") or str(f).endswith("_grade.py"): # We only touch .py files. + if os.path.isdir(f) or not str(f).endswith(".py") or str(f).endswith("_grade.py"): continue - # f_dir = os.path.normpath(f if os.path.isdir(f) else os.path.dirname(f)) if accept: - # slist = hw['solutions'] if not CE else [os.path.basename(f)[:-3]] - # base = None - # if students_irlc_tools is not None: - # base = os.path.relpath(str(f), students_irlc_tools + "/..") - # base = base.replace("\\", "/") - - # if "assignments" in str(f) and "_grade.py" in str(f): - # continue - # info = {'new_references': [], 'code_copyright': 'Example student code. This file is automatically generated from the files in the instructor-directory'} - # paths = {} solution_list = [] kwargs = {} - # cut_files = True - # copyright() - - # run_files = True - nrem = censor_file(f, run_files=run_files, run_out_dirs=output_dir, cut_files=cut_files, solution_list=solution_list, + nrem = censor_file(f, run_files=run_files, run_out_dirs=output_dir, cut_files=cut_files, + solution_list=solution_list, base_path=dest_dir, references=references, license_head=license_head, @@ -105,7 +83,6 @@ def snip_dir(source_dir, # Sources if os.path.isfile(rm_file): os.remove(rm_file) else: - if os.path.isdir(rm_file + "\\"): + if os.path.isdir(rm_file+"/"): shutil.rmtree(rm_file) - return n diff --git a/src/snipper/snipper_main.py b/src/snipper/snipper_main.py index b2f5803f5f52ee03e15a652fa91ad0df6e964a2c..762bfbcbfd7fade3b61ff78540ba6400998037fc 100644 --- a/src/snipper/snipper_main.py +++ b/src/snipper/snipper_main.py @@ -1,12 +1,11 @@ import os import re - from snipper.block_parsing import full_strip from snipper.fix_i import run_i from snipper.fix_r import fix_r from snipper.fix_s import save_s from snipper.fix_cite import fix_citations -from snipper.fix_bf import fix_f, fix_b2 +from snipper.fix_bf import fix_f, fix_b from snipper.fix_o import run_o @@ -23,12 +22,11 @@ def rem_nonprintable_ctrl_chars(txt): def censor_code(lines, keep=True): dbug = True - lines = fix_f(lines, dbug) - lines, nB, cut = fix_b2(lines, keep=True) + lines = fix_f(lines, dbug, keep=keep) + lines, nB, cut = fix_b(lines, keep=keep) return lines - def censor_file(file, run_files=True, run_out_dirs=None, cut_files=True, solution_list=None, censor_files=True, base_path=None, @@ -66,36 +64,36 @@ def censor_file(file, run_files=True, run_out_dirs=None, cut_files=True, solutio run_o(lines, file=file, output=ofiles[0]) run_i(lines, file=file, output=ofiles[0]) if cut_files: - save_s(lines, file_path=os.path.relpath(file, base_path), output_dir=run_out_dirs) # save file snips to disk + save_s(lines, file_path=os.path.relpath(file, base_path), output_dir=run_out_dirs) lines = full_strip(lines, ["#!s", "#!o", '#!i']) if censor_files: lines = fix_f(lines, dbug) - lines, nB, cut = fix_b2(lines) + lines, nB, cut = fix_b(lines) else: nB = 0 lines = fix_r(lines) - if censor_files and len(cut) > 0 and solution_list is not None: - fname = file.__str__() - i = fname.find("irlc") - # wk = fname[i+5:fname.find("\\", i+6)] - # sp = paths['02450students'] +"/solutions/" - # if not os.path.exists(sp): - # os.mkdir(sp) - # sp = sp + wk - # if not os.path.exists(sp): - # os.mkdir(sp) - sols = [] - stext = ["\n".join(lines) for lines in cut] - for i,sol in enumerate(stext): - sols.append( (sol,) ) - # sout = sp + f"/{os.path.basename(fname)[:-3]}_TODO_{i+1}.py" - # wsol = any([True for s in solution_list if os.path.basename(sout).startswith(s)]) - # print(sout, "(published)" if wsol else "") - # if wsol: - # with open(sout, "w") as f: - # f.write(sol) + # if censor_files and len(cut) > 0 and solution_list is not None: + # # fname = file.__str__() + # # i = fname.find("irlc") + # # wk = fname[i+5:fname.find("\\", i+6)] + # # sp = paths['02450students'] +"/solutions/" + # # if not os.path.exists(sp): + # # os.mkdir(sp) + # # sp = sp + wk + # # if not os.path.exists(sp): + # # os.mkdir(sp) + # sols = [] + # stext = ["\n".join(lines) for lines in cut] + # # for i,sol in enumerate(stext): + # # # sols.append( (sol,) ) + # # # sout = sp + f"/{os.path.basename(fname)[:-3]}_TODO_{i+1}.py" + # # # wsol = any([True for s in solution_list if os.path.basename(sout).startswith(s)]) + # # # print(sout, "(published)" if wsol else "") + # # # if wsol: + # # # with open(sout, "w") as f: + # # # f.write(sol) if len(lines) > 0 and len(lines[-1])>0: lines.append("") @@ -104,7 +102,6 @@ def censor_file(file, run_files=True, run_out_dirs=None, cut_files=True, solutio if license_head is not None: s2 = fix_copyright(s2, license_head) - with open(file, 'w', encoding='utf-8') as f: f.write(s2) return nB @@ -112,5 +109,4 @@ def censor_file(file, run_files=True, run_out_dirs=None, cut_files=True, solutio def fix_copyright(s, license_head): return "\n".join( ["# " + l.strip() for l in license_head.splitlines()] ) +"\n" + s - -# lines: 294, 399, 420, 116 \ No newline at end of file +# lines: 294, 399, 420, 116 diff --git a/src/snipper/version.py b/src/snipper/version.py new file mode 100644 index 0000000000000000000000000000000000000000..283b03a075bdc493860ebb799f4b2f3ee2a72d87 --- /dev/null +++ b/src/snipper/version.py @@ -0,0 +1 @@ +__version__ = "0.1.7" \ No newline at end of file