Refactor dir snip

example/output/load_references_a.py → examples/output/load_references_a.py

 # load_references.py
-    from snipper.citations import get_bibtex, get_aux 
+    from snipper.load_citations import get_bibtex, get_aux
     bibfile = "latex/library.bib"
     auxfile = 'latex/index.aux'
     bibtex = get_bibtex(bibfile)

examples/process_cs101.py

+def main():
+    from snipper.snip_dir import snip_dir
+    from snipper.load_citations import get_aux, get_bibtex
+    bibfile = get_bibtex('latex/library.bib')
+    auxfile = get_aux('latex/index.aux')
+
+    references = dict(bibtex=bibfile,
+                      aux=auxfile,
+                      commands=[
+                          dict(command='\\aref2', output="(Assignment 2, \\ref{%s})", aux=auxfile),
+                          dict(command='\\href', output="\cite[\\ref{%s}]{herlau}", aux=auxfile),
+                        ]
+                      )
+
+
+    output = snip_dir(source_dir="./cs101_instructorm", dest_dir="./cs101_students", output_dir="./cs101_output",
+             references=references,
+             )
+
+    a = 234
+
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

src/codesnipper.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: codesnipper
-Version: 0.0.1
+Version: 0.0.2
 Summary: A lightweight framework for censoring student solutions files and extracting code + output
 Home-page: https://lab.compute.dtu.dk/tuhe/snipper
 Author: Tue Herlau
 Author-email: tuhe@dtu.dk
-License: UNKNOWN
+License: MIT
 Project-URL: Bug Tracker, https://lab.compute.dtu.dk/tuhe/snipper/issues
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
@@ -15,8 +15,158 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 
-# Snip
-A lightweight framework for removing code from student solutions. Currently used at DTU. 
+# Snipper
+A lightweight framework for removing code from student solutions.
+## Installation
+```console
+pip install codesnipper
+```
+## What it does
+This project address the following three challenges for administering a python-based course
 
+ - You need to maintain a (working) version for debugging as well as a version handed out to students (with code missing)
+ - You ideally want to make references in source code to course material *"(see equation 2.1 in exercise 5)"* but these tend to go out of date
+ - You want to include code snippets and code output in lectures notes/exercises/beamer slides
+ - You want to automatically create student solutions
 
+This framework address these problems and allow you to maintain a **single**, working project repository. 
+
+The project is currently used in **02465** at DTU. An example of student code can be found at:
+ - https://gitlab.gbar.dtu.dk/02465material/02465students/blob/master/irlc/ex02/dp.py
+
+A set of lectures notes where all code examples/output are automatically generated from the working repository can be found a
+- https://lab.compute.dtu.dk/tuhe/books (see **Sequential decision making**)
+ 
+## How it works
+The basic functionality is quite simple. You start with your working script in your private repository and add special tags to the script. 
+In this case I have added the tags `#!b` (cut a block) and `#!f` (cut function scope). 
+```python
+def myfun(): #!f The error I am going to raise
+    """ The function docstring will not be removed"""
+    print("This is a function")
+    return 42
+    
+def a_long_function():
+    a = 234
+    print("a line")
+    print("a line") #!b
+    print("a line")
+    print("a line") #!b Insert three missing print statements. 
+    print("a line")
+    return a
+    
+if __name__ == "__main__":
+    myfun()
+```
+This will produce the following file:
+```python
+def myfun():
+    """ The function docstring will not be removed"""
+    # TODO: 2 lines missing.
+    raise NotImplementedError("The error I am going to raise")
+    
+def a_long_function():
+    a = 234
+    print("a line")
+    # TODO: 3 lines missing.
+    raise NotImplementedError("Insert three missing print statements.")
+    print("a line")
+    return a
+    
+if __name__ == "__main__":
+    myfun()
+```
+You can also use the framework to capture code snippets, outputs and interactive python output. 
+To do this, save the following in `foo.py`
+```python
+def myfun(): #!s This snippet will be saved to foo.py in the output directory. 
+    print("Hello") #!s 
+
+print("Do not capture me") 
+for i in range(4): #!o
+    print("Output", i)
+print("Goodbuy world") #!o
+print("don't capture me")
+
+# Interactive pythong example
+print("Hello World") #!i #!i # this is a single-line cutout.
+````
+These block-tags will create a file `foo.py` (in the output directory) containing
+```python
+def myfun():
+    print("Hello") 
+```
+A file `foo.txt` containing the captured output
+```txt
+Output 0
+Output 1
+Output 2
+Output 3
+Goodbuy world
+```
+and a typeset version of an interactive python session in `foo.pyi` (use `pycon` in minted; this gitlab server appears not to support `pycon`)
+```pycon
+>>> print("hello world")
+Hello World"
+```
+All these files can be directly imported into `LaTeX` using e.g. `minted`: You never need to mix `LaTeX` code and python again!
+
+
+## References: 
+Bibliography references can be loaded from `references.bib`-files and in-document references from the `.aux` file. 
+For this example, we will insert references shown in the `examples/latex/index.tex`-document. To do so, we can use these tags:
+```python
+def myfun(): #!s
+    """
+    To solve this exercise, look at \ref{eq1} in \ref{sec1}.
+    You can also look at \cite{bertsekasII} and \cite{herlau}
+    More specifically, look at \cite[Equation 117]{bertsekasII} and \cite[\ref{fig1}]{herlau}
+
+    We can also write a special tag to reduce repetition: \nref{fig1} and \nref{sec1}.
+    """
+    return 42 #!s
+
+```
+We can manually compile this example by first loading the aux-files and the bibliographies as follows:
+```python 
+# load_references.py
+    from snipper.citations import get_bibtex, get_aux 
+    bibfile = "latex/library.bib"
+    auxfile = 'latex/index.aux'
+    bibtex = get_bibtex(bibfile)
+    aux = get_aux(auxfile) 
+```
+Next, we load the python file containing the reference code and fix all references based on the aux and bibliography data. 
+```python 
+# load_references.py
+    file = "citations.py" 
+    with open(file, 'r') as f:
+        lines = f.read().splitlines()
+    lines = fix_aux(lines, aux=aux)
+    lines = fix_aux_special(lines, aux=aux, command='\\nref', bibref='herlau')
+    lines = fix_bibtex(lines, bibtex=bibtex)
+    with open('output/citations.py', 'w') as f:
+        f.write("\n".join(lines)) 
+```
+The middle command is a convenience feature: It allows us to specify a special citation command `\nref{..}` which always compiles to `\cite[\ref{...}]{herlau}`. This is useful if e.g. `herlau` is the bibtex key for your lecture notes. The result is as follows:
+```python 
+"""
+References:
+  [Ber07] Dimitri P. Bertsekas. Dynamic Programming and Optimal Control, Vol. II. Athena Scientific, 3rd edition, 2007. ISBN 1886529302.
+  [Her21] Tue Herlau. Sequential decision making. (See 02465_Notes.pdf), 2021.
+"""
+def myfun(): #!s
+    """
+    To solve this exercise, look at eq. (1) in Section 1.
+    You can also look at (Ber07) and (Her21)
+    More specifically, look at (Ber07, Equation 117) and (Her21, Figure 1)
+
+    We can also write a special tag to reduce repetition: (Her21, Figure 1) and (Her21, Section 1).
+    """
+    return 42 #!s
+```
+Note this example uses the low-level api. Normally you would just pass the bibtex and aux-file to the main censor-file command.
+
+## Additional features:
+-  You can name tags using `#!s=bar` to get a `foo_bar.py` snippet. This is useful when you need to cut multiple sessions. This also works for the other tags. 
 

src/codesnipper.egg-info/SOURCES.txt

@@ -9,5 +9,8 @@ src/codesnipper.egg-info/dependency_links.txt
 src/codesnipper.egg-info/requires.txt
 src/codesnipper.egg-info/top_level.txt
 src/snipper/__init__.py
+src/snipper/citations.py
+src/snipper/fix_cite.py
+src/snipper/fix_s.py
 src/snipper/snip_dir.py
-src/snipper/snipper.py
\ No newline at end of file
+src/snipper/snipper_main.py
\ No newline at end of file

src/codesnipper.egg-info/requires.txt

-jinja2
+pexpect
+wexpect
+pybtex

src/snipper/block_parsing.py

+def f2(lines, tag, i=0, j=0):
+    for k in range(i, len(lines)):
+        index = lines[k].find(tag, j if k == i else 0)
+        if index >= 0:
+            return k, index
+    return None, None
+
+
+def block_iterate(lines, tag):
+    contents = {'joined': lines}
+    while True:
+        contents = block_split(contents['joined'], tag)
+        if contents is None:
+            break
+
+        yield  contents
+
+
+def block_split(lines, tag):
+    stag = tag[:2]  # Start of any next tag.
+
+    def join(contents):
+        return contents['first'] + [contents['block'][0] + contents['post1']] + contents['block'][1:-1] \
+                + [contents['block'][-1] + contents['post2']] + contents['last']
+    contents = {}
+    i, j = f2(lines, tag)
+
+    def get_tag_args(line):
+        k = line.find(" ")
+        tag_args = (line[:k + 1] if k >= 0 else line)[len(tag):]
+        if len(tag_args) == 0:
+            return {'': ''}  # No name.
+        tag_args = dict([t.split("=") for t in tag_args.split(";")])
+        return tag_args
+
+    if i is None:
+        return None
+    else:
+        start_tag_args = get_tag_args(lines[i][j:])
+        START_TAG = f"{tag}={start_tag_args['']}" if '' in start_tag_args else tag
+        END_TAG = START_TAG
+        i2, j2 = f2(lines, END_TAG, i=i, j=j+1)
+        if i2 == None:
+            END_TAG = tag
+            i2, j2 = f2(lines, END_TAG, i=i, j=j+1)
+        if i2 == None:
+            print("\n".join( lines[i:]))
+            raise Exception("Did not find matching tag", tag)
+
+
+    if i == i2:
+        # Splitting a single line. To reduce confusion, this will be treated slightly differently:
+        l2 = lines[:i] + [lines[i][:j2], lines[i][j2:]] + lines[i2+1:]
+        c2 = block_split(l2, tag=tag)
+        c2['block'].pop()
+        c2['joined'] = join(c2)
+        return c2
+    else:
+        contents['first'] = lines[:i]
+        contents['last'] = lines[i2+1:]
+
+        def argpost(line, j):
+            nx_tag = line.find(stag, j+1)
+            arg1 = line[j+len(tag):nx_tag]
+            if nx_tag >= 0:
+                post = line[nx_tag:]
+            else:
+                post = ''
+            return arg1, post
+
+        contents['arg1'], contents['post1'] = argpost(lines[i], j)
+        contents['arg2'], contents['post2'] = argpost(lines[i2], j2)
+        blk = [lines[i][:j]] + lines[i+1:i2] + [lines[i2][:j2]]
+        contents['block'] = blk
+        contents['joined'] = join(contents)
+        contents['start_tag_args'] = start_tag_args
+        contents['name'] = start_tag_args['']
+        return contents
\ No newline at end of file

src/snipper/fix_cite.py

-from snipper.citations import find_tex_cite
+from snipper.load_citations import find_tex_cite
 from snipper.snipper_main import COMMENT
 
-def fix_references(lines, info, strict=True):
-    for cmd in info['new_references']:
-        lines = fix_single_reference(lines, cmd, info['new_references'][cmd], strict=strict)
-    return lines
 
-def fix_single_reference(lines, cmd, aux, strict=True):
-    references = aux
-    s = "\n".join(lines)
-    i = 0
-    while True:
-        (i, j), reference, txt = find_tex_cite(s, start=i, key=cmd)
-        if i < 0:
-            break
-        if reference not in references:
-            er = "cref label not found for label: " + reference
-            if strict:
-                raise IndexError(er)
-            else:
-                print(er)
-                continue
-        r = references[reference]
-        rtxt = r['nicelabel']
-        s = s[:i] + rtxt + s[j + 1:]
-        i = i + len(rtxt)
-        print(cmd, rtxt)
-
-    lines = s.splitlines(keepends=False)
-    return lines
+def fix_citations():
+    # This should be the master function.
+    pass
 
 
 def fix_aux_special(lines, aux, command='\\nref', bibref='herlau'):
@@ -71,3 +47,34 @@ def fix_bibtex(lines, bibtex):
 
     # s = s.replace(cpr, info['code_copyright'])
     return s.splitlines()
+
+
+
+def fix_references(lines, info, strict=True):
+    for cmd in info['new_references']:
+        lines = fix_single_reference(lines, cmd, info['new_references'][cmd], strict=strict)
+    return lines
+
+def fix_single_reference(lines, cmd, aux, strict=True):
+    references = aux
+    s = "\n".join(lines)
+    i = 0
+    while True:
+        (i, j), reference, txt = find_tex_cite(s, start=i, key=cmd)
+        if i < 0:
+            break
+        if reference not in references:
+            er = "cref label not found for label: " + reference
+            if strict:
+                raise IndexError(er)
+            else:
+                print(er)
+                continue
+        r = references[reference]
+        rtxt = r['nicelabel']
+        s = s[:i] + rtxt + s[j + 1:]
+        i = i + len(rtxt)
+        print(cmd, rtxt)
+
+    lines = s.splitlines(keepends=False)
+    return lines
\ No newline at end of file

src/snipper/fix_s.py

-import functools
 from collections import defaultdict
-from snipper.snipper_main import full_strip, block_process
-
-def f2(lines, tag, i=0, j=0):
-    for k in range(i, len(lines)):
-        index = lines[k].find(tag, j if k == i else 0)
-        if index >= 0:
-            return k, index
-    return None, None
-
-def block_iterate(lines, tag):
-    contents = {'joined': lines}
-    while True:
-        contents = block_split(contents['joined'], tag)
-        if contents is None:
-            break
-
-        yield  contents
-
-def block_split(lines, tag):
-    stag = tag[:2]  # Start of any next tag.
-
-    def join(contents):
-        return contents['first'] + [contents['block'][0] + contents['post1']] + contents['block'][1:-1] \
-                + [contents['block'][-1] + contents['post2']] + contents['last']
-    contents = {}
-    i, j = f2(lines, tag)
-
-    def get_tag_args(line):
-        k = line.find(" ")
-        tag_args = (line[:k + 1] if k >= 0 else line)[len(tag):]
-        if len(tag_args) == 0:
-            return {'': ''}  # No name.
-        tag_args = dict([t.split("=") for t in tag_args.split(";")])
-        return tag_args
-
-    if i is None:
-        return None
-    else:
-        start_tag_args = get_tag_args(lines[i][j:])
-        START_TAG = f"{tag}={start_tag_args['']}" if '' in start_tag_args else tag
-        END_TAG = START_TAG
-        i2, j2 = f2(lines, END_TAG, i=i, j=j+1)
-        if i2 == None:
-            END_TAG = tag
-            i2, j2 = f2(lines, END_TAG, i=i, j=j+1)
-
-    if i == i2:
-        # Splitting a single line. To reduce confusion, this will be treated slightly differently:
-        l2 = lines[:i] + [lines[i][:j2], lines[i][j2:]] + lines[i2+1:]
-        c2 = block_split(l2, tag=tag)
-        c2['block'].pop()
-        c2['joined'] = join(c2)
-        return c2
-    else:
-        contents['first'] = lines[:i]
-        contents['last'] = lines[i2+1:]
-
-        def argpost(line, j):
-            nx_tag = line.find(stag, j+1)
-            arg1 = line[j+len(tag):nx_tag]
-            if nx_tag >= 0:
-                post = line[nx_tag:]
-            else:
-                post = ''
-            return arg1, post
-
-        contents['arg1'], contents['post1'] = argpost(lines[i], j)
-        contents['arg2'], contents['post2'] = argpost(lines[i2], j2)
-        blk = [lines[i][:j]] + lines[i+1:i2] + [lines[i2][:j2]]
-        contents['block'] = blk
-        contents['joined'] = join(contents)
-        contents['start_tag_args'] = start_tag_args
-        contents['name'] = start_tag_args['']
-        return contents
+import os
+from snipper.block_parsing import block_iterate
 
 
 def get_s(lines):
@@ -86,8 +13,6 @@ def get_s(lines):
         output[name] = [l for c in co for l in c['block']]
     return output
 
-import os
-
 def save_s(lines, output_dir, file_path): # save file snips to disk
     content = get_s(lines)
     if not os.path.isdir(output_dir):
@@ -99,30 +24,6 @@ def save_s(lines, output_dir, file_path): # save file snips to disk
         with open(output_dir + "/" + os.path.basename(file_path)[:-3] + ("_" + name if len(name) > 0 else name) + ".py", 'w') as f:
             f.write(out)
 
-        # out = "\n".join([f"# {include_path_base}"] + ["\n".join(v[1]) for v in c if v[0] == outf])
-        # with open(outf, 'w') as f:
-        #     f.write(out)
-
-    # def block_fun(lines, start_extra, end_extra, art, output, **kwargs):
-    #     outf = output + ("_" + art if art is not None and len(art) > 0 else "") + ".py"
-    #     lines = full_strip(lines)
-    #     return lines, [outf, lines]
-    # try:
-    #     a,b,c,_ = block_process(lines, tag="#!s", block_fun=functools.partial(block_fun, output=output))
-    #     if len(c)>0:
-    #         kvs= { v[0] for v in c}
-    #         for outf in kvs:
-    #             out = "\n".join([f"# {include_path_base}"]  + ["\n".join(v[1]) for v in c if v[0] == outf] )
-    #             with open(outf, 'w') as f:
-    #                 f.write(out)
-    #
-    # except Exception as e:
-    #     print("lines are")
-    #     print("\n".join(lines))
-    #     print("Bad thing in #!s command in file", file)
-    #     raise e
-    # return lines
-
 s1 = """
 L1
 L2 #!s=a
@@ -147,5 +48,3 @@ if __name__ == "__main__":
     # contents = block_split(s1.splitlines(), tag="#!s")
     # contents = block_split(contents['joined'], tag="#!s")
     # lines2 = contents['first'] +
-    a = 234
-    pass
\ No newline at end of file

src/snipper/snip_dir.py

@@ -5,7 +5,15 @@ import time
 import fnmatch
 
 
-def snip_dir(source_dir, dest_dir, exclude=None, clean_destination_dir=True):
+def snip_dir(source_dir,  # Sources
+             dest_dir,  # Will write to this directory
+             output_dir=None, # Where snippets are going to be stored
+             references=None, # Reference database
+             exclude=None, clean_destination_dir=True):
+
+    if references == None:
+        references = dict(aux=None, bibtex=None,  commands=[])
+
     if exclude == None:
         exclude = []
     if not os.path.exists(dest_dir):

src/snipper/snipper_main.py

@@ -315,7 +315,6 @@ def censor_code(lines, keep=True):
 
 
 
-
 def censor_file(file, info, paths, run_files=True, run_out_dirs=None, cut_files=True, solution_list=None,
                 censor_files=True,
                 include_path_base=None,