Refactor citation + tags and update README.md

examples/b_example.py

@@ -17,12 +17,15 @@ def primes_sieve(limit):
 
 # Example use: print(primes_sieve(42)) #!s
 
-
 def wspace(l):
     whitespace = " " * (len(l) - len(l.lstrip()))
     return whitespace
 
-
+def cmnt(lines):
+    whitespace = " " * (len(lines[0]) - len(lines[0].lstrip()))
+    lines = textwrap.dedent("\n".join(lines)).splitlines()
+    lines = ["# " + l for l in lines]
+    return lines, whitespace
 
 def obscure(blk, fun, outfile):
     blok = block_split(blk, "#!b")
@@ -35,27 +38,24 @@ def obscure(blk, fun, outfile):
 
 # driver program
 if __name__ == '__main__':
+    # Read the code using the #!s-block-tag.
     with open(__file__, 'r') as f:
         s = f.read().splitlines()
     blk = get_s(s)['']
 
+    # Save it in output for the readme.
     with open("cs101_output/sieve.py", 'w') as f:
         f.write('\n'.join(blk))
 
-    def cmnt(lines):
-        whitespace = " " * (len(lines[0]) - len(lines[0].lstrip()))
-        lines = textwrap.dedent("\n".join(lines)).splitlines()
-        lines = ["# " + l for l in lines]
-        return lines, whitespace
-
+    # Example 1: Simply permute the lines
     def f1(lines):
         lines, whitespace = cmnt(lines)
         lines = [lines[i] for i in np.random.permutation(len(lines))]
         lines = textwrap.indent("\n".join(lines), whitespace).splitlines()
         return lines
-
     obscure(blk, f1, 'cs101_output/obscure_1.py')
 
+    # Example 2: Try to preserve keywords and special syntax symbols
     def f2(lines):
         lines, whitespace = cmnt(lines)
         kp = """#'"[](){},.+-012345679:="""
@@ -72,14 +72,13 @@ if __name__ == '__main__':
         lines = l2
         lines = textwrap.indent("\n".join(lines), whitespace).splitlines()
         return lines
-
     obscure(blk, f2, 'cs101_output/obscure_2.py')
 
+    # Example 3: keep half of the lines
     def f3(lines):
         lines = [ (l.strip(), len(l.strip()), wspace(l)) for l in lines ]
         lines = [ wp + l[:k//2] + "?"*(k-k//2) for l, k, wp in lines]
         lines, whitespace = cmnt(lines)
         lines = textwrap.indent("\n".join(lines), whitespace).splitlines()
         return lines
-
     obscure(blk, f3, 'cs101_output/obscure_3.py')

setup.py

@@ -14,7 +14,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setuptools.setup(
     name="codesnipper",
-    version="0.0.2",
+    version="0.1.0",
     author="Tue Herlau",
     author_email="tuhe@dtu.dk",
     description="A lightweight framework for censoring student solutions files and extracting code + output",

src/codesnipper.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: codesnipper
-Version: 0.0.2
+Version: 0.1.0
 Summary: A lightweight framework for censoring student solutions files and extracting code + output
 Home-page: https://lab.compute.dtu.dk/tuhe/snipper
 Author: Tue Herlau
@@ -37,136 +37,356 @@ The project is currently used in **02465** at DTU. An example of student code ca
 A set of lectures notes where all code examples/output are automatically generated from the working repository can be found a
 - https://lab.compute.dtu.dk/tuhe/books (see **Sequential decision making**)
  
-## How it works
-The basic functionality is quite simple. You start with your working script in your private repository and add special tags to the script. 
-In this case I have added the tags `#!b` (cut a block) and `#!f` (cut function scope). 
-```python
-def myfun(): #!f The error I am going to raise
-    """ The function docstring will not be removed"""
-    print("This is a function")
-    return 42
-    
-def a_long_function():
-    a = 234
-    print("a line")
-    print("a line") #!b
-    print("a line")
-    print("a line") #!b Insert three missing print statements. 
-    print("a line")
-    return a
+# Usage
+All examples can be found in the `/examples` directory. The idea is all our (complete) files are found in the instructor directory and snipper keeps everything up-to-date:
+```text
+examples/cs101_instructor  # This directory contains the (hidden) instructor files. You edit these
+examples/cs101_students    # This directory contains the (processed) student files. Don't edit these
+examples/cs101_output      # This contains automatically generated contents (snippets, etc.).
+```
+The basic functionality is you insert special comment tags in your source, such as `#!b` or `#!s` and the script then process
+the sources based on the tags.  The following will show most basic usages:
 
+## The #f!-tag
+Let's start with the simplest example, blocking out a function (see `examples/cs101_instructor/f_tag.py`; actually it will work for any scope)
+You insert a comment like: `#!f <exception message>` like so:
+```python
+def myfun(a,b): #!f return the sum of a and b
+    """ The doc-string is not removed. """
+    sm = a+b
+    return sm
+```
+To compile this (and all other examples) use the script `examples/process_cs101.py`
+```python
 if __name__ == "__main__":
-    myfun()
+    from snipper.snip_dir import snip_dir
+    snip_dir("./cs101_instructor", "./cs101_students", output_dir="./cs101_output")
 ```
-This will produce the following file:
+The output can be found in `examples/students/f_tag.py`. It will cut out the body of the function but leave any return statement and docstrings. It will also raise an exception (and print how many lines are missing) to help students.
 ```python
-def myfun():
-    """ The function docstring will not be removed"""
-    # TODO: 2 lines missing.
-    raise NotImplementedError("The error I am going to raise")
-    
-def a_long_function():
-    a = 234
-    print("a line")
-    # TODO: 3 lines missing.
-    raise NotImplementedError("Insert three missing print statements.")
-    print("a line")
-    return a
+def myfun(a,b): 
+    """ The doc-string is not removed. """
+    # TODO: 1 lines missing.
+    raise NotImplementedError("return the sum of a and b")
+    return sm
+```
 
-if __name__ == "__main__":
-    myfun()
+## The #b!-tag
+The #!b-tag allows you more control over what is cut out. The instructor file:
+```python
+def primes_sieve(limit):
+    limitn = limit+1 #!b
+    primes = range(2, limitn)
+    for i in primes:
+        factors = list(range(i, limitn, i))
+        for f in factors[1:]:
+            if f in primes:
+                primes.remove(f) #!b Compute the list `primes` here of all primes up to `limit`
+    return primes
+width, height = 2, 4
+print("Area of square of width", width, "and height", height, "is:")
+print(width*height) #!b #!b Compute and print area here
+print("and that is a fact!")
 ```
-You can also use the framework to capture code snippets, outputs and interactive python output. 
-To do this, save the following in `foo.py`
+Is compiled into:
 ```python 
-def myfun(): #!s This snippet will be saved to foo.py in the output directory. 
-    print("Hello") #!s 
+def primes_sieve(limit):
+    # TODO: 8 lines missing.
+    raise NotImplementedError("Compute the list `primes` here of all primes up to `limit`")
+    return primes
+width, height = 2, 4
+print("Area of square of width", width, "and height", height, "is:")
+# TODO: 1 lines missing.
+raise NotImplementedError("Compute and print area here")
+print("and that is a fact!")
+```
+This allows you to cut out text across scopes, but still allows you to insert exceptions. 
+
 
-print("Do not capture me") 
-for i in range(4): #!o
-    print("Output", i)
-print("Goodbuy world") #!o
-print("don't capture me")
 
-# Interactive pythong example
-print("Hello World") #!i #!i # this is a single-line cutout.
-````
-These block-tags will create a file `foo.py` (in the output directory) containing
+## The #s!-tag
+The #!s-tag is useful for making examples to include in exercises and lecture notes. The #!s (snip) tag cuts out the text between 
+tags and places it in files found in the output-directory. As an example, here is the instructor file:
 ```python
-def myfun():
-    print("Hello") 
+width, height = 2, 4
+print("Area of square of width", width, "and height", height, "is:") #!s
+print(width*height)  #!s  # This is an example of a simple cutout
+print("and that is a fact!")
+print("An extra cutout") #!s #!s  # This will be added to the above cutout
+def primes_sieve(limit): #!s=a # A named cutout
+    limitn = limit+1
+    primes = range(2, limitn)
+    for i in primes: #!s=b A nested/named cutout.
+        factors = list(range(i, limitn, i))
+        for f in factors[1:]:
+            if f in primes:
+                primes.remove(f)  #!s=b
+    return primes #!s=a
 ```
-A file `foo.txt` containing the captured output
-```txt
-Output 0
-Output 1
-Output 2
-Output 3
-Goodbuy world
+Note it allows 
+ - naming using the #!s=<name> command 
+ - automatically join snippets with the same name (useful to cut out details)
+ - The named tags will be matched, and do not have to strictly contained in each other
+
+This example will produce three files
+`cs101_output/s_tag.py`, `cs101_output/s_tag_a.py`, and `cs101_output/s_tag_b.py` containing the output:
+```python 
+# s_tag.py
+print("Area of square of width", width, "and height", height, "is:") 
+print(width*height)  
+print("An extra cutout")
 ```
-and a typeset version of an interactive python session in `foo.pyi` (use `pycon` in minted; this gitlab server appears not to support `pycon`)
-```pycon
->>> print("hello world")
-Hello World"
+and 
+```python 
+# s_tag.py
+def primes_sieve(limit): 
+    limitn = limit+1
+    primes = range(2, limitn)
+    for i in primes: #!s=b A nested/named cutout.
+        factors = list(range(i, limitn, i))
+        for f in factors[1:]:
+            if f in primes:
+                primes.remove(f)  #!s=b
+    return primes
 ```
-All these files can be directly imported into `LaTeX` using e.g. `minted`: You never need to mix `LaTeX` code and python again!
+and finally:
+```python 
+# s_tag.py
+    for i in primes: 
+        factors = list(range(i, limitn, i))
+        for f in factors[1:]:
+            if f in primes:
+                primes.remove(f)
+```
+I recommend using `\inputminted{filename}` to insert the cutouts in LaTeX. 
+
 
+## The #o!-tag
+The #!o-tag allows you to capture output from the code, which can be useful when showing students the expected 
+behavior of their scripts. Like the #!s-tag, the #!o-tags can be named. 
 
-## References: 
-Bibliography references can be loaded from `references.bib`-files and in-document references from the `.aux` file. 
-For this example, we will insert references shown in the `examples/latex/index.tex`-document. To do so, we can use these tags:
+As an example, Consider the instructor file
 ```python
-def myfun(): #!s
-    """
-    To solve this exercise, look at \ref{eq1} in \ref{sec1}.
-    You can also look at \cite{bertsekasII} and \cite{herlau}
-    More specifically, look at \cite[Equation 117]{bertsekasII} and \cite[\ref{fig1}]{herlau}
+if __name__ == "__main__":
+    print("Here are the first 4 square numbers") #!o=a
+    for k in range(1,5):
+        print(k*k, "is a square")
+    #!o=a
+    print("This line will not be part of a cutout.")
+    width, height = 2, 4 #!o=b
+    print("Area of square of width", width, "and height", height, "is:")
+    print(width*height)
+    print("and that is a fact!") #!o=b
+```
+This example will produce two files `cs101_output/o_tag_a.txt`, `cs101_output/o_tag_b.txt`:
+```terminal 
+Here are the first 4 square numbers
+1 is a square
+4 is a square
+9 is a square
+16 is a square
+```
+and 
+```terminal 
+Area of square of width 2 and height 4 is:
+8
+and that is a fact!
+```
 
-    We can also write a special tag to reduce repetition: \nref{fig1} and \nref{sec1}.
-    """
-    return 42 #!s
+## The #i!-tag
+The #!i-tag allows you to create interactive python shell-snippets that can be imported using 
+ the minted `pycon` environment (`\inputminted{python}{input.shell}`).
+ As an example, consider the instructor file
+```python
+for animal in ["Dog", "cat", "wolf"]: #!i=a
+    print("An example of a four legged animal is", animal) #!i=a
+#!i=b
+def myfun(a,b):
+    return a+b
+myfun(3,4) #!i=b
+# Snipper will automatically insert an 'enter' after the function definition.
+```
+This example will produce two files `cs101_output/i_tag_a.shell`, `cs101_output/i_tag_b.shell`:
+```pycon
+>>> for animal in ["Dog", "cat", "wolf"]:
+...     print("An example of a four legged animal is", animal)
+... 
+An example of a four legged animal is Dog
+An example of a four legged animal is cat
+An example of a four legged animal is wolf
+```
+and 
+```pycon 
+>>> def myfun(a,b):
+...     return a+b
+...
+>>> myfun(3,4)
+7
+```
+Note that apparently there 
+ is no library for converting python code to shell sessions so I had to write it myself, which means it can properly get confused with multi-line statements (lists, etc.). On the plus-side, it will automatically insert newlines after the end of scopes. 
+ My parse is also known to be a bit confused if your code outputs `...` since it has to manually parse the interactive python session and this normally indicates a new line. 
+
+## References and citations (`\ref` and `\cite`)
+One of the most annoying parts of maintaining student code is to constantly write "see equation on slide 41 bottom" only to have the reference go stale because slide 23 got removed. Well now anymore, now you can direcly refence anything with a bibtex or aux file!
+Let's consider the following example of a simple document with a couple of references: (see `examples/latex/index.pdf`):
+
+![LaTeX sample](https://gitlab.compute.dtu.dk/tuhe/snipper/-/raw/main/docs/index.png)
 
+The code for this document is:
+```latex
+\documentclass{article}
+\usepackage{url,graphics,rotating,hyperref}
+\usepackage{cleveref}
+\usepackage{showlabels}
+\begin{document}
+\section{First section}\label{sec1}
+Math is hard \cite{bertsekasII,rosolia2018data,herlau}, see also \cref{eq1} and \cref{fig1}.
+\begin{equation}
+	2+2 = 4 \label{eq1}
+\end{equation}
+\begin{figure}\centering
+\includegraphics[width=.8\linewidth]{br}\caption{A figure}\label{fig1}
+\end{figure}
+\bibliographystyle{alpha}
+\bibliography{library}						
+\end{document}
 ``` 
-We can manually compile this example by first loading the aux-files and the bibliographies as follows:
+
+To use the references in code we first have to load the `references.bib` file and the `index.aux`-file and then:
+ - Snipper allows you to directly insert this information using `\cite` and `\ref`
+ - You can also define custom citation command which allows you to reference common sources like 
+   - Lecture notes
+    - Exercise sheets
+    - Slides for a particular week
+
+Let's look at all of these in turn. The file we will consider in the instructor-version looks like this: (`examples/cs101_instructor/references.py`):
 ```python
-# load_references.py
-    from snipper.citations import get_bibtex, get_aux 
-    bibfile = "latex/library.bib"
-    auxfile = 'latex/index.aux'
-    bibtex = get_bibtex(bibfile)
-    aux = get_aux(auxfile) 
+def myfun():
+    """
+    Simple aux references \ref{eq1} in \ref{sec1}.
+    Simple bibtex citations: \cite{bertsekasII} and \cite[Somewhere around the middle]{herlau}
+    Example of custom command (reference notes)
+    > \nref{fig1}
+    Other example of custom command (reference assignment)
+    > \aref2{sec1}
+    """
+    print("See \ref{sec1}")  # Also works.
+    return 42
 ```
-Next, we load the python file containing the reference code and fix all references based on the aux and bibliography data. 
+Note the last parts of the file contains the special commands `\nref` (references to lecture notes) and `\aref2` (assignment 2) which I want to define. 
+This can be done by changing the call to snipper as follows (`examples/process_cs101_references.py`)
 ```python
-# load_references.py
-    file = "citations.py" 
-    with open(file, 'r') as f:
-        lines = f.read().splitlines()
-    lines = fix_aux(lines, aux=aux)
-    lines = fix_aux_special(lines, aux=aux, command='\\nref', bibref='herlau')
-    lines = fix_bibtex(lines, bibtex=bibtex)
-    with open('output/citations.py', 'w') as f:
-        f.write("\n".join(lines)) 
+from snipper.snip_dir import snip_dir
+from snipper.load_citations import get_aux, get_bibtex
+def main():
+    bibfile = get_bibtex('latex/library.bib')
+    auxfile = get_aux('latex/index.aux')
+    references = dict(bibtex=bibfile,
+                      aux=auxfile,
+                      commands=[dict(command='\\aref2', output="(Assignment 2, %s)", aux=auxfile),
+                                dict(command='\\nref', output="\cite[%s]{herlau}", aux=auxfile),
+                               ])
+    snip_dir("./cs101_instructor", "./cs101_students", output_dir="./cs101_output", references=references)
+if __name__ == "__main__":
+    main()
 ```
-The middle command is a convenience feature: It allows us to specify a special citation command `\nref{..}` which always compiles to `\cite[\ref{...}]{herlau}`. This is useful if e.g. `herlau` is the bibtex key for your lecture notes. The result is as follows:
+And this then produce the output:
 ```python
 """
 References:
   [Ber07] Dimitri P. Bertsekas. Dynamic Programming and Optimal Control, Vol. II. Athena Scientific, 3rd edition, 2007. ISBN 1886529302.
   [Her21] Tue Herlau. Sequential decision making. (See 02465_Notes.pdf), 2021.
 """
-def myfun(): #!s
+def myfun():
     """
-    To solve this exercise, look at eq. (1) in Section 1.
-    You can also look at (Ber07) and (Her21)
-    More specifically, look at (Ber07, Equation 117) and (Her21, Figure 1)
-
-    We can also write a special tag to reduce repetition: (Her21, Figure 1) and (Her21, Section 1).
+    Simple aux references eq. (1) in Section 1.
+    Simple bibtex citations: (Ber07) and (Her21, Somewhere around the middle)
+    Example of custom command (reference notes)
+    > (Her21, Figure 1)
+    Other example of custom command (reference assignment)
+    > (Assignment 2, Section 1)
     """
-    return 42 #!s
+    print("See Section 1")  # Also works.
+    return 42
 ```
-Note this example uses the low-level api. Normally you would just pass the bibtex and aux-file to the main censor-file command.
+Since the aux/bibtex databases are just dictionaries it is easy to join them together from different sources. 
+ I have written reference tags to lecture and exercise material as well as my notes and it makes reference management very easy. 
+
 
-## Additional features:
--  You can name tags using `#!s=bar` to get a `foo_bar.py` snippet. This is useful when you need to cut multiple sessions. This also works for the other tags. 
+## Advanced block processing
+The default behavior for code removal (#!b and #!f-tags) is to simply remove the code and insert the number of missing lines.
+We can easily create more interesting behavior. The code for the following example can be found in `examples/b_example.py` and will deal with the following problem:
+```python
+import numpy as np  
+# Implement a sieve here.
+def primes_sieve(limit):
+    limitn = limit+1 #!b
+    primes = range(2, limitn)
+    for i in primes:
+        factors = list(range(i, limitn, i))
+        for f in factors[1:]:
+            if f in primes:
+                primes.remove(f)
+    return primes #!b
+# Example use: print(primes_sieve(42))
+```
+The example shows how we can easily define custom functions for processing the code which is to be removed. 
+All such a function need is to take a list of lines (to be obfuscated) and return a new list of lines (the obfuscated code). 
+A couple of short examples:
+### Permute lines
+This example simple randomly permutes the line and prefix them with a comment tag to ensure the code still compiles
+```python
+import numpy as np  
+# Implement a sieve here.
+def primes_sieve(limit):
+    #             primes.remove(f)
+    #         if f in primes:
+    #     factors = list(range(i, limitn, i))
+    # for i in primes:
+    # 
+    # limitn = limit+1 
+    # return primes 
+    # primes = range(2, limitn)
+    #     for f in factors[1:]:
+# Example use: print(primes_sieve(42)) 
+raise NotImplementedError('Complete the above program')
+```
+### Partial replacement
+This example replaces non-keyword, non-special-symbol parts of the lines:
+```python
+import numpy as np  
+# Implement a sieve here.
+def primes_sieve(limit):
+    # ?????? = ?????+1 
+    # ?????? = ?????(2, ??????)
+    # 
+    # for ? in ??????:
+    #     ??????? = ????(?????(?, ??????, ?))
+    #     for ? in ???????[1:]:
+    #         if ? in ??????:
+    #             ??????.??????(?)
+    # return ?????? 
+# Example use: print(primes_sieve(42)) 
+raise NotImplementedError('Complete the above program')
+```
+
+### Half of the solution
+The final solution display half of the proposed solution:
+```python
+import numpy as np  
+# Implement a sieve here.
+def primes_sieve(limit):
+    # limitn =????????
+    # primes = ran?????????????
+    # 
+    # for i in????????
+    #     factors = list(ra??????????????????
+    #     for f in f???????????
+    #         if f in????????
+    #             primes.r????????
+    # return???????
+# Example use: print(primes_sieve(42)) 
+raise NotImplementedError('Complete the above program')
+```
 

src/codesnipper.egg-info/SOURCES.txt

@@ -9,8 +9,13 @@ src/codesnipper.egg-info/dependency_links.txt
 src/codesnipper.egg-info/requires.txt
 src/codesnipper.egg-info/top_level.txt
 src/snipper/__init__.py
-src/snipper/citations.py
+src/snipper/block_parsing.py
+src/snipper/fix_bf.py
 src/snipper/fix_cite.py
+src/snipper/fix_i.py
+src/snipper/fix_o.py
 src/snipper/fix_s.py
+src/snipper/legacy.py
+src/snipper/load_citations.py
 src/snipper/snip_dir.py
 src/snipper/snipper_main.py
\ No newline at end of file