Updates

src/snipper/block_parsing.py

@@ -2,6 +2,7 @@
 Module for new-style block parsing. Make sure all functions are up-to-date and used.
 """
 
+
 def f2(lines, tag, i=0, j=0):
     for k in range(i, len(lines)):
         index = lines[k].find(tag, j if k == i else 0)
@@ -21,16 +22,6 @@ def block_iterate(lines, tag):
 
 
 def block_split(lines, tag):
-    # print("\n".join(lines))
-    # if tag != "#!s":
-    #     return {}
-
-    if tag =="#!s" and "#!s" in "".join(lines):
-        print("tag is", tag)
-    # else:
-    #     return dict(name="asdfasdf", joined='')
-
-    z = 234
     stag = tag[:2]  # Start of any next tag.
 
     def join(contents):
@@ -47,8 +38,6 @@ def block_split(lines, tag):
         if len(tag_args) == 0:
             return {'': ''}  # No name.
 
-        # print("TAG ARGS")
-        # print(tag_args)
         tag_args = dict([t.split("=") for t in tag_args.split(";")])
         return tag_args
 
@@ -68,7 +57,6 @@ def block_split(lines, tag):
             print("\n".join( lines[i:]))
             raise Exception("Did not find matching tag", tag)
 
-
     if i == i2:
         # Splitting a single line. To reduce confusion, this will be treated slightly differently:
         l2 = lines[:i] + [lines[i][:j2], lines[i][j2:]] + lines[i2+1:]
@@ -97,3 +85,28 @@ def block_split(lines, tag):
         contents['start_tag_args'] = start_tag_args
         contents['name'] = start_tag_args['']
         return contents
+
+
+def indent(l):
+    return l[:len(l) - len(l.lstrip())]
+
+
+def full_strip(lines, tags=None):
+    if tags is None:
+        tags = ["#!s", "#!o", "#!f", "#!b"]
+    for t in tags:
+        lines = strip_tag(lines, t)
+    return lines
+
+
+def strip_tag(lines, tag):
+    lines2 = []
+    for l in lines:
+        dx = l.find(tag)
+        if dx > 0:
+            l = l[:dx]
+            if len(l.strip()) == 0:
+                l = None
+        if l is not None:
+            lines2.append(l)
+    return lines2
\ No newline at end of file

src/snipper/fix_bf.py

 import functools
-
-from snipper.legacy import indent, gcoms, block_process
+from snipper.legacy import gcoms, block_process
+from snipper.block_parsing import indent
 
 
 def fix_f(lines, debug):

src/snipper/fix_cite.py

@@ -51,13 +51,11 @@ def fix_bibtex(lines, bibtex):
     # s = s.replace(cpr, info['code_copyright'])
     return s.splitlines()
 
-
-
-def fix_references(lines, info, strict=True):
-    assert False
-    for cmd in info['new_references']:
-        lines = fix_single_reference(lines, cmd, info['new_references'][cmd], strict=strict)
-    return lines
+# def fix_references(lines, info, strict=True):
+#     assert False
+#     for cmd in info['new_references']:
+#         lines = fix_single_reference(lines, cmd, info['new_references'][cmd], strict=strict)
+#     return lines
 
 def fix_single_reference(lines, cmd, aux, strict=True):
     references = aux

src/snipper/fix_i.py

+import functools
+import textwrap
+from snipper.legacy import block_process
+from snipper.block_parsing import full_strip
+
+
+def run_i(lines, file, output):
+    extra = dict(python=None, output=output, evaluated_lines=0)
+    def block_fun(lines, start_extra, end_extra, art, head="", tail="", output=None, extra=None):
+        outf = output + ("_" + art if art is not None and len(art) > 0 else "") + ".shell"
+        lines = full_strip(lines)
+        s = "\n".join(lines)
+        s.replace("...", "..") # passive-aggressively truncate ... because of #issues.
+        lines = textwrap.dedent(s).strip().splitlines()
+
+        if extra['python'] is None:
+            import os
+            if os.name == 'nt':
+                import wexpect as we
+            else:
+                import pexpect as we
+            an = we.spawn("python", encoding="utf-8", timeout=20)
+            an.expect([">>>"])
+            extra['python'] = an
+
+        analyzer = extra['python']
+        def rsession(analyzer, lines):
+            l2 = []
+            for i, l in enumerate(lines):
+                l2.append(l)
+                if l.startswith(" ") and i < len(lines)-1 and not lines[i+1].startswith(" "):
+                    if not lines[i+1].strip().startswith("else:") and not lines[i+1].strip().startswith("elif") :
+                        l2.append("\n")
+
+            lines = l2
+            alines = []
+
+            # indented = False
+            in_dot_mode = False
+            if len(lines[-1]) > 0 and (lines[-1].startswith(" ") or lines[-1].startswith("\t")):
+                lines += [""]
+
+            for i, word in enumerate(lines):
+                analyzer.sendline(word)
+                before = ""
+                while True:
+                    analyzer.expect_exact([">>>", "..."])
+                    before += analyzer.before
+                    if analyzer.before.endswith("\n"):
+                        break
+                    else:
+                        before += analyzer.after
+
+                dotmode = analyzer.after == "..."
+                if 'dir(s)' in word:
+                    pass
+                if 'help(s.find)' in word:
+                    pass
+                if dotmode:
+                    # alines.append("..." + word)
+                    alines.append(">>>" + analyzer.before.rstrip() if not in_dot_mode else "..." + analyzer.before.rstrip())
+                    in_dot_mode = True
+                    # if i < len(lines) - 1 and not lines[i + 1].startswith(" "):
+                    #     analyzer.sendline("\n")  # going out of indentation mode .
+                    #     analyzer.expect_exact([">>>", "..."])
+                    #     alines.append("..." + analyzer.after.rstrip())
+                    #     pass
+                else:
+                    alines.append( ("..." if in_dot_mode else ">>>") + analyzer.before.rstrip())
+                    in_dot_mode = False
+            return alines
+
+        for l in (head[extra['evaluated_lines']:] + ["\n"]):
+            analyzer.sendline(l)
+            analyzer.expect_exact([">>>", "..."])
+
+
+        alines = rsession(analyzer, lines)
+        extra['evaluated_lines'] += len(head) + len(lines)
+        lines = alines
+        return lines, [outf, lines]
+    try:
+        a,b,c,_ = block_process(lines, tag="#!i", block_fun=functools.partial(block_fun, output=output, extra=extra))
+        if extra['python'] is not None:
+            extra['python'].close()
+
+        if len(c)>0:
+            kvs= { v[0] for v in c}
+            for outf in kvs:
+                out = "\n".join( ["\n".join(v[1]) for v in c if v[0] == outf] )
+                out = out.replace("\r", "")
+
+                with open(outf, 'w') as f:
+                    f.write(out)
+
+    except Exception as e:
+        print("lines are")
+        print("\n".join(lines))
+        print("Bad thing in #!i command in file", file)
+        raise e
+    return lines
\ No newline at end of file

src/snipper/fix_o.py

 import functools
 import os
-
-from snipper.legacy import indent, block_process
+from snipper.legacy import block_process
+from snipper.block_parsing import indent
 
 
 def run_o(lines, file, output):
@@ -11,7 +11,6 @@ def run_o(lines, file, output):
         l2 = []
         l2 += [id + "import sys", id + f"sys.stdout = open('{outf}', 'w')"]
         l2 += lines
-        # l2 += [indent(lines[-1]) + "sys.stdout.close()"]
         l2 += [indent(lines[-1]) + "sys.stdout = sys.__stdout__"]
         return l2, None
     try:

src/snipper/fix_r.py

+from snipper.block_parsing import indent
+
+
+def fix_r(lines):
+    for i,l in enumerate(lines):
+        if "#!r" in l:
+            lines[i] = indent(l) + l[l.find("#!r") + 3:].lstrip()
+    return lines

src/snipper/fix_s.py

@@ -2,16 +2,12 @@ from collections import defaultdict
 import os
 from snipper.block_parsing import block_iterate
 
-
 def get_s(lines):
     """ Return snips from 'lines' """
     blocks = defaultdict(list)
     for c in block_iterate(lines, "#!s"):
         blocks[c['name']].append(c)
-    if len(blocks) > 0:
-        print("\n".join(lines))
 
-        print("asdf")
     output = {}
     for name, co in blocks.items():
         output[name] = [l for c in co for l in c['block']]
@@ -28,6 +24,7 @@ def save_s(lines, output_dir, file_path): # save file snips to disk
         with open(output_dir + "/" + os.path.basename(file_path)[:-3] + ("_" + name if len(name) > 0 else name) + ".py", 'w') as f:
             f.write(out)
 
+
 s1 = """
 L1
 L2 #!s=a
@@ -40,7 +37,6 @@ L8
 L9 #!s=b
 went
 """
-
 if __name__ == "__main__":
     # for c in block_iterate(s1.splitlines(), tag="#!s"):
     #     print(c['block'])
@@ -49,6 +45,4 @@ if __name__ == "__main__":
         print("name:", k)
         print("\n".join(v))
         print("="*10)
-    # contents = block_split(s1.splitlines(), tag="#!s")
-    # contents = block_split(contents['joined'], tag="#!s")
-    # lines2 = contents['first'] +
+

src/snipper/legacy.py

-def indent(l):
-    v = len(l) - len(l.lstrip())
-    return l[:v]
-
-
 COMMENT = '"""'
 
 

src/snipper/snip_dir.py

@@ -32,7 +32,6 @@ def snip_dir(source_dir,  # Sources
         shutil.rmtree(dest_dir)
     os.makedirs(dest_dir)
 
-
     out = dest_dir
     hw = {'base': source_dir,
           'exclusion': exclude}
@@ -41,9 +40,7 @@ def snip_dir(source_dir,  # Sources
     if os.path.exists(dest_dir):
         shutil.rmtree(dest_dir)
 
-    base = hw['base']
-
-    shutil.copytree(base, dest_dir)
+    shutil.copytree(source_dir, dest_dir)
     time.sleep(0.2)
 
     ls = list(Path(dest_dir).glob('**/*.*'))
@@ -63,7 +60,7 @@ def snip_dir(source_dir,  # Sources
     for f, accept in acceptable:
         if os.path.isdir(f) or not str(f).endswith(".py"): # We only touch .py files.
             continue
-        f_dir = os.path.normpath(f if os.path.isdir(f) else os.path.dirname(f))
+        # f_dir = os.path.normpath(f if os.path.isdir(f) else os.path.dirname(f))
         if accept:
             # slist = hw['solutions'] if not CE else [os.path.basename(f)[:-3]]
             # base = None
@@ -74,13 +71,12 @@ def snip_dir(source_dir,  # Sources
             # if "assignments" in str(f) and "_grade.py" in str(f):
             #     continue
             info = {'new_references': [], 'code_copyright': 'Example student code. This file is automatically generated from the files in the instructor-directory'}
-            paths = {}
+            # paths = {}
             solution_list = []
             kwargs = {}
             cut_files = True
             run_files = True
-            nrem = censor_file(f, info, run_files=run_files, run_out_dirs=output_dir, cut_files=cut_files, solution_list=solution_list,
-                               include_path_base=base,
+            nrem = censor_file(f, run_files=run_files, run_out_dirs=output_dir, cut_files=cut_files, solution_list=solution_list,
                                base_path=dest_dir,
                                references=references,
                                **kwargs)
@@ -88,7 +84,6 @@ def snip_dir(source_dir,  # Sources
                 print(f"{nrem}> {f}")
             n += nrem
     for rm_file, accept in acceptable:
-        # rm_file = l
         rm_file = os.path.abspath(rm_file)
         if not accept:
             if os.path.isfile(rm_file):

src/snipper/snipper_main.py

 import os
-import functools
-import textwrap
 import re
+
+from snipper.block_parsing import full_strip
+from snipper.fix_i import run_i
+from snipper.fix_r import fix_r
 from snipper.fix_s import save_s
 from snipper.fix_cite import fix_citations
 from snipper.fix_bf import fix_f, fix_b2
 from snipper.fix_o import run_o
-from snipper.legacy import indent, block_process
-
-def fix_r(lines):
-    for i,l in enumerate(lines):
-        if "#!r" in l:
-            lines[i] = indent(l) + l[l.find("#!r") + 3:].lstrip()
-    return lines
-
-
-def strip_tag(lines, tag):
-    lines2 = []
-    for l in lines:
-        dx = l.find(tag)
-        if dx > 0:
-            l = l[:dx]
-            if len(l.strip()) == 0:
-                l = None
-        if l is not None:
-            lines2.append(l)
-    return lines2
 
 
 def rem_nonprintable_ctrl_chars(txt):
@@ -39,111 +21,6 @@ def rem_nonprintable_ctrl_chars(txt):
     return txt
 
 
-def run_i(lines, file, output):
-    extra = dict(python=None, output=output, evaluated_lines=0)
-    def block_fun(lines, start_extra, end_extra, art, head="", tail="", output=None, extra=None):
-        outf = output + ("_" + art if art is not None and len(art) > 0 else "") + ".shell"
-        lines = full_strip(lines)
-        s = "\n".join(lines)
-        s.replace("...", "..") # passive-aggressively truncate ... because of #issues.
-        lines = textwrap.dedent(s).strip().splitlines()
-
-        if extra['python'] is None:
-            import os
-            if os.name == 'nt':
-                import wexpect as we
-            else:
-                import pexpect as we
-            an = we.spawn("python", encoding="utf-8", timeout=20)
-            an.expect([">>>"])
-            extra['python'] = an
-
-        analyzer = extra['python']
-        def rsession(analyzer, lines):
-            l2 = []
-            for i, l in enumerate(lines):
-                l2.append(l)
-                if l.startswith(" ") and i < len(lines)-1 and not lines[i+1].startswith(" "):
-                    if not lines[i+1].strip().startswith("else:") and not lines[i+1].strip().startswith("elif") :
-                        l2.append("\n")
-
-            lines = l2
-            alines = []
-
-            # indented = False
-            in_dot_mode = False
-            if len(lines[-1]) > 0 and (lines[-1].startswith(" ") or lines[-1].startswith("\t")):
-                lines += [""]
-
-            for i, word in enumerate(lines):
-                analyzer.sendline(word)
-                before = ""
-                while True:
-                    analyzer.expect_exact([">>>", "..."])
-                    before += analyzer.before
-                    if analyzer.before.endswith("\n"):
-                        break
-                    else:
-                        before += analyzer.after
-
-                dotmode = analyzer.after == "..."
-                if 'dir(s)' in word:
-                    pass
-                if 'help(s.find)' in word:
-                    pass
-                if dotmode:
-                    # alines.append("..." + word)
-                    alines.append(">>>" + analyzer.before.rstrip() if not in_dot_mode else "..." + analyzer.before.rstrip())
-                    in_dot_mode = True
-                    # if i < len(lines) - 1 and not lines[i + 1].startswith(" "):
-                    #     analyzer.sendline("\n")  # going out of indentation mode .
-                    #     analyzer.expect_exact([">>>", "..."])
-                    #     alines.append("..." + analyzer.after.rstrip())
-                    #     pass
-                else:
-                    alines.append( ("..." if in_dot_mode else ">>>") + analyzer.before.rstrip())
-                    in_dot_mode = False
-            return alines
-
-        for l in (head[extra['evaluated_lines']:] + ["\n"]):
-            analyzer.sendline(l)
-            analyzer.expect_exact([">>>", "..."])
-
-
-        alines = rsession(analyzer, lines)
-        extra['evaluated_lines'] += len(head) + len(lines)
-        lines = alines
-        return lines, [outf, lines]
-    try:
-        a,b,c,_ = block_process(lines, tag="#!i", block_fun=functools.partial(block_fun, output=output, extra=extra))
-        if extra['python'] is not None:
-            extra['python'].close()
-
-        if len(c)>0:
-            kvs= { v[0] for v in c}
-            for outf in kvs:
-                out = "\n".join( ["\n".join(v[1]) for v in c if v[0] == outf] )
-                out = out.replace("\r", "")
-
-                with open(outf, 'w') as f:
-                    f.write(out)
-
-    except Exception as e:
-        print("lines are")
-        print("\n".join(lines))
-        print("Bad thing in #!i command in file", file)
-        raise e
-    return lines
-
-
-def full_strip(lines, tags=None):
-    if tags is None:
-        tags = ["#!s", "#!o", "#!f", "#!b"]
-    for t in tags:
-        lines = strip_tag(lines, t)
-    return lines
-
-
 def censor_code(lines, keep=True):
     dbug = True
     lines = fix_f(lines, dbug)
@@ -152,17 +29,15 @@ def censor_code(lines, keep=True):
 
 
 
-def censor_file(file, info, run_files=True, run_out_dirs=None, cut_files=True, solution_list=None,
+def censor_file(file, run_files=True, run_out_dirs=None, cut_files=True, solution_list=None,
                 censor_files=True,
                 base_path=None,
-                include_path_base=None,
                 strict=True,
                 references=None):
 
     if references == None:
         references = {}
 
-
     dbug = False
     with open(file, 'r', encoding='utf8') as f:
         s = f.read()
@@ -175,17 +50,14 @@ def censor_file(file, info, run_files=True, run_out_dirs=None, cut_files=True, s
 
         try:
             lines = fix_citations(lines, references, strict=strict)
-            # lines = s.split("\n")
         except IndexError as e:
             print(e)
-            print("Fuckup in file, cite/reference tag not found!>", file)
+            print("Error in file, cite/reference tag not found!>", file)
             raise e
 
         if run_files or cut_files:
             ofiles = []
             for rod in [run_out_dirs]:
-                # if not os.path.isdir(rod):
-                #     os.mkdir(rod)
                 ofiles.append(os.path.join(rod, os.path.basename(file).split(".")[0]) )
             ofiles[0] = ofiles[0].replace("\\", "/")
 
@@ -193,11 +65,9 @@ def censor_file(file, info, run_files=True, run_out_dirs=None, cut_files=True, s
                 run_o(lines, file=file, output=ofiles[0])
                 run_i(lines, file=file, output=ofiles[0])
             if cut_files:
-
                 save_s(lines, file_path=os.path.relpath(file, base_path), output_dir=run_out_dirs)  # save file snips to disk
         lines = full_strip(lines, ["#!s", "#!o", '#!i'])
 
-        # lines = fix_c(lines)
         if censor_files:
             lines = fix_f(lines, dbug)
             lines, nB, cut = fix_b2(lines)
@@ -208,7 +78,7 @@ def censor_file(file, info, run_files=True, run_out_dirs=None, cut_files=True, s
         if censor_files and len(cut) > 0 and solution_list is not None:
             fname = file.__str__()
             i = fname.find("irlc")
-            wk = fname[i+5:fname.find("\\", i+6)]
+            # wk = fname[i+5:fname.find("\\", i+6)]
             # sp = paths['02450students'] +"/solutions/"
             # if not os.path.exists(sp):
             #     os.mkdir(sp)