v0.1.20

docs/mkdocs.py

@@ -8,7 +8,7 @@ if __name__ == "__main__":
     bibtex = make_bibliography("../setup.py", "./")
 
     out = subprocess.check_output("python --version").decode("utf-8")
-    fn = unitgrade_private.__path__[0] + "/../../examples/02631/instructor/programs/report1intro.py"
+    fn = unitgrade_private.__path__[0] + "/../../examples/02631/instructor/week5/report1intro.py"
 
     out = subprocess.check_output(f"cd {os.path.dirname(fn)} && python {os.path.basename(fn)} --noprogress", shell=True, encoding='utf8', errors='strict')
     out = out.replace("", "")

docs/unitgrade.bib

 @online{unitgrade,
-	title={Unitgrade (0.0.3): \texttt{pip install unitgrade}},
+	title={Unitgrade (0.1.18): \texttt{pip install unitgrade}},
 	url={https://lab.compute.dtu.dk/tuhe/unitgrade},
-	urldate = {2021-09-07}, 
+	urldate = {2021-09-20}, 
 	month={9},
 	publisher={Technical University of Denmark (DTU)},
 	author={Tue Herlau},

src/unitgrade.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: unitgrade
-Version: 0.1.17
+Version: 0.1.20
 Summary: A student homework/exam evaluation framework build on pythons unittest framework.
 Home-page: https://lab.compute.dtu.dk/tuhe/unitgrade
 Author: Tue Herlau

src/unitgrade/__init__.py

 from unitgrade.version import __version__
-from unitgrade.utils import myround, msum, mfloor, Capturing, ActiveProgress, cache, hide
+from unitgrade.utils import myround, msum, mfloor, Capturing, ActiveProgress, cache, hide, Capturing2
 # from unitgrade import hide
 from unitgrade.framework import Report, UTestCase, NotebookTestCase
 from unitgrade.evaluate import evaluate_report_student

src/unitgrade/evaluate.py

@@ -136,7 +136,11 @@ def evaluate_report(report, question=None, qitem=None, passall=False, verbose=Fa
             continue
         suite = loader.loadTestsFromTestCase(q)
         qtitle = q.question_title() if hasattr(q, 'question_title') else q.__qualname__
+        if not report.abbreviate_questions:
             q_title_print = "Question %i: %s"%(n+1, qtitle)
+        else:
+            q_title_print = "q%i) %s" % (n + 1, qtitle)
+
         print(q_title_print, end="")
         q.possible = 0
         q.obtained = 0
@@ -148,10 +152,20 @@ def evaluate_report(report, question=None, qitem=None, passall=False, verbose=Fa
         UTextResult.unmute = unmute # Hacky as well.
         UTextResult.setUpClass_time = q._cache.get(((q.__name__, 'setUpClass'), 'time'), 3) if hasattr(q, '_cache') and q._cache is not None else 3
 
+
         res = UTextTestRunner(verbosity=2, resultclass=UTextResult).run(suite)
         details = {}
         for s, msg in res.successes + res.failures + res.errors:
-            key = (q.__name__, s._testMethodName) # cannot use the cache_id method bc. it is not compatible with plain unittest.
+            # from unittest.suite import _ErrorHolder
+            # from unittest import _Err
+            # if isinstance(s, _ErrorHolder)
+            if hasattr(s, '_testMethodName'):
+                key = (q.__name__, s._testMethodName)
+            else:
+                # In case s is an _ErrorHolder (unittest.suite)
+                key = (q.__name__, s.id())
+            # key = (q.__name__, s._testMethodName) # cannot use the cache_id method bc. it is not compatible with plain unittest.
+
             detail = {}
             if (s,msg) in res.successes:
                 detail['status'] = "pass"
@@ -174,7 +188,7 @@ def evaluate_report(report, question=None, qitem=None, passall=False, verbose=Fa
         possible = res.testsRun
         obtained = len(res.successes)
 
-        assert len(res.successes) +  len(res.errors) + len(res.failures) == res.testsRun
+        # assert len(res.successes) +  len(res.errors) + len(res.failures) == res.testsRun
 
         obtained = int(w * obtained * 1.0 / possible ) if possible > 0 else 0
         score[n] = {'w': w, 'possible': w, 'obtained': obtained, 'items': details, 'title': qtitle, 'name': q.__name__,

src/unitgrade/framework.py

@@ -11,6 +11,8 @@ import textwrap
 from unitgrade.runners import UTextResult
 from unitgrade.utils import gprint, Capturing2, Capturing
 colorama.init(autoreset=True)  # auto resets your settings after every output
+import numpy
+numpy.seterr(all='raise')
 
 def setup_dir_by_class(C, base_dir):
     name = C.__class__.__name__
@@ -19,6 +21,8 @@ def setup_dir_by_class(C, base_dir):
 
 class Report:
     title = "report title"
+    abbreviate_questions = False # Should the test items start with 'Question ...' or just be q1).
+
     version = None
     questions = []
     pack_imports = []
@@ -41,7 +45,15 @@ class Report:
 
     def _import_base_relative(self):
         if hasattr(self.pack_imports[0], '__path__'):
-            root_dir = self.pack_imports[0].__path__._path[0]
+            # im = self.pack_imports[0]
+            # print("im!!!!")
+            # print(im.__path__)
+            # if isinstance(im, list):
+            #     print("im is", im)
+            # __path__ was originally not a list; but now it seems like it behaves like a list. Add check if problem persists and treat it as either list or string.
+            # root_dir = self.pack_imports[0].__path__[0]._path[0]
+            root_dir = self.pack_imports[0].__path__[0]
+
         else:
             root_dir = self.pack_imports[0].__file__
 
@@ -75,6 +87,8 @@ class Report:
             for q, _ in self.questions:
                 q._with_coverage = True
                 q._report = self
+        for q, _ in self.questions:
+            q._setup_answers_mode = True
 
         from unitgrade import evaluate_report_student
         evaluate_report_student(self, unmute=True)
@@ -129,6 +143,10 @@ class UTestCase(unittest.TestCase):
     _covcache = None # Coverage cache. Written to if _with_coverage is true.
     _report = None  # The report used. This is very, very hacky and should always be None. Don't rely on it!
 
+    # If true, the tests will not fail when cache is used. This is necesary since otherwise the cache will not be updated
+    # during setup, and the deploy script must be run many times.
+    _setup_answers_mode = False
+
 
     def capture(self):
         if hasattr(self, '_stdout') and self._stdout is not None:
@@ -174,6 +192,7 @@ class UTestCase(unittest.TestCase):
             try:
                 # print("Stoppping coverage...")
                 self.cov.stop()
+                # print("Coverage was stopped")
                 # self.cov.html_report()
                 # print("Success!")
             except Exception as e:
@@ -216,12 +235,18 @@ class UTestCase(unittest.TestCase):
                     assert len(lines) == len(lines2)
                     # print("In file ", file, "context by lineno", data.contexts_by_lineno(file))
                     for ll in data.contexts_by_lineno(file):
+                        # For empty files (e.g. __init__) there is a potential bug where coverage will return the file but lines2 will be = [].
+                        # print("loop B: ll is", ll)
                         l = ll-1
                         # print(l)
-                        l1 = (lines[l] + " "*1000)[:80]
-                        l2 = (lines2[l] + " "*1000)[:80]
+                        # l1 = (lines[l] + " "*1000)[:80]
+                        # l2 = (lines2[l] + " "*1000)[:80]
                         # print("l is", l, l1, " " + l2, "file", file)
-                        if lines2[l].strip() == garb:
+                        # print("Checking if statement: ")
+                        # print(l, lines2)
+                        # print(">> ", lines2[l])
+                        # print(">", lines2[l].strip(), garb)
+                        if l < len(lines2) and lines2[l].strip() == garb:
                             # print("Got a hit at l", l)
                             rel = os.path.relpath(child, root)
                             cc = self._covcache
@@ -237,8 +262,9 @@ class UTestCase(unittest.TestCase):
                             cc[rel][fun] = (l, "\n".join(comments))
                             # print("found", rel, fun)
                             self._cache_put((self.cache_id(), 'coverage'), self._covcache)
-        # import sys
-        # sys.exit()
+        #                 print("ending loop B")
+        #         print("At end of outer loop A")
+        # print("-------------------------------------------- Tear down called")
 
     def shortDescriptionStandard(self):
         sd = super().shortDescription()
@@ -305,6 +331,13 @@ class UTestCase(unittest.TestCase):
         self._ensure_cache_exists()
         return key in self.__class__._cache
 
+    def get_expected_test_value(self):
+        key = (self.cache_id(), 'assert')
+        id = self._assert_cache_index
+        cache = self._cache_get(key)
+        _expected = cache.get(id, f"Key {id} not found in cache; framework files missing. Please run deploy()")
+        return _expected
+
     def wrap_assert(self, assert_fun, first, *args, **kwargs):
         key = (self.cache_id(), 'assert')
         if not self._cache_contains(key):
@@ -313,15 +346,27 @@ class UTestCase(unittest.TestCase):
                 key] = {}  # A new dict. We manually insert it because we have to use that the dict is mutable.
         cache = self._cache_get(key)
         id = self._assert_cache_index
-        if not id in cache:
-            print("Warning, framework missing cache index", key, "id =", id)
         _expected = cache.get(id, f"Key {id} not found in cache; framework files missing. Please run deploy()")
+        if not id in cache:
+            print("Warning, framework missing cache index", key, "id =", id, " - The test will be skipped for now.")
+            if self._setup_answers_mode:
+                _expected = first # Bypass by setting equal to first. This is in case multiple self.assertEqualC's are run in a row and have to be set.
 
         # The order of these calls is important. If the method assert fails, we should still store the correct result in cache.
         cache[id] = first
         self._cache_put(key, cache)
         self._assert_cache_index += 1
+        if not self._setup_answers_mode:
             assert_fun(first, _expected, *args, **kwargs)
+        else:
+            try:
+                assert_fun(first, _expected, *args, **kwargs)
+            except Exception as e:
+                print("Mumble grumble. Cache function failed during class setup. Most likely due to old cache. Re-run deploy to check it pass.", id)
+                print("> first", first)
+                print("> expected", _expected)
+                print(e)
+
 
     def assertEqualC(self, first, msg=None):
         self.wrap_assert(self.assertEqual, first, msg)
@@ -334,7 +379,13 @@ class UTestCase(unittest.TestCase):
         if len(msg) > 0:
             msg += "\n"
         self.assertEqual(a1.shape, a2.shape, msg=msg + "Dimensions of input data does not agree.")
+        assert(np.all(np.isinf(a1) == np.isinf(a2)))  # Check infinite part.
+        a1[np.isinf(a1)] = 0
+        a2[np.isinf(a2)] = 0
+
         diff = np.abs(a1 - a2)
+
+        # print(a1, a2, diff)
         return diff
 
 
@@ -344,18 +395,26 @@ class UTestCase(unittest.TestCase):
         else:
             diff = self._shape_equal(first, second)
             if max(diff.flat) >= tol:
-                self.assertEqual(first, second, msg=msg + f"Not equal within tolerance {tol}")
+                from unittest.util import safe_repr
+                # msg = f'{safe_repr(first)} != {safe_repr(second)} : Not equal within tolerance {tol}'
+                # print(msg)
+                self.assertEqual(first, second, msg=f'Not equal within tolerance {tol}')
 
-    def assertL2(self, first, second=None, tol=1e-5, msg=None):
+    def assertL2(self, first, second=None, tol=1e-5, msg=None, relative=False):
         if second is None:
-            return self.wrap_assert(self.assertL2, first, tol=tol, msg=msg)
+            return self.wrap_assert(self.assertL2, first, tol=tol, msg=msg, relative=relative)
         else:
             # We first test using numpys build-in testing method to see if one coordinate deviates a great deal.
             # This gives us better output, and we know that the coordinate wise difference is lower than the norm difference.
+            if not relative:
                 np.testing.assert_allclose(first, second, atol=tol)
             diff = self._shape_equal(first, second)
             diff = ( ( np.asarray( diff.flatten() )**2).sum() )**.5
-            if max(diff.flat) >= tol:
+
+            scale = (2/(np.linalg.norm(np.asarray(first).flat) + np.linalg.norm(np.asarray(second).flat)) ) if relative else 1
+            if max(diff.flat)*scale >= tol:
+                msg = "" if msg is None else msg
+                print(f"|first - second|_2 = {diff} > {tol} ")
                 self.assertEqual(first, second, msg=msg + f"Not equal within tolerance {tol}")
 
     def _cache_file(self):

src/unitgrade/runners.py

@@ -27,12 +27,22 @@ class UTextResult(unittest.TextTestResult):
     def addError(self, test, err):
         super(unittest.TextTestResult, self).addError(test, err)
         err = self.errors[-1][1]
+        if hasattr(sys.stdout, 'log'):
             stdout = sys.stdout.log.readlines()  # Only works because we set sys.stdout to a unitgrade.Logger
+        else:
+            stdout = ""
         self.errors[-1] = (self.errors[-1][0], {'return': None,
                                 'stderr': err,
                                 'stdout': stdout
                                 })
 
+        if not hasattr(self, 'item_title_print'):
+            # In case setUpClass() fails with an error the short description may not be set. This will fix that problem.
+            self.item_title_print = test.shortDescription()
+            if self.item_title_print is None:  # In case the short description is not set either...
+                self.item_title_print = test.id()
+
+
         self.cc_terminate(success=False)
 
     def addFailure(self, test, err):

src/unitgrade/utils.py

@@ -70,7 +70,7 @@ class Capturing2(Capturing):
     def __exit__(self, *args):
         lines = self._stringio.getvalue().splitlines()
         txt = "\n".join(lines)
-        numbers = extract_numbers(txt)
+        numbers = extract_numbers(rm_progress_bar(txt))
         self.extend(lines)
         del self._stringio  # free up some memory
         sys.stdout = self._stdout
@@ -121,7 +121,9 @@ class ActiveProgress():
 
     def terminate(self):
         if not self._running:
-            raise Exception("Stopping a stopped progress bar. ")
+            print("Stopping a progress bar which is not running (class unitgrade.utils.ActiveProgress")
+            pass
+            # raise Exception("Stopping a stopped progress bar. ")
         self._running = False
         if self.show_progress_bar:
             self.thread.join()
@@ -205,11 +207,16 @@ def cache(foo, typed=False):
     def wrapper(self, *args, **kwargs):
         key = (self.cache_id(), ("@cache", foo.__name__, _make_key(args, kwargs, typed)))
         # print(self._cache.keys())
+        # for k in self._cache:
+        #     print(k)
         if not self._cache_contains(key):
             value = foo(self, *args, **kwargs)
             self._cache_put(key, value)
         else:
             value = self._cache_get(key)
+            # This appears to be required since there are two caches. Otherwise, when deploy method is run twice,
+            # the cache will not be set correctly.
+            self._cache_put(key, value)
         return value
 
     return wrapper

src/unitgrade/version.py

-__version__ = "0.1.17"
\ No newline at end of file
+__version__ = "0.1.20"
\ No newline at end of file