From 81f30ddce2968bfa4ea1984d70e94a43b21a5930 Mon Sep 17 00:00:00 2001
From: Tue Herlau <tuhe@dtu.dk>
Date: Fri, 16 Sep 2022 11:04:02 +0200
Subject: [PATCH] Removed pupdb as a dependency, using diskcache
---
requirements.txt | 5 +
setup.py | 4 +-
src/unitgrade.egg-info/PKG-INFO | 4 +-
src/unitgrade.egg-info/SOURCES.txt | 2 +
src/unitgrade.egg-info/requires.txt | 15 +-
src/unitgrade/artifacts.py | 75 ++---
src/unitgrade/dashboard/app.py | 315 ++++++++++--------
src/unitgrade/dashboard/app_helpers.py | 71 ++--
src/unitgrade/dashboard/dashboard_cli.py | 11 +-
src/unitgrade/dashboard/dbwatcher.py | 51 +++
.../dashboard/file_change_handler.py | 20 +-
src/unitgrade/dashboard/static/unitgrade.js | 73 ++--
src/unitgrade/dashboard/static/wz_js.js | 4 +-
src/unitgrade/dashboard/templates/base.html | 4 +
src/unitgrade/dashboard/templates/index3.html | 99 +++---
src/unitgrade/dashboard/templates/info.html | 38 +++
src/unitgrade/dashboard/watcher.py | 16 +-
src/unitgrade/framework.py | 248 +++++++-------
src/unitgrade/pupdb.py | 151 +++++++++
src/unitgrade/utils.py | 53 ++-
src/unitgrade/version.py | 2 +-
21 files changed, 778 insertions(+), 483 deletions(-)
create mode 100644 src/unitgrade/dashboard/dbwatcher.py
create mode 100644 src/unitgrade/dashboard/templates/info.html
create mode 100644 src/unitgrade/pupdb.py
diff --git a/requirements.txt b/requirements.txt
index ab94e1b..baf2a47 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,8 @@ numpy
scikit_learn
importnb # Experimental notebook inclusion feature. May not be required.
requests # To read remote files for automatic updating.
+diskcache # dashboard
+watchdog # dashboard
+flask_socketio # dashboard
+flask # dashboard
+Werkzeug # dashboard
diff --git a/setup.py b/setup.py
index ff12795..4111242 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,9 @@ setuptools.setup(
packages=setuptools.find_packages(where="src"),
python_requires=">=3.8",
license="MIT",
- install_requires=['numpy', 'tabulate', "pyfiglet", "coverage", "colorama", 'tqdm', 'importnb', 'requests'],
+ install_requires=['numpy', 'tabulate', "pyfiglet", "coverage", "colorama", 'tqdm', 'importnb', 'requests',
+ 'watchdog', 'flask_socketio', 'flask', 'Werkzeug', 'diskcache', # These are for the dashboard.
+ ],
include_package_data=True,
package_data={'': ['dashboard/static/*', 'dashboard/templates/*'],}, # so far no Manifest.in.
entry_points={
diff --git a/src/unitgrade.egg-info/PKG-INFO b/src/unitgrade.egg-info/PKG-INFO
index 64b0ff8..f0b3f81 100644
--- a/src/unitgrade.egg-info/PKG-INFO
+++ b/src/unitgrade.egg-info/PKG-INFO
@@ -1,12 +1,13 @@
Metadata-Version: 2.1
Name: unitgrade
-Version: 0.1.28.2
+Version: 0.1.28.8
Summary: A student homework/exam evaluation framework build on pythons unittest framework.
Home-page: https://lab.compute.dtu.dk/tuhe/unitgrade
Author: Tue Herlau
Author-email: tuhe@dtu.dk
License: MIT
Project-URL: Bug Tracker, https://lab.compute.dtu.dk/tuhe/unitgrade/issues
+Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
@@ -190,3 +191,4 @@ Please contact me and we can discuss your specific concerns.
year={2022},
}
```
+
diff --git a/src/unitgrade.egg-info/SOURCES.txt b/src/unitgrade.egg-info/SOURCES.txt
index aae28ff..93c7366 100644
--- a/src/unitgrade.egg-info/SOURCES.txt
+++ b/src/unitgrade.egg-info/SOURCES.txt
@@ -6,6 +6,7 @@ src/unitgrade/__init__.py
src/unitgrade/artifacts.py
src/unitgrade/evaluate.py
src/unitgrade/framework.py
+src/unitgrade/pupdb.py
src/unitgrade/runners.py
src/unitgrade/utils.py
src/unitgrade/version.py
@@ -26,5 +27,6 @@ src/unitgrade/dashboard/static/wz_style_modified.css
src/unitgrade/dashboard/templates/base.html
src/unitgrade/dashboard/templates/bootstrap.html
src/unitgrade/dashboard/templates/index3.html
+src/unitgrade/dashboard/templates/info.html
src/unitgrade/dashboard/templates/terminal.html
src/unitgrade/dashboard/templates/wz.html
\ No newline at end of file
diff --git a/src/unitgrade.egg-info/requires.txt b/src/unitgrade.egg-info/requires.txt
index 88a0855..be7c360 100644
--- a/src/unitgrade.egg-info/requires.txt
+++ b/src/unitgrade.egg-info/requires.txt
@@ -1,8 +1,13 @@
-numpy
-tabulate
-pyfiglet
-coverage
+Werkzeug
colorama
-tqdm
+coverage
+flask
+flask_socketio
importnb
+numpy
+pupdb
+pyfiglet
requests
+tabulate
+tqdm
+watchdog
diff --git a/src/unitgrade/artifacts.py b/src/unitgrade/artifacts.py
index e9bd92d..7b00750 100644
--- a/src/unitgrade/artifacts.py
+++ b/src/unitgrade/artifacts.py
@@ -1,12 +1,8 @@
-
-
-import os, time
import threading
-from queue import Queue
-
from colorama import Fore
+
class WorkerThread(threading.Thread):
""" A worker thread that takes directory names from a queue, finds all
files in them recursively and reports the result.
@@ -39,7 +35,7 @@ class WorkerThread(threading.Thread):
if len(ss) > 0:
# print(ss)
cq = self.db.get('stdout')
- self.db.set('stdout', cq + [(len(cq), ss)])
+ self.db.set('stdout', cq + [[len(cq), ss]])
def run(self):
# As long as we weren't asked to stop, try to take new tasks from the
@@ -67,7 +63,10 @@ class DummyPipe:
def write(self, message):
if not self.mute:
self.std_.write(message)
- self.queue.put( (self.type, message) )
+ self.queue.put((self.type, message))
+
+ def write_mute(self, message):
+ self.queue.put((self.type, message))
def flush(self):
self.std_.flush()
@@ -91,18 +90,6 @@ class StdCapturing():
self.thread = WorkerThread(self.queu, db=db) # threading.Thread(target=self.consume_queue, args=self.lifo)
self.thread.start()
- # def write(self, message):
- # if not self.mute:
- # self.stdout.write(message)
- # self.queu.put(message)
- #
- # if self.recording:
- # self.recordings[-1] += message
-
- # def flush(self):
- # if not self.mute:
- # self.stdout.flush()
-
def close(self):
try:
self.thread.join()
@@ -111,29 +98,27 @@ class StdCapturing():
pass
-class ArtifactMapper:
- def __init__(self):
- self.artifact_output_json = ''
- from threading import Lock
- self.lock = Lock()
- import pupdb
- self.db = pupdb.Pupdb(self.artifact_output_json)
- run = self.db.get("run_number", 0)
-
- def add_stack_trace(self, e):
- # Add an error.
- pass
-
- def print_stdout(self, msg, timestamp):
- pass
-
- def print_stderr(self, msg, timestamp):
- pass
-
- def restart_test(self):
- pass
-
- def register_outcome(self, did_fail=False, did_pass=False, did_error=False):
- pass
-
-# gummiandlichessthibault42.
\ No newline at end of file
+# class ArtifactMapper:
+# def __init__(self):
+# self.artifact_output_json = ''
+# from threading import Lock
+# self.lock = Lock()
+# import pupdb
+# self.db = pupdb.Pupdb(self.artifact_output_json)
+# run = self.db.get("run_number", 0)
+#
+# def add_stack_trace(self, e):
+# # Add an error.
+# pass
+#
+# def print_stdout(self, msg, timestamp):
+# pass
+#
+# def print_stderr(self, msg, timestamp):
+# pass
+#
+# def restart_test(self):
+# pass
+#
+# def register_outcome(self, did_fail=False, did_pass=False, did_error=False):
+# pass
diff --git a/src/unitgrade/dashboard/app.py b/src/unitgrade/dashboard/app.py
index 1e8bfd4..666ef46 100644
--- a/src/unitgrade/dashboard/app.py
+++ b/src/unitgrade/dashboard/app.py
@@ -1,26 +1,27 @@
import fnmatch
from threading import Lock
-import argparse
+import time
import datetime
import os
-import subprocess
import logging
import sys
import glob
from pathlib import Path
from flask import Flask, render_template
from flask_socketio import SocketIO
-from pupdb.core import PupDB
-from unitgrade_private.hidden_gather_upload import dict2picklestring, picklestring2dict
-from unitgrade.dashboard.app_helpers import get_available_reports
+from unitgrade.utils import picklestring2dict, load_token
+from unitgrade.dashboard.app_helpers import get_available_reports, _run_test_cmd
from unitgrade.dashboard.watcher import Watcher
from unitgrade.dashboard.file_change_handler import FileChangeHandler
-from unitgrade_private import load_token
+from unitgrade.framework import DKPupDB
+from unitgrade.dashboard.dbwatcher import DBWatcher
+from diskcache import Cache
logging.getLogger('werkzeug').setLevel("WARNING")
-def mkapp(base_dir="./"):
+def mkapp(base_dir="./", use_command_line=True):
+ print("BUILDING THE APPLICATION ONCE!")
app = Flask(__name__, template_folder="templates", static_folder="static", static_url_path="/static")
- x = {'watcher': None, 'handler': None} # maintain program state across functions.
+ x = {'watcher': None, 'handler': None, 'db_watcher': None} # maintain program state across functions.
app.config["SECRET_KEY"] = "secret!"
app.config["fd"] = None
app.config["TEMPLATES_AUTO_RELOAD"] = True
@@ -31,6 +32,24 @@ def mkapp(base_dir="./"):
watched_files_lock = Lock()
watched_files_dictionary = {}
+ def test_handle_function(key, db):
+ state = db.get('state')
+ wz = db.get('wz_stacktrace') if 'wz_stacktrace' in db.keys() else None
+ if wz is not None:
+ wz = wz.replace('<div class="traceback">', f'<div class="traceback"><div class="{key}-traceback">')
+ wz += "</div>"
+ coverage_files_changed = db.get('coverage_files_changed') if 'coverage_files_changed' in db.keys() else None
+ if state == "fail":
+ pass
+
+ print("Emitting update of", key, "to", state)
+ def callb(*args, **kwargs):
+ print("Hi I am the callback function")
+ socketio.emit('testupdate', {"id": key, 'state': state, 'stacktrace': wz, 'stdout': db.get('stdout'),
+ 'run_id': db.get('run_id'),
+ 'coverage_files_changed': coverage_files_changed}, namespace="/status", to=x.get("client_id", None), callback=callb)
+ z = 234
+
def do_something(file_pattern):
"""
Oh crap, `file` has changed on disk. We need to open it, look at it, and then do stuff based on what is in it.
@@ -46,45 +65,46 @@ def mkapp(base_dir="./"):
if type == 'question_json': # file.endswith(".json"):
if file is None:
return # There is nothing to do, the file does not exist.
- db = PupDB(file)
- if "state" not in db.keys(): # Test has not really been run yet. There is no reason to submit this change to the UI.
- return
- state = db.get('state')
- key = os.path.basename(file)[:-5]
- wz = db.get('wz_stacktrace') if 'wz_stacktrace' in db.keys() else None
- if wz is not None:
- wz = wz.replace('<div class="traceback">', f'<div class="traceback"><div class="{key}-traceback">')
- wz += "</div>"
- coverage_files_changed = db.get('coverage_files_changed') if 'coverage_files_changed' in db.keys() else None
- if state == "fail":
- print("State is fail, I am performing update", state, key)
- socketio.emit('testupdate', {"id": key, 'state': state, 'stacktrace': wz, 'stdout': db.get('stdout'), 'run_id': db.get('run_id'),
- 'coverage_files_changed': coverage_files_changed}, namespace="/status")
+ # try:
+ # db = DKPupDB(file)
+ # if "state" not in db.keys(): # Test has not really been run yet. There is no reason to submit this change to the UI.
+ # return
+ # except Exception as e:
+ # print(e)
+ # os.remove(file) # Delete the file. This is a bad database file, so we trash it and restart.
+ # return
+ #
+ # state = db.get('state')
+ # key = os.path.basename(file)[:-5]
+ # # print("updating", file, key)
+ # wz = db.get('wz_stacktrace') if 'wz_stacktrace' in db.keys() else None
+ # if wz is not None:
+ # wz = wz.replace('<div class="traceback">', f'<div class="traceback"><div class="{key}-traceback">')
+ # wz += "</div>"
+ # coverage_files_changed = db.get('coverage_files_changed') if 'coverage_files_changed' in db.keys() else None
+ # if state == "fail":
+ # pass
+ # print("State is fail, I am performing update", state, key)
+ # socketio.emit('testupdate', {"id": key, 'state': state, 'stacktrace': wz, 'stdout': db.get('stdout'), 'run_id': db.get('run_id'),
+ # 'coverage_files_changed': coverage_files_changed}, namespace="/status")
elif type =='coverage':
if lrc is None: # Program startup. We don't care about this.
return
- db = get_report_database()
- for q in db['questions']:
- for i in db['questions'][q]['tests']:
- # key = '-'.join(i)
+ # db = get_report_database()
+ for q in current_report['questions']:
+ for i in current_report['questions'][q]['tests']:
test_invalidated = False
-
- for f in db['questions'][q]['tests'][i]['coverage_files']:
-
- # fnmatch.fnmatch(f, file_pattern)
+ for f in current_report['questions'][q]['tests'][i]['coverage_files']:
if fnmatch.fnmatch(file, "**/" + f):
# This file has been matched. The question is now invalid.
test_invalidated = True
break
-
if test_invalidated:
- # Why not simply write this bitch into the db?
dbf = current_report['root_dir'] + "/" + current_report['questions'][q]['tests'][i]['artifact_file']
- db = PupDB(dbf)
- print("A test has been invalidated. Setting coverage files", file)
- db.set('coverage_files_changed', [file])
- # print("dbf", dbf)
- # print("marking a test as invalidated: ", db)
+ db2 = DKPupDB(dbf)
+ print("A test has been invalidated. Setting coverage files", file, dbf)
+ db2.set('coverage_files_changed', [file])
+
elif type =="token":
a, b = load_token(file)
rs = {}
@@ -96,67 +116,67 @@ def mkapp(base_dir="./"):
else:
raise Exception("Bad type: " + type)
- def get_json_base(jobfolder):
- return current_report['json']
-
- def get_report_database():
- dbjson = get_json_base(base_dir)
- db = PupDB(dbjson)
- from unitgrade_private.hidden_gather_upload import picklestring2dict
- rs = {}
- for k in db.keys():
- if k == 'questions':
- qenc, _ = picklestring2dict(db.get("questions"))
- rs['questions'] = qenc # This feels like a good place to find the test-file stuff.
- else:
- rs[k] = db.get(k)
-
- lpath_full = Path(os.path.normpath(os.path.dirname(dbjson) + "/../" + os.path.basename(dbjson)[12:-5] + ".py"))
- rpath = Path(db.get('relative_path'))
- base = lpath_full.parts[:-len(rpath.parts)]
-
- rs['local_base_dir_for_test_module'] = str(Path(*base))
- rs['test_module'] = ".".join(db.get('modules'))
-
- del rs['root_dir'] # Don't overwrite this one.
- return rs
+ # def get_report_database():
+ # assert False
+ # import pickle
+ # dbjson = current_report['json']
+ # with open(current_report['json'], 'rb') as f:
+ # rs = pickle.load(f)
+ #
+ # # db = DKPupDB(dbjson)
+ # # from unitgrade_private.hidden_gather_upload import picklestring2dict
+ # # rs = {}
+ # # for k in db.keys():
+ # # if k == 'questions':
+ # # qenc, _ = picklestring2dict(db.get("questions"))
+ # # rs['questions'] = qenc # This feels like a good place to find the test-file stuff.
+ # # else:
+ # # rs[k] = db.get(k)
+ #
+ # lpath_full = Path(os.path.normpath(os.path.dirname(dbjson) + "/../" + os.path.basename(dbjson)[12:].split(".")[0] + ".py"))
+ # rpath = Path(rs['relative_path'])
+ # base = lpath_full.parts[:-len(rpath.parts)]
+ #
+ # rs['local_base_dir_for_test_module'] = str(Path(*base))
+ # rs['test_module'] = ".".join(rs['modules'])
+ #
+ # del rs['root_dir'] # Don't overwrite this one.
+ # return rs
def select_report_file(json):
current_report.clear()
for k, v in available_reports[json].items():
current_report[k] = v
- print(f"{current_report['root_dir']=}")
-
- for k, v in get_report_database().items():
- current_report[k] = v
-
- print(f"{current_report['root_dir']=}")
+ # for k, v in get_report_database().items():
+ # current_report[k] = v
def mkempty(pattern, type):
fls = glob.glob(current_report['root_dir'] + pattern)
fls.sort(key=os.path.getmtime)
- # if type == 'token':
- # fls.sort(key= lambda s: int(s[:-6].split("_")[-3]) ) # Sort by points obtained.
f = None if len(fls) == 0 else fls[-1] # Bootstrap with the given best matched file.
return {'type': type, 'last_recorded_change': None, 'last_handled_change': None, 'file': f}
+
+ watched_blocks = []
with watched_files_lock:
watched_files_dictionary.clear()
- db = PupDB(json)
- dct = picklestring2dict(db.get('questions'))[0]
+ # db = PupDB(json)
+ dct = current_report['questions'] # picklestring2dict(db.get('questions'))[0]
for q in dct.values():
for i in q['tests'].values():
file = "*/"+i['artifact_file']
+ watched_blocks.append(os.path.basename( i['artifact_file'])[:-5])
watched_files_dictionary[file] = mkempty(file, 'question_json') # when the file was last changed and when that change was last handled.
for c in i['coverage_files']:
file = "*/"+c
watched_files_dictionary[file] = mkempty(file, "coverage")
+
# tdir = "*/"+os.path.dirname(current_report['relative_path_token']) + "/" + os.path.basename(current_report['relative_path'])[:-3] + "*.token"
tdir = "*/"+current_report['token_stub'] + "*.token"
watched_files_dictionary[tdir] = mkempty(tdir, 'token')
- for l in ['watcher', 'handler']:
+ for l in ['watcher', 'handler', 'db_watcher']:
if x[l] is not None: x[l].close()
x['watcher'] = Watcher(current_report['root_dir'], watched_files_dictionary, watched_files_lock)
@@ -165,25 +185,36 @@ def mkapp(base_dir="./"):
x['handler'] = FileChangeHandler(watched_files_dictionary, watched_files_lock, do_something)
x['handler'].start()
+ x['db_watcher'] = DBWatcher(os.path.dirname( current_report['json'] ), watched_blocks, test_handle_function=test_handle_function)
+ x['db_watcher'].start()
+
if len(available_reports) == 0:
print("Unitgrade was launched in the directory")
print(">", base_dir)
- print("But this directory does not contain any reports. Run unitgrade from a directory containing report files.")
+ print("But this directory does not contain any reports. Please run unitgrade from a directory which contains report files.")
sys.exit()
# x['current_report'] =
select_report_file(list(available_reports.keys()).pop())
- @app.route("/app.js")
- def appjs():
- return render_template("app.js")
+ # @app.route("/app.js") # Unclear if used
+ # def appjs():
+ # return render_template("app.js")
- @socketio.on("ping", namespace="/status")
+ @socketio.on("ping", namespace="/status") # Unclear if used.
def ping():
- json = get_json_base(jobfolder=base_dir)[0]
+ json = current_report['json']
socketio.emit("pong", {'base_json': json})
+ @app.route("/info")
+ def info_page():
+ # Print an info page.
+
+ # db = Cache(self)
+ db = Cache( os.path.dirname( current_report['json'] ) )
+ info = {k: db[k] for k in db}
+ return render_template("info.html", **current_report, available_reports=available_reports, db=info)
@app.route("/")
def index_bare():
@@ -192,14 +223,13 @@ def mkapp(base_dir="./"):
@app.route("/report/<report>")
def index(report):
-
if report != current_report['menu_name']:
for k, r in available_reports.items():
if report == r['menu_name']:
select_report_file(k)
raise Exception("Bad report selected", report)
- rs = get_report_database()
+ rs = current_report
qenc = rs['questions']
x = {}
for k, v in current_report.items():
@@ -207,11 +237,15 @@ def mkapp(base_dir="./"):
x['questions'] = {}
for q in qenc:
+ x['questions'][q] = current_report['questions'][q].copy()
items = {}
- for it_key, it_value in qenc[q]['tests'].items():
+ for it_key, it_value in current_report['questions'][q]['tests'].items():
it_key_js = "-".join(it_key)
# do a quick formatting of the hints. Split into list by breaking at *.
hints = it_value['hints']
+
+ hints = [] if hints is None else hints.copy()
+
for k in range(len(hints)):
ahints = []
for h in hints[k][0].split("\n"):
@@ -220,9 +254,9 @@ def mkapp(base_dir="./"):
h = h.strip()[1:]
ahints[-1] += "\n" + h
hints[k] = (ahints, hints[k][1], hints[k][2])
-
- items[it_key_js] = {'title': it_value['title'], 'hints': hints}
- x['questions'][q] = {'title': qenc[q]['title'], 'tests': items}
+ # items[it_key_js] =
+ items[it_key_js] = {'title': it_value['title'], 'hints': hints, 'runable': it_value['title'] != 'setUpClass', "coverage_files": it_value['coverage_files']}
+ x['questions'][q]['tests'] = items # = current_report['questions'][q] # {'title': qenc[q]['title'], 'tests': items}
run_cmd_grade = '.'.join(x['modules']) + "_grade"
x['grade_script'] = x['modules'][-1] + "_grade.py"
@@ -232,25 +266,31 @@ def mkapp(base_dir="./"):
@socketio.on("rerun", namespace="/status")
def rerun(data):
+
+ t0 = time.time()
"""write to the child pty. The pty sees this as if you are typing in a real
terminal.
"""
- db = get_report_database()
+ # db = get_report_database()
targs = ".".join( data['test'].split("-") )
- m = '.'.join(db['modules'])
+ m = '.'.join(current_report['modules'])
# cmd = f"python -m {m} {targs}"
- cmd = f"python -m unittest {m}.{targs}"
- try:
- out = subprocess.run(cmd, cwd=db['local_base_dir_for_test_module'], shell=True, check=True, capture_output=True, text=True)
- except Exception as e: # I think this is related to simple exceptions being treated as errors.
- print(e)
- pass
+ # cmd = f"python -m unittest {m}.{targs}"
+ # import unittest
+ _run_test_cmd(dir=current_report['root_dir'], module_name=m, test_spec=targs, use_command_line=use_command_line)
+ # try:
+ # pass
+ # # out = subprocess.run(cmd, cwd=db['local_base_dir_for_test_module'], shell=True, check=True, capture_output=True, text=True)
+ # except Exception as e: # I think this is related to simple exceptions being treated as errors.
+ # print(e)
+ # pass
# print("oh dear.")
- for q in db['questions']:
- for i in db['questions'][q]['tests']:
+ for q in current_report['questions']:
+ for i in current_report['questions'][q]['tests']:
if "-".join(i) == data['test']:
with watched_files_lock:
- watched_files_dictionary["*/"+db['questions'][q]['tests'][i]['artifact_file']]['last_recorded_change'] = datetime.datetime.now()
+ watched_files_dictionary["*/"+current_report['questions'][q]['tests'][i]['artifact_file']]['last_recorded_change'] = datetime.datetime.now()
+ print("rerun tests took", time.time()-t0)
@socketio.on("rerun_all", namespace="/status")
@@ -258,25 +298,10 @@ def mkapp(base_dir="./"):
"""write to the child pty. The pty sees this as if you are typing in a real
terminal.
"""
- db = get_report_database()
- # targs = ".".join( data['test'].split("-") )
- m = '.'.join(db['modules'])
- # cmd = f"python -m {m}"
- cmd = f"python -m unittest {m}"
- print(cmd)
- try:
- out = subprocess.run(cmd, cwd=db['local_base_dir_for_test_module'], shell=True, check=True, capture_output=True, text=True)
- except Exception as e:
- print(e)
-
- @socketio.on("pty-input", namespace="/pty")
- def pty_input(data):
- """write to the child pty. The pty sees this as if you are typing in a real
- terminal.
- """
- if app.config["fd"]:
- logging.debug("received input from browser: %s" % data["input"])
- os.write(app.config["fd"], data["input"].encode())
+ # db = get_report_database()
+ # db = current_report
+ m = '.'.join(current_report['modules'])
+ _run_test_cmd(dir=current_report['root_dir'], module_name=m, test_spec="", use_command_line=use_command_line)
@app.route("/crash")
def navbar():
@@ -286,12 +311,22 @@ def mkapp(base_dir="./"):
def wz():
return render_template('wz.html')
+ @socketio.event
+ def connect(sid, environ):
+ print(environ)
+ print(sid)
+ # username = authenticate_user(environ)
+ # socketio.save_session(sid, {'username': 'bobthebob'})
+
+
@socketio.on("reconnected", namespace="/status")
def client_reconnected(data):
"""write to the child pty. The pty sees this as if you are typing in a real
terminal.
"""
- print("Client has reconnected...")
+ print("--------Client has reconnected----------")
+ sid = 45;
+ print(f"{sid=}, {data=}")
with watched_files_lock:
for k in watched_files_dictionary:
if watched_files_dictionary[k]['type'] in ['token', 'question_json']:
@@ -300,53 +335,39 @@ def mkapp(base_dir="./"):
pass
else:
raise Exception()
+ x['client_id'] = data['id']
+ x['db_watcher'].mark_all_as_fresh()
- closeables = [x['watcher'], x['handler']]
+ closeables = [x['watcher'], x['handler'], x['db_watcher']]
return app, socketio, closeables
def main():
- parser = argparse.ArgumentParser(
- description=(
- "A fully functional terminal in your browser. "
- "https://github.com/cs01/pyxterm.js"
- ),
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
- )
- parser.add_argument("-p", "--port", default=5000, help="port to run server on")
- parser.add_argument("--host",default="127.0.0.1", help="host to run server on (use 0.0.0.0 to allow access from other hosts)",)
- parser.add_argument("--debug", action="store_true", help="debug the server")
- parser.add_argument("--version", action="store_true", help="print version and exit")
- # parser.add_argument("--command", default="bash", help="Command to run in the terminal")
- # parser.add_argument("--cmd-args",default="", help="arguments to pass to command (i.e. --cmd-args='arg1 arg2 --flag')",)
- args = parser.parse_args()
+ from cs108 import deploy
+ from cs108.report_devel import mk_bad
args_port = 5000
args_host = "127.0.0.1"
- from cs108 import deploy
- deploy.main(with_coverage=True)
- import subprocess
- from cs108.report_devel import mk_bad
+ # Deploy local files for debug.
+ deploy.main(with_coverage=False)
mk_bad()
- bdir = os.path.dirname(deploy.__file__)
+ bdir = os.path.dirname(deploy.__file__)
app, socketio, closeables = mkapp(base_dir=bdir)
- # app.config["cmd"] = [args.command] + shlex.split(args.cmd_args)
green = "\033[92m"
end = "\033[0m"
log_format = green + "pyxtermjs > " + end + "%(levelname)s (%(funcName)s:%(lineno)s) %(message)s"
- logging.basicConfig(
- format=log_format,
- stream=sys.stdout,
- level=logging.DEBUG if args.debug else logging.INFO,
- )
- logging.info(f"serving on http://{args.host}:{args.port}")
- debug = args.debug
- debug = True
- # os.environ["WERKZEUG_DEBUG_PIN"] = "off"
- socketio.run(app, debug=debug, port=args_port, host=args.host, allow_unsafe_werkzeug=debug)
+ debug = False
+ # logging.basicConfig(
+ # format=log_format,
+ # stream=sys.stdout,
+ # level=logging.DEBUG if True else logging.INFO,
+ # )
+ logging.info(f"serving on http://{args_host}:{args_port}")
+ os.environ["WERKZEUG_DEBUG_PIN"] = "off"
+ socketio.run(app, debug=debug, port=args_port, host=args_host, allow_unsafe_werkzeug=True)
for c in closeables:
c.close()
sys.exit()
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main() # 386
\ No newline at end of file
diff --git a/src/unitgrade/dashboard/app_helpers.py b/src/unitgrade/dashboard/app_helpers.py
index 9043179..02b7b1e 100644
--- a/src/unitgrade/dashboard/app_helpers.py
+++ b/src/unitgrade/dashboard/app_helpers.py
@@ -1,51 +1,78 @@
#!/usr/bin/env python3
-# from queue import Queue
-# from threading import Lock
-# import argparse
-# import datetime
-# import subprocess
-# from flask import Flask, render_template
-# from flask_socketio import SocketIO
-# import pty
+import sys
+import subprocess
+import unittest
import os
import glob
-from pupdb.core import PupDB
-# from unitgrade_private.hidden_gather_upload import picklestring2dict
-# from unitgrade_private.hidden_gather_upload import dict2picklestring, picklestring2dict
+import pickle
+# from pupdb.core import PupDB
from pathlib import Path
def get_available_reports(jobfolder):
bdir = os.path.abspath(jobfolder)
available_reports = {}
if os.path.isdir(bdir):
- fls = glob.glob(bdir + "/**/main_config_*.json", recursive=True)
+ fls = glob.glob(bdir + "/**/main_config_*.artifacts.pkl", recursive=True)
elif os.path.isfile(bdir):
- fls = glob.glob(os.path.dirname(bdir) + "/**/main_config_*.json", recursive=True)
+ fls = glob.glob(os.path.dirname(bdir) + "/**/main_config_*.artifacts.pkl", recursive=True)
else:
raise Exception("No report files found in the given directory. Start the dashboard in a folder which contains a report test file.")
for f in fls:
- db = PupDB(f)
+ with open(f, 'rb') as file:
+ db = pickle.load(file)
- report_py = db.get('relative_path')
+ # db = PupDB(f)
+
+ report_py = db['relative_path']
lpath_full = Path(os.path.normpath(os.path.dirname(f) + f"/../{os.path.basename(report_py)}"))
# rpath =
base = lpath_full.parts[:-len(Path(report_py).parts)]
# rs['local_base_dir_for_test_module'] = str(Path(*base))
- print("root_dir", base)
- print(f"{lpath_full=}")
+ # print("root_dir", base)
+ # print(f"{lpath_full=}")
root_dir = str(Path(*base))
- print(f"{root_dir=}")
- print(f"{base=}")
- print(f"{report_py=}")
+ # print(f"{root_dir=}")
+ # print(f"{base=}")
+ # print(f"{report_py=}")
+
+ # lpath_full = Path(os.path.normpath(os.path.dirname(dbjson) + "/../" + os.path.basename(dbjson)[12:].split(".")[0] + ".py"))
+ # rpath = Path(rs['relative_path'])
+ # base = lpath_full.parts[:-len(rpath.parts)]
+
+ # rs['local_base_dir_for_test_module'] = str(Path(*base))
+ # rs['test_module'] =
+
token = report_py[:-3] + "_grade.py"
available_reports[f] = {'json': f,
+ 'questions': db['questions'],
+ 'token_stub': db['token_stub'],
+ 'modules': db['modules'],
'relative_path': report_py,
'root_dir': root_dir,
- 'title': db.get('title'),
+ 'title': db.get('title', 'untitled report'),
'relative_path_token': None if not os.path.isfile(root_dir + "/" + token) else token,
'menu_name': os.path.basename(report_py),
+ 'rest_module': ".".join(db['modules'])
}
- return available_reports
\ No newline at end of file
+ return available_reports
+
+def _run_test_cmd(dir, module_name, test_spec="", use_command_line=False):
+ """
+ Example: run_test_cmd('/home/tuhe/../base', 'cs108.my_test_file', test_spect='Numpy.test_something')
+ """
+ if use_command_line:
+ try:
+ cmd = f"python -m unittest {module_name}{'.'+test_spec if test_spec is not None and len(test_spec)> 0 else ''}"
+ out = subprocess.run(cmd, cwd=dir, shell=True, check=True, capture_output=True, text=True)
+ print("running command", cmd, "output\n", out)
+ except Exception as e:
+ print(e)
+ else:
+ # If you use this, you have to do reliable refresh of the library to re-import files. Not sure that is easy or not.
+ dir = os.path.normpath(dir)
+ if dir not in sys.path:
+ sys.path.append(dir)
+ test = unittest.main(module=module_name, exit=False, argv=['', test_spec])
diff --git a/src/unitgrade/dashboard/dashboard_cli.py b/src/unitgrade/dashboard/dashboard_cli.py
index fb07564..0df8b87 100644
--- a/src/unitgrade/dashboard/dashboard_cli.py
+++ b/src/unitgrade/dashboard/dashboard_cli.py
@@ -24,17 +24,9 @@ def main():
if args.version:
print(__version__)
exit(0)
- # from cs108 import deploy
- # deploy.main(with_coverage=True)
- # import subprocess
- # subprocess.run("python ", cwd="")
- # from cs108.report_devel import mk_bad
- # mk_bad()
- # bdir = os.path.dirname(deploy.__file__)
app, socketio, closeables = mkapp(base_dir=args.dir)
- # app.config["cmd"] = [args.command] + shlex.split(args.cmd_args)
green = "\033[92m"
end = "\033[0m"
log_format = green + "pyxtermjs > " + end + "%(levelname)s (%(funcName)s:%(lineno)s) %(message)s"
@@ -47,11 +39,10 @@ def main():
logging.info(f"Starting unitgrade dashboard version {__version__}")
logging.info(f"Serving dashboard on: {url}")
- # debug = args.debug
debug = False
os.environ["WERKZEUG_DEBUG_PIN"] = "off"
+
import webbrowser
- # url = 'https://codefather.tech/blog/'
webbrowser.open(url)
socketio.run(app, debug=debug, port=args.port, host=args.host) # , allow_unsafe_werkzeug=True )
for c in closeables:
diff --git a/src/unitgrade/dashboard/dbwatcher.py b/src/unitgrade/dashboard/dbwatcher.py
new file mode 100644
index 0000000..f5416b5
--- /dev/null
+++ b/src/unitgrade/dashboard/dbwatcher.py
@@ -0,0 +1,51 @@
+import threading
+from threading import Thread
+import datetime
+import time
+from datetime import timedelta
+import os
+
+class DBWatcher(Thread):
+ def __init__(self, unitgrade_data_dir, watched_blocks_list, test_handle_function):
+ super().__init__()
+ self.stoprequest = threading.Event()
+ # self.unitgrade_data = unitgrade_data_dir
+ self.watched_blocks_list = watched_blocks_list
+ from diskcache import Cache
+ self.db = Cache(unitgrade_data_dir)
+ self.test_handle_function = test_handle_function
+
+ def mark_all_as_fresh(self):
+ for k in self.watched_blocks_list:
+ self.db[k + "-updated"] = True
+
+
+ def run(self):
+ print("A DB WATCHER INSTANCE HAS BEEN STARTED!")
+ # As long as we weren't asked to stop, try to take new tasks from the
+ # queue. The tasks are taken with a blocking 'get', so no CPU
+ # cycles are wasted while waiting.
+ # Also, 'get' is given a timeout, so stoprequest is always checked,
+ # even if there's nothing in the queue.
+ while not self.stoprequest.is_set():
+ ct = datetime.datetime.now()
+ d = None
+ k = "undef"
+ for k in self.watched_blocks_list:
+ ukey = k + "-updated"
+ with self.db.transact():
+ if ukey in self.db and self.db[ukey] and k in self.db:
+ d = self.db[k] # Dict of all values.
+ self.db[ukey] = False
+ break
+ time.sleep(max(0.2, (datetime.datetime.now()-ct).seconds ) )
+ if d is not None:
+ self.test_handle_function(k, d)
+
+ def join(self, timeout=None):
+ self.stoprequest.set()
+ super().join(timeout)
+
+ def close(self):
+ self.join()
+ print("Stopped DB watcher.")
diff --git a/src/unitgrade/dashboard/file_change_handler.py b/src/unitgrade/dashboard/file_change_handler.py
index cbc3cb6..7bdf69b 100644
--- a/src/unitgrade/dashboard/file_change_handler.py
+++ b/src/unitgrade/dashboard/file_change_handler.py
@@ -1,7 +1,7 @@
from threading import Thread
-from queue import Queue, Empty
import threading
import datetime
+from datetime import timedelta
import time
@@ -22,8 +22,6 @@ class FileChangeHandler(Thread):
# even if there's nothing in the queue.
while not self.stoprequest.is_set():
ct = datetime.datetime.now()
- # try:
-
file_to_handle = None
with self.watched_files_lock:
for k, v in self.watched_files_dictionary.items():
@@ -33,7 +31,7 @@ class FileChangeHandler(Thread):
else:
# This file has been handled recently. Check last change to the file.
if v['last_recorded_change'] is not None:
- from datetime import timedelta
+
if (v['last_recorded_change'] - v['last_handled_change'] ) > timedelta(seconds=0):
file_to_handle = k
break
@@ -45,23 +43,13 @@ class FileChangeHandler(Thread):
with self.watched_files_lock:
self.watched_files_dictionary[file_to_handle]['last_handled_change'] = datetime.datetime.now()
- time.sleep(min(0.1, (datetime.datetime.now()-ct).seconds ) )
+ time.sleep(max(0.1, 0.1 - (datetime.datetime.now()-ct).seconds ) )
def join(self, timeout=None):
- # print("Got a stop")
self.stoprequest.set()
super().join(timeout)
def close(self):
- print("Closing change handler..")
self.join()
- print("Closed.")
-
- #
- # q = Queue()
- # try:
- # e = q.get_nowait()
- # except Empty as e:
- # print("empty queue. ")
- # # menial tasks if required.
+ print("Stopped file change handler.")
diff --git a/src/unitgrade/dashboard/static/unitgrade.js b/src/unitgrade/dashboard/static/unitgrade.js
index a862da7..87c4496 100644
--- a/src/unitgrade/dashboard/static/unitgrade.js
+++ b/src/unitgrade/dashboard/static/unitgrade.js
@@ -1,26 +1,30 @@
const socket = io.connect("/status"); // Status of the tests.
socket.on("connect", () => {
+ console.log("socker.on('connect'){... }")
$("#status-connected").show(); // className = "badge rounded-pill bg-success"
$("#status-connecting").hide(); // className = "badge rounded-pill bg-success"
+ reconnect()
});
socket.on("disconnect", () => {
$("#status-connected").hide(); // className = "badge rounded-pill bg-success"
$("#status-connecting").show(); // className = "badge rounded-pill bg-success"
+
});
+function _rerun_test_term(test){
+ ui_set_state(test, 'running', {'stacktrace': 'Test is running'});
+ terminals[test][0].reset();
+ terminals[test][0].writeln('Dashboard> Rerunning test..');
+}
function re_run_all_tests(){
socket.emit("rerun_all", {});
for(test in terminals){
- ui_set_state(test, 'running', {'stacktrace': 'Test is running'});
- terminals[test][0].reset();
- terminals[test][0].writeln('Rerunning test...');
+ _rerun_test_term(test);
}
}
function re_run_test(test){
socket.emit("rerun", {'test': test});
- ui_set_state(test, 'running', {'stacktrace': 'Test is running'});
- terminals[test][0].reset();
- terminals[test][0].writeln('Rerunning test...');
+ _rerun_test_term(test);
}
function tests_and_state(){
/** This function update the token/test results. **/
@@ -43,8 +47,8 @@ socket.on("token_update", function(data){
e.innerHTML = data.token;
}
for(k in data.results){
- console.log(k);
- console.log(data.results[k]);
+// console.log(k);
+// console.log(data.results[k]);
state = data.results[k];
$("#tbl-"+k+"-token").removeClass();
$("#tbl-"+k+"-token").addClass(td_classes[state]);
@@ -69,8 +73,10 @@ function ui_set_state(test_id, state, data){
$("#tbl-"+test_id+"-title").addClass(td_classes[state]);
$("#tbl-"+test_id+"-unit").removeClass();
$("#tbl-"+test_id+"-unit").addClass(td_classes[state]);
- $("#tbl-"+test_id+"-unit")[0].querySelector("span").innerHTML = state;
-
+ tbl = $("#tbl-"+test_id+"-unit")[0];
+ if (tbl != null){
+ tbl.querySelector("span").innerHTML = state;
+ }
for(const e of $("." + test_id + "-status")){
var icon = e.querySelector("#" + test_id + "-icon")
if (icon != null){
@@ -79,7 +85,7 @@ function ui_set_state(test_id, state, data){
var icon = e.querySelector("#" + test_id + "-status")
if (icon != null){
nc = state_classes[state]
-
+// console.log("coverage files changeD?", data.coverage_files_changed);
if(data.coverage_files_changed != null){
nc = nc + " text-decoration-line-through";
}
@@ -104,7 +110,7 @@ function ui_set_state(test_id, state, data){
$('#'+test_id+'-stacktrace').html(data.stacktrace+js).ready(
function(){
$('.'+test_id +"-traceback").ready( function() {
- console.log('in 200ms, I will call: do_call_doc_ready("' + test_id+'")');
+// console.log('in 200ms, I will call: do_call_doc_ready("' + test_id+'")');
setTimeout(function(){
do_call_doc_ready(test_id)
}, 200);
@@ -138,18 +144,17 @@ function ui_set_state(test_id, state, data){
**/
socket.on("testupdate", function(data){
- console.log('> ', data.state, ': updating test with with id', data.id);
+ console.log('> ', data.id, "updating to state", data.state)
+
ui_set_state(data.id, data.state, data);
const targetNode = document.getElementById(''+data.id+'-stacktrace');
const callback = (mutationList, observer) => {
for (const mutation of mutationList) {
}
};
- //console.log(data.stdout);
if(data.run_id != terminals[data.id][2]['run_id']){
terminals[data.id][0].reset();
-// terminals[data.id][0].writeln('> Tes');
-
+// terminals[data.id][0].writeln('Dashboard> New run, resetting terminal');
terminals[data.id][2]['run_id'] = data.run_id;
terminals[data.id][2]['last_chunk_id'] = -1;
}
@@ -160,7 +165,12 @@ function ui_set_state(test_id, state, data){
terminals[data.id][2]['last_chunk_id'] = o[0]
}
}
+// if(data.state == 'pass' || data.state == 'fail' || data.state == 'error'){
+// terminals[data.id][0].writeln("Dashboard> Test execution completed.")
+// }
+
}
+ return "function called on client side."
});
function debounce(func, wait_ms) {
@@ -171,28 +181,17 @@ function ui_set_state(test_id, state, data){
timeout = setTimeout(() => func.apply(context, args), wait_ms);
};
}
- $("#status-connected").hide();
- function reconnect(){
- console.log("> Reconnected to server...");
- socket.emit("reconnected", {'hello': 'world'});
- // <span class="badge rounded-pill bg-success">Success</span>
- // $('#status').innerHTML = '<span style="background-color: lightgreen;">connected tp tje server.</span>';
- $("#status-connected").show(); // className = "badge rounded-pill bg-success"
- $("#status-connecting").hide(); // className = "badge rounded-pill bg-success"
-// $("#status").text('Connected')
- // console.log("changed html");
- /**
- socket.on("connect", () => {
- fitToscreen();
- status.innerHTML =
- '<span style="background-color: lightgreen;">connected</span>';
- });
- **/
- }
- const wait_ms = 50;
+$("#status-connected").hide();
+function reconnect(){
+ console.log("> Reconnected to server...");
+ socket.emit("reconnected", {'hello': 'world', 'id': socket.id});
+ $("#status-connected").show(); // className = "badge rounded-pill bg-success"
+ $("#status-connecting").hide(); // className = "badge rounded-pill bg-success"
+}
+const wait_ms = 50;
// window.onresize = debounce(fitToscreen, wait_ms);
//reconnect();
-window.onload = debounce(reconnect, wait_ms);
+//window.onload = debounce(reconnect, wait_ms);
/** This block of code is responsible for managing the terminals */
//console.log(terminals);
@@ -220,7 +219,7 @@ for (var key in terminals) {
for(k in terminals){
e = mpt.querySelector("#"+k + "-pane");
if ( e.classList.contains("active") ){
- console.log("Fitting the terminal given by ", k)
+// console.log("Fitting the terminal given by ", k)
terminals[k][1].fit();
}
}
diff --git a/src/unitgrade/dashboard/static/wz_js.js b/src/unitgrade/dashboard/static/wz_js.js
index 1a49a9a..c848bdc 100644
--- a/src/unitgrade/dashboard/static/wz_js.js
+++ b/src/unitgrade/dashboard/static/wz_js.js
@@ -25,9 +25,9 @@ function do_call_doc_ready(id){
function addToggleFrameTraceback(frames) {
frames.forEach((frame) => { // the problem may be the event listener is added multiple times...
- console.log("Adding event listener to frame ", frame);
+// console.log("Adding event listener to frame ", frame);
frame.addEventListener("click", () => {
- console.log("Now the element has been clicked. " + frame + " " + frame.getElementsByTagName("pre")[0].parentElement);
+// console.log("Now the element has been clicked. " + frame + " " + frame.getElementsByTagName("pre")[0].parentElement);
frame.getElementsByTagName("pre")[0].parentElement.classList.toggle("expanded");
});
})
diff --git a/src/unitgrade/dashboard/templates/base.html b/src/unitgrade/dashboard/templates/base.html
index ed8774e..845c287 100644
--- a/src/unitgrade/dashboard/templates/base.html
+++ b/src/unitgrade/dashboard/templates/base.html
@@ -61,10 +61,13 @@
<div class="d-flex flex-column" style="min-height: calc(100vh - 30px);">
<header>
<a href="/" class="d-flex align-items-center pb-1 mb-1 link-dark text-decoration-none">
+ <!---
<svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="bi me-2" width="30" height="24" viewBox="0 0 16 16">
<path d="M8.211 2.047a.5.5 0 0 0-.422 0l-7.5 3.5a.5.5 0 0 0 .025.917l7.5 3a.5.5 0 0 0 .372 0L14 7.14V13a1 1 0 0 0-1 1v2h3v-2a1 1 0 0 0-1-1V6.739l.686-.275a.5.5 0 0 0 .025-.917l-7.5-3.5Z"/>
<path d="M4.176 9.032a.5.5 0 0 0-.656.327l-.5 1.7a.5.5 0 0 0 .294.605l4.5 1.8a.5.5 0 0 0 .372 0l4.5-1.8a.5.5 0 0 0 .294-.605l-.5-1.7a.5.5 0 0 0-.656-.327L8 10.466 4.176 9.032Z"/>
</svg>
+ -->
+ <img src="/static/favicon.ico" width="30px">
<span class="fs-5 fw-semibold"> Unitgrade</span>
<span class="badge rounded-pill bg-success" id="status-connected">Connected</span>
<span class="badge rounded-pill bg-warning text-dark" id="status-connecting">Connecting</span>
@@ -80,6 +83,7 @@
{% endfor %}
</ul>
</div>
+
</header>
<content>
diff --git a/src/unitgrade/dashboard/templates/index3.html b/src/unitgrade/dashboard/templates/index3.html
index 758bab2..7497949 100644
--- a/src/unitgrade/dashboard/templates/index3.html
+++ b/src/unitgrade/dashboard/templates/index3.html
@@ -1,10 +1,6 @@
{% extends 'base.html' %}
-{% macro build_question_body(hi) %}
-{{hi}}
-{% endmacro %}
-{% block head %}
-{% endblock %}
-
+{% macro build_question_body(hi) %} {{hi}} {% endmacro %}
+{% block head %}{% endblock %}
{% block content %}
<div class="tab-content container-fluid" id="main_page_tabs">
<script>
@@ -12,7 +8,7 @@ var terminals = {};
</script>
{% set count=0 %}
{% for qkey, qbody in questions.items() %}
- {% set outer_loop = loop %}
+ {% set outer_loop = loop %}
{% for ikey, ibody in qbody.tests.items() %}
<script>terminals["{{ikey}}"] = null; </script>
<div class="tab-pane fade {{ 'show active' if outer_loop.index == 1 and loop.index == 1 else ''}}" id="{{ikey}}-pane" role="tabpanel" aria-labelledby="{{ikey}}-pane-tab">
@@ -20,9 +16,9 @@ var terminals = {};
<h1>{{qbody.title}}</h1>
<h4>
<span class="{{ikey}}-status">
- <span id="{{ikey}}-status"><i id="{{ikey}}-icon" class="bi bi-emoji-neutral"></i> <span class="text-left">{{ibody.title}}</span></span>
+ <span id="{{ikey}}-status"><i id="{{ikey}}-icon" class="bi bi-emoji-neutral"></i> <span class="text-left">{{ibody.title}}</span></span>
</span>
- <a onclick="re_run_test('{{ikey}}');" type="button" class="btn btn-primary">Rerun</a>
+ {% if ibody.runable %}<a onclick="re_run_test('{{ikey}}');" type="button" class="btn btn-primary">Rerun</a>{% endif %}
</h4>
<div class="card shadow mb-3 bg-white rounded">
@@ -33,9 +29,7 @@ var terminals = {};
<div style="padding-left: 5px; padding-top: 5px; background-color: black;">
<div style="width: 100%; height: 20%;" id="{{ikey}}"></div>
</div>
- <p class="card-text">
-
- </p>
+ <!-- <p class="card-text"></p> -->
</div>
</div>
@@ -52,7 +46,21 @@ var terminals = {};
<div class="card shadow mb-5 bg-white rounded">
<div class="card-header"> Hints </div>
<div class="card-body">
- <dl>
+ {% if ibody.coverage_files != None and ibody.coverage_files|length > 0 %}
+This test depends (directly or indirectly) on the following files which contains code you need to write:
+<ul>
+ {% for cov in ibody.coverage_files %}
+ <li> <pre>{{cov}}</pre></li>
+<ul>
+{% for m in ibody.coverage_files[cov] %}
+ <li><code> {{m + " (...)" }}</code> </li>
+{% endfor %}
+</ul>
+{% endfor %}
+</ul>
+{% endif %}
+ {% if ibody.hints|length > 0 %}
+<dl>
{% for h in ibody.hints %}
<dt>{% if not h[1] %} Overall hints: {% else %} From the file <emph>{{ h[1] }}</emph> {% endif %}</dt>
<dd>
@@ -64,7 +72,10 @@ var terminals = {};
</dd>
{% endfor %}
</dl>
-
+{% endif %}
+{% if ibody.hints|length == 0 and ibody.coverage_files|length == 0 %}
+This test does not appear to depend on any files that students have to edit.
+{% endif %}
</div>
</div>
</div>
@@ -106,41 +117,38 @@ var terminals = {};
{% endif %}
</tr>
{% for ikey, ibody in qbody.tests.items() %}
+ {% if ibody.runable %}
<tr style="line-height: 10px; height: 10px;">
<td id="tbl-{{ikey}}-title">{{ibody.title}}</td>
<td id="tbl-{{ikey}}-unit"><span class="test-state">Test has not been run</span></td>
<td id="tbl-{{ikey}}-token"><span class="test-state">Test has not been run</span></td>
</tr>
- <!--
- <div class="tab-pane fade" id="{{ikey}}-pane" role="tabpanel" aria-labelledby="{{ikey}}-pane-tab">
- -->
- <!-- begin tab card -->
- <!--
- <h1>{{qbody.title}}</h1>-->
-
+ {% endif %}
{% endfor %}
- </table>
-
{% endfor %}
+ </table>
</div></div>
<div class="card shadow mb-5 bg-white rounded">
<div class="card-header">Hand-in instructions:</div>
<div class="card-body">
<p>
-To hand in your results, you should run the file <code>{{grade_script}}</code>. You can either do this from your IDE, or by going to the directory:
-<pre>
- <code>{{root_dir}}</code>
-</pre>
-and from there run the command:<pre><code>
- {{run_cmd_grade}}</code>
-</pre>
-This will generate a <code>.token</code> file which contains your answers and you should upload to DTU learn.
+To hand in your results, you should run the file <code>{{grade_script}}</code>. You can either do this from your IDE, or by running the command:</p>
+<p style="text-indent: 25px">
+<code>{{run_cmd_grade}}</code>
+</p>
+<p>
+from the directory:</p>
+<p style="text-indent: 25px">
+<code>{{root_dir}}</code>
+</p>
+<p>
+This will generate a <code>.token</code> file which contains your answers and which you should upload to DTU learn. The outcome of your current token file can be seen above.
</p>
</div>
</div>
</div>
</div>
- </div>
+</div>
{% endblock %}
<!---------------------------------- NAVIGATION SECTION -------------------->
@@ -163,45 +171,30 @@ This will generate a <code>.token</code> file which contains your answers and yo
<div class="col col-lg-11 text-truncate" style="background-color: white;">
<button class="btn rounded collapsed nav-link {{ 'active' if outer_loop.index == 1 and loop.index == 1 else ''}} text-left" style="width: 100%;" id="{{ikey}}-pane-tab" data-bs-toggle="pill" data-bs-target="#{{ikey}}-pane" type="button" role="tab" aria-controls="{{ikey}}-pane" aria-selected="false" data-toggle="tab">
<span class="{{ikey}}-status">
- <span id="{{ikey}}-status"><i style="height: 16px;width: 16px; id="{{ikey}}-icon" class="bi bi-emoji-neutral"></i> <span class="text-left">{{ibody.title}}</span></span>
+ <span id="{{ikey}}-status"><i style="height: 16px;width: 16px;" id="{{ikey}}-icon" class="bi bi-emoji-neutral"></i> <span class="text-left">{{ibody.title}}</span></span>
</span>
</button>
</div>
+
+ {% if ibody.runable %}
<div class="col col-lg-auto" style="padding: 0px; backgrund-color: white;">
<a onclick="re_run_test('{{ikey}}');" type="button" class="btn btn-primary btn-sm" style="padding: 0px; margin: 0px;"><i class="bi bi-arrow-clockwise"></i></a>
</div>
+ {% endif %}
</div>
</div>
-
-
- <!--
- <button class="nav-link" id="v-pills-settings-tab" data-bs-toggle="pill" data-bs-target="#v-pills-settings" type="button" role="tab" aria-controls="v-pills-settings" aria-selected="false">Settings</button>
- -->
-
</li>
- {% endfor %} <!--
- <li><a href="#" class="link-dark rounded">Updates</a></li>
- <li><a href="#" class="link-dark rounded">Reports</a></li> -->
+ {% endfor %}
</ul>
</div>
</li>
{% endfor %}
</ul>
- <hr/>
+
{% endblock %}
{% block navigation_footer %}
<div class="btn-group-vertical" style="width: 100%">
<a onclick="re_run_all_tests();" class="btn btn-primary btn-sm" style="width: 100%;" type="button">Rerun all tests</a>
<button class="btn btn-success btn-sm" style="width: 100%;" id="token-pane-tab" data-bs-toggle="pill" data-bs-target="#token-pane" type="button" role="tab" aria-controls="token-pane" aria-selected="false" data-toggle="tab">Submit results</button>
</div>
-<!--
- <p>
- <a onclick="re_run_all_tests();" class="btn btn-primary btn-sm" style="width: 100%;" type="button">
- Rerun all tests
- </a></p>
- <p>
- <button class="btn btn-success btn-sm" style="width: 100%;" id="token-pane-tab" data-bs-toggle="pill" data-bs-target="#token-pane" type="button" role="tab" aria-controls="token-pane" aria-selected="false" data-toggle="tab">
- Submit results
- </button>
- </p>-->
{% endblock %}
\ No newline at end of file
diff --git a/src/unitgrade/dashboard/templates/info.html b/src/unitgrade/dashboard/templates/info.html
new file mode 100644
index 0000000..b714b59
--- /dev/null
+++ b/src/unitgrade/dashboard/templates/info.html
@@ -0,0 +1,38 @@
+{% extends 'base.html' %}
+{% block content %}
+
+{% macro write_dict(d) %}
+<dl>
+{% for k in d %}
+<dt style="margin-left: 20px">{{k}}</dt><dd style="margin-left: 20px">{% if d[k] is mapping %} {{write_dict(d[k])}} {% else %} {{ d[k] }} {% endif %}</dd>
+{% endfor %}
+</dl>
+{% endmacro %}
+
+<table>
+{% for q in questions %}
+<tr><td>
+<hr>
+ <table>
+ {% for t in questions[q]['tests'] %}
+
+ <tr>
+<td> {{t}}</td><td>
+{% set test = questions[q]['tests'][t] %}
+<table border="1">
+{% for k in test %}
+<tr><td>{{k}} </td><td>{{ test[k]}} </td></tr>
+ {% endfor %}
+ </table>
+</td>
+</tr>
+ {% endfor %}
+ </table>
+ </td></tr>
+{% endfor %}
+</table>
+<hr>
+<h4> Content of diskcache DB file</h4>
+{{ write_dict(db) }}
+
+{% endblock %}
\ No newline at end of file
diff --git a/src/unitgrade/dashboard/watcher.py b/src/unitgrade/dashboard/watcher.py
index 5ff3413..ef6903b 100644
--- a/src/unitgrade/dashboard/watcher.py
+++ b/src/unitgrade/dashboard/watcher.py
@@ -1,14 +1,9 @@
-import os
from watchdog.events import FileSystemEventHandler
-import time
from watchdog.observers import Observer
import datetime
import fnmatch
-import os
class Watcher:
- # DIRECTORY_TO_WATCH = "/path/to/my/directory"
-
def __init__(self, base_directory, watched_files_dictionary, watched_files_lock):
self.base_directory = base_directory
self.watched_files_dictionary = watched_files_dictionary
@@ -17,19 +12,16 @@ class Watcher:
def run(self):
event_handler = Handler(self.watched_files_dictionary, self.watched_files_lock)
- # directory = os.path.commonpath([os.path.dirname(f) for f in self.watched_files_dictionary.keys()])
- print("self.base_directory", self.base_directory)
+ # print("self.base_directory", self.base_directory)
self.observer.schedule(event_handler, self.base_directory, recursive=True)
self.observer.start()
def close(self):
- print("Closing watcher..")
self.observer.stop()
self.observer.join()
- print("closed")
+ print("Closed file change watcher.")
def __del__(self):
- print("Stopping watcher...")
self.close()
@@ -54,8 +46,8 @@ class Handler(FileSystemEventHandler):
if event.src_path.endswith("json"):
from pupdb.core import PupDB
db = PupDB(event.src_path)
- if db.get("state") == "fail":
- print("File watcher state:", db.get("state"), "in", event.src_path)
+ # if db.get("state") == "fail":
+ # print("File watcher state:", db.get("state"), "in", event.src_path)
self.watched_files_dictionary[fnd_]['last_recorded_change'] = datetime.datetime.now()
self.watched_files_dictionary[fnd_]['file'] = event.src_path
diff --git a/src/unitgrade/framework.py b/src/unitgrade/framework.py
index 2020cc6..8656b09 100644
--- a/src/unitgrade/framework.py
+++ b/src/unitgrade/framework.py
@@ -14,6 +14,8 @@ import ast
import numpy
from unitgrade.runners import UTextResult
from unitgrade.utils import gprint, Capturing2, Capturing
+from unitgrade.artifacts import StdCapturing
+from diskcache import Cache
colorama.init(autoreset=True) # auto resets your settings after every output
numpy.seterr(all='raise')
@@ -22,15 +24,97 @@ def setup_dir_by_class(C, base_dir):
name = C.__class__.__name__
return base_dir, name
-# def dash(func):
-# if isinstance(func, classmethod):
-# raise Exception("the @dash-decorator was used in the wrong order. The right order is: @dash\n@classmethod\ndef setUpClass(cls):")
-#
-# def wrapper(*args, **kwargs):
-# print("Something is happening before the function is called.")
-# func(*args, **kwargs)
-# print("Something is happening after the function is called.")
-# return wrapper
+
+class DKPupDB:
+ def __init__(self, artifact_file, use_pupdb=True):
+ # Make a double-headed disk cache thingy.
+ self.dk = Cache(os.path.dirname(artifact_file)) # Start in this directory.
+ self.name_ = os.path.basename(artifact_file[:-5])
+ if self.name_ not in self.dk:
+ self.dk[self.name_] = dict()
+ self.use_pupdb = use_pupdb
+ if self.use_pupdb:
+ from pupdb.core import PupDB
+ self.db_ = PupDB(artifact_file)
+
+ def __setitem__(self, key, value):
+ if self.use_pupdb:
+ self.db_.set(key, value)
+ with self.dk.transact():
+ d = self.dk[self.name_]
+ d[key] = value
+ self.dk[self.name_] = d
+ self.dk[self.name_ + "-updated"] = True
+
+ def __getitem__(self, item):
+ v = self.dk[self.name_][item]
+ if self.use_pupdb:
+ v2 = self.db_.get(item)
+ if v != v2:
+ print("Mismatch v1, v2 for ", item)
+ return v
+
+ def keys(self): # This one is also deprecated.
+ return tuple(self.dk[self.name_].keys()) #.iterkeys())
+ # return self.db_.keys()
+
+ def set(self, item, value): # This one is deprecated.
+ self[item] = value
+
+ def get(self, item, default=None):
+ return self[item] if item in self else default
+
+ def __contains__(self, item):
+ return item in self.dk[self.name_] #keys()
+ # return item in self.dk
+
+
+_DASHBOARD_COMPLETED_MESSAGE = "Dashboard> Evaluation completed."
+
+# Consolidate this code.
+class classmethod_dashboard(classmethod):
+ def __init__(self, f):
+ def dashboard_wrap(cls: UTestCase):
+ if not cls._generate_artifacts:
+ f(cls)
+ return
+
+ db = DKPupDB(cls._artifact_file_for_setUpClass())
+ r = np.random.randint(1000 * 1000)
+ db.set('run_id', r)
+ db.set('coverage_files_changed', None)
+
+ state_ = 'fail'
+ try:
+ _stdout = sys.stdout
+ _stderr = sys.stderr
+ std_capture = StdCapturing(stdout=sys.stdout, stderr=sys.stderr, db=db, mute=False)
+
+ # Run this unittest and record all of the output.
+ # This is probably where we should hijack the stdout output and save it -- after all, this is where the test is actually run.
+ # sys.stdout = stdout_capture
+ sys.stderr = std_capture.dummy_stderr
+ sys.stdout = std_capture.dummy_stdout
+ db.set("state", "running")
+ f(cls)
+ state_ = 'pass'
+ except Exception as e:
+ from werkzeug.debug.tbtools import DebugTraceback, _process_traceback
+ state_ = 'fail'
+ db.set('state', state_)
+ exi = e
+ dbt = DebugTraceback(exi)
+ sys.stderr.write(dbt.render_traceback_text())
+ html = dbt.render_traceback_html(include_title="hello world")
+ db.set('wz_stacktrace', html)
+ raise e
+ finally:
+ db.set('state', state_)
+ std_capture.dummy_stdout.write_mute(_DASHBOARD_COMPLETED_MESSAGE)
+ sys.stdout = _stdout
+ sys.stderr = _stderr
+ std_capture.close()
+ super().__init__(dashboard_wrap)
class Report:
title = "report title"
@@ -63,8 +147,7 @@ class Report:
def _artifact_file(self):
""" File for the artifacts DB (thread safe). This file is optinal. Note that it is a pupdb database file.
Note the file is shared between all sub-questions. """
- return os.path.join(os.path.dirname(self._file()), "unitgrade_data/main_config_"+ os.path.basename(self._file()[:-3]) + ".json")
-
+ return os.path.join(os.path.dirname(self._file()), "unitgrade_data/main_config_"+ os.path.basename(self._file()[:-3]) + ".artifacts.pkl")
def _is_run_in_grade_mode(self):
""" True if this report is being run as part of a grade run. """
@@ -80,7 +163,6 @@ class Report:
relative_path = os.path.relpath(self._file(), root_dir)
modules = os.path.normpath(relative_path[:-3]).split(os.sep)
relative_path = relative_path.replace("\\", "/")
-
return root_dir, relative_path, modules
def __init__(self, strict=False, payload=None):
@@ -128,6 +210,7 @@ class Report:
if with_coverage:
for q, _ in self.questions:
q._with_coverage = False
+
# report_cache is saved on a per-question basis.
# it could also contain additional information such as runtime metadata etc. This may not be appropriate to store with the invidivual questions(?).
# In this case, the function should be re-defined.
@@ -145,15 +228,11 @@ class Report:
if not url.endswith("/"):
url += "/"
snapshot_file = os.path.dirname(self._file()) + "/unitgrade_data/.snapshot"
- # print("Sanity checking time using snapshot", snapshot_file)
- # print("and using self-identified file", self._file())
-
if os.path.isfile(snapshot_file):
with open(snapshot_file, 'r') as f:
t = f.read()
if (time.time() - float(t)) < self._remote_check_cooldown_seconds:
return
- # print("Is this file run in local mode?", self._is_run_in_grade_mode())
if self.url.startswith("https://gitlab"):
# Try to turn url into a 'raw' format.
@@ -180,18 +259,6 @@ class Report:
print("You can find the most recent code here:")
print(self.url)
raise Exception(f"Version of grade script does not match the remote version. Please update using git pull")
- #
- # # node = ast.parse(text)
- # # classes = [n for n in node.body if isinstance(n, ast.ClassDef) if n.name == self.__class__.__name__][0]
- #
- # # for b in classes.body:
- # # print(b.)
- # # if b.targets[0].id == "version":
- # # print(b)
- # # print(b.value)
- # version_remote = b.value.value
- # break
- # if version_remote != self.version:
else:
text = requests.get(raw_url).text
node = ast.parse(text)
@@ -215,7 +282,7 @@ class Report:
for (q,_) in self.questions:
qq = q(skip_remote_check=True)
- cfile = qq._cache_file()
+ cfile = q._cache_file()
relpath = os.path.relpath(cfile, os.path.dirname(self._file()))
relpath = relpath.replace("\\", "/")
@@ -293,11 +360,6 @@ class UTestCase(unittest.TestCase):
file = sys.stdout
return Capturing2(stdout=file)
- # def __call__(self, *args, **kwargs):
- # a = '234'
- # pass
-
-
@classmethod
def question_title(cls):
""" Return the question title """
@@ -308,18 +370,19 @@ class UTestCase(unittest.TestCase):
return cls.__qualname__
def run(self, result):
+ # print("Run called in test framework...", self._generate_artifacts)
if not self._generate_artifacts:
return super().run(result)
from unitgrade.artifacts import StdCapturing
from unittest.case import TestCase
- from pupdb.core import PupDB
- db = PupDB(self._artifact_file())
- db.set('run_id', np.random.randint(1000*1000))
+
+ db = DKPupDB(self._artifact_file())
db.set("state", "running")
+ db.set('run_id', np.random.randint(1000*1000))
db.set('coverage_files_changed', None)
- # print("Re-running test")
+
_stdout = sys.stdout
_stderr = sys.stderr
@@ -327,7 +390,7 @@ class UTestCase(unittest.TestCase):
# stderr_capture = StdCapturing(sys.stderr, db=db)
# std_err_capture = StdCapturing(sys.stderr, "stderr", db=db)
-
+ state_ = None
try:
# Run this unittest and record all of the output.
# This is probably where we should hijack the stdout output and save it -- after all, this is where the test is actually run.
@@ -355,69 +418,23 @@ class UTestCase(unittest.TestCase):
sys.stderr.write(dbt.render_traceback_text())
html = dbt.render_traceback_html(include_title="hello world")
db.set('wz_stacktrace', html)
- db.set('state', 'fail')
+ # db.set('state', 'fail')
+ state_ = "fail"
else:
- db.set('state', 'pass')
+ state_ = "pass"
except Exception as e:
- print("-----------------.///////////////////////////////////////////////////////////////")
- # print(e)
+ state_ = "fail"
import traceback
traceback.print_exc()
raise e
finally:
+ db.set('state', state_)
+ std_capture.dummy_stdout.write_mute(_DASHBOARD_COMPLETED_MESSAGE)
sys.stdout = _stdout
sys.stderr = _stderr
std_capture.close()
return result_
- @classmethod
- def before_setup_called(cls):
- print("hi")
- # print("I am called before the fucking class is fucking made. setUpClass has been broken!")
- pass
-
- _setUpClass_not_overwritten = False
- @classmethod
- def setUpClass(cls) -> None:
- cls._setUpClass_not_overwritten = True
-
- @classmethod
- def __new__(cls, *args, **kwargs):
- old_setup = cls.setUpClass
- def new_setup():
- raise Exception("Bad")
- cls.before_setup_called()
- if cls.setUpClass == UTestCase.setUpClass:
- print("Setup class not overwritten")
- else:
- print("Setup class is overwritten")
-
- try:
- old_setup()
- except Exception as e:
- raise e
- finally:
- pass
-
- # cls.setUpClass = new_setup
- ci = super().__new__(cls)
- ci.setUpClass = new_setup
- return ci
-
- # def inheritors(klass):
- # import new
- # z.q = new.instancemethod(method, z, None)
-
- # def __getattr__(self, item):
- # # print("hi there ", item)
- # return super().__getattr__(item)
- #
- # def __getattribute__(self, item):
- # # print("oh hello sexy. ", item)
- # return super().__getattribute__(item)
-
-
-
def _callSetUp(self):
if self._with_coverage:
if self._covcache is None:
@@ -675,22 +692,25 @@ class UTestCase(unittest.TestCase):
self.assertFalse(max_diff >= tol, msg=f'Input arrays are not equal within tolerance {tol}')
# self.assertEqual(first, second, msg=msg + f"Not equal within tolerance {tol}")
- def _cache_file(self):
- return os.path.dirname(inspect.getabsfile(type(self))) + "/unitgrade_data/" + self.__class__.__name__ + ".pkl"
+ @classmethod
+ def _cache_file(cls):
+ return os.path.dirname(inspect.getabsfile(cls)) + "/unitgrade_data/" + cls.__name__ + ".pkl"
@classmethod
def _artifact_file_for_setUpClass(cls):
- cf = os.path.dirname(inspect.getabsfile(cls)) + "/unitgrade_data/" + cls.__name__
- return os.path.join(os.path.dirname(self._cache_file()), "-setUpClass.json")
+ file = os.path.join(os.path.dirname(cls._cache_file()), ""+cls.__name__+"-setUpClass.json")
+ print("_artifact_file_for_setUpClass(cls): will return", file, "__class__", cls)
+ # cf = os.path.dirname(inspect.getabsfile(cls)) + "/unitgrade_data/" + cls.__name__
+ return file
def _artifact_file(self):
""" File for the artifacts DB (thread safe). This file is optinal. Note that it is a pupdb database file.
Note the file is shared between all sub-questions. """
- return os.path.join(os.path.dirname(self._cache_file()), '-'.join(self.cache_id()) + ".json")
+ return os.path.join(os.path.dirname(self.__class__._cache_file()), '-'.join(self.cache_id()) + ".json")
def _save_cache(self):
# get the class name (i.e. what to save to).
- cfile = self._cache_file()
+ cfile = self.__class__._cache_file()
if not os.path.isdir(os.path.dirname(cfile)):
os.makedirs(os.path.dirname(cfile))
@@ -703,7 +723,7 @@ class UTestCase(unittest.TestCase):
if self._cache is not None: # Cache already loaded. We will not load it twice.
return
# raise Exception("Loaded cache which was already set. What is going on?!")
- cfile = self._cache_file()
+ cfile = self.__class__._cache_file()
if os.path.exists(cfile):
try:
with open(cfile, 'rb') as f:
@@ -719,7 +739,7 @@ class UTestCase(unittest.TestCase):
key = (self.cache_id(), 'coverage')
# CC = None
# if self._cache_contains(key):
- return self._cache_get(key, None)
+ return self._cache_get(key, []) # Anything wrong with the empty list?
# return CC
def _get_hints(self):
@@ -815,22 +835,6 @@ class UTestCase(unittest.TestCase):
def startTestRun(self):
super().startTestRun()
-# subclasses = set()
-# work = [UTestCase]
-# while work:
-# parent = work.pop()
-# for child in parent.__subclasses__():
-# if child not in subclasses:
-# subclasses.add(child)
-# work.append(child)
-# return subclasses
-# import builtins
-# ga = builtins.getattr
-# def my_funky_getatt(a,b,c=None):
-# print("ga", a, b, c)
-# return ga(a,b,c)
-# builtins.getattr = my_funky_getatt
-
class Required:
pass
@@ -857,16 +861,4 @@ class NotebookTestCase(UTestCase):
@property
def nb(self):
return self.__class__._nb
-
-# import __builtin__
-# all subclasses which are known at this point.
-# def get_all_subclasses(cls):
-# all_subclasses = []
-#
-# for subclass in cls.__subclasses__():
-# all_subclasses.append(subclass)
-# all_subclasses.extend(get_all_subclasses(subclass))
-#
-# return all_subclasses
-#
-# a = 234
\ No newline at end of file
+ # 870.
\ No newline at end of file
diff --git a/src/unitgrade/pupdb.py b/src/unitgrade/pupdb.py
new file mode 100644
index 0000000..b0b067e
--- /dev/null
+++ b/src/unitgrade/pupdb.py
@@ -0,0 +1,151 @@
+# """
+# Core module containing entrypoint functions for PupDB.
+#
+# Tue: This is my copy of this module to avoid project dependencies. The long-term plan is to switch to
+# a more up-to-date key/value store such as:
+#
+# https://grantjenks.com/docs/diskcache/
+#
+# """
+#
+# import logging
+# import os
+# import json
+# import traceback
+#
+# from filelock import FileLock
+#
+# logging.basicConfig(
+# level=logging.INFO,
+# format='%(asctime)s | %(process)d | %(levelname)s | %(message)s'
+# )
+#
+#
+# # pylint: disable=useless-object-inheritance
+# class PupDB(object):
+# """ This class represents the core of the PupDB database. """
+#
+# def __init__(self, db_file_path):
+# """ Initializes the PupDB database instance. """
+#
+# self.db_file_path = db_file_path
+# self.process_lock_path = '{}.lock'.format(db_file_path)
+# self.process_lock = FileLock(self.process_lock_path, timeout=-1)
+# self.init_db()
+#
+# def __repr__(self):
+# """ String representation of this class instance. """
+#
+# return str(self._get_database())
+#
+# def __len__(self):
+# """ Function to return the size of iterable. """
+#
+# return len(self._get_database())
+#
+# def init_db(self):
+# """ Initializes the database file. """
+#
+# with self.process_lock:
+# if not os.path.exists(self.db_file_path):
+# with open(self.db_file_path, 'w') as db_file:
+# db_file.write(json.dumps({}))
+# return True
+#
+# def _get_database(self):
+# """ Returns the database json object. """
+#
+# with self.process_lock:
+# with open(self.db_file_path, 'r') as db_file:
+# database = json.loads(db_file.read())
+# return database
+#
+# def _flush_database(self, database):
+# """ Flushes/Writes the database changes to disk. """
+#
+# with self.process_lock:
+# with open(self.db_file_path, 'w') as db_file:
+# db_file.write(json.dumps(database))
+# return True
+#
+# def set(self, key, val):
+# """
+# Sets the value to a key in the database.
+# Overwrites the value if the key already exists.
+# """
+#
+# try:
+# database = self._get_database()
+# database[key] = val
+# self._flush_database(database)
+# except Exception:
+# logging.error(
+# 'Error while writing to DB: %s', traceback.format_exc())
+# return False
+# return True
+#
+# def get(self, key):
+# """
+# Gets the value of a key from the database.
+# Returns None if the key is not found in the database.
+# """
+#
+# key = str(key)
+# database = self._get_database()
+# return database.get(key, None)
+#
+# def remove(self, key):
+# """
+# Removes a key from the database.
+# """
+#
+# key = str(key)
+# database = self._get_database()
+# if key not in database:
+# raise KeyError(
+# 'Non-existent Key {} in database'.format(key)
+# )
+# del database[key]
+#
+# try:
+# self._flush_database(database)
+# except Exception:
+# logging.error(
+# 'Error while writing to DB: %s', traceback.format_exc())
+# return False
+# return True
+#
+# def keys(self):
+# """
+# Returns a list (py27) or iterator (py3) of all the keys
+# in the database.
+# """
+#
+# return self._get_database().keys()
+#
+# def values(self):
+# """
+# Returns a list (py27) or iterator (py3) of all the values
+# in the database.
+# """
+#
+# return self._get_database().values()
+#
+# def items(self):
+# """
+# Returns a list (py27) or iterator (py3) of all the items i.e.
+# (key, val) pairs in the database.
+# """
+#
+# return self._get_database().items()
+#
+# def dumps(self):
+# """ Returns a string dump of the entire database sorted by key. """
+#
+# return json.dumps(self._get_database(), sort_keys=True)
+#
+# def truncate_db(self):
+# """ Truncates the entire database (makes it empty). """
+#
+# self._flush_database({})
+# return True
\ No newline at end of file
diff --git a/src/unitgrade/utils.py b/src/unitgrade/utils.py
index 8398c01..1c05e48 100644
--- a/src/unitgrade/utils.py
+++ b/src/unitgrade/utils.py
@@ -2,6 +2,10 @@ import re
import sys
import threading
import time
+import lzma
+import hashlib
+import pickle
+import base64
from collections import namedtuple
from io import StringIO
import numpy as np
@@ -21,8 +25,9 @@ msum = lambda x: sum(x)
mfloor = lambda x: np.floor(x)
-
-
+"""
+Clean up the various output-related helper classes.
+"""
class Logger(object):
def __init__(self, buffer, write_to_stdout=True):
# assert False
@@ -256,4 +261,46 @@ def methodsWithDecorator(cls, decorator):
if hasattr(maybeDecorated, 'decorator'):
if maybeDecorated.decorator == decorator:
print(maybeDecorated)
- yield maybeDecorated
\ No newline at end of file
+ yield maybeDecorated
+
+
+""" Methods responsible for turning a dictionary into a string that can be pickled or put into a json file. """
+def dict2picklestring(dd):
+ """
+ Turns a dictionary into a string with some compression.
+
+ :param dd:
+ :return:
+ """
+ b = lzma.compress(pickle.dumps(dd))
+ b_hash = hashlib.blake2b(b).hexdigest()
+ return base64.b64encode(b).decode("utf-8"), b_hash
+
+def picklestring2dict(picklestr):
+ """ Reverse of the above method: Turns the string back into a dictionary. """
+ b = base64.b64decode(picklestr)
+ hash = hashlib.blake2b(b).hexdigest()
+ dictionary = pickle.loads(lzma.decompress(b))
+ return dictionary, hash
+
+token_sep = "-"*70 + " ..ooO0Ooo.. " + "-"*70
+def load_token(file_in):
+ """ We put this one here to allow loading of token files for the dashboard. """
+ with open(file_in, 'r') as f:
+ s = f.read()
+ splt = s.split(token_sep)
+ data = splt[-1]
+ info = splt[-2]
+ head = token_sep.join(splt[:-2])
+ plain_text=head.strip()
+ hash, l1 = info.split(" ")
+ data = "".join( data.strip()[1:-1].splitlines() )
+ l1 = int(l1)
+ dictionary, b_hash = picklestring2dict(data)
+ assert len(data) == l1
+ assert b_hash == hash.strip()
+ return dictionary, plain_text
+
+
+
+## Key/value store related.
diff --git a/src/unitgrade/version.py b/src/unitgrade/version.py
index 4aa214f..effd49d 100644
--- a/src/unitgrade/version.py
+++ b/src/unitgrade/version.py
@@ -1 +1 @@
-__version__ = "0.1.28.2"
\ No newline at end of file
+__version__ = "0.1.29.0"
\ No newline at end of file
--
GitLab