Skip to content
Snippets Groups Projects
Commit f9797101 authored by fima's avatar fima :beers:
Browse files

Merge branch 'memory_utils' into 'main'

Completed memory_utils + memory info in load()

See merge request !19
parents d29492a6 a030867d
No related branches found
No related tags found
1 merge request!19Completed memory_utils + memory info in load()
......@@ -10,6 +10,7 @@ from pathlib import Path
import qim3d
from qim3d.io.logger import log
from qim3d.utils.internal_tools import sizeof
from qim3d.utils.system import Memory
class DataLoader:
......@@ -66,13 +67,11 @@ class DataLoader:
"""
if self.virtual_stack:
log.info("Using virtual stack")
vol = tifffile.memmap(path)
else:
vol = tifffile.imread(path)
log.info("Loaded shape: %s", vol.shape)
log.info("Using %s of memory", sizeof(sys.getsizeof(vol)))
return vol
......@@ -148,12 +147,9 @@ class DataLoader:
if not self.virtual_stack:
vol = vol[()] # Load dataset into memory
f.close()
else:
log.info("Using virtual stack")
log.info("Loaded the following dataset: %s", name)
log.info("Loaded shape: %s", vol.shape)
log.info("Using %s of memory", sizeof(sys.getsizeof(vol)))
if self.return_metadata:
return vol, metadata
......@@ -207,12 +203,9 @@ class DataLoader:
if not self.virtual_stack:
vol = np.copy(vol) # Copy to memory
else:
log.info("Using virtual stack")
log.info("Found %s file(s)", len(tiff_stack))
log.info("Loaded shape: %s", vol.shape)
log.info("Using %s of memory", sizeof(sys.getsizeof(vol)))
return vol
......@@ -243,7 +236,6 @@ class DataLoader:
) # In case of an XRM file, the third redundant dimension is removed
log.info("Loaded shape: %s", vol.shape)
log.info("Using %s of memory", sizeof(sys.getsizeof(vol)))
if self.virtual_stack:
raise NotImplementedError(
......@@ -351,7 +343,20 @@ def load(
**kwargs,
)
return loader.load(path)
data = loader.load(path)
if not virtual_stack:
mem = Memory()
log.info(
"Volume using %s of memory\n",
sizeof(data[0].nbytes if return_metadata else data.nbytes),
)
mem.report()
else:
log.info("Using virtual stack")
return data
class ImgExamples:
......
import qim3d
def test_memory():
mem = qim3d.utils.Memory()
assert all([
mem.total>0,
mem.used>0,
mem.free>0])
assert mem.used + mem.free == mem.total
......@@ -4,3 +4,4 @@ from .augmentations import Augmentation
from .data import Dataset, prepare_datasets, prepare_dataloaders
#from .doi import get_bibtex, get_reference
from . import doi
from .system import Memory
\ No newline at end of file
"""Provides tools for obtaining informaion about the system."""
import psutil
from qim3d.utils.internal_tools import sizeof
from qim3d.io.logger import log
class Memory:
"""Class for obtaining current memory information
Attributes:
total (int): Total system memory in bytes
free (int): Free system memory in bytes
used (int): Used system memory in bytes
used_pct (float): Used system memory in percentage
"""
def __init__(self):
mem = psutil.virtual_memory()
self.total = mem.total
self.free = mem.available
self.free_pct = (mem.free / mem.total) * 100
self.used = mem.total - mem.available
self.used_pct = mem.percent
def report(self):
log.info(
"System memory:\n • Total.: %s\n • Used..: %s (%s%%)\n • Free..: %s (%s%%)",
sizeof(self.total),
sizeof(self.used),
round(self.used_pct, 2),
sizeof(self.free),
round(self.free_pct, 2),
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment