Coverage for C:\Program Files\JetBrains\PyCharm Community Edition 2020.3.2\plugins\python-ce\helpers\pydev\_pydevd_bundle\pydevd_utils.py : 1%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import nested_scopes
3import os
4import traceback
5import warnings
7import pydevd_file_utils
9try:
10 from urllib import quote
11except:
12 from urllib.parse import quote # @UnresolvedImport
14try:
15 from collections import OrderedDict
16except:
17 OrderedDict = dict
19import inspect
20from _pydevd_bundle.pydevd_constants import BUILTINS_MODULE_NAME, IS_PY38_OR_GREATER, dict_iter_items, get_global_debugger, IS_PY3K, LOAD_VALUES_POLICY, \
21 ValuesPolicy
22import sys
23from _pydev_bundle import pydev_log
24from _pydev_imps._pydev_saved_modules import threading
27def _normpath(filename):
28 return pydevd_file_utils.get_abs_path_real_path_and_base_from_file(filename)[0]
31def save_main_module(file, module_name):
32 # patch provided by: Scott Schlesier - when script is run, it does not
33 # use globals from pydevd:
34 # This will prevent the pydevd script from contaminating the namespace for the script to be debugged
35 # pretend pydevd is not the main module, and
36 # convince the file to be debugged that it was loaded as main
37 sys.modules[module_name] = sys.modules['__main__']
38 sys.modules[module_name].__name__ = module_name
40 with warnings.catch_warnings():
41 warnings.simplefilter("ignore", category=DeprecationWarning)
42 warnings.simplefilter("ignore", category=PendingDeprecationWarning)
43 from imp import new_module
45 m = new_module('__main__')
46 sys.modules['__main__'] = m
47 if hasattr(sys.modules[module_name], '__loader__'):
48 m.__loader__ = getattr(sys.modules[module_name], '__loader__')
49 m.__file__ = file
51 return m
54def to_number(x):
55 if is_string(x):
56 try:
57 n = float(x)
58 return n
59 except ValueError:
60 pass
62 l = x.find('(')
63 if l != -1:
64 y = x[0:l - 1]
65 # print y
66 try:
67 n = float(y)
68 return n
69 except ValueError:
70 pass
71 return None
74def compare_object_attrs_key(x):
75 if '__len__' == x:
76 # __len__ should appear after other attributes in a list.
77 num = 99999999
78 else:
79 num = to_number(x)
80 if num is not None:
81 return 1, num
82 else:
83 return -1, to_string(x)
87if IS_PY3K:
89 def is_string(x):
90 return isinstance(x, str)
92else:
94 def is_string(x):
95 return isinstance(x, basestring)
98def to_string(x):
99 if is_string(x):
100 return x
101 else:
102 return str(x)
105def print_exc():
106 if traceback:
107 traceback.print_exc()
110if IS_PY3K:
112 def quote_smart(s, safe='/'):
113 return quote(s, safe)
115else:
117 def quote_smart(s, safe='/'):
118 if isinstance(s, unicode):
119 s = s.encode('utf-8')
121 return quote(s, safe)
124def get_clsname_for_code(code, frame):
125 clsname = None
126 if len(code.co_varnames) > 0:
127 # We are checking the first argument of the function
128 # (`self` or `cls` for methods).
129 first_arg_name = code.co_varnames[0]
130 if first_arg_name in frame.f_locals:
131 first_arg_obj = frame.f_locals[first_arg_name]
132 if inspect.isclass(first_arg_obj): # class method
133 first_arg_class = first_arg_obj
134 else: # instance method
135 first_arg_class = first_arg_obj.__class__
136 func_name = code.co_name
137 if hasattr(first_arg_class, func_name):
138 method = getattr(first_arg_class, func_name)
139 func_code = None
140 if hasattr(method, 'func_code'): # Python2
141 func_code = method.func_code
142 elif hasattr(method, '__code__'): # Python3
143 func_code = method.__code__
144 if func_code and func_code == code:
145 clsname = first_arg_class.__name__
147 return clsname
150_PROJECT_ROOTS_CACHE = []
151_LIBRARY_ROOTS_CACHE = []
152_FILENAME_TO_IN_SCOPE_CACHE = {}
155def _convert_to_str_and_clear_empty(roots):
156 if sys.version_info[0] <= 2:
157 # In py2 we need bytes for the files.
158 roots = [
159 root if not isinstance(root, unicode) else root.encode(sys.getfilesystemencoding())
160 for root in roots
161 ]
163 new_roots = []
164 for root in roots:
165 assert isinstance(root, str), '%s not str (found: %s)' % (root, type(root))
166 if root:
167 new_roots.append(root)
168 return new_roots
171def _clear_caches_related_to_scope_changes():
172 # Clear related caches.
173 _FILENAME_TO_IN_SCOPE_CACHE.clear()
174 debugger = get_global_debugger()
175 if debugger is not None:
176 debugger.clear_skip_caches()
179def _set_roots(roots, cache):
180 roots = _convert_to_str_and_clear_empty(roots)
181 new_roots = []
182 for root in roots:
183 new_roots.append(_normpath(root))
184 cache.append(new_roots)
185 # Leave only the last one added.
186 del cache[:-1]
187 _clear_caches_related_to_scope_changes()
188 return new_roots
191def _get_roots(cache, env_var, set_when_not_cached, get_default_val=None):
192 if not cache:
193 roots = os.getenv(env_var, None)
194 if roots is not None:
195 roots = roots.split(os.pathsep)
196 else:
197 if not get_default_val:
198 roots = []
199 else:
200 roots = get_default_val()
201 if not roots:
202 pydev_log.warn('%s being set to empty list.' % (env_var,))
203 set_when_not_cached(roots)
204 return cache[-1] # returns the roots with case normalized
207def _get_default_library_roots():
208 # Provide sensible defaults if not in env vars.
209 import site
210 roots = [sys.prefix]
211 if hasattr(sys, 'base_prefix'):
212 roots.append(sys.base_prefix)
213 if hasattr(sys, 'real_prefix'):
214 roots.append(sys.real_prefix)
216 if hasattr(site, 'getusersitepackages'):
217 site_paths = site.getusersitepackages()
218 if isinstance(site_paths, (list, tuple)):
219 for site_path in site_paths:
220 roots.append(site_path)
221 else:
222 roots.append(site_paths)
224 if hasattr(site, 'getsitepackages'):
225 site_paths = site.getsitepackages()
226 if isinstance(site_paths, (list, tuple)):
227 for site_path in site_paths:
228 roots.append(site_path)
229 else:
230 roots.append(site_paths)
232 for path in sys.path:
233 if os.path.exists(path) and os.path.basename(path) == 'site-packages':
234 roots.append(path)
236 return sorted(set(roots))
239# --- Project roots
240def set_project_roots(project_roots):
241 project_roots = _set_roots(project_roots, _PROJECT_ROOTS_CACHE)
242 pydev_log.debug("IDE_PROJECT_ROOTS %s\n" % project_roots)
245def _get_project_roots(project_roots_cache=_PROJECT_ROOTS_CACHE):
246 return _get_roots(project_roots_cache, 'IDE_PROJECT_ROOTS', set_project_roots)
249# --- Library roots
250def set_library_roots(roots):
251 roots = _set_roots(roots, _LIBRARY_ROOTS_CACHE)
252 pydev_log.debug("LIBRARY_ROOTS %s\n" % roots)
255def _get_library_roots(library_roots_cache=_LIBRARY_ROOTS_CACHE):
256 return _get_roots(library_roots_cache, 'LIBRARY_ROOTS', set_library_roots, _get_default_library_roots)
259def in_project_roots(filename, filename_to_in_scope_cache=_FILENAME_TO_IN_SCOPE_CACHE):
260 # Note: the filename_to_in_scope_cache is the same instance among the many calls to the method
261 try:
262 return filename_to_in_scope_cache[filename]
263 except:
264 project_roots = _get_project_roots()
265 original_filename = filename
266 if not filename.endswith('>'):
267 filename = _normpath(filename)
269 found_in_project = []
270 for root in project_roots:
271 if root and filename.startswith(root):
272 found_in_project.append(root)
274 found_in_library = []
275 library_roots = _get_library_roots()
276 for root in library_roots:
277 if root and filename.startswith(root):
278 found_in_library.append(root)
280 if not project_roots:
281 # If we have no project roots configured, consider it being in the project
282 # roots if it's not found in site-packages (because we have defaults for those
283 # and not the other way around).
284 if filename.endswith('>'):
285 in_project = False
286 else:
287 in_project = not found_in_library
288 else:
289 in_project = False
290 if found_in_project:
291 if not found_in_library:
292 in_project = True
293 else:
294 # Found in both, let's see which one has the bigger path matched.
295 if max(len(x) for x in found_in_project) > max(len(x) for x in found_in_library):
296 in_project = True
298 filename_to_in_scope_cache[original_filename] = in_project
299 return in_project
302def is_exception_trace_in_project_scope(trace):
303 if trace is None:
304 return False
305 elif in_project_roots(trace.tb_frame.f_code.co_filename):
306 return True
307 else:
308 while trace is not None:
309 if not in_project_roots(trace.tb_frame.f_code.co_filename):
310 return False
311 trace = trace.tb_next
312 return True
315def is_top_level_trace_in_project_scope(trace):
316 if trace is not None and trace.tb_next is not None:
317 return is_exception_trace_in_project_scope(trace) and not is_exception_trace_in_project_scope(trace.tb_next)
318 return is_exception_trace_in_project_scope(trace)
321def is_test_item_or_set_up_caller(trace):
322 """Check if the frame is the test item or set up caller.
324 A test function caller is a function that calls actual test code which can be, for example,
325 `unittest.TestCase` test method or function `pytest` assumes to be a test. A caller function
326 is the one we want to trace to catch failed test events. Tracing test functions
327 themselves is not possible because some exceptions can be caught in the test code, and
328 we are interested only in exceptions that are propagated to the test framework level.
329 """
330 if not trace:
331 return False
333 frame = trace.tb_frame
335 abs_path, _, _ = pydevd_file_utils.get_abs_path_real_path_and_base_from_frame(frame)
336 if in_project_roots(abs_path):
337 # We are interested in exceptions made it to the test framework scope.
338 return False
340 if not trace.tb_next:
341 # This can happen when the exception has been raised inside a test item or set up caller.
342 return False
344 if not _is_next_stack_trace_in_project_roots(trace):
345 # The next stack frame must be the frame of a project scope function, otherwise we risk stopping
346 # at a line a few times since multiple test framework functions we are looking for may appear in the stack.
347 return False
349 # Set up and tear down methods can be checked immediately, since they are shared by both `pytest` and `unittest`.
350 unittest_set_up_and_tear_down_methods = ('_callSetUp', '_callTearDown')
351 if frame.f_code.co_name in unittest_set_up_and_tear_down_methods:
352 return True
354 # It is important to check if the tests are run with `pytest` first because it can run `unittest` code
355 # internally. This may lead to stopping on broken tests twice: one in the `pytest` test runner
356 # and second in the `unittest` runner.
357 is_pytest = False
359 f = frame
360 while f:
361 # noinspection SpellCheckingInspection
362 if f.f_code.co_name == 'pytest_cmdline_main':
363 is_pytest = True
364 f = f.f_back
366 unittest_caller_names = ['_callTestMethod', 'runTest', 'run']
367 if IS_PY3K:
368 unittest_caller_names.append('subTest')
370 if is_pytest:
371 # noinspection SpellCheckingInspection
372 if frame.f_code.co_name in ('pytest_pyfunc_call', 'call_fixture_func', '_eval_scope_callable', '_teardown_yield_fixture'):
373 return True
374 else:
375 return frame.f_code.co_name in unittest_caller_names
377 else:
378 import unittest
379 test_case_obj = frame.f_locals.get('self')
381 # Check for `_FailedTest` is important to detect cases when tests cannot be run on the first place,
382 # e.g. there was an import error in the test module. Can happen both in Python 3.8 and earlier versions.
383 if isinstance(test_case_obj, getattr(getattr(unittest, 'loader', None), '_FailedTest', None)):
384 return False
386 if frame.f_code.co_name in unittest_caller_names:
387 # unittest and nose
388 return True
390 return False
393def _is_next_stack_trace_in_project_roots(trace):
394 if trace and trace.tb_next and trace.tb_next.tb_frame:
395 frame = trace.tb_next.tb_frame
396 return in_project_roots(pydevd_file_utils.get_abs_path_real_path_and_base_from_frame(frame)[0])
397 return False
400# noinspection SpellCheckingInspection
401def should_stop_on_failed_test(exc_info):
402 """Check if the debugger should stop on failed test. Some failed tests can be marked as expected failures
403 and should be ignored because of that.
405 :param exc_info: exception type, value, and traceback
406 :return: `False` if test is marked as an expected failure, ``True`` otherwise.
407 """
408 exc_type, _, trace = exc_info
410 # unittest
411 test_item = trace.tb_frame.f_locals.get('method') if IS_PY38_OR_GREATER else trace.tb_frame.f_locals.get('testMethod')
412 if test_item:
413 return not getattr(test_item, '__unittest_expecting_failure__', False)
415 # pytest
416 testfunction = trace.tb_frame.f_locals.get('testfunction')
417 if testfunction and hasattr(testfunction, 'pytestmark'):
418 # noinspection PyBroadException
419 try:
420 for attr in testfunction.pytestmark:
421 # noinspection PyUnresolvedReferences
422 if attr.name == 'xfail':
423 # noinspection PyUnresolvedReferences
424 exc_to_ignore = attr.kwargs.get('raises')
425 if not exc_to_ignore:
426 # All exceptions should be ignored, if no type is specified.
427 return False
428 elif hasattr(exc_to_ignore, '__iter__'):
429 return exc_type not in exc_to_ignore
430 else:
431 return exc_type is not exc_to_ignore
432 except BaseException:
433 pass
434 return True
437def is_exception_in_test_unit_can_be_ignored(exception):
438 return exception.__name__ == 'SkipTest'
441def get_top_level_trace_in_project_scope(trace):
442 while trace:
443 if is_top_level_trace_in_project_scope(trace):
444 break
445 trace = trace.tb_next
446 return trace
449def is_filter_enabled():
450 return os.getenv('PYDEVD_FILTERS') is not None
453def is_filter_libraries():
454 is_filter = os.getenv('PYDEVD_FILTER_LIBRARIES') is not None
455 pydev_log.debug("PYDEVD_FILTER_LIBRARIES %s\n" % is_filter)
456 return is_filter
459def _get_stepping_filters(filters_cache=[]):
460 if not filters_cache:
461 filters = os.getenv('PYDEVD_FILTERS', '').split(';')
462 pydev_log.debug("PYDEVD_FILTERS %s\n" % filters)
463 new_filters = []
464 for new_filter in filters:
465 new_filters.append(new_filter)
466 filters_cache.append(new_filters)
467 return filters_cache[-1]
470def is_ignored_by_filter(filename, filename_to_ignored_by_filters_cache={}):
471 try:
472 return filename_to_ignored_by_filters_cache[filename]
473 except:
474 import fnmatch
475 for stepping_filter in _get_stepping_filters():
476 if fnmatch.fnmatch(filename, stepping_filter):
477 pydev_log.debug("File %s ignored by filter %s" % (filename, stepping_filter))
478 filename_to_ignored_by_filters_cache[filename] = True
479 break
480 else:
481 filename_to_ignored_by_filters_cache[filename] = False
483 return filename_to_ignored_by_filters_cache[filename]
486def get_non_pydevd_threads():
487 threads = threading.enumerate()
488 return [t for t in threads if t and not getattr(t, 'is_pydev_daemon_thread', False)]
491def dump_threads(stream=None):
492 '''
493 Helper to dump thread info.
494 '''
495 if stream is None:
496 stream = sys.stderr
497 thread_id_to_name = {}
498 try:
499 for t in threading.enumerate():
500 thread_id_to_name[t.ident] = '%s (daemon: %s, pydevd thread: %s)' % (
501 t.name, t.daemon, getattr(t, 'is_pydev_daemon_thread', False))
502 except:
503 pass
505 from _pydevd_bundle.pydevd_additional_thread_info_regular import _current_frames
507 stream.write('===============================================================================\n')
508 stream.write('Threads running\n')
509 stream.write('================================= Thread Dump =================================\n')
510 stream.flush()
512 for thread_id, stack in _current_frames().items():
513 stream.write('\n-------------------------------------------------------------------------------\n')
514 stream.write(" Thread %s" % thread_id_to_name.get(thread_id, thread_id))
515 stream.write('\n\n')
517 for i, (filename, lineno, name, line) in enumerate(traceback.extract_stack(stack)):
519 stream.write(' File "%s", line %d, in %s\n' % (filename, lineno, name))
520 if line:
521 stream.write(" %s\n" % (line.strip()))
523 if i == 0 and 'self' in stack.f_locals:
524 stream.write(' self: ')
525 try:
526 stream.write(str(stack.f_locals['self']))
527 except:
528 stream.write('Unable to get str of: %s' % (type(stack.f_locals['self']),))
529 stream.write('\n')
530 stream.flush()
532 stream.write('\n=============================== END Thread Dump ===============================')
533 stream.flush()
536def take_first_n_coll_elements(coll, n):
537 if coll.__class__ in (list, tuple):
538 return coll[:n]
539 elif coll.__class__ in (set, frozenset):
540 buf = []
541 for i, x in enumerate(coll):
542 if i >= n:
543 break
544 buf.append(x)
545 return type(coll)(buf)
546 elif coll.__class__ in (dict, OrderedDict):
547 ret = type(coll)()
548 for i, (k, v) in enumerate(dict_iter_items(coll)):
549 if i >= n:
550 break
551 ret[k] = v
552 return ret
553 else:
554 raise TypeError("Unsupported collection type: '%s'" % str(coll.__class__))
557class VariableWithOffset(object):
558 def __init__(self, data, offset):
559 self.data, self.offset = data, offset
562def get_var_and_offset(var):
563 if isinstance(var, VariableWithOffset):
564 return var.data, var.offset
565 return var, 0
568def is_pandas_container(type_qualifier, var_type, var):
569 return var_type in ("DataFrame", "Series") and type_qualifier.startswith("pandas") and hasattr(var, "shape")
572def is_numpy_container(type_qualifier, var_type, var):
573 return var_type == "ndarray" and type_qualifier == "numpy" and hasattr(var, "shape")
576def is_numeric_container(type_qualifier, var_type, var):
577 return is_numpy_container(type_qualifier, var_type, var) or is_pandas_container(type_qualifier, var_type, var)
580def is_builtin(x):
581 return getattr(x, '__module__', None) == BUILTINS_MODULE_NAME
584def is_numpy(x):
585 if not getattr(x, '__module__', None) == 'numpy':
586 return False
587 type_name = x.__name__
588 return type_name == 'dtype' or type_name == 'bool_' or type_name == 'str_' or 'int' in type_name or 'uint' in type_name \
589 or 'float' in type_name or 'complex' in type_name
592def should_evaluate_full_value(val):
593 return LOAD_VALUES_POLICY == ValuesPolicy.SYNC \
594 or ((is_builtin(type(val)) or is_numpy(type(val))) and not isinstance(val, (list, tuple, dict, set, frozenset))) \
595 or (is_in_unittests_debugging_mode() and isinstance(val, Exception))
598def should_evaluate_shape():
599 return LOAD_VALUES_POLICY != ValuesPolicy.ON_DEMAND
602def _series_to_str(s, max_items):
603 res = []
604 s = s[:max_items]
605 for item in s.iteritems():
606 # item: (index, value)
607 res.append(str(item))
608 return ' '.join(res)
611def _df_to_str(value):
612 # Avoid using df.iteritems() or df.values[i], because it works very slow for large data frames
613 # df.__str__() is already optimised and works fast enough
614 res = []
615 rows = value.split('\n')
616 for (i, r) in enumerate(rows):
617 if i == 0:
618 res.append(r.strip())
619 else:
620 res.append("[%s]" % r)
621 return ' '.join(res)
624def pandas_to_str(df, type_name, str_value, max_items):
625 try:
626 if type_name == "Series":
627 return _series_to_str(df, max_items)
628 elif type_name == "DataFrame":
629 return _df_to_str(str_value)
630 else:
631 return str(df)
632 except Exception as e:
633 pydev_log.warn("Failed to format pandas variable: " + str(e))
634 return str(df)
637def format_numpy_array(num_array, max_items):
638 return str(num_array[:max_items]).replace('\n', ',').strip()
641def is_in_unittests_debugging_mode():
642 debugger = get_global_debugger()
643 if debugger:
644 return debugger.stop_on_failed_tests