Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import nested_scopes 

2 

3import os 

4import traceback 

5import warnings 

6 

7import pydevd_file_utils 

8 

9try: 

10 from urllib import quote 

11except: 

12 from urllib.parse import quote # @UnresolvedImport 

13 

14try: 

15 from collections import OrderedDict 

16except: 

17 OrderedDict = dict 

18 

19import inspect 

20from _pydevd_bundle.pydevd_constants import BUILTINS_MODULE_NAME, IS_PY38_OR_GREATER, dict_iter_items, get_global_debugger, IS_PY3K, LOAD_VALUES_POLICY, \ 

21 ValuesPolicy 

22import sys 

23from _pydev_bundle import pydev_log 

24from _pydev_imps._pydev_saved_modules import threading 

25 

26 

27def _normpath(filename): 

28 return pydevd_file_utils.get_abs_path_real_path_and_base_from_file(filename)[0] 

29 

30 

31def save_main_module(file, module_name): 

32 # patch provided by: Scott Schlesier - when script is run, it does not 

33 # use globals from pydevd: 

34 # This will prevent the pydevd script from contaminating the namespace for the script to be debugged 

35 # pretend pydevd is not the main module, and 

36 # convince the file to be debugged that it was loaded as main 

37 sys.modules[module_name] = sys.modules['__main__'] 

38 sys.modules[module_name].__name__ = module_name 

39 

40 with warnings.catch_warnings(): 

41 warnings.simplefilter("ignore", category=DeprecationWarning) 

42 warnings.simplefilter("ignore", category=PendingDeprecationWarning) 

43 from imp import new_module 

44 

45 m = new_module('__main__') 

46 sys.modules['__main__'] = m 

47 if hasattr(sys.modules[module_name], '__loader__'): 

48 m.__loader__ = getattr(sys.modules[module_name], '__loader__') 

49 m.__file__ = file 

50 

51 return m 

52 

53 

54def to_number(x): 

55 if is_string(x): 

56 try: 

57 n = float(x) 

58 return n 

59 except ValueError: 

60 pass 

61 

62 l = x.find('(') 

63 if l != -1: 

64 y = x[0:l - 1] 

65 # print y 

66 try: 

67 n = float(y) 

68 return n 

69 except ValueError: 

70 pass 

71 return None 

72 

73 

74def compare_object_attrs_key(x): 

75 if '__len__' == x: 

76 # __len__ should appear after other attributes in a list. 

77 num = 99999999 

78 else: 

79 num = to_number(x) 

80 if num is not None: 

81 return 1, num 

82 else: 

83 return -1, to_string(x) 

84 

85 

86 

87if IS_PY3K: 

88 

89 def is_string(x): 

90 return isinstance(x, str) 

91 

92else: 

93 

94 def is_string(x): 

95 return isinstance(x, basestring) 

96 

97 

98def to_string(x): 

99 if is_string(x): 

100 return x 

101 else: 

102 return str(x) 

103 

104 

105def print_exc(): 

106 if traceback: 

107 traceback.print_exc() 

108 

109 

110if IS_PY3K: 

111 

112 def quote_smart(s, safe='/'): 

113 return quote(s, safe) 

114 

115else: 

116 

117 def quote_smart(s, safe='/'): 

118 if isinstance(s, unicode): 

119 s = s.encode('utf-8') 

120 

121 return quote(s, safe) 

122 

123 

124def get_clsname_for_code(code, frame): 

125 clsname = None 

126 if len(code.co_varnames) > 0: 

127 # We are checking the first argument of the function 

128 # (`self` or `cls` for methods). 

129 first_arg_name = code.co_varnames[0] 

130 if first_arg_name in frame.f_locals: 

131 first_arg_obj = frame.f_locals[first_arg_name] 

132 if inspect.isclass(first_arg_obj): # class method 

133 first_arg_class = first_arg_obj 

134 else: # instance method 

135 first_arg_class = first_arg_obj.__class__ 

136 func_name = code.co_name 

137 if hasattr(first_arg_class, func_name): 

138 method = getattr(first_arg_class, func_name) 

139 func_code = None 

140 if hasattr(method, 'func_code'): # Python2 

141 func_code = method.func_code 

142 elif hasattr(method, '__code__'): # Python3 

143 func_code = method.__code__ 

144 if func_code and func_code == code: 

145 clsname = first_arg_class.__name__ 

146 

147 return clsname 

148 

149 

150_PROJECT_ROOTS_CACHE = [] 

151_LIBRARY_ROOTS_CACHE = [] 

152_FILENAME_TO_IN_SCOPE_CACHE = {} 

153 

154 

155def _convert_to_str_and_clear_empty(roots): 

156 if sys.version_info[0] <= 2: 

157 # In py2 we need bytes for the files. 

158 roots = [ 

159 root if not isinstance(root, unicode) else root.encode(sys.getfilesystemencoding()) 

160 for root in roots 

161 ] 

162 

163 new_roots = [] 

164 for root in roots: 

165 assert isinstance(root, str), '%s not str (found: %s)' % (root, type(root)) 

166 if root: 

167 new_roots.append(root) 

168 return new_roots 

169 

170 

171def _clear_caches_related_to_scope_changes(): 

172 # Clear related caches. 

173 _FILENAME_TO_IN_SCOPE_CACHE.clear() 

174 debugger = get_global_debugger() 

175 if debugger is not None: 

176 debugger.clear_skip_caches() 

177 

178 

179def _set_roots(roots, cache): 

180 roots = _convert_to_str_and_clear_empty(roots) 

181 new_roots = [] 

182 for root in roots: 

183 new_roots.append(_normpath(root)) 

184 cache.append(new_roots) 

185 # Leave only the last one added. 

186 del cache[:-1] 

187 _clear_caches_related_to_scope_changes() 

188 return new_roots 

189 

190 

191def _get_roots(cache, env_var, set_when_not_cached, get_default_val=None): 

192 if not cache: 

193 roots = os.getenv(env_var, None) 

194 if roots is not None: 

195 roots = roots.split(os.pathsep) 

196 else: 

197 if not get_default_val: 

198 roots = [] 

199 else: 

200 roots = get_default_val() 

201 if not roots: 

202 pydev_log.warn('%s being set to empty list.' % (env_var,)) 

203 set_when_not_cached(roots) 

204 return cache[-1] # returns the roots with case normalized 

205 

206 

207def _get_default_library_roots(): 

208 # Provide sensible defaults if not in env vars. 

209 import site 

210 roots = [sys.prefix] 

211 if hasattr(sys, 'base_prefix'): 

212 roots.append(sys.base_prefix) 

213 if hasattr(sys, 'real_prefix'): 

214 roots.append(sys.real_prefix) 

215 

216 if hasattr(site, 'getusersitepackages'): 

217 site_paths = site.getusersitepackages() 

218 if isinstance(site_paths, (list, tuple)): 

219 for site_path in site_paths: 

220 roots.append(site_path) 

221 else: 

222 roots.append(site_paths) 

223 

224 if hasattr(site, 'getsitepackages'): 

225 site_paths = site.getsitepackages() 

226 if isinstance(site_paths, (list, tuple)): 

227 for site_path in site_paths: 

228 roots.append(site_path) 

229 else: 

230 roots.append(site_paths) 

231 

232 for path in sys.path: 

233 if os.path.exists(path) and os.path.basename(path) == 'site-packages': 

234 roots.append(path) 

235 

236 return sorted(set(roots)) 

237 

238 

239# --- Project roots 

240def set_project_roots(project_roots): 

241 project_roots = _set_roots(project_roots, _PROJECT_ROOTS_CACHE) 

242 pydev_log.debug("IDE_PROJECT_ROOTS %s\n" % project_roots) 

243 

244 

245def _get_project_roots(project_roots_cache=_PROJECT_ROOTS_CACHE): 

246 return _get_roots(project_roots_cache, 'IDE_PROJECT_ROOTS', set_project_roots) 

247 

248 

249# --- Library roots 

250def set_library_roots(roots): 

251 roots = _set_roots(roots, _LIBRARY_ROOTS_CACHE) 

252 pydev_log.debug("LIBRARY_ROOTS %s\n" % roots) 

253 

254 

255def _get_library_roots(library_roots_cache=_LIBRARY_ROOTS_CACHE): 

256 return _get_roots(library_roots_cache, 'LIBRARY_ROOTS', set_library_roots, _get_default_library_roots) 

257 

258 

259def in_project_roots(filename, filename_to_in_scope_cache=_FILENAME_TO_IN_SCOPE_CACHE): 

260 # Note: the filename_to_in_scope_cache is the same instance among the many calls to the method 

261 try: 

262 return filename_to_in_scope_cache[filename] 

263 except: 

264 project_roots = _get_project_roots() 

265 original_filename = filename 

266 if not filename.endswith('>'): 

267 filename = _normpath(filename) 

268 

269 found_in_project = [] 

270 for root in project_roots: 

271 if root and filename.startswith(root): 

272 found_in_project.append(root) 

273 

274 found_in_library = [] 

275 library_roots = _get_library_roots() 

276 for root in library_roots: 

277 if root and filename.startswith(root): 

278 found_in_library.append(root) 

279 

280 if not project_roots: 

281 # If we have no project roots configured, consider it being in the project 

282 # roots if it's not found in site-packages (because we have defaults for those 

283 # and not the other way around). 

284 if filename.endswith('>'): 

285 in_project = False 

286 else: 

287 in_project = not found_in_library 

288 else: 

289 in_project = False 

290 if found_in_project: 

291 if not found_in_library: 

292 in_project = True 

293 else: 

294 # Found in both, let's see which one has the bigger path matched. 

295 if max(len(x) for x in found_in_project) > max(len(x) for x in found_in_library): 

296 in_project = True 

297 

298 filename_to_in_scope_cache[original_filename] = in_project 

299 return in_project 

300 

301 

302def is_exception_trace_in_project_scope(trace): 

303 if trace is None: 

304 return False 

305 elif in_project_roots(trace.tb_frame.f_code.co_filename): 

306 return True 

307 else: 

308 while trace is not None: 

309 if not in_project_roots(trace.tb_frame.f_code.co_filename): 

310 return False 

311 trace = trace.tb_next 

312 return True 

313 

314 

315def is_top_level_trace_in_project_scope(trace): 

316 if trace is not None and trace.tb_next is not None: 

317 return is_exception_trace_in_project_scope(trace) and not is_exception_trace_in_project_scope(trace.tb_next) 

318 return is_exception_trace_in_project_scope(trace) 

319 

320 

321def is_test_item_or_set_up_caller(trace): 

322 """Check if the frame is the test item or set up caller. 

323 

324 A test function caller is a function that calls actual test code which can be, for example, 

325 `unittest.TestCase` test method or function `pytest` assumes to be a test. A caller function 

326 is the one we want to trace to catch failed test events. Tracing test functions 

327 themselves is not possible because some exceptions can be caught in the test code, and 

328 we are interested only in exceptions that are propagated to the test framework level. 

329 """ 

330 if not trace: 

331 return False 

332 

333 frame = trace.tb_frame 

334 

335 abs_path, _, _ = pydevd_file_utils.get_abs_path_real_path_and_base_from_frame(frame) 

336 if in_project_roots(abs_path): 

337 # We are interested in exceptions made it to the test framework scope. 

338 return False 

339 

340 if not trace.tb_next: 

341 # This can happen when the exception has been raised inside a test item or set up caller. 

342 return False 

343 

344 if not _is_next_stack_trace_in_project_roots(trace): 

345 # The next stack frame must be the frame of a project scope function, otherwise we risk stopping 

346 # at a line a few times since multiple test framework functions we are looking for may appear in the stack. 

347 return False 

348 

349 # Set up and tear down methods can be checked immediately, since they are shared by both `pytest` and `unittest`. 

350 unittest_set_up_and_tear_down_methods = ('_callSetUp', '_callTearDown') 

351 if frame.f_code.co_name in unittest_set_up_and_tear_down_methods: 

352 return True 

353 

354 # It is important to check if the tests are run with `pytest` first because it can run `unittest` code 

355 # internally. This may lead to stopping on broken tests twice: one in the `pytest` test runner 

356 # and second in the `unittest` runner. 

357 is_pytest = False 

358 

359 f = frame 

360 while f: 

361 # noinspection SpellCheckingInspection 

362 if f.f_code.co_name == 'pytest_cmdline_main': 

363 is_pytest = True 

364 f = f.f_back 

365 

366 unittest_caller_names = ['_callTestMethod', 'runTest', 'run'] 

367 if IS_PY3K: 

368 unittest_caller_names.append('subTest') 

369 

370 if is_pytest: 

371 # noinspection SpellCheckingInspection 

372 if frame.f_code.co_name in ('pytest_pyfunc_call', 'call_fixture_func', '_eval_scope_callable', '_teardown_yield_fixture'): 

373 return True 

374 else: 

375 return frame.f_code.co_name in unittest_caller_names 

376 

377 else: 

378 import unittest 

379 test_case_obj = frame.f_locals.get('self') 

380 

381 # Check for `_FailedTest` is important to detect cases when tests cannot be run on the first place, 

382 # e.g. there was an import error in the test module. Can happen both in Python 3.8 and earlier versions. 

383 if isinstance(test_case_obj, getattr(getattr(unittest, 'loader', None), '_FailedTest', None)): 

384 return False 

385 

386 if frame.f_code.co_name in unittest_caller_names: 

387 # unittest and nose 

388 return True 

389 

390 return False 

391 

392 

393def _is_next_stack_trace_in_project_roots(trace): 

394 if trace and trace.tb_next and trace.tb_next.tb_frame: 

395 frame = trace.tb_next.tb_frame 

396 return in_project_roots(pydevd_file_utils.get_abs_path_real_path_and_base_from_frame(frame)[0]) 

397 return False 

398 

399 

400# noinspection SpellCheckingInspection 

401def should_stop_on_failed_test(exc_info): 

402 """Check if the debugger should stop on failed test. Some failed tests can be marked as expected failures 

403 and should be ignored because of that. 

404 

405 :param exc_info: exception type, value, and traceback 

406 :return: `False` if test is marked as an expected failure, ``True`` otherwise. 

407 """ 

408 exc_type, _, trace = exc_info 

409 

410 # unittest 

411 test_item = trace.tb_frame.f_locals.get('method') if IS_PY38_OR_GREATER else trace.tb_frame.f_locals.get('testMethod') 

412 if test_item: 

413 return not getattr(test_item, '__unittest_expecting_failure__', False) 

414 

415 # pytest 

416 testfunction = trace.tb_frame.f_locals.get('testfunction') 

417 if testfunction and hasattr(testfunction, 'pytestmark'): 

418 # noinspection PyBroadException 

419 try: 

420 for attr in testfunction.pytestmark: 

421 # noinspection PyUnresolvedReferences 

422 if attr.name == 'xfail': 

423 # noinspection PyUnresolvedReferences 

424 exc_to_ignore = attr.kwargs.get('raises') 

425 if not exc_to_ignore: 

426 # All exceptions should be ignored, if no type is specified. 

427 return False 

428 elif hasattr(exc_to_ignore, '__iter__'): 

429 return exc_type not in exc_to_ignore 

430 else: 

431 return exc_type is not exc_to_ignore 

432 except BaseException: 

433 pass 

434 return True 

435 

436 

437def is_exception_in_test_unit_can_be_ignored(exception): 

438 return exception.__name__ == 'SkipTest' 

439 

440 

441def get_top_level_trace_in_project_scope(trace): 

442 while trace: 

443 if is_top_level_trace_in_project_scope(trace): 

444 break 

445 trace = trace.tb_next 

446 return trace 

447 

448 

449def is_filter_enabled(): 

450 return os.getenv('PYDEVD_FILTERS') is not None 

451 

452 

453def is_filter_libraries(): 

454 is_filter = os.getenv('PYDEVD_FILTER_LIBRARIES') is not None 

455 pydev_log.debug("PYDEVD_FILTER_LIBRARIES %s\n" % is_filter) 

456 return is_filter 

457 

458 

459def _get_stepping_filters(filters_cache=[]): 

460 if not filters_cache: 

461 filters = os.getenv('PYDEVD_FILTERS', '').split(';') 

462 pydev_log.debug("PYDEVD_FILTERS %s\n" % filters) 

463 new_filters = [] 

464 for new_filter in filters: 

465 new_filters.append(new_filter) 

466 filters_cache.append(new_filters) 

467 return filters_cache[-1] 

468 

469 

470def is_ignored_by_filter(filename, filename_to_ignored_by_filters_cache={}): 

471 try: 

472 return filename_to_ignored_by_filters_cache[filename] 

473 except: 

474 import fnmatch 

475 for stepping_filter in _get_stepping_filters(): 

476 if fnmatch.fnmatch(filename, stepping_filter): 

477 pydev_log.debug("File %s ignored by filter %s" % (filename, stepping_filter)) 

478 filename_to_ignored_by_filters_cache[filename] = True 

479 break 

480 else: 

481 filename_to_ignored_by_filters_cache[filename] = False 

482 

483 return filename_to_ignored_by_filters_cache[filename] 

484 

485 

486def get_non_pydevd_threads(): 

487 threads = threading.enumerate() 

488 return [t for t in threads if t and not getattr(t, 'is_pydev_daemon_thread', False)] 

489 

490 

491def dump_threads(stream=None): 

492 ''' 

493 Helper to dump thread info. 

494 ''' 

495 if stream is None: 

496 stream = sys.stderr 

497 thread_id_to_name = {} 

498 try: 

499 for t in threading.enumerate(): 

500 thread_id_to_name[t.ident] = '%s (daemon: %s, pydevd thread: %s)' % ( 

501 t.name, t.daemon, getattr(t, 'is_pydev_daemon_thread', False)) 

502 except: 

503 pass 

504 

505 from _pydevd_bundle.pydevd_additional_thread_info_regular import _current_frames 

506 

507 stream.write('===============================================================================\n') 

508 stream.write('Threads running\n') 

509 stream.write('================================= Thread Dump =================================\n') 

510 stream.flush() 

511 

512 for thread_id, stack in _current_frames().items(): 

513 stream.write('\n-------------------------------------------------------------------------------\n') 

514 stream.write(" Thread %s" % thread_id_to_name.get(thread_id, thread_id)) 

515 stream.write('\n\n') 

516 

517 for i, (filename, lineno, name, line) in enumerate(traceback.extract_stack(stack)): 

518 

519 stream.write(' File "%s", line %d, in %s\n' % (filename, lineno, name)) 

520 if line: 

521 stream.write(" %s\n" % (line.strip())) 

522 

523 if i == 0 and 'self' in stack.f_locals: 

524 stream.write(' self: ') 

525 try: 

526 stream.write(str(stack.f_locals['self'])) 

527 except: 

528 stream.write('Unable to get str of: %s' % (type(stack.f_locals['self']),)) 

529 stream.write('\n') 

530 stream.flush() 

531 

532 stream.write('\n=============================== END Thread Dump ===============================') 

533 stream.flush() 

534 

535 

536def take_first_n_coll_elements(coll, n): 

537 if coll.__class__ in (list, tuple): 

538 return coll[:n] 

539 elif coll.__class__ in (set, frozenset): 

540 buf = [] 

541 for i, x in enumerate(coll): 

542 if i >= n: 

543 break 

544 buf.append(x) 

545 return type(coll)(buf) 

546 elif coll.__class__ in (dict, OrderedDict): 

547 ret = type(coll)() 

548 for i, (k, v) in enumerate(dict_iter_items(coll)): 

549 if i >= n: 

550 break 

551 ret[k] = v 

552 return ret 

553 else: 

554 raise TypeError("Unsupported collection type: '%s'" % str(coll.__class__)) 

555 

556 

557class VariableWithOffset(object): 

558 def __init__(self, data, offset): 

559 self.data, self.offset = data, offset 

560 

561 

562def get_var_and_offset(var): 

563 if isinstance(var, VariableWithOffset): 

564 return var.data, var.offset 

565 return var, 0 

566 

567 

568def is_pandas_container(type_qualifier, var_type, var): 

569 return var_type in ("DataFrame", "Series") and type_qualifier.startswith("pandas") and hasattr(var, "shape") 

570 

571 

572def is_numpy_container(type_qualifier, var_type, var): 

573 return var_type == "ndarray" and type_qualifier == "numpy" and hasattr(var, "shape") 

574 

575 

576def is_numeric_container(type_qualifier, var_type, var): 

577 return is_numpy_container(type_qualifier, var_type, var) or is_pandas_container(type_qualifier, var_type, var) 

578 

579 

580def is_builtin(x): 

581 return getattr(x, '__module__', None) == BUILTINS_MODULE_NAME 

582 

583 

584def is_numpy(x): 

585 if not getattr(x, '__module__', None) == 'numpy': 

586 return False 

587 type_name = x.__name__ 

588 return type_name == 'dtype' or type_name == 'bool_' or type_name == 'str_' or 'int' in type_name or 'uint' in type_name \ 

589 or 'float' in type_name or 'complex' in type_name 

590 

591 

592def should_evaluate_full_value(val): 

593 return LOAD_VALUES_POLICY == ValuesPolicy.SYNC \ 

594 or ((is_builtin(type(val)) or is_numpy(type(val))) and not isinstance(val, (list, tuple, dict, set, frozenset))) \ 

595 or (is_in_unittests_debugging_mode() and isinstance(val, Exception)) 

596 

597 

598def should_evaluate_shape(): 

599 return LOAD_VALUES_POLICY != ValuesPolicy.ON_DEMAND 

600 

601 

602def _series_to_str(s, max_items): 

603 res = [] 

604 s = s[:max_items] 

605 for item in s.iteritems(): 

606 # item: (index, value) 

607 res.append(str(item)) 

608 return ' '.join(res) 

609 

610 

611def _df_to_str(value): 

612 # Avoid using df.iteritems() or df.values[i], because it works very slow for large data frames 

613 # df.__str__() is already optimised and works fast enough 

614 res = [] 

615 rows = value.split('\n') 

616 for (i, r) in enumerate(rows): 

617 if i == 0: 

618 res.append(r.strip()) 

619 else: 

620 res.append("[%s]" % r) 

621 return ' '.join(res) 

622 

623 

624def pandas_to_str(df, type_name, str_value, max_items): 

625 try: 

626 if type_name == "Series": 

627 return _series_to_str(df, max_items) 

628 elif type_name == "DataFrame": 

629 return _df_to_str(str_value) 

630 else: 

631 return str(df) 

632 except Exception as e: 

633 pydev_log.warn("Failed to format pandas variable: " + str(e)) 

634 return str(df) 

635 

636 

637def format_numpy_array(num_array, max_items): 

638 return str(num_array[:max_items]).replace('\n', ',').strip() 

639 

640 

641def is_in_unittests_debugging_mode(): 

642 debugger = get_global_debugger() 

643 if debugger: 

644 return debugger.stop_on_failed_tests