Coverage for C:\Program Files\JetBrains\PyCharm Community Edition 2020.3.2\plugins\python-ce\helpers\pydev\pydevd_file_utils.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1r'''
2 This module provides utilities to get the absolute filenames so that we can be sure that:
3 - The case of a file will match the actual file in the filesystem (otherwise breakpoints won't be hit).
4 - Providing means for the user to make path conversions when doing a remote debugging session in
5 one machine and debugging in another.
7 To do that, the PATHS_FROM_ECLIPSE_TO_PYTHON constant must be filled with the appropriate paths.
9 @note:
10 in this context, the server is where your python process is running
11 and the client is where eclipse is running.
13 E.g.:
14 If the server (your python process) has the structure
15 /user/projects/my_project/src/package/module1.py
17 and the client has:
18 c:\my_project\src\package\module1.py
20 the PATHS_FROM_ECLIPSE_TO_PYTHON would have to be:
21 PATHS_FROM_ECLIPSE_TO_PYTHON = [(r'c:\my_project\src', r'/user/projects/my_project/src')]
23 alternatively, this can be set with an environment variable from the command line:
24 set PATHS_FROM_ECLIPSE_TO_PYTHON=[['c:\my_project\src','/user/projects/my_project/src']]
26 @note: DEBUG_CLIENT_SERVER_TRANSLATION can be set to True to debug the result of those translations
28 @note: the case of the paths is important! Note that this can be tricky to get right when one machine
29 uses a case-independent filesystem and the other uses a case-dependent filesystem (if the system being
30 debugged is case-independent, 'normcase()' should be used on the paths defined in PATHS_FROM_ECLIPSE_TO_PYTHON).
32 @note: all the paths with breakpoints must be translated (otherwise they won't be found in the server)
34 @note: to enable remote debugging in the target machine (pydev extensions in the eclipse installation)
35 import pydevd;pydevd.settrace(host, stdoutToServer, stderrToServer, port, suspend)
37 see parameter docs on pydevd.py
39 @note: for doing a remote debugging session, all the pydevd_ files must be on the server accessible
40 through the PYTHONPATH (and the PATHS_FROM_ECLIPSE_TO_PYTHON only needs to be set on the target
41 machine for the paths that'll actually have breakpoints).
42'''
44from _pydev_bundle import pydev_log
45from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding
46from _pydevd_bundle.pydevd_constants import IS_PY2, IS_PY3K, DebugInfoHolder, IS_WINDOWS, IS_JYTHON
47from _pydevd_bundle.pydevd_comm_constants import file_system_encoding, filesystem_encoding_is_utf8
48import json
49import os.path
50import sys
51import traceback
53_os_normcase = os.path.normcase
54basename = os.path.basename
55exists = os.path.exists
56join = os.path.join
58try:
59 rPath = os.path.realpath # @UndefinedVariable
60except:
61 # jython does not support os.path.realpath
62 # realpath is a no-op on systems without islink support
63 rPath = os.path.abspath
65# defined as a list of tuples where the 1st element of the tuple is the path in the client machine
66# and the 2nd element is the path in the server machine.
67# see module docstring for more details.
68try:
69 PATHS_FROM_ECLIPSE_TO_PYTHON = json.loads(os.environ.get('PATHS_FROM_ECLIPSE_TO_PYTHON', '[]'))
70except Exception:
71 sys.stderr.write('Error loading PATHS_FROM_ECLIPSE_TO_PYTHON from environment variable.\n')
72 traceback.print_exc()
73 PATHS_FROM_ECLIPSE_TO_PYTHON = []
74else:
75 if not isinstance(PATHS_FROM_ECLIPSE_TO_PYTHON, list):
76 sys.stderr.write('Expected PATHS_FROM_ECLIPSE_TO_PYTHON loaded from environment variable to be a list.\n')
77 PATHS_FROM_ECLIPSE_TO_PYTHON = []
78 else:
79 # Converting json lists to tuple
80 PATHS_FROM_ECLIPSE_TO_PYTHON = [tuple(x) for x in PATHS_FROM_ECLIPSE_TO_PYTHON]
82# example:
83# PATHS_FROM_ECLIPSE_TO_PYTHON = [
84# (r'd:\temp\temp_workspace_2\test_python\src\yyy\yyy',
85# r'd:\temp\temp_workspace_2\test_python\src\hhh\xxx')
86# ]
88convert_to_long_pathname = lambda filename:filename
89convert_to_short_pathname = lambda filename:filename
90get_path_with_real_case = lambda filename:filename
92if sys.platform == 'win32':
93 try:
94 import ctypes
95 from ctypes.wintypes import MAX_PATH, LPCWSTR, LPWSTR, DWORD
97 GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
98 GetLongPathName.argtypes = [LPCWSTR, LPWSTR, DWORD]
99 GetLongPathName.restype = DWORD
101 GetShortPathName = ctypes.windll.kernel32.GetShortPathNameW
102 GetShortPathName.argtypes = [LPCWSTR, LPWSTR, DWORD]
103 GetShortPathName.restype = DWORD
105 def _convert_to_long_pathname(filename):
106 buf = ctypes.create_unicode_buffer(MAX_PATH)
108 if IS_PY2 and isinstance(filename, str):
109 filename = filename.decode(getfilesystemencoding())
110 rv = GetLongPathName(filename, buf, MAX_PATH)
111 if rv != 0 and rv <= MAX_PATH:
112 filename = buf.value
114 if IS_PY2:
115 filename = filename.encode(getfilesystemencoding())
116 return filename
118 def _convert_to_short_pathname(filename):
119 buf = ctypes.create_unicode_buffer(MAX_PATH)
121 if IS_PY2 and isinstance(filename, str):
122 filename = filename.decode(getfilesystemencoding())
123 rv = GetShortPathName(filename, buf, MAX_PATH)
124 if rv != 0 and rv <= MAX_PATH:
125 filename = buf.value
127 if IS_PY2:
128 filename = filename.encode(getfilesystemencoding())
129 return filename
131 def _get_path_with_real_case(filename):
132 ret = convert_to_long_pathname(convert_to_short_pathname(filename))
133 # This doesn't handle the drive letter properly (it'll be unchanged).
134 # Make sure the drive letter is always uppercase.
135 if len(ret) > 1 and ret[1] == ':' and ret[0].islower():
136 return ret[0].upper() + ret[1:]
137 return ret
139 # Check that it actually works
140 _get_path_with_real_case(__file__)
141 except:
142 # Something didn't quite work out, leave no-op conversions in place.
143 if DebugInfoHolder.DEBUG_TRACE_LEVEL > 2:
144 traceback.print_exc()
145 else:
146 convert_to_long_pathname = _convert_to_long_pathname
147 convert_to_short_pathname = _convert_to_short_pathname
148 get_path_with_real_case = _get_path_with_real_case
151elif IS_JYTHON and IS_WINDOWS:
153 def get_path_with_real_case(filename):
154 from java.io import File
155 f = File(filename)
156 ret = f.getCanonicalPath()
157 if IS_PY2 and not isinstance(ret, str):
158 return ret.encode(getfilesystemencoding())
159 return ret
162if IS_WINDOWS:
164 if IS_JYTHON:
166 def normcase(filename):
167 return filename.lower()
169 else:
171 def normcase(filename):
172 # `normcase` doesn't lower case on Python 2 for non-English locale, but Java
173 # side does it, so we should do it manually.
174 if '~' in filename:
175 filename = convert_to_long_pathname(filename)
177 filename = _os_normcase(filename)
178 return filename.lower()
180else:
182 def normcase(filename):
183 return filename # no-op
185_ide_os = 'WINDOWS' if IS_WINDOWS else 'UNIX'
188def set_ide_os(os):
189 '''
190 We need to set the IDE os because the host where the code is running may be
191 actually different from the client (and the point is that we want the proper
192 paths to translate from the client to the server).
194 :param os:
195 'UNIX' or 'WINDOWS'
196 '''
197 global _ide_os
198 prev = _ide_os
199 if os == 'WIN': # Apparently PyCharm uses 'WIN' (https://github.com/fabioz/PyDev.Debugger/issues/116)
200 os = 'WINDOWS'
202 assert os in ('WINDOWS', 'UNIX')
204 if prev != os:
205 _ide_os = os
206 # We need to (re)setup how the client <-> server translation works to provide proper separators.
207 setup_client_server_paths(_last_client_server_paths_set)
210DEBUG_CLIENT_SERVER_TRANSLATION = os.environ.get('DEBUG_PYDEVD_PATHS_TRANSLATION', 'False').lower() in ('1', 'true')
212# Caches filled as requested during the debug session.
213NORM_PATHS_CONTAINER = {}
214NORM_PATHS_AND_BASE_CONTAINER = {}
217def _NormFile(filename):
218 abs_path, real_path = _NormPaths(filename)
219 return real_path
222def _AbsFile(filename):
223 abs_path, real_path = _NormPaths(filename)
224 return abs_path
227# Returns tuple of absolute path and real path for given filename
228def _NormPaths(filename):
229 try:
230 return NORM_PATHS_CONTAINER[filename]
231 except KeyError:
232 if filename.__class__ != str:
233 filename = _path_to_expected_str(filename)
234 if filename.__class__ != str:
235 pydev_log.warn('Failed to convert filename to str: %s (%s)' % (filename, type(filename)))
236 return '', ''
237 abs_path = _NormPath(filename, os.path.abspath)
238 real_path = _NormPath(filename, rPath)
240 # cache it for fast access later
241 NORM_PATHS_CONTAINER[filename] = abs_path, real_path
242 return abs_path, real_path
245def _NormPath(filename, normpath):
246 r = normpath(filename)
247 ind = r.find('.zip')
248 if ind == -1:
249 ind = r.find('.egg')
250 if ind != -1:
251 ind += 4
252 zip_path = r[:ind]
253 inner_path = r[ind:]
254 if inner_path.startswith('!'):
255 # Note (fabioz): although I can replicate this by creating a file ending as
256 # .zip! or .egg!, I don't really know what's the real-world case for this
257 # (still kept as it was added by @jetbrains, but it should probably be reviewed
258 # later on).
259 # Note 2: it goes hand-in-hand with 'exists'.
260 inner_path = inner_path[1:]
261 zip_path = zip_path + '!'
263 if inner_path.startswith('/') or inner_path.startswith('\\'):
264 inner_path = inner_path[1:]
265 if inner_path:
266 r = join(normcase(zip_path), inner_path)
267 return r
269 r = normcase(r)
270 return r
273_ZIP_SEARCH_CACHE = {}
274_NOT_FOUND_SENTINEL = object()
277def exists(file):
278 if os.path.exists(file):
279 return file
281 ind = file.find('.zip')
282 if ind == -1:
283 ind = file.find('.egg')
285 if ind != -1:
286 ind += 4
287 zip_path = file[:ind]
288 inner_path = file[ind:]
289 if inner_path.startswith("!"):
290 # Note (fabioz): although I can replicate this by creating a file ending as
291 # .zip! or .egg!, I don't really know what's the real-world case for this
292 # (still kept as it was added by @jetbrains, but it should probably be reviewed
293 # later on).
294 # Note 2: it goes hand-in-hand with '_NormPath'.
295 inner_path = inner_path[1:]
296 zip_path = zip_path + '!'
298 zip_file_obj = _ZIP_SEARCH_CACHE.get(zip_path, _NOT_FOUND_SENTINEL)
299 if zip_file_obj is None:
300 return False
301 elif zip_file_obj is _NOT_FOUND_SENTINEL:
302 try:
303 import zipfile
304 zip_file_obj = zipfile.ZipFile(zip_path, 'r')
305 _ZIP_SEARCH_CACHE[zip_path] = zip_file_obj
306 except:
307 _ZIP_SEARCH_CACHE[zip_path] = _NOT_FOUND_SENTINEL
308 return False
310 try:
311 if inner_path.startswith('/') or inner_path.startswith('\\'):
312 inner_path = inner_path[1:]
314 _info = zip_file_obj.getinfo(inner_path.replace('\\', '/'))
316 return join(zip_path, inner_path)
317 except KeyError:
318 return None
319 return None
322# Now, let's do a quick test to see if we're working with a version of python that has no problems
323# related to the names generated...
324try:
325 try:
326 code = rPath.func_code
327 except AttributeError:
328 code = rPath.__code__
329 if not exists(_NormFile(code.co_filename)):
330 sys.stderr.write('-------------------------------------------------------------------------------\n')
331 sys.stderr.write('pydev debugger: CRITICAL WARNING: This version of python seems to be incorrectly compiled (internal generated filenames are not absolute)\n')
332 sys.stderr.write('pydev debugger: The debugger may still function, but it will work slower and may miss breakpoints.\n')
333 sys.stderr.write('pydev debugger: Related bug: http://bugs.python.org/issue1666807\n')
334 sys.stderr.write('-------------------------------------------------------------------------------\n')
335 sys.stderr.flush()
337 NORM_SEARCH_CACHE = {}
339 initial_norm_paths = _NormPaths
341 def _NormPaths(filename): # Let's redefine _NormPaths to work with paths that may be incorrect
342 try:
343 return NORM_SEARCH_CACHE[filename]
344 except KeyError:
345 abs_path, real_path = initial_norm_paths(filename)
346 if not exists(real_path):
347 # We must actually go on and check if we can find it as if it was a relative path for some of the paths in the pythonpath
348 for path in sys.path:
349 abs_path, real_path = initial_norm_paths(join(path, filename))
350 if exists(real_path):
351 break
352 else:
353 sys.stderr.write('pydev debugger: Unable to find real location for: %s\n' % (filename,))
354 abs_path = filename
355 real_path = filename
357 NORM_SEARCH_CACHE[filename] = abs_path, real_path
358 return abs_path, real_path
360except:
361 # Don't fail if there's something not correct here -- but at least print it to the user so that we can correct that
362 traceback.print_exc()
364# Note: as these functions may be rebound, users should always import
365# pydevd_file_utils and then use:
366#
367# pydevd_file_utils.norm_file_to_client
368# pydevd_file_utils.norm_file_to_server
369#
370# instead of importing any of those names to a given scope.
373def _path_to_expected_str(filename):
374 if IS_PY2:
375 if not filesystem_encoding_is_utf8 and hasattr(filename, "decode"):
376 # filename_in_utf8 is a byte string encoded using the file system encoding
377 # convert it to utf8
378 filename = filename.decode(file_system_encoding)
380 if not isinstance(filename, bytes):
381 filename = filename.encode('utf-8')
383 else: # py3
384 if isinstance(filename, bytes):
385 filename = filename.decode(file_system_encoding)
387 return filename
390def _original_file_to_client(filename, cache={}):
391 try:
392 return cache[filename]
393 except KeyError:
394 cache[filename] = get_path_with_real_case(_AbsFile(filename))
395 return cache[filename]
397_original_file_to_server = _NormFile
399norm_file_to_client = _original_file_to_client
400norm_file_to_server = _original_file_to_server
403def _fix_path(path, sep):
404 if path.endswith('/') or path.endswith('\\'):
405 path = path[:-1]
407 if sep != '/':
408 path = path.replace('/', sep)
409 return path
412_last_client_server_paths_set = []
415def setup_client_server_paths(paths):
416 '''paths is the same format as PATHS_FROM_ECLIPSE_TO_PYTHON'''
418 global norm_file_to_client
419 global norm_file_to_server
420 global _last_client_server_paths_set
421 _last_client_server_paths_set = paths[:]
423 # Work on the client and server slashes.
424 python_sep = '\\' if IS_WINDOWS else '/'
425 eclipse_sep = '\\' if _ide_os == 'WINDOWS' else '/'
427 norm_filename_to_server_container = {}
428 norm_filename_to_client_container = {}
429 initial_paths = list(paths)
430 paths_from_eclipse_to_python = initial_paths[:]
432 # Apply normcase to the existing paths to follow the os preferences.
434 for i, (path0, path1) in enumerate(paths_from_eclipse_to_python[:]):
435 if IS_PY2:
436 if isinstance(path0, unicode):
437 path0 = path0.encode(sys.getfilesystemencoding())
438 if isinstance(path1, unicode):
439 path1 = path1.encode(sys.getfilesystemencoding())
441 path0 = _fix_path(path0, eclipse_sep)
442 path1 = _fix_path(path1, python_sep)
443 initial_paths[i] = (path0, path1)
445 paths_from_eclipse_to_python[i] = (normcase(path0), normcase(path1))
447 if not paths_from_eclipse_to_python:
448 # no translation step needed (just inline the calls)
449 norm_file_to_client = _original_file_to_client
450 norm_file_to_server = _original_file_to_server
451 return
453 # only setup translation functions if absolutely needed!
454 def _norm_file_to_server(filename, cache=norm_filename_to_server_container):
455 # Eclipse will send the passed filename to be translated to the python process
456 # So, this would be 'NormFileFromEclipseToPython'
457 try:
458 return cache[filename]
459 except KeyError:
460 if eclipse_sep != python_sep:
461 # Make sure that the separators are what we expect from the IDE.
462 filename = filename.replace(python_sep, eclipse_sep)
464 # used to translate a path from the client to the debug server
465 translated = normcase(filename)
466 for eclipse_prefix, server_prefix in paths_from_eclipse_to_python:
467 if translated.startswith(eclipse_prefix):
468 if DEBUG_CLIENT_SERVER_TRANSLATION:
469 sys.stderr.write('pydev debugger: replacing to server: %s\n' % (translated,))
470 translated = translated.replace(eclipse_prefix, server_prefix)
471 if DEBUG_CLIENT_SERVER_TRANSLATION:
472 sys.stderr.write('pydev debugger: sent to server: %s\n' % (translated,))
473 break
474 else:
475 if DEBUG_CLIENT_SERVER_TRANSLATION:
476 sys.stderr.write('pydev debugger: to server: unable to find matching prefix for: %s in %s\n' % \
477 (translated, [x[0] for x in paths_from_eclipse_to_python]))
479 # Note that when going to the server, we do the replace first and only later do the norm file.
480 if eclipse_sep != python_sep:
481 translated = translated.replace(eclipse_sep, python_sep)
482 translated = _NormFile(translated)
484 cache[filename] = translated
485 return translated
487 def _norm_file_to_client(filename, cache=norm_filename_to_client_container):
488 # The result of this method will be passed to eclipse
489 # So, this would be 'NormFileFromPythonToEclipse'
490 try:
491 return cache[filename]
492 except KeyError:
493 # used to translate a path from the debug server to the client
494 translated = _NormFile(filename)
496 # After getting the real path, let's get it with the path with
497 # the real case and then obtain a new normalized copy, just in case
498 # the path is different now.
499 translated_proper_case = get_path_with_real_case(translated)
500 translated = _NormFile(translated_proper_case)
502 if IS_WINDOWS:
503 if translated.lower() != translated_proper_case.lower():
504 translated_proper_case = translated
505 if DEBUG_CLIENT_SERVER_TRANSLATION:
506 sys.stderr.write(
507 'pydev debugger: _NormFile changed path (from: %s to %s)\n' % (
508 translated_proper_case, translated))
510 for i, (eclipse_prefix, python_prefix) in enumerate(paths_from_eclipse_to_python):
511 if translated.startswith(python_prefix):
512 if DEBUG_CLIENT_SERVER_TRANSLATION:
513 sys.stderr.write('pydev debugger: replacing to client: %s\n' % (translated,))
515 # Note: use the non-normalized version.
516 eclipse_prefix = initial_paths[i][0]
517 translated = eclipse_prefix + translated_proper_case[len(python_prefix):]
518 if DEBUG_CLIENT_SERVER_TRANSLATION:
519 sys.stderr.write('pydev debugger: sent to client: %s\n' % (translated,))
520 break
521 else:
522 if DEBUG_CLIENT_SERVER_TRANSLATION:
523 sys.stderr.write('pydev debugger: to client: unable to find matching prefix for: %s in %s\n' % \
524 (translated, [x[1] for x in paths_from_eclipse_to_python]))
525 translated = translated_proper_case
527 if eclipse_sep != python_sep:
528 translated = translated.replace(python_sep, eclipse_sep)
530 # The resulting path is not in the python process, so, we cannot do a _NormFile here,
531 # only at the beginning of this method.
532 cache[filename] = translated
533 return translated
535 norm_file_to_server = _norm_file_to_server
536 norm_file_to_client = _norm_file_to_client
539setup_client_server_paths(PATHS_FROM_ECLIPSE_TO_PYTHON)
542def _is_int(filename):
543 # isdigit() doesn't support negative numbers
544 try:
545 int(filename)
546 return True
547 except:
548 return False
550def is_real_file(filename):
551 # Check for Jupyter cells
552 return not _is_int(filename) and not filename.startswith("<ipython-input")
554# For given file f returns tuple of its absolute path, real path and base name
555def get_abs_path_real_path_and_base_from_file(f):
556 try:
557 return NORM_PATHS_AND_BASE_CONTAINER[f]
558 except:
559 if _NormPaths is None: # Interpreter shutdown
560 return f
562 if f is not None:
563 if f.endswith('.pyc'):
564 f = f[:-1]
565 elif f.endswith('$py.class'):
566 f = f[:-len('$py.class')] + '.py'
568 if not is_real_file(f):
569 abs_path, real_path, base = f, f, f
570 else:
571 abs_path, real_path = _NormPaths(f)
572 base = basename(real_path)
573 ret = abs_path, real_path, base
574 NORM_PATHS_AND_BASE_CONTAINER[f] = ret
575 return ret
578def get_abs_path_real_path_and_base_from_frame(frame):
579 try:
580 return NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename]
581 except:
582 # This one is just internal (so, does not need any kind of client-server translation)
583 f = frame.f_code.co_filename
584 if f is not None and f.startswith (('build/bdist.', 'build\\bdist.')):
585 # files from eggs in Python 2.7 have paths like build/bdist.linux-x86_64/egg/<path-inside-egg>
586 f = frame.f_globals['__file__']
587 if get_abs_path_real_path_and_base_from_file is None: # Interpreter shutdown
588 return f
590 ret = get_abs_path_real_path_and_base_from_file(f)
591 # Also cache based on the frame.f_code.co_filename (if we had it inside build/bdist it can make a difference).
592 NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename] = ret
593 return ret
596def get_fullname(mod_name):
597 if IS_PY3K:
598 import pkgutil
599 else:
600 from _pydev_imps import _pydev_pkgutil_old as pkgutil
601 try:
602 loader = pkgutil.get_loader(mod_name)
603 except:
604 return None
605 if loader is not None:
606 for attr in ("get_filename", "_get_filename"):
607 meth = getattr(loader, attr, None)
608 if meth is not None:
609 return meth(mod_name)
610 return None
613def get_package_dir(mod_name):
614 for path in sys.path:
615 mod_path = join(path, mod_name.replace('.', '/'))
616 if os.path.isdir(mod_path):
617 return mod_path
618 return None