1# Copyright 2014 Altera Corporation. All Rights Reserved. 2# Copyright 2015-2017 John McGehee 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""This module provides a base class derived from `unittest.TestClass` 17for unit tests using the :py:class:`pyfakefs` module. 18 19`fake_filesystem_unittest.TestCase` searches `sys.modules` for modules 20that import the `os`, `io`, `path` `shutil`, and `pathlib` modules. 21 22The `setUpPyfakefs()` method binds these modules to the corresponding fake 23modules from `pyfakefs`. Further, the `open()` built-in is bound to a fake 24`open()`. 25 26It is expected that `setUpPyfakefs()` be invoked at the beginning of the 27derived class' `setUp()` method. There is no need to add anything to the 28derived class' `tearDown()` method. 29 30During the test, everything uses the fake file system and modules. This means 31that even in your test fixture, familiar functions like `open()` and 32`os.makedirs()` manipulate the fake file system. 33 34Existing unit tests that use the real file system can be retrofitted to use 35pyfakefs by simply changing their base class from `:py:class`unittest.TestCase` 36to `:py:class`pyfakefs.fake_filesystem_unittest.TestCase`. 37""" 38import _io # type:ignore[import] 39import builtins 40import doctest 41import functools 42import genericpath 43import inspect 44import io 45import linecache 46import os 47import shutil 48import sys 49import tempfile 50import tokenize 51from importlib.abc import Loader, MetaPathFinder 52from types import ModuleType, TracebackType, FunctionType 53from typing import ( 54 Any, 55 Callable, 56 Dict, 57 List, 58 Set, 59 Tuple, 60 Optional, 61 Union, 62 Type, 63 Iterator, 64 cast, 65 ItemsView, 66 Sequence, 67) 68import unittest 69import warnings 70from unittest import TestSuite 71 72from pyfakefs.fake_filesystem import ( 73 set_uid, 74 set_gid, 75 reset_ids, 76 PatchMode, 77 FakeFilesystem, 78) 79from pyfakefs.helpers import IS_PYPY 80from pyfakefs.mox3_stubout import StubOutForTesting 81 82from importlib.machinery import ModuleSpec 83from importlib import reload 84 85from pyfakefs import fake_filesystem, fake_io, fake_os, fake_open, fake_path, fake_file 86from pyfakefs import fake_filesystem_shutil 87from pyfakefs import fake_pathlib 88from pyfakefs import mox3_stubout 89from pyfakefs.extra_packages import pathlib2, use_scandir 90 91if use_scandir: 92 from pyfakefs import fake_scandir 93 94OS_MODULE = "nt" if sys.platform == "win32" else "posix" 95PATH_MODULE = "ntpath" if sys.platform == "win32" else "posixpath" 96 97 98def patchfs( 99 _func: Optional[Callable] = None, 100 *, 101 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 102 modules_to_reload: Optional[List[ModuleType]] = None, 103 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 104 allow_root_user: bool = True, 105 use_known_patches: bool = True, 106 patch_open_code: PatchMode = PatchMode.OFF, 107 patch_default_args: bool = False, 108 use_cache: bool = True 109) -> Callable: 110 """Convenience decorator to use patcher with additional parameters in a 111 test function. 112 113 Usage:: 114 115 @patchfs 116 def test_my_function(fake_fs): 117 fake_fs.create_file('foo') 118 119 @patchfs(allow_root_user=False) 120 def test_with_patcher_args(fs): 121 os.makedirs('foo/bar') 122 """ 123 124 def wrap_patchfs(f: Callable) -> Callable: 125 @functools.wraps(f) 126 def wrapped(*args, **kwargs): 127 with Patcher( 128 additional_skip_names=additional_skip_names, 129 modules_to_reload=modules_to_reload, 130 modules_to_patch=modules_to_patch, 131 allow_root_user=allow_root_user, 132 use_known_patches=use_known_patches, 133 patch_open_code=patch_open_code, 134 patch_default_args=patch_default_args, 135 use_cache=use_cache, 136 ) as p: 137 args = list(args) 138 args.append(p.fs) 139 return f(*args, **kwargs) 140 141 return wrapped 142 143 if _func: 144 if not callable(_func): 145 raise TypeError( 146 "Decorator argument is not a function.\n" 147 "Did you mean `@patchfs(additional_skip_names=...)`?" 148 ) 149 if hasattr(_func, "patchings"): 150 _func.nr_patches = len(_func.patchings) # type: ignore 151 return wrap_patchfs(_func) 152 153 return wrap_patchfs 154 155 156DOCTEST_PATCHER = None 157 158 159def load_doctests( 160 loader: Any, 161 tests: TestSuite, 162 ignore: Any, 163 module: ModuleType, 164 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 165 modules_to_reload: Optional[List[ModuleType]] = None, 166 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 167 allow_root_user: bool = True, 168 use_known_patches: bool = True, 169 patch_open_code: PatchMode = PatchMode.OFF, 170 patch_default_args: bool = False, 171) -> TestSuite: # pylint:disable=unused-argument 172 """Load the doctest tests for the specified module into unittest. 173 Args: 174 loader, tests, ignore : arguments passed in from `load_tests()` 175 module: module under test 176 remaining args: see :py:class:`TestCase` for an explanation 177 178 File `example_test.py` in the pyfakefs release provides a usage example. 179 """ 180 has_patcher = Patcher.DOC_PATCHER is not None 181 if not has_patcher: 182 Patcher.DOC_PATCHER = Patcher( 183 additional_skip_names=additional_skip_names, 184 modules_to_reload=modules_to_reload, 185 modules_to_patch=modules_to_patch, 186 allow_root_user=allow_root_user, 187 use_known_patches=use_known_patches, 188 patch_open_code=patch_open_code, 189 patch_default_args=patch_default_args, 190 is_doc_test=True, 191 ) 192 assert Patcher.DOC_PATCHER is not None 193 globs = Patcher.DOC_PATCHER.replace_globs(vars(module)) 194 tests.addTests( 195 doctest.DocTestSuite( 196 module, 197 globs=globs, 198 setUp=Patcher.DOC_PATCHER.setUp, 199 tearDown=Patcher.DOC_PATCHER.tearDown, 200 ) 201 ) 202 return tests 203 204 205class TestCaseMixin: 206 """Test case mixin that automatically replaces file-system related 207 modules by fake implementations. 208 209 Attributes: 210 additional_skip_names: names of modules inside of which no module 211 replacement shall be performed, in addition to the names in 212 :py:attr:`fake_filesystem_unittest.Patcher.SKIPNAMES`. 213 Instead of the module names, the modules themselves may be used. 214 modules_to_reload: A list of modules that need to be reloaded 215 to be patched dynamically; may be needed if the module 216 imports file system modules under an alias 217 218 .. caution:: Reloading modules may have unwanted side effects. 219 modules_to_patch: A dictionary of fake modules mapped to the 220 fully qualified patched module names. Can be used to add patching 221 of modules not provided by `pyfakefs`. 222 223 If you specify some of these attributes here and you have DocTests, 224 consider also specifying the same arguments to :py:func:`load_doctests`. 225 226 Example usage in derived test classes:: 227 228 from unittest import TestCase 229 from fake_filesystem_unittest import TestCaseMixin 230 231 class MyTestCase(TestCase, TestCaseMixin): 232 def __init__(self, methodName='runTest'): 233 super(MyTestCase, self).__init__( 234 methodName=methodName, 235 additional_skip_names=['posixpath']) 236 237 import sut 238 239 class AnotherTestCase(TestCase, TestCaseMixin): 240 def __init__(self, methodName='runTest'): 241 super(MyTestCase, self).__init__( 242 methodName=methodName, modules_to_reload=[sut]) 243 """ 244 245 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None 246 modules_to_reload: Optional[List[ModuleType]] = None 247 modules_to_patch: Optional[Dict[str, ModuleType]] = None 248 249 @property 250 def patcher(self): 251 if hasattr(self, "_patcher"): 252 return self._patcher or Patcher.PATCHER 253 return Patcher.PATCHER 254 255 @property 256 def fs(self) -> FakeFilesystem: 257 return cast(FakeFilesystem, self.patcher.fs) 258 259 def setUpPyfakefs( 260 self, 261 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 262 modules_to_reload: Optional[List[ModuleType]] = None, 263 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 264 allow_root_user: bool = True, 265 use_known_patches: bool = True, 266 patch_open_code: PatchMode = PatchMode.OFF, 267 patch_default_args: bool = False, 268 use_cache: bool = True, 269 ) -> None: 270 """Bind the file-related modules to the :py:class:`pyfakefs` fake file 271 system instead of the real file system. Also bind the fake `open()` 272 function. 273 274 Invoke this at the beginning of the `setUp()` method in your unit test 275 class. 276 For the arguments, see the `TestCaseMixin` attribute description. 277 If any of the arguments is not None, it overwrites the settings for 278 the current test case. Settings the arguments here may be a more 279 convenient way to adapt the setting than overwriting `__init__()`. 280 """ 281 # if the class has already a patcher setup, we use this one 282 if Patcher.PATCHER is not None: 283 return 284 285 if additional_skip_names is None: 286 additional_skip_names = self.additional_skip_names 287 if modules_to_reload is None: 288 modules_to_reload = self.modules_to_reload 289 if modules_to_patch is None: 290 modules_to_patch = self.modules_to_patch 291 self._patcher = Patcher( 292 additional_skip_names=additional_skip_names, 293 modules_to_reload=modules_to_reload, 294 modules_to_patch=modules_to_patch, 295 allow_root_user=allow_root_user, 296 use_known_patches=use_known_patches, 297 patch_open_code=patch_open_code, 298 patch_default_args=patch_default_args, 299 use_cache=use_cache, 300 ) 301 302 self._patcher.setUp() 303 cast(TestCase, self).addCleanup(self._patcher.tearDown) 304 305 @classmethod 306 def setUpClassPyfakefs( 307 cls, 308 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 309 modules_to_reload: Optional[List[ModuleType]] = None, 310 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 311 allow_root_user: bool = True, 312 use_known_patches: bool = True, 313 patch_open_code: PatchMode = PatchMode.OFF, 314 patch_default_args: bool = False, 315 use_cache: bool = True, 316 ) -> None: 317 """Similar to :py:func:`setUpPyfakefs`, but as a class method that 318 can be used in `setUpClass` instead of in `setUp`. 319 The fake filesystem will live in all test methods in the test class 320 and can be used in the usual way. 321 Note that using both :py:func:`setUpClassPyfakefs` and 322 :py:func:`setUpPyfakefs` in the same class will not work correctly. 323 324 .. note:: This method is only available from Python 3.8 onwards. 325 """ 326 if sys.version_info < (3, 8): 327 raise NotImplementedError( 328 "setUpClassPyfakefs is only available in " 329 "Python versions starting from 3.8" 330 ) 331 332 # if the class has already a patcher setup, we use this one 333 if Patcher.PATCHER is not None: 334 return 335 336 if additional_skip_names is None: 337 additional_skip_names = cls.additional_skip_names 338 if modules_to_reload is None: 339 modules_to_reload = cls.modules_to_reload 340 if modules_to_patch is None: 341 modules_to_patch = cls.modules_to_patch 342 Patcher.PATCHER = Patcher( 343 additional_skip_names=additional_skip_names, 344 modules_to_reload=modules_to_reload, 345 modules_to_patch=modules_to_patch, 346 allow_root_user=allow_root_user, 347 use_known_patches=use_known_patches, 348 patch_open_code=patch_open_code, 349 patch_default_args=patch_default_args, 350 use_cache=use_cache, 351 ) 352 353 Patcher.PATCHER.setUp() 354 cast(TestCase, cls).addClassCleanup(Patcher.PATCHER.tearDown) 355 356 @classmethod 357 def fake_fs(cls): 358 """Convenience class method for accessing the fake filesystem. 359 For use inside `setUpClass`, after :py:func:`setUpClassPyfakefs` 360 has been called. 361 """ 362 if Patcher.PATCHER: 363 return Patcher.PATCHER.fs 364 return None 365 366 def pause(self) -> None: 367 """Pause the patching of the file system modules until `resume` is 368 called. After that call, all file system calls are executed in the 369 real file system. 370 Calling pause() twice is silently ignored. 371 372 """ 373 self.patcher.pause() 374 375 def resume(self) -> None: 376 """Resume the patching of the file system modules if `pause` has 377 been called before. After that call, all file system calls are 378 executed in the fake file system. 379 Does nothing if patching is not paused. 380 """ 381 self.patcher.resume() 382 383 384class TestCase(unittest.TestCase, TestCaseMixin): 385 """Test case class that automatically replaces file-system related 386 modules by fake implementations. Inherits :py:class:`TestCaseMixin`. 387 388 The arguments are explained in :py:class:`TestCaseMixin`. 389 """ 390 391 def __init__( 392 self, 393 methodName: str = "runTest", 394 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 395 modules_to_reload: Optional[List[ModuleType]] = None, 396 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 397 ): 398 """Creates the test class instance and the patcher used to stub out 399 file system related modules. 400 401 Args: 402 methodName: The name of the test method (same as in 403 unittest.TestCase) 404 """ 405 super().__init__(methodName) 406 407 self.additional_skip_names = additional_skip_names 408 self.modules_to_reload = modules_to_reload 409 self.modules_to_patch = modules_to_patch 410 411 def tearDownPyfakefs(self) -> None: 412 """This method is deprecated and exists only for backward 413 compatibility. It does nothing. 414 """ 415 416 417class Patcher: 418 """ 419 Instantiate a stub creator to bind and un-bind the file-related modules to 420 the :py:mod:`pyfakefs` fake modules. 421 422 The arguments are explained in :py:class:`TestCaseMixin`. 423 424 :py:class:`Patcher` is used in :py:class:`TestCaseMixin`. 425 :py:class:`Patcher` also works as a context manager for other tests:: 426 427 with Patcher(): 428 doStuff() 429 """ 430 431 """Stub nothing that is imported within these modules. 432 `sys` is included to prevent `sys.path` from being stubbed with the fake 433 `os.path`. 434 The `linecache` module is used to read the test file in case of test 435 failure to get traceback information before test tear down. 436 In order to make sure that reading the test file is not faked, 437 we skip faking the module. 438 We also have to set back the cached open function in tokenize. 439 """ 440 SKIPMODULES = { 441 None, 442 fake_filesystem, 443 fake_filesystem_shutil, 444 fake_os, 445 fake_io, 446 fake_open, 447 fake_path, 448 fake_file, 449 sys, 450 linecache, 451 tokenize, 452 os, 453 io, 454 _io, 455 genericpath, 456 os.path, 457 } 458 if sys.platform == "win32": 459 import nt # type:ignore[import] 460 import ntpath 461 462 SKIPMODULES.add(nt) 463 SKIPMODULES.add(ntpath) 464 else: 465 import posix 466 import posixpath 467 import fcntl 468 469 SKIPMODULES.add(posix) 470 SKIPMODULES.add(posixpath) 471 SKIPMODULES.add(fcntl) 472 473 # caches all modules that do not have file system modules or function 474 # to speed up _find_modules 475 CACHED_MODULES: Set[ModuleType] = set() 476 FS_MODULES: Dict[str, Set[Tuple[ModuleType, str]]] = {} 477 FS_FUNCTIONS: Dict[Tuple[str, str, str], Set[ModuleType]] = {} 478 FS_DEFARGS: List[Tuple[FunctionType, int, Callable[..., Any]]] = [] 479 SKIPPED_FS_MODULES: Dict[str, Set[Tuple[ModuleType, str]]] = {} 480 481 assert None in SKIPMODULES, "sys.modules contains 'None' values;" " must skip them." 482 483 IS_WINDOWS = sys.platform in ("win32", "cygwin") 484 485 SKIPNAMES: Set[str] = set() 486 487 # hold values from last call - if changed, the cache has to be invalidated 488 PATCHED_MODULE_NAMES: Set[str] = set() 489 ADDITIONAL_SKIP_NAMES: Set[str] = set() 490 PATCH_DEFAULT_ARGS = False 491 PATCHER: Optional["Patcher"] = None 492 DOC_PATCHER: Optional["Patcher"] = None 493 REF_COUNT = 0 494 DOC_REF_COUNT = 0 495 496 def __new__(cls, *args, **kwargs): 497 if kwargs.get("is_doc_test", False): 498 if cls.DOC_PATCHER is None: 499 cls.DOC_PATCHER = super().__new__(cls) 500 return cls.DOC_PATCHER 501 if cls.PATCHER is None: 502 cls.PATCHER = super().__new__(cls) 503 return cls.PATCHER 504 505 def __init__( 506 self, 507 additional_skip_names: Optional[List[Union[str, ModuleType]]] = None, 508 modules_to_reload: Optional[List[ModuleType]] = None, 509 modules_to_patch: Optional[Dict[str, ModuleType]] = None, 510 allow_root_user: bool = True, 511 use_known_patches: bool = True, 512 patch_open_code: PatchMode = PatchMode.OFF, 513 patch_default_args: bool = False, 514 use_cache: bool = True, 515 is_doc_test: bool = False, 516 ) -> None: 517 """ 518 Args: 519 additional_skip_names: names of modules inside of which no module 520 replacement shall be performed, in addition to the names in 521 :py:attr:`fake_filesystem_unittest.Patcher.SKIPNAMES`. 522 Instead of the module names, the modules themselves 523 may be used. 524 modules_to_reload: A list of modules that need to be reloaded 525 to be patched dynamically; may be needed if the module 526 imports file system modules under an alias 527 528 .. caution:: Reloading modules may have unwanted side effects. 529 modules_to_patch: A dictionary of fake modules mapped to the 530 fully qualified patched module names. Can be used to add 531 patching of modules not provided by `pyfakefs`. 532 allow_root_user: If True (default), if the test is run as root 533 user, the user in the fake file system is also considered a 534 root user, otherwise it is always considered a regular user. 535 use_known_patches: If True (the default), some patches for commonly 536 used packages are applied which make them usable with pyfakefs. 537 patch_open_code: If True, `io.open_code` is patched. The default 538 is not to patch it, as it mostly is used to load compiled 539 modules that are not in the fake file system. 540 patch_default_args: If True, default arguments are checked for 541 file system functions, which are patched. This check is 542 expansive, so it is off by default. 543 use_cache: If True (default), patched and non-patched modules are 544 cached between tests for performance reasons. As this is a new 545 feature, this argument allows to turn it off in case it 546 causes any problems. 547 """ 548 self.is_doc_test = is_doc_test 549 if is_doc_test: 550 if self.DOC_REF_COUNT > 0: 551 return 552 elif self.REF_COUNT > 0: 553 return 554 if not allow_root_user: 555 # set non-root IDs even if the real user is root 556 set_uid(1) 557 set_gid(1) 558 559 self._skip_names = self.SKIPNAMES.copy() 560 # save the original open function for use in pytest plugin 561 self.original_open = open 562 self.patch_open_code = patch_open_code 563 self.fake_open: fake_open.FakeFileOpen 564 565 if additional_skip_names is not None: 566 skip_names = [ 567 cast(ModuleType, m).__name__ if inspect.ismodule(m) else cast(str, m) 568 for m in additional_skip_names 569 ] 570 self._skip_names.update(skip_names) 571 572 self._fake_module_classes: Dict[str, Any] = {} 573 self._unfaked_module_classes: Dict[str, Any] = {} 574 self._class_modules: Dict[str, List[str]] = {} 575 self._init_fake_module_classes() 576 577 # reload tempfile under posix to patch default argument 578 self.modules_to_reload: List[ModuleType] = ( 579 [] if sys.platform == "win32" else [tempfile] 580 ) 581 if modules_to_reload is not None: 582 self.modules_to_reload.extend(modules_to_reload) 583 self.patch_default_args = patch_default_args 584 self.use_cache = use_cache 585 586 if use_known_patches: 587 from pyfakefs.patched_packages import ( 588 get_modules_to_patch, 589 get_classes_to_patch, 590 get_fake_module_classes, 591 ) 592 593 modules_to_patch = modules_to_patch or {} 594 modules_to_patch.update(get_modules_to_patch()) 595 self._class_modules.update(get_classes_to_patch()) 596 self._fake_module_classes.update(get_fake_module_classes()) 597 598 if modules_to_patch is not None: 599 for name, fake_module in modules_to_patch.items(): 600 self._fake_module_classes[name] = fake_module 601 patched_module_names = set(modules_to_patch) 602 else: 603 patched_module_names = set() 604 clear_cache = not use_cache 605 if use_cache: 606 if patched_module_names != self.PATCHED_MODULE_NAMES: 607 self.__class__.PATCHED_MODULE_NAMES = patched_module_names 608 clear_cache = True 609 if self._skip_names != self.ADDITIONAL_SKIP_NAMES: 610 self.__class__.ADDITIONAL_SKIP_NAMES = self._skip_names 611 clear_cache = True 612 if patch_default_args != self.PATCH_DEFAULT_ARGS: 613 self.__class__.PATCH_DEFAULT_ARGS = patch_default_args 614 clear_cache = True 615 616 if clear_cache: 617 self.clear_cache() 618 self._fake_module_functions: Dict[str, Dict] = {} 619 self._init_fake_module_functions() 620 621 # Attributes set by _refresh() 622 self._stubs: Optional[StubOutForTesting] = None 623 self.fs: Optional[FakeFilesystem] = None 624 self.fake_modules: Dict[str, Any] = {} 625 self.unfaked_modules: Dict[str, Any] = {} 626 627 # _isStale is set by tearDown(), reset by _refresh() 628 self._isStale = True 629 self._dyn_patcher: Optional[DynamicPatcher] = None 630 self._patching = False 631 632 @classmethod 633 def clear_fs_cache(cls) -> None: 634 """Clear the module cache.""" 635 cls.CACHED_MODULES = set() 636 cls.FS_MODULES = {} 637 cls.FS_FUNCTIONS = {} 638 cls.FS_DEFARGS = [] 639 cls.SKIPPED_FS_MODULES = {} 640 641 def clear_cache(self) -> None: 642 """Clear the module cache (convenience instance method).""" 643 self.__class__.clear_fs_cache() 644 645 def _init_fake_module_classes(self) -> None: 646 # IMPORTANT TESTING NOTE: Whenever you add a new module below, test 647 # it by adding an attribute in fixtures/module_with_attributes.py 648 # and a test in fake_filesystem_unittest_test.py, class 649 # TestAttributesWithFakeModuleNames. 650 self._fake_module_classes = { 651 "os": fake_os.FakeOsModule, 652 "shutil": fake_filesystem_shutil.FakeShutilModule, 653 "io": fake_io.FakeIoModule, 654 "pathlib": fake_pathlib.FakePathlibModule, 655 } 656 if IS_PYPY: 657 # in PyPy io.open, the module is referenced as _io 658 self._fake_module_classes["_io"] = fake_io.FakeIoModule 659 if sys.platform == "win32": 660 self._fake_module_classes["nt"] = fake_path.FakeNtModule 661 else: 662 self._fake_module_classes["fcntl"] = fake_filesystem.FakeFcntlModule 663 664 # class modules maps class names against a list of modules they can 665 # be contained in - this allows for alternative modules like 666 # `pathlib` and `pathlib2` 667 self._class_modules["Path"] = ["pathlib"] 668 self._unfaked_module_classes["pathlib"] = fake_pathlib.RealPathlibModule 669 if pathlib2: 670 self._fake_module_classes["pathlib2"] = fake_pathlib.FakePathlibModule 671 self._class_modules["Path"].append("pathlib2") 672 self._unfaked_module_classes["pathlib2"] = fake_pathlib.RealPathlibModule 673 self._fake_module_classes["Path"] = fake_pathlib.FakePathlibPathModule 674 self._unfaked_module_classes["Path"] = fake_pathlib.RealPathlibPathModule 675 if use_scandir: 676 self._fake_module_classes["scandir"] = fake_scandir.FakeScanDirModule 677 678 def _init_fake_module_functions(self) -> None: 679 # handle patching function imported separately like 680 # `from os import stat` 681 # each patched function name has to be looked up separately 682 for mod_name, fake_module in self._fake_module_classes.items(): 683 if hasattr(fake_module, "dir"): 684 module_dir = fake_module.dir 685 if inspect.isfunction(module_dir): 686 for fct_name in fake_module.dir(): 687 module_attr = (getattr(fake_module, fct_name), mod_name) 688 self._fake_module_functions.setdefault(fct_name, {})[ 689 mod_name 690 ] = module_attr 691 if mod_name == "os": 692 self._fake_module_functions.setdefault(fct_name, {})[ 693 OS_MODULE 694 ] = module_attr 695 696 # special handling for functions in os.path 697 fake_module = fake_filesystem.FakePathModule 698 for fct_name in fake_module.dir(): 699 module_attr = (getattr(fake_module, fct_name), PATH_MODULE) 700 self._fake_module_functions.setdefault(fct_name, {})[ 701 "genericpath" 702 ] = module_attr 703 self._fake_module_functions.setdefault(fct_name, {})[ 704 PATH_MODULE 705 ] = module_attr 706 707 def __enter__(self) -> "Patcher": 708 """Context manager for usage outside of 709 fake_filesystem_unittest.TestCase. 710 Ensure that all patched modules are removed in case of an 711 unhandled exception. 712 """ 713 self.setUp() 714 return self 715 716 def __exit__( 717 self, 718 exc_type: Optional[Type[BaseException]], 719 exc_val: Optional[BaseException], 720 exc_tb: Optional[TracebackType], 721 ) -> None: 722 self.tearDown() 723 724 def _is_fs_module( 725 self, mod: ModuleType, name: str, module_names: List[str] 726 ) -> bool: 727 try: 728 return ( 729 inspect.ismodule(mod) 730 and mod.__name__ in module_names 731 or inspect.isclass(mod) 732 and mod.__module__ in self._class_modules.get(name, []) 733 ) 734 except Exception: 735 # handle cases where the module has no __name__ or __module__ 736 # attribute - see #460, and any other exception triggered 737 # by inspect functions 738 return False 739 740 def _is_fs_function(self, fct: FunctionType) -> bool: 741 try: 742 return ( 743 (inspect.isfunction(fct) or inspect.isbuiltin(fct)) 744 and fct.__name__ in self._fake_module_functions 745 and fct.__module__ in self._fake_module_functions[fct.__name__] 746 ) 747 except Exception: 748 # handle cases where the function has no __name__ or __module__ 749 # attribute, or any other exception in inspect functions 750 return False 751 752 def _def_values( 753 self, item: FunctionType 754 ) -> Iterator[Tuple[FunctionType, int, Any]]: 755 """Find default arguments that are file-system functions to be 756 patched in top-level functions and members of top-level classes.""" 757 # check for module-level functions 758 try: 759 if item.__defaults__ and inspect.isfunction(item): 760 for i, d in enumerate(item.__defaults__): 761 if self._is_fs_function(d): 762 yield item, i, d 763 except Exception: 764 pass 765 try: 766 if inspect.isclass(item): 767 # check for methods in class 768 # (nested classes are ignored for now) 769 # inspect.getmembers is very expansive! 770 for m in inspect.getmembers(item, predicate=inspect.isfunction): 771 f = cast(FunctionType, m[1]) 772 if f.__defaults__: 773 for i, d in enumerate(f.__defaults__): 774 if self._is_fs_function(d): 775 yield f, i, d 776 except Exception: 777 # Ignore any exception, examples: 778 # ImportError: No module named '_gdbm' 779 # _DontDoThat() (see #523) 780 pass 781 782 def _find_def_values(self, module_items: ItemsView[str, FunctionType]) -> None: 783 for _, fct in module_items: 784 for f, i, d in self._def_values(fct): 785 self.__class__.FS_DEFARGS.append((f, i, d)) 786 787 def _find_modules(self) -> None: 788 """Find and cache all modules that import file system modules. 789 Later, `setUp()` will stub these with the fake file system 790 modules. 791 """ 792 module_names = list(self._fake_module_classes.keys()) + [PATH_MODULE] 793 for name, module in list(sys.modules.items()): 794 try: 795 if ( 796 self.use_cache 797 and module in self.CACHED_MODULES 798 or not inspect.ismodule(module) 799 ): 800 continue 801 except Exception: 802 # workaround for some py (part of pytest) versions 803 # where py.error has no __name__ attribute 804 # see https://github.com/pytest-dev/py/issues/73 805 # and any other exception triggered by inspect.ismodule 806 if self.use_cache: 807 self.__class__.CACHED_MODULES.add(module) 808 continue 809 skipped = module in self.SKIPMODULES or any( 810 [sn.startswith(module.__name__) for sn in self._skip_names] 811 ) 812 module_items = module.__dict__.copy().items() 813 814 modules = { 815 name: mod 816 for name, mod in module_items 817 if self._is_fs_module(mod, name, module_names) 818 } 819 820 if skipped: 821 for name, mod in modules.items(): 822 self.__class__.SKIPPED_FS_MODULES.setdefault(name, set()).add( 823 (module, mod.__name__) 824 ) 825 else: 826 for name, mod in modules.items(): 827 self.__class__.FS_MODULES.setdefault(name, set()).add( 828 (module, mod.__name__) 829 ) 830 functions = { 831 name: fct for name, fct in module_items if self._is_fs_function(fct) 832 } 833 834 for name, fct in functions.items(): 835 self.__class__.FS_FUNCTIONS.setdefault( 836 (name, fct.__name__, fct.__module__), set() 837 ).add(module) 838 839 # find default arguments that are file system functions 840 if self.patch_default_args: 841 self._find_def_values(module_items) 842 843 if self.use_cache: 844 self.__class__.CACHED_MODULES.add(module) 845 846 def _refresh(self) -> None: 847 """Renew the fake file system and set the _isStale flag to `False`.""" 848 if self._stubs is not None: 849 self._stubs.smart_unset_all() 850 self._stubs = mox3_stubout.StubOutForTesting() 851 852 self.fs = fake_filesystem.FakeFilesystem(patcher=self, create_temp_dir=True) 853 self.fs.patch_open_code = self.patch_open_code 854 self.fake_open = fake_open.FakeFileOpen(self.fs) 855 for name in self._fake_module_classes: 856 self.fake_modules[name] = self._fake_module_classes[name](self.fs) 857 if hasattr(self.fake_modules[name], "skip_names"): 858 self.fake_modules[name].skip_names = self._skip_names 859 self.fake_modules[PATH_MODULE] = self.fake_modules["os"].path 860 for name in self._unfaked_module_classes: 861 self.unfaked_modules[name] = self._unfaked_module_classes[name]() 862 863 self._isStale = False 864 865 def setUp(self, doctester: Any = None) -> None: 866 """Bind the file-related modules to the :py:mod:`pyfakefs` fake 867 modules real ones. Also bind the fake `file()` and `open()` functions. 868 """ 869 if self.is_doc_test: 870 self.__class__.DOC_REF_COUNT += 1 871 if self.__class__.DOC_REF_COUNT > 1: 872 return 873 else: 874 self.__class__.REF_COUNT += 1 875 if self.__class__.REF_COUNT > 1: 876 return 877 self.has_fcopy_file = ( 878 sys.platform == "darwin" 879 and hasattr(shutil, "_HAS_FCOPYFILE") 880 and shutil._HAS_FCOPYFILE 881 ) 882 if self.has_fcopy_file: 883 shutil._HAS_FCOPYFILE = False # type: ignore[attr-defined] 884 885 with warnings.catch_warnings(): 886 # ignore warnings, see #542 and #614 887 warnings.filterwarnings("ignore") 888 self._find_modules() 889 890 self._refresh() 891 892 if doctester is not None: 893 doctester.globs = self.replace_globs(doctester.globs) 894 895 self.start_patching() 896 linecache.open = self.original_open # type: ignore[attr-defined] 897 tokenize._builtin_open = self.original_open # type: ignore 898 899 def start_patching(self) -> None: 900 if not self._patching: 901 self._patching = True 902 903 self.patch_modules() 904 self.patch_functions() 905 self.patch_defaults() 906 907 self._dyn_patcher = DynamicPatcher(self) 908 sys.meta_path.insert(0, self._dyn_patcher) 909 for module in self.modules_to_reload: 910 if sys.modules.get(module.__name__) is module: 911 reload(module) 912 913 def patch_functions(self) -> None: 914 assert self._stubs is not None 915 for (name, ft_name, ft_mod), modules in self.FS_FUNCTIONS.items(): 916 method, mod_name = self._fake_module_functions[ft_name][ft_mod] 917 fake_module = self.fake_modules[mod_name] 918 attr = method.__get__( 919 fake_module, fake_module.__class__ 920 ) # pytype: disable=attribute-error 921 for module in modules: 922 self._stubs.smart_set(module, name, attr) 923 924 def patch_modules(self) -> None: 925 assert self._stubs is not None 926 for name, modules in self.FS_MODULES.items(): 927 for module, attr in modules: 928 self._stubs.smart_set(module, name, self.fake_modules[attr]) 929 for name, modules in self.SKIPPED_FS_MODULES.items(): 930 for module, attr in modules: 931 if attr in self.unfaked_modules: 932 self._stubs.smart_set(module, name, self.unfaked_modules[attr]) 933 if sys.version_info >= (3, 12): 934 # workaround for patching open - does not work with skip modules 935 self._stubs.smart_set(builtins, "open", self.fake_open) 936 937 def patch_defaults(self) -> None: 938 for fct, idx, ft in self.FS_DEFARGS: 939 method, mod_name = self._fake_module_functions[ft.__name__][ft.__module__] 940 fake_module = self.fake_modules[mod_name] 941 attr = method.__get__( 942 fake_module, fake_module.__class__ 943 ) # pytype: disable=attribute-error 944 new_defaults = [] 945 assert fct.__defaults__ is not None 946 for i, d in enumerate(fct.__defaults__): 947 if i == idx: 948 new_defaults.append(attr) 949 else: 950 new_defaults.append(d) 951 fct.__defaults__ = tuple(new_defaults) 952 953 def replace_globs(self, globs_: Dict[str, Any]) -> Dict[str, Any]: 954 globs = globs_.copy() 955 if self._isStale: 956 self._refresh() 957 for name in self._fake_module_classes: 958 if name in globs: 959 globs[name] = self._fake_module_classes[name](self.fs) 960 return globs 961 962 def tearDown(self, doctester: Any = None): 963 """Clear the fake filesystem bindings created by `setUp()`.""" 964 if self.is_doc_test: 965 self.__class__.DOC_REF_COUNT -= 1 966 if self.__class__.DOC_REF_COUNT > 0: 967 return 968 else: 969 self.__class__.REF_COUNT -= 1 970 if self.__class__.REF_COUNT > 0: 971 return 972 self.stop_patching() 973 if self.has_fcopy_file: 974 shutil._HAS_FCOPYFILE = True # type: ignore[attr-defined] 975 976 reset_ids() 977 if self.is_doc_test: 978 self.__class__.DOC_PATCHER = None 979 else: 980 self.__class__.PATCHER = None 981 982 def stop_patching(self) -> None: 983 if self._patching: 984 self._isStale = True 985 self._patching = False 986 if self._stubs: 987 self._stubs.smart_unset_all() 988 self.unset_defaults() 989 if self._dyn_patcher: 990 self._dyn_patcher.cleanup() 991 sys.meta_path.pop(0) 992 993 def unset_defaults(self) -> None: 994 for fct, idx, ft in self.FS_DEFARGS: 995 new_defaults = [] 996 for i, d in enumerate(cast(Tuple, fct.__defaults__)): 997 if i == idx: 998 new_defaults.append(ft) 999 else: 1000 new_defaults.append(d) 1001 fct.__defaults__ = tuple(new_defaults) 1002 1003 def pause(self) -> None: 1004 """Pause the patching of the file system modules until `resume` is 1005 called. After that call, all file system calls are executed in the 1006 real file system. 1007 Calling pause() twice is silently ignored. 1008 1009 """ 1010 self.stop_patching() 1011 1012 def resume(self) -> None: 1013 """Resume the patching of the file system modules if `pause` has 1014 been called before. After that call, all file system calls are 1015 executed in the fake file system. 1016 Does nothing if patching is not paused. 1017 """ 1018 self.start_patching() 1019 1020 1021class Pause: 1022 """Simple context manager that allows to pause/resume patching the 1023 filesystem. Patching is paused in the context manager, and resumed after 1024 going out of it's scope. 1025 """ 1026 1027 def __init__(self, caller: Union[Patcher, TestCaseMixin, FakeFilesystem]): 1028 """Initializes the context manager with the fake filesystem. 1029 1030 Args: 1031 caller: either the FakeFilesystem instance, the Patcher instance 1032 or the pyfakefs test case. 1033 """ 1034 if isinstance(caller, (Patcher, TestCaseMixin)): 1035 assert caller.fs is not None 1036 self._fs: FakeFilesystem = caller.fs 1037 elif isinstance(caller, FakeFilesystem): 1038 self._fs = caller 1039 else: 1040 raise ValueError( 1041 "Invalid argument - should be of type " 1042 '"fake_filesystem_unittest.Patcher", ' 1043 '"fake_filesystem_unittest.TestCase" ' 1044 'or "fake_filesystem.FakeFilesystem"' 1045 ) 1046 1047 def __enter__(self) -> FakeFilesystem: 1048 self._fs.pause() 1049 return self._fs 1050 1051 def __exit__(self, *args: Any) -> None: 1052 self._fs.resume() 1053 1054 1055class DynamicPatcher(MetaPathFinder, Loader): 1056 """A file loader that replaces file system related modules by their 1057 fake implementation if they are loaded after calling `setUpPyfakefs()`. 1058 Implements the protocol needed for import hooks. 1059 """ 1060 1061 def __init__(self, patcher: Patcher) -> None: 1062 self._patcher = patcher 1063 self.sysmodules = {} 1064 self.modules = self._patcher.fake_modules 1065 self._loaded_module_names: Set[str] = set() 1066 1067 # remove all modules that have to be patched from `sys.modules`, 1068 # otherwise the find_... methods will not be called 1069 for name in self.modules: 1070 if self.needs_patch(name) and name in sys.modules: 1071 self.sysmodules[name] = sys.modules[name] 1072 del sys.modules[name] 1073 1074 for name, module in self.modules.items(): 1075 sys.modules[name] = module 1076 1077 def cleanup(self) -> None: 1078 for module_name in self.sysmodules: 1079 sys.modules[module_name] = self.sysmodules[module_name] 1080 for module in self._patcher.modules_to_reload: 1081 if module.__name__ in sys.modules: 1082 reload(module) 1083 reloaded_module_names = [ 1084 module.__name__ for module in self._patcher.modules_to_reload 1085 ] 1086 # Dereference all modules loaded during the test so they will reload on 1087 # the next use, ensuring that no faked modules are referenced after the 1088 # test. 1089 for name in self._loaded_module_names: 1090 if name in sys.modules and name not in reloaded_module_names: 1091 del sys.modules[name] 1092 1093 def needs_patch(self, name: str) -> bool: 1094 """Check if the module with the given name shall be replaced.""" 1095 if name not in self.modules: 1096 self._loaded_module_names.add(name) 1097 return False 1098 if name in sys.modules and type(sys.modules[name]) is self.modules[name]: 1099 return False 1100 return True 1101 1102 def find_spec( 1103 self, 1104 fullname: str, 1105 path: Optional[Sequence[Union[bytes, str]]], 1106 target: Optional[ModuleType] = None, 1107 ) -> Optional[ModuleSpec]: 1108 """Module finder.""" 1109 if self.needs_patch(fullname): 1110 return ModuleSpec(fullname, self) 1111 return None 1112 1113 def load_module(self, fullname: str) -> ModuleType: 1114 """Replaces the module by its fake implementation.""" 1115 sys.modules[fullname] = self.modules[fullname] 1116 return self.modules[fullname] 1117