1# Owner(s): ["module: autograd"] 2 3import importlib 4import inspect 5import json 6import logging 7import os 8import pkgutil 9import unittest 10from typing import Callable 11 12import torch 13from torch._utils_internal import get_file_path_2 14from torch.testing._internal.common_utils import ( 15 IS_JETSON, 16 IS_MACOS, 17 IS_WINDOWS, 18 run_tests, 19 skipIfTorchDynamo, 20 TestCase, 21) 22 23 24log = logging.getLogger(__name__) 25 26 27class TestPublicBindings(TestCase): 28 def test_no_new_reexport_callables(self): 29 """ 30 This test aims to stop the introduction of new re-exported callables into 31 torch whose names do not start with _. Such callables are made available as 32 torch.XXX, which may not be desirable. 33 """ 34 reexported_callables = sorted( 35 k 36 for k, v in vars(torch).items() 37 if callable(v) and not v.__module__.startswith("torch") 38 ) 39 self.assertTrue( 40 all(k.startswith("_") for k in reexported_callables), reexported_callables 41 ) 42 43 def test_no_new_bindings(self): 44 """ 45 This test aims to stop the introduction of new JIT bindings into torch._C 46 whose names do not start with _. Such bindings are made available as 47 torch.XXX, which may not be desirable. 48 49 If your change causes this test to fail, add your new binding to a relevant 50 submodule of torch._C, such as torch._C._jit (or other relevant submodule of 51 torch._C). If your binding really needs to be available as torch.XXX, add it 52 to torch._C and add it to the allowlist below. 53 54 If you have removed a binding, remove it from the allowlist as well. 55 """ 56 57 # This allowlist contains every binding in torch._C that is copied into torch at 58 # the time of writing. It was generated with 59 # 60 # {elem for elem in dir(torch._C) if not elem.startswith("_")} 61 torch_C_allowlist_superset = { 62 "AggregationType", 63 "AliasDb", 64 "AnyType", 65 "Argument", 66 "ArgumentSpec", 67 "AwaitType", 68 "autocast_decrement_nesting", 69 "autocast_increment_nesting", 70 "AVG", 71 "BenchmarkConfig", 72 "BenchmarkExecutionStats", 73 "Block", 74 "BoolType", 75 "BufferDict", 76 "StorageBase", 77 "CallStack", 78 "Capsule", 79 "ClassType", 80 "clear_autocast_cache", 81 "Code", 82 "CompilationUnit", 83 "CompleteArgumentSpec", 84 "ComplexType", 85 "ConcreteModuleType", 86 "ConcreteModuleTypeBuilder", 87 "cpp", 88 "CudaBFloat16TensorBase", 89 "CudaBoolTensorBase", 90 "CudaByteTensorBase", 91 "CudaCharTensorBase", 92 "CudaComplexDoubleTensorBase", 93 "CudaComplexFloatTensorBase", 94 "CudaDoubleTensorBase", 95 "CudaFloatTensorBase", 96 "CudaHalfTensorBase", 97 "CudaIntTensorBase", 98 "CudaLongTensorBase", 99 "CudaShortTensorBase", 100 "DeepCopyMemoTable", 101 "default_generator", 102 "DeserializationStorageContext", 103 "device", 104 "DeviceObjType", 105 "DictType", 106 "DisableTorchFunction", 107 "DisableTorchFunctionSubclass", 108 "DispatchKey", 109 "DispatchKeySet", 110 "dtype", 111 "EnumType", 112 "ErrorReport", 113 "ExcludeDispatchKeyGuard", 114 "ExecutionPlan", 115 "FatalError", 116 "FileCheck", 117 "finfo", 118 "FloatType", 119 "fork", 120 "FunctionSchema", 121 "Future", 122 "FutureType", 123 "Generator", 124 "GeneratorType", 125 "get_autocast_cpu_dtype", 126 "get_autocast_dtype", 127 "get_autocast_ipu_dtype", 128 "get_default_dtype", 129 "get_num_interop_threads", 130 "get_num_threads", 131 "Gradient", 132 "Graph", 133 "GraphExecutorState", 134 "has_cuda", 135 "has_cudnn", 136 "has_lapack", 137 "has_mkl", 138 "has_mkldnn", 139 "has_mps", 140 "has_openmp", 141 "has_spectral", 142 "iinfo", 143 "import_ir_module_from_buffer", 144 "import_ir_module", 145 "InferredType", 146 "init_num_threads", 147 "InterfaceType", 148 "IntType", 149 "SymFloatType", 150 "SymBoolType", 151 "SymIntType", 152 "IODescriptor", 153 "is_anomaly_enabled", 154 "is_anomaly_check_nan_enabled", 155 "is_autocast_cache_enabled", 156 "is_autocast_cpu_enabled", 157 "is_autocast_ipu_enabled", 158 "is_autocast_enabled", 159 "is_grad_enabled", 160 "is_inference_mode_enabled", 161 "JITException", 162 "layout", 163 "ListType", 164 "LiteScriptModule", 165 "LockingLogger", 166 "LoggerBase", 167 "memory_format", 168 "merge_type_from_type_comment", 169 "ModuleDict", 170 "Node", 171 "NoneType", 172 "NoopLogger", 173 "NumberType", 174 "OperatorInfo", 175 "OptionalType", 176 "OutOfMemoryError", 177 "ParameterDict", 178 "parse_ir", 179 "parse_schema", 180 "parse_type_comment", 181 "PyObjectType", 182 "PyTorchFileReader", 183 "PyTorchFileWriter", 184 "qscheme", 185 "read_vitals", 186 "RRefType", 187 "ScriptClass", 188 "ScriptClassFunction", 189 "ScriptDict", 190 "ScriptDictIterator", 191 "ScriptDictKeyIterator", 192 "ScriptList", 193 "ScriptListIterator", 194 "ScriptFunction", 195 "ScriptMethod", 196 "ScriptModule", 197 "ScriptModuleSerializer", 198 "ScriptObject", 199 "ScriptObjectProperty", 200 "SerializationStorageContext", 201 "set_anomaly_enabled", 202 "set_autocast_cache_enabled", 203 "set_autocast_cpu_dtype", 204 "set_autocast_dtype", 205 "set_autocast_ipu_dtype", 206 "set_autocast_cpu_enabled", 207 "set_autocast_ipu_enabled", 208 "set_autocast_enabled", 209 "set_flush_denormal", 210 "set_num_interop_threads", 211 "set_num_threads", 212 "set_vital", 213 "Size", 214 "StaticModule", 215 "Stream", 216 "StreamObjType", 217 "Event", 218 "StringType", 219 "SUM", 220 "SymFloat", 221 "SymInt", 222 "TensorType", 223 "ThroughputBenchmark", 224 "TracingState", 225 "TupleType", 226 "Type", 227 "unify_type_list", 228 "UnionType", 229 "Use", 230 "Value", 231 "set_autocast_gpu_dtype", 232 "get_autocast_gpu_dtype", 233 "vitals_enabled", 234 "wait", 235 "Tag", 236 "set_autocast_xla_enabled", 237 "set_autocast_xla_dtype", 238 "get_autocast_xla_dtype", 239 "is_autocast_xla_enabled", 240 } 241 242 torch_C_bindings = {elem for elem in dir(torch._C) if not elem.startswith("_")} 243 244 # torch.TensorBase is explicitly removed in torch/__init__.py, so included here (#109940) 245 explicitly_removed_torch_C_bindings = {"TensorBase"} 246 247 torch_C_bindings = torch_C_bindings - explicitly_removed_torch_C_bindings 248 249 # Check that the torch._C bindings are all in the allowlist. Since 250 # bindings can change based on how PyTorch was compiled (e.g. with/without 251 # CUDA), the two may not be an exact match but the bindings should be 252 # a subset of the allowlist. 253 difference = torch_C_bindings.difference(torch_C_allowlist_superset) 254 msg = f"torch._C had bindings that are not present in the allowlist:\n{difference}" 255 self.assertTrue(torch_C_bindings.issubset(torch_C_allowlist_superset), msg) 256 257 @staticmethod 258 def _is_mod_public(modname): 259 split_strs = modname.split(".") 260 for elem in split_strs: 261 if elem.startswith("_"): 262 return False 263 return True 264 265 @unittest.skipIf( 266 IS_WINDOWS or IS_MACOS, 267 "Inductor/Distributed modules hard fail on windows and macos", 268 ) 269 @skipIfTorchDynamo("Broken and not relevant for now") 270 def test_modules_can_be_imported(self): 271 failures = [] 272 273 def onerror(modname): 274 failures.append( 275 (modname, ImportError("exception occurred importing package")) 276 ) 277 278 for mod in pkgutil.walk_packages(torch.__path__, "torch.", onerror=onerror): 279 modname = mod.name 280 try: 281 # TODO: fix "torch/utils/model_dump/__main__.py" 282 # which calls sys.exit() when we try to import it 283 if "__main__" in modname: 284 continue 285 importlib.import_module(modname) 286 except Exception as e: 287 # Some current failures are not ImportError 288 log.exception("import_module failed") 289 failures.append((modname, e)) 290 291 # It is ok to add new entries here but please be careful that these modules 292 # do not get imported by public code. 293 private_allowlist = { 294 "torch._inductor.codegen.cuda.cuda_kernel", 295 # TODO(#133647): Remove the onnx._internal entries after 296 # onnx and onnxscript are installed in CI. 297 "torch.onnx._internal.exporter", 298 "torch.onnx._internal.exporter._analysis", 299 "torch.onnx._internal.exporter._building", 300 "torch.onnx._internal.exporter._capture_strategies", 301 "torch.onnx._internal.exporter._compat", 302 "torch.onnx._internal.exporter._core", 303 "torch.onnx._internal.exporter._decomp", 304 "torch.onnx._internal.exporter._dispatching", 305 "torch.onnx._internal.exporter._fx_passes", 306 "torch.onnx._internal.exporter._ir_passes", 307 "torch.onnx._internal.exporter._isolated", 308 "torch.onnx._internal.exporter._onnx_program", 309 "torch.onnx._internal.exporter._registration", 310 "torch.onnx._internal.exporter._reporting", 311 "torch.onnx._internal.exporter._schemas", 312 "torch.onnx._internal.exporter._tensors", 313 "torch.onnx._internal.exporter._verification", 314 "torch.onnx._internal.fx._pass", 315 "torch.onnx._internal.fx.analysis", 316 "torch.onnx._internal.fx.analysis.unsupported_nodes", 317 "torch.onnx._internal.fx.decomposition_skip", 318 "torch.onnx._internal.fx.diagnostics", 319 "torch.onnx._internal.fx.fx_onnx_interpreter", 320 "torch.onnx._internal.fx.fx_symbolic_graph_extractor", 321 "torch.onnx._internal.fx.onnxfunction_dispatcher", 322 "torch.onnx._internal.fx.op_validation", 323 "torch.onnx._internal.fx.passes", 324 "torch.onnx._internal.fx.passes._utils", 325 "torch.onnx._internal.fx.passes.decomp", 326 "torch.onnx._internal.fx.passes.functionalization", 327 "torch.onnx._internal.fx.passes.modularization", 328 "torch.onnx._internal.fx.passes.readability", 329 "torch.onnx._internal.fx.passes.type_promotion", 330 "torch.onnx._internal.fx.passes.virtualization", 331 "torch.onnx._internal.fx.type_utils", 332 "torch.testing._internal.common_distributed", 333 "torch.testing._internal.common_fsdp", 334 "torch.testing._internal.dist_utils", 335 "torch.testing._internal.distributed.common_state_dict", 336 "torch.testing._internal.distributed._shard.sharded_tensor", 337 "torch.testing._internal.distributed._shard.test_common", 338 "torch.testing._internal.distributed._tensor.common_dtensor", 339 "torch.testing._internal.distributed.ddp_under_dist_autograd_test", 340 "torch.testing._internal.distributed.distributed_test", 341 "torch.testing._internal.distributed.distributed_utils", 342 "torch.testing._internal.distributed.fake_pg", 343 "torch.testing._internal.distributed.multi_threaded_pg", 344 "torch.testing._internal.distributed.nn.api.remote_module_test", 345 "torch.testing._internal.distributed.rpc.dist_autograd_test", 346 "torch.testing._internal.distributed.rpc.dist_optimizer_test", 347 "torch.testing._internal.distributed.rpc.examples.parameter_server_test", 348 "torch.testing._internal.distributed.rpc.examples.reinforcement_learning_rpc_test", 349 "torch.testing._internal.distributed.rpc.faulty_agent_rpc_test", 350 "torch.testing._internal.distributed.rpc.faulty_rpc_agent_test_fixture", 351 "torch.testing._internal.distributed.rpc.jit.dist_autograd_test", 352 "torch.testing._internal.distributed.rpc.jit.rpc_test", 353 "torch.testing._internal.distributed.rpc.jit.rpc_test_faulty", 354 "torch.testing._internal.distributed.rpc.rpc_agent_test_fixture", 355 "torch.testing._internal.distributed.rpc.rpc_test", 356 "torch.testing._internal.distributed.rpc.tensorpipe_rpc_agent_test_fixture", 357 "torch.testing._internal.distributed.rpc_utils", 358 "torch._inductor.codegen.cuda.cuda_template", 359 "torch._inductor.codegen.cuda.gemm_template", 360 "torch._inductor.codegen.cpp_template", 361 "torch._inductor.codegen.cpp_gemm_template", 362 "torch._inductor.codegen.cpp_micro_gemm", 363 "torch._inductor.codegen.cpp_template_kernel", 364 "torch._inductor.runtime.triton_helpers", 365 "torch.ao.pruning._experimental.data_sparsifier.lightning.callbacks.data_sparsity", 366 "torch.backends._coreml.preprocess", 367 "torch.contrib._tensorboard_vis", 368 "torch.distributed._composable", 369 "torch.distributed._functional_collectives", 370 "torch.distributed._functional_collectives_impl", 371 "torch.distributed._shard", 372 "torch.distributed._sharded_tensor", 373 "torch.distributed._sharding_spec", 374 "torch.distributed._spmd.api", 375 "torch.distributed._spmd.batch_dim_utils", 376 "torch.distributed._spmd.comm_tensor", 377 "torch.distributed._spmd.data_parallel", 378 "torch.distributed._spmd.distribute", 379 "torch.distributed._spmd.experimental_ops", 380 "torch.distributed._spmd.parallel_mode", 381 "torch.distributed._tensor", 382 "torch.distributed.algorithms._checkpoint.checkpoint_wrapper", 383 "torch.distributed.algorithms._optimizer_overlap", 384 "torch.distributed.rpc._testing.faulty_agent_backend_registry", 385 "torch.distributed.rpc._utils", 386 "torch.ao.pruning._experimental.data_sparsifier.benchmarks.dlrm_utils", 387 "torch.ao.pruning._experimental.data_sparsifier.benchmarks.evaluate_disk_savings", 388 "torch.ao.pruning._experimental.data_sparsifier.benchmarks.evaluate_forward_time", 389 "torch.ao.pruning._experimental.data_sparsifier.benchmarks.evaluate_model_metrics", 390 "torch.ao.pruning._experimental.data_sparsifier.lightning.tests.test_callbacks", 391 "torch.csrc.jit.tensorexpr.scripts.bisect", 392 "torch.csrc.lazy.test_mnist", 393 "torch.distributed._shard.checkpoint._fsspec_filesystem", 394 "torch.distributed._tensor.examples.visualize_sharding_example", 395 "torch.distributed.checkpoint._fsspec_filesystem", 396 "torch.distributed.examples.memory_tracker_example", 397 "torch.testing._internal.distributed.rpc.fb.thrift_rpc_agent_test_fixture", 398 "torch.utils._cxx_pytree", 399 "torch.utils.tensorboard._convert_np", 400 "torch.utils.tensorboard._embedding", 401 "torch.utils.tensorboard._onnx_graph", 402 "torch.utils.tensorboard._proto_graph", 403 "torch.utils.tensorboard._pytorch_graph", 404 "torch.utils.tensorboard._utils", 405 } 406 407 # No new entries should be added to this list. 408 # All public modules should be importable on all platforms. 409 public_allowlist = { 410 "torch.distributed.algorithms.ddp_comm_hooks", 411 "torch.distributed.algorithms.model_averaging.averagers", 412 "torch.distributed.algorithms.model_averaging.hierarchical_model_averager", 413 "torch.distributed.algorithms.model_averaging.utils", 414 "torch.distributed.checkpoint", 415 "torch.distributed.constants", 416 "torch.distributed.distributed_c10d", 417 "torch.distributed.elastic.agent.server", 418 "torch.distributed.elastic.rendezvous", 419 "torch.distributed.fsdp", 420 "torch.distributed.launch", 421 "torch.distributed.launcher", 422 "torch.distributed.nn", 423 "torch.distributed.nn.api.remote_module", 424 "torch.distributed.optim", 425 "torch.distributed.optim.optimizer", 426 "torch.distributed.rendezvous", 427 "torch.distributed.rpc.api", 428 "torch.distributed.rpc.backend_registry", 429 "torch.distributed.rpc.constants", 430 "torch.distributed.rpc.internal", 431 "torch.distributed.rpc.options", 432 "torch.distributed.rpc.rref_proxy", 433 "torch.distributed.elastic.rendezvous.etcd_rendezvous", 434 "torch.distributed.elastic.rendezvous.etcd_rendezvous_backend", 435 "torch.distributed.elastic.rendezvous.etcd_store", 436 "torch.distributed.rpc.server_process_global_profiler", 437 "torch.distributed.run", 438 "torch.distributed.tensor.parallel", 439 "torch.distributed.utils", 440 "torch.utils.tensorboard", 441 "torch.utils.tensorboard.summary", 442 "torch.utils.tensorboard.writer", 443 "torch.ao.quantization.experimental.fake_quantize", 444 "torch.ao.quantization.experimental.linear", 445 "torch.ao.quantization.experimental.observer", 446 "torch.ao.quantization.experimental.qconfig", 447 } 448 449 errors = [] 450 for mod, exc in failures: 451 if mod in public_allowlist: 452 # TODO: Ensure this is the right error type 453 454 continue 455 if mod in private_allowlist: 456 continue 457 errors.append( 458 f"{mod} failed to import with error {type(exc).__qualname__}: {str(exc)}" 459 ) 460 self.assertEqual("", "\n".join(errors)) 461 462 # AttributeError: module 'torch.distributed' has no attribute '_shard' 463 @unittest.skipIf(IS_WINDOWS or IS_JETSON or IS_MACOS, "Distributed Attribute Error") 464 @skipIfTorchDynamo("Broken and not relevant for now") 465 def test_correct_module_names(self): 466 """ 467 An API is considered public, if its `__module__` starts with `torch.` 468 and there is no name in `__module__` or the object itself that starts with "_". 469 Each public package should either: 470 - (preferred) Define `__all__` and all callables and classes in there must have their 471 `__module__` start with the current submodule's path. Things not in `__all__` should 472 NOT have their `__module__` start with the current submodule. 473 - (for simple python-only modules) Not define `__all__` and all the elements in `dir(submod)` must have their 474 `__module__` that start with the current submodule. 475 """ 476 477 failure_list = [] 478 with open( 479 get_file_path_2(os.path.dirname(__file__), "allowlist_for_publicAPI.json") 480 ) as json_file: 481 # no new entries should be added to this allow_dict. 482 # New APIs must follow the public API guidelines. 483 484 allow_dict = json.load(json_file) 485 # Because we want minimal modifications to the `allowlist_for_publicAPI.json`, 486 # we are adding the entries for the migrated modules here from the original 487 # locations. 488 489 for modname in allow_dict["being_migrated"]: 490 if modname in allow_dict: 491 allow_dict[allow_dict["being_migrated"][modname]] = allow_dict[ 492 modname 493 ] 494 495 def test_module(modname): 496 try: 497 if "__main__" in modname: 498 return 499 mod = importlib.import_module(modname) 500 except Exception: 501 # It is ok to ignore here as we have a test above that ensures 502 # this should never happen 503 504 return 505 if not self._is_mod_public(modname): 506 return 507 # verifies that each public API has the correct module name and naming semantics 508 509 def check_one_element(elem, modname, mod, *, is_public, is_all): 510 obj = getattr(mod, elem) 511 512 # torch.dtype is not a class nor callable, so we need to check for it separately 513 if not ( 514 isinstance(obj, (Callable, torch.dtype)) or inspect.isclass(obj) 515 ): 516 return 517 elem_module = getattr(obj, "__module__", None) 518 519 # Only used for nice error message below 520 why_not_looks_public = "" 521 if elem_module is None: 522 why_not_looks_public = ( 523 "because it does not have a `__module__` attribute" 524 ) 525 526 # If a module is being migrated from foo.a to bar.a (that is entry {"foo": "bar"}), 527 # the module's starting package would be referred to as the new location even 528 # if there is a "from foo import a" inside the "bar.py". 529 modname = allow_dict["being_migrated"].get(modname, modname) 530 elem_modname_starts_with_mod = ( 531 elem_module is not None 532 and elem_module.startswith(modname) 533 and "._" not in elem_module 534 ) 535 if not why_not_looks_public and not elem_modname_starts_with_mod: 536 why_not_looks_public = ( 537 f"because its `__module__` attribute (`{elem_module}`) is not within the " 538 f"torch library or does not start with the submodule where it is defined (`{modname}`)" 539 ) 540 541 # elem's name must NOT begin with an `_` and it's module name 542 # SHOULD start with it's current module since it's a public API 543 looks_public = not elem.startswith("_") and elem_modname_starts_with_mod 544 if not why_not_looks_public and not looks_public: 545 why_not_looks_public = f"because it starts with `_` (`{elem}`)" 546 if is_public != looks_public: 547 if modname in allow_dict and elem in allow_dict[modname]: 548 return 549 if is_public: 550 why_is_public = ( 551 f"it is inside the module's (`{modname}`) `__all__`" 552 if is_all 553 else "it is an attribute that does not start with `_` on a module that " 554 "does not have `__all__` defined" 555 ) 556 fix_is_public = ( 557 f"remove it from the modules's (`{modname}`) `__all__`" 558 if is_all 559 else f"either define a `__all__` for `{modname}` or add a `_` at the beginning of the name" 560 ) 561 else: 562 assert is_all 563 why_is_public = ( 564 f"it is not inside the module's (`{modname}`) `__all__`" 565 ) 566 fix_is_public = ( 567 f"add it from the modules's (`{modname}`) `__all__`" 568 ) 569 if looks_public: 570 why_looks_public = ( 571 "it does look public because it follows the rules from the doc above " 572 "(does not start with `_` and has a proper `__module__`)." 573 ) 574 fix_looks_public = "make its name start with `_`" 575 else: 576 why_looks_public = why_not_looks_public 577 if not elem_modname_starts_with_mod: 578 fix_looks_public = ( 579 "make sure the `__module__` is properly set and points to a submodule " 580 f"of `{modname}`" 581 ) 582 else: 583 fix_looks_public = ( 584 "remove the `_` at the beginning of the name" 585 ) 586 failure_list.append(f"# {modname}.{elem}:") 587 is_public_str = "" if is_public else " NOT" 588 failure_list.append( 589 f" - Is{is_public_str} public: {why_is_public}" 590 ) 591 looks_public_str = "" if looks_public else " NOT" 592 failure_list.append( 593 f" - Does{looks_public_str} look public: {why_looks_public}" 594 ) 595 # Swap the str below to avoid having to create the NOT again 596 failure_list.append( 597 " - You can do either of these two things to fix this problem:" 598 ) 599 failure_list.append( 600 f" - To make it{looks_public_str} public: {fix_is_public}" 601 ) 602 failure_list.append( 603 f" - To make it{is_public_str} look public: {fix_looks_public}" 604 ) 605 606 if hasattr(mod, "__all__"): 607 public_api = mod.__all__ 608 all_api = dir(mod) 609 for elem in all_api: 610 check_one_element( 611 elem, modname, mod, is_public=elem in public_api, is_all=True 612 ) 613 else: 614 all_api = dir(mod) 615 for elem in all_api: 616 if not elem.startswith("_"): 617 check_one_element( 618 elem, modname, mod, is_public=True, is_all=False 619 ) 620 621 for mod in pkgutil.walk_packages(torch.__path__, "torch."): 622 modname = mod.name 623 test_module(modname) 624 test_module("torch") 625 626 msg = ( 627 "All the APIs below do not meet our guidelines for public API from " 628 "https://github.com/pytorch/pytorch/wiki/Public-API-definition-and-documentation.\n" 629 ) 630 msg += ( 631 "Make sure that everything that is public is expected (in particular that the module " 632 "has a properly populated `__all__` attribute) and that everything that is supposed to be public " 633 "does look public (it does not start with `_` and has a `__module__` that is properly populated)." 634 ) 635 636 msg += "\n\nFull list:\n" 637 msg += "\n".join(map(str, failure_list)) 638 639 # empty lists are considered false in python 640 self.assertTrue(not failure_list, msg) 641 642 643if __name__ == "__main__": 644 run_tests() 645