From cb3be64cf6ec2a1dc4f9e2f9849820520cde7d60 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 20:44:44 +0900 Subject: [PATCH 01/10] Show affected tests for given module names --- scripts/update_lib/deps.py | 144 ++++++++++++++++++++++++++ scripts/update_lib/show_deps.py | 43 +++++++- scripts/update_lib/tests/test_deps.py | 130 +++++++++++++++++++++++ 3 files changed, 315 insertions(+), 2 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 566f5ae5f0..71d59f9139 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -504,3 +504,147 @@ def resolve_all_paths( result["data"].append(data_path) return result + + +def _build_import_graph(lib_prefix: str = "Lib") -> dict[str, set[str]]: + """Build a graph of module imports from lib_prefix directory. + + Args: + lib_prefix: RustPython Lib directory (default: "Lib") + + Returns: + Dict mapping module_name -> set of modules it imports + """ + lib_dir = pathlib.Path(lib_prefix) + if not lib_dir.exists(): + return {} + + import_graph: dict[str, set[str]] = {} + + # Scan all .py files in lib_prefix (excluding test/ directory for module imports) + for entry in lib_dir.iterdir(): + if entry.name.startswith(("_", ".")): + continue + if entry.name == "test": + continue + + module_name = None + if entry.is_file() and entry.suffix == ".py": + module_name = entry.stem + elif entry.is_dir() and (entry / "__init__.py").exists(): + module_name = entry.name + + if module_name: + # Parse imports from this module + imports = set() + for _, content in read_python_files(entry): + imports.update(parse_lib_imports(content)) + # Remove self-imports + imports.discard(module_name) + import_graph[module_name] = imports + + return import_graph + + +def _build_reverse_graph(import_graph: dict[str, set[str]]) -> dict[str, set[str]]: + """Build reverse dependency graph (who imports this module). + + Args: + import_graph: Forward import graph (module -> imports) + + Returns: + Reverse graph (module -> imported_by) + """ + reverse_graph: dict[str, set[str]] = {} + + for module, imports in import_graph.items(): + for imported in imports: + if imported not in reverse_graph: + reverse_graph[imported] = set() + reverse_graph[imported].add(module) + + return reverse_graph + + +@functools.cache +def get_transitive_imports( + module_name: str, + lib_prefix: str = "Lib", +) -> frozenset[str]: + """Get all modules that transitively depend on module_name. + + Args: + module_name: Target module + lib_prefix: RustPython Lib directory (default: "Lib") + + Returns: + Frozenset of module names that import module_name (directly or indirectly) + """ + import_graph = _build_import_graph(lib_prefix) + reverse_graph = _build_reverse_graph(import_graph) + + # BFS from module_name following reverse edges + visited: set[str] = set() + queue = list(reverse_graph.get(module_name, set())) + + while queue: + current = queue.pop(0) + if current in visited: + continue + visited.add(current) + # Add modules that import current module + for importer in reverse_graph.get(current, set()): + if importer not in visited: + queue.append(importer) + + return frozenset(visited) + + +@functools.cache +def find_tests_importing_module( + module_name: str, + lib_prefix: str = "Lib", + include_transitive: bool = True, +) -> frozenset[pathlib.Path]: + """Find all test files that import the given module (directly or transitively). + + Args: + module_name: Module to search for (e.g., "datetime") + lib_prefix: RustPython Lib directory (default: "Lib") + include_transitive: Whether to include transitive dependencies + + Returns: + Frozenset of test file paths that depend on this module + """ + lib_dir = pathlib.Path(lib_prefix) + test_dir = lib_dir / "test" + + if not test_dir.exists(): + return frozenset() + + # Build set of modules to search for + target_modules = {module_name} + if include_transitive: + # Add all modules that transitively depend on module_name + target_modules.update(get_transitive_imports(module_name, lib_prefix)) + + # Excluded test file for this module (test_.py) + excluded_test = f"test_{module_name}.py" + + # Scan test directory for files that import any of the target modules + result: set[pathlib.Path] = set() + + for test_file in test_dir.glob("*.py"): + if test_file.name == excluded_test: + continue + + content = safe_read_text(test_file) + if content is None: + continue + + imports = parse_lib_imports(content) + # Check if any target module is imported + if imports & target_modules: + result.add(test_file) + + return frozenset(result) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index b6beacacaa..5d1f0584e0 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -145,6 +145,7 @@ def format_deps( lib_prefix: str = "Lib", max_depth: int = 10, _visited: set[str] | None = None, + show_impact: bool = False, ) -> list[str]: """Format all dependency information for a module. @@ -154,14 +155,17 @@ def format_deps( lib_prefix: Local Lib directory prefix max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules + show_impact: Whether to show reverse dependencies (tests that import this module) Returns: List of formatted lines """ from update_lib.deps import ( DEPENDENCIES, + find_tests_importing_module, get_lib_paths, get_test_paths, + get_transitive_imports, ) if _visited is None: @@ -194,6 +198,33 @@ def format_deps( ) ) + # Show impact (reverse dependencies) if requested + if show_impact: + impacted_tests = find_tests_importing_module(name, lib_prefix) + transitive_importers = get_transitive_imports(name, lib_prefix) + + if impacted_tests: + lines.append(f"[+] impact: ({len(impacted_tests)} tests depend on {name})") + # Sort tests and show with dependency info + for test_path in sorted(impacted_tests, key=lambda p: p.name): + # Determine if direct or via which module + test_content = test_path.read_text(errors="ignore") + from update_lib.deps import parse_lib_imports + + test_imports = parse_lib_imports(test_content) + if name in test_imports: + lines.append(f" - {test_path.name} (direct)") + else: + # Find which transitive module is imported + via_modules = test_imports & transitive_importers + if via_modules: + via_str = ", ".join(sorted(via_modules)) + lines.append(f" - {test_path.name} (via {via_str})") + else: + lines.append(f" - {test_path.name}") + else: + lines.append(f"[+] impact: (no tests depend on {name})") + return lines @@ -202,6 +233,7 @@ def show_deps( cpython_prefix: str = "cpython", lib_prefix: str = "Lib", max_depth: int = 10, + show_impact: bool = False, ) -> None: """Show all dependency information for modules.""" # Expand "all" to all module names @@ -218,7 +250,9 @@ def show_deps( for i, name in enumerate(expanded_names): if i > 0: print() # blank line between modules - for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited): + for line in format_deps( + name, cpython_prefix, lib_prefix, max_depth, visited, show_impact + ): print(line) @@ -248,11 +282,16 @@ def main(argv: list[str] | None = None) -> int: default=10, help="Maximum recursion depth for soft_deps tree (default: 10)", ) + parser.add_argument( + "--impact", + action="store_true", + help="Show tests that import this module (reverse dependencies)", + ) args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth) + show_deps(args.names, args.cpython, args.lib, args.depth, args.impact) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index bc70925348..56665187d7 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -5,11 +5,13 @@ import unittest from update_lib.deps import ( + find_tests_importing_module, get_data_paths, get_lib_paths, get_soft_deps, get_test_dependencies, get_test_paths, + get_transitive_imports, parse_lib_imports, parse_test_imports, resolve_all_paths, @@ -422,5 +424,133 @@ def test_nested_different(self): self.assertFalse(_dircmp_is_same(dcmp)) +class TestGetTransitiveImports(unittest.TestCase): + """Tests for get_transitive_imports function.""" + + def test_direct_dependency(self): + """A imports B → B's transitive importers include A.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + (lib_dir / "a.py").write_text("import b\n") + (lib_dir / "b.py").write_text("# b module") + + get_transitive_imports.cache_clear() + result = get_transitive_imports("b", lib_prefix=str(lib_dir)) + self.assertIn("a", result) + + def test_chain_dependency(self): + """A imports B, B imports C → C's transitive importers include A and B.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + (lib_dir / "a.py").write_text("import b\n") + (lib_dir / "b.py").write_text("import c\n") + (lib_dir / "c.py").write_text("# c module") + + get_transitive_imports.cache_clear() + result = get_transitive_imports("c", lib_prefix=str(lib_dir)) + self.assertIn("a", result) + self.assertIn("b", result) + + def test_cycle_handling(self): + """Handle circular imports without infinite loop.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + (lib_dir / "a.py").write_text("import b\n") + (lib_dir / "b.py").write_text("import a\n") # cycle + + get_transitive_imports.cache_clear() + # Should not hang or raise + result = get_transitive_imports("a", lib_prefix=str(lib_dir)) + self.assertIn("b", result) + + +class TestFindTestsImportingModule(unittest.TestCase): + """Tests for find_tests_importing_module function.""" + + def test_direct_import(self): + """Test finding tests that directly import a module.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + # Create target module + (lib_dir / "bar.py").write_text("# bar module") + + # Create test that imports bar + (test_dir / "test_foo.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + self.assertIn(test_dir / "test_foo.py", result) + + def test_excludes_test_module_itself(self): + """Test that test_.py is excluded from results.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "test_bar.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + # test_bar.py should NOT be in results (it's the primary test) + self.assertNotIn(test_dir / "test_bar.py", result) + + def test_transitive_import(self): + """Test finding tests with transitive (indirect) imports.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + # bar.py (target module) + (lib_dir / "bar.py").write_text("# bar module") + + # baz.py imports bar + (lib_dir / "baz.py").write_text("import bar\n") + + # test_foo.py imports baz (not bar directly) + (test_dir / "test_foo.py").write_text("import baz\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + # test_foo.py should be found via transitive dependency + self.assertIn(test_dir / "test_foo.py", result) + + def test_empty_when_no_importers(self): + """Test returns empty when no tests import the module.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "test_unrelated.py").write_text("import os\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + self.assertEqual(result, frozenset()) + + if __name__ == "__main__": unittest.main() From 74ffff5b444fe78207484866123e61c0004e9a51 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 21:59:44 +0900 Subject: [PATCH 02/10] Support --exclude option to deps subcommand to exclude dependencies --- scripts/update_lib/deps.py | 4 + scripts/update_lib/show_deps.py | 18 ++++- scripts/update_lib/tests/test_deps.py | 101 ++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 71d59f9139..79e6769b96 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -605,6 +605,7 @@ def find_tests_importing_module( module_name: str, lib_prefix: str = "Lib", include_transitive: bool = True, + exclude_imports: frozenset[str] = frozenset(), ) -> frozenset[pathlib.Path]: """Find all test files that import the given module (directly or transitively). @@ -612,6 +613,7 @@ def find_tests_importing_module( module_name: Module to search for (e.g., "datetime") lib_prefix: RustPython Lib directory (default: "Lib") include_transitive: Whether to include transitive dependencies + exclude_imports: Modules to exclude from test file imports when checking Returns: Frozenset of test file paths that depend on this module @@ -643,6 +645,8 @@ def find_tests_importing_module( continue imports = parse_lib_imports(content) + # Remove excluded modules from imports + imports = imports - exclude_imports # Check if any target module is imported if imports & target_modules: result.add(test_file) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 5d1f0584e0..5119b89445 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -146,6 +146,7 @@ def format_deps( max_depth: int = 10, _visited: set[str] | None = None, show_impact: bool = False, + exclude_imports: frozenset[str] = frozenset(), ) -> list[str]: """Format all dependency information for a module. @@ -156,6 +157,7 @@ def format_deps( max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules show_impact: Whether to show reverse dependencies (tests that import this module) + exclude_imports: Modules to exclude from impact analysis Returns: List of formatted lines @@ -200,7 +202,7 @@ def format_deps( # Show impact (reverse dependencies) if requested if show_impact: - impacted_tests = find_tests_importing_module(name, lib_prefix) + impacted_tests = find_tests_importing_module(name, lib_prefix, exclude_imports=exclude_imports) transitive_importers = get_transitive_imports(name, lib_prefix) if impacted_tests: @@ -212,6 +214,8 @@ def format_deps( from update_lib.deps import parse_lib_imports test_imports = parse_lib_imports(test_content) + # Remove excluded modules from display (consistent with matching) + test_imports = test_imports - exclude_imports if name in test_imports: lines.append(f" - {test_path.name} (direct)") else: @@ -234,6 +238,7 @@ def show_deps( lib_prefix: str = "Lib", max_depth: int = 10, show_impact: bool = False, + exclude_imports: frozenset[str] = frozenset(), ) -> None: """Show all dependency information for modules.""" # Expand "all" to all module names @@ -251,7 +256,7 @@ def show_deps( if i > 0: print() # blank line between modules for line in format_deps( - name, cpython_prefix, lib_prefix, max_depth, visited, show_impact + name, cpython_prefix, lib_prefix, max_depth, visited, show_impact, exclude_imports ): print(line) @@ -287,11 +292,18 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Show tests that import this module (reverse dependencies)", ) + parser.add_argument( + "--exclude", + action="append", + default=[], + help="Modules to exclude from impact analysis (can be repeated: --exclude unittest --exclude doctest)", + ) args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth, args.impact) + exclude_imports = frozenset(args.exclude) + show_deps(args.names, args.cpython, args.lib, args.depth, args.impact, exclude_imports) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 56665187d7..06fc9ef2ff 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -552,5 +552,106 @@ def test_empty_when_no_importers(self): self.assertEqual(result, frozenset()) +class TestFindTestsImportingModuleExclude(unittest.TestCase): + """Tests for exclude_imports parameter.""" + + def test_exclude_single_module(self): + """Test excluding a single module from import analysis.""" + # Given: + # bar.py (target module) + # unittest.py (module to exclude) + # test_foo.py imports: bar, unittest + # test_qux.py imports: unittest (only) + # When: find_tests_importing_module("bar", exclude_imports=frozenset({"unittest"})) + # Then: test_foo.py is included (bar matches), test_qux.py is excluded + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (lib_dir / "unittest.py").write_text("# unittest module") + + # test_foo imports both bar and unittest + (test_dir / "test_foo.py").write_text("import bar\nimport unittest\n") + # test_qux imports only unittest + (test_dir / "test_qux.py").write_text("import unittest\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module( + "bar", + lib_prefix=str(lib_dir), + exclude_imports=frozenset({"unittest"}) + ) + + # test_foo.py should be included (imports bar) + self.assertIn(test_dir / "test_foo.py", result) + # test_qux.py should be excluded (only imports unittest) + self.assertNotIn(test_dir / "test_qux.py", result) + + def test_exclude_transitive_via_excluded_module(self): + """Test that transitive dependencies via excluded modules are also excluded.""" + # Given: + # bar.py (target) + # baz.py imports bar + # unittest.py imports baz (so unittest transitively depends on bar) + # test_foo.py imports unittest (only) + # When: find_tests_importing_module("bar", exclude_imports=frozenset({"unittest"})) + # Then: test_foo.py is NOT included (unittest is excluded, no other path to bar) + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (lib_dir / "baz.py").write_text("import bar\n") + (lib_dir / "unittest.py").write_text("import baz\n") # unittest -> baz -> bar + + # test_foo imports only unittest + (test_dir / "test_foo.py").write_text("import unittest\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module( + "bar", + lib_prefix=str(lib_dir), + exclude_imports=frozenset({"unittest"}) + ) + + # test_foo.py should NOT be included + # (even though unittest -> baz -> bar, unittest is excluded) + self.assertNotIn(test_dir / "test_foo.py", result) + + def test_exclude_empty_set_same_as_default(self): + """Test that empty exclude set behaves same as no exclusion.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "test_foo.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + + result_default = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + find_tests_importing_module.cache_clear() + result_empty = find_tests_importing_module( + "bar", + lib_prefix=str(lib_dir), + exclude_imports=frozenset() + ) + + self.assertEqual(result_default, result_empty) + + if __name__ == "__main__": unittest.main() From 9423b87d0e6caa4878cbdaf0f2c717875d35993f Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 22:35:51 +0900 Subject: [PATCH 03/10] Exclude non-test files from deps subcommand impact output --- scripts/update_lib/deps.py | 79 +++++++++++++++----- scripts/update_lib/tests/test_deps.py | 100 ++++++++++++++++++++++++-- 2 files changed, 157 insertions(+), 22 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 79e6769b96..6cdf7c8c3b 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -600,6 +600,37 @@ def get_transitive_imports( return frozenset(visited) +def _build_test_import_graph(test_dir: pathlib.Path) -> dict[str, set[str]]: + """Build import graph for files within test directory. + + Args: + test_dir: Path to Lib/test/ directory + + Returns: + Dict mapping filename (stem) -> set of test modules it imports + """ + import_graph: dict[str, set[str]] = {} + + for py_file in test_dir.glob("*.py"): + content = safe_read_text(py_file) + if content is None: + continue + + imports = set() + # Parse "from test import X" style imports + imports.update(parse_test_imports(content)) + # Also check direct imports of test modules + all_imports = parse_lib_imports(content) + # Filter to only test modules that exist in test_dir + for imp in all_imports: + if (test_dir / f"{imp}.py").exists(): + imports.add(imp) + + import_graph[py_file.stem] = imports + + return import_graph + + @functools.cache def find_tests_importing_module( module_name: str, @@ -609,6 +640,9 @@ def find_tests_importing_module( ) -> frozenset[pathlib.Path]: """Find all test files that import the given module (directly or transitively). + Only returns test_*.py files. Support files (like pickletester.py, string_tests.py) + are used for transitive dependency calculation but not included in the result. + Args: module_name: Module to search for (e.g., "datetime") lib_prefix: RustPython Lib directory (default: "Lib") @@ -616,7 +650,7 @@ def find_tests_importing_module( exclude_imports: Modules to exclude from test file imports when checking Returns: - Frozenset of test file paths that depend on this module + Frozenset of test_*.py file paths that depend on this module """ lib_dir = pathlib.Path(lib_prefix) test_dir = lib_dir / "test" @@ -624,31 +658,40 @@ def find_tests_importing_module( if not test_dir.exists(): return frozenset() - # Build set of modules to search for + # Build set of modules to search for (Lib/ modules) target_modules = {module_name} if include_transitive: # Add all modules that transitively depend on module_name target_modules.update(get_transitive_imports(module_name, lib_prefix)) - # Excluded test file for this module (test_.py) - excluded_test = f"test_{module_name}.py" - - # Scan test directory for files that import any of the target modules - result: set[pathlib.Path] = set() - - for test_file in test_dir.glob("*.py"): - if test_file.name == excluded_test: - continue + # Build test directory import graph for transitive analysis within test/ + test_import_graph = _build_test_import_graph(test_dir) - content = safe_read_text(test_file) + # First pass: find all files (by stem) that directly import target modules + directly_importing: set[str] = set() + for py_file in test_dir.glob("*.py"): + content = safe_read_text(py_file) if content is None: continue - - imports = parse_lib_imports(content) - # Remove excluded modules from imports - imports = imports - exclude_imports - # Check if any target module is imported + imports = parse_lib_imports(content) - exclude_imports if imports & target_modules: - result.add(test_file) + directly_importing.add(py_file.stem) + + # Second pass: find files that transitively import via support files within test/ + # BFS to find all files that import any file in all_importing + all_importing = set(directly_importing) + queue = list(directly_importing) + while queue: + current = queue.pop(0) + for file_stem, imports in test_import_graph.items(): + if current in imports and file_stem not in all_importing: + all_importing.add(file_stem) + queue.append(file_stem) + + # Filter to only test_*.py files and build result paths + result: set[pathlib.Path] = set() + for file_stem in all_importing: + if file_stem.startswith("test_"): + result.add(test_dir / f"{file_stem}.py") return frozenset(result) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 06fc9ef2ff..70eeeeb7fa 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -495,8 +495,8 @@ def test_direct_import(self): result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) self.assertIn(test_dir / "test_foo.py", result) - def test_excludes_test_module_itself(self): - """Test that test_.py is excluded from results.""" + def test_includes_test_module_itself(self): + """Test that test_.py IS included in results.""" with tempfile.TemporaryDirectory() as tmpdir: tmpdir = pathlib.Path(tmpdir) lib_dir = tmpdir / "Lib" @@ -509,8 +509,8 @@ def test_excludes_test_module_itself(self): get_transitive_imports.cache_clear() find_tests_importing_module.cache_clear() result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - # test_bar.py should NOT be in results (it's the primary test) - self.assertNotIn(test_dir / "test_bar.py", result) + # test_bar.py IS now included (module's own test is part of impact) + self.assertIn(test_dir / "test_bar.py", result) def test_transitive_import(self): """Test finding tests with transitive (indirect) imports.""" @@ -653,5 +653,97 @@ def test_exclude_empty_set_same_as_default(self): self.assertEqual(result_default, result_empty) +class TestFindTestsOnlyTestFiles(unittest.TestCase): + """Tests for filtering to only test_*.py files in output.""" + + def test_support_file_not_in_output(self): + """Support files should not appear in output even if they import target.""" + # Given: + # bar.py (target module in Lib/) + # helper.py (support file in test/, imports bar) + # test_foo.py (test file, imports bar) + # When: find_tests_importing_module("bar") + # Then: test_foo.py is included, helper.py is NOT included + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + # helper.py imports bar directly but doesn't start with test_ + (test_dir / "helper.py").write_text("import bar\n") + # test_foo.py also imports bar + (test_dir / "test_foo.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # Only test_foo.py should be in results + self.assertIn(test_dir / "test_foo.py", result) + # helper.py should be excluded + self.assertNotIn(test_dir / "helper.py", result) + + def test_transitive_via_support_file(self): + """Test file importing support file that imports target should be included.""" + # Given: + # bar.py (target module in Lib/) + # helper.py (support file in test/, imports bar) + # test_foo.py (test file, imports helper - NOT bar directly) + # When: find_tests_importing_module("bar") + # Then: test_foo.py IS included (via helper.py), helper.py is NOT + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + # helper.py imports bar + (test_dir / "helper.py").write_text("import bar\n") + # test_foo.py imports only helper (not bar directly) + (test_dir / "test_foo.py").write_text("from test import helper\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # test_foo.py depends on bar via helper, so it should be included + self.assertIn(test_dir / "test_foo.py", result) + # helper.py should be excluded from output + self.assertNotIn(test_dir / "helper.py", result) + + def test_chain_through_multiple_support_files(self): + """Test transitive chain through multiple support files.""" + # Given: + # bar.py (target) + # helper_a.py imports bar + # helper_b.py imports helper_a + # test_foo.py imports helper_b + # Then: test_foo.py IS included, helper_a/b are NOT + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "helper_a.py").write_text("import bar\n") + (test_dir / "helper_b.py").write_text("from test import helper_a\n") + (test_dir / "test_foo.py").write_text("from test import helper_b\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + self.assertIn(test_dir / "test_foo.py", result) + self.assertNotIn(test_dir / "helper_a.py", result) + self.assertNotIn(test_dir / "helper_b.py", result) + + if __name__ == "__main__": unittest.main() From d35e0003692c43949372adb458ea14a335f5ee05 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:00:36 +0900 Subject: [PATCH 04/10] Recursively search test module directories for deps impact Extend find_tests_importing_module() and _build_test_import_graph() to recursively search Lib/test/test_*/ directories using **/*.py glob pattern instead of just *.py. This fixes incomplete dependency analysis that was missing test files inside module directories like test_json/, test_importlib/, etc. Changes: - Add _parse_test_submodule_imports() to handle "from test.X import Y" - Update _build_test_import_graph() with recursive glob and submodule import handling - Update find_tests_importing_module() to use relative paths and handle __init__.py files correctly - Update show_deps.py to display relative paths (e.g., test_json/test_decode.py) - Add TestFindTestsInModuleDirectories test class with 3 tests Co-Authored-By: Claude --- scripts/update_lib/deps.py | 92 ++++++++++++++++++++----- scripts/update_lib/show_deps.py | 14 ++-- scripts/update_lib/tests/test_deps.py | 96 +++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 19 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 6cdf7c8c3b..3d1d7bf267 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -600,18 +600,51 @@ def get_transitive_imports( return frozenset(visited) +def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: + """Parse 'from test.X import Y' to get submodule imports. + + Args: + content: Python file content + + Returns: + Dict mapping submodule (e.g., "test_bar") -> set of imported names (e.g., {"helper"}) + """ + import ast + + tree = safe_parse_ast(content) + if tree is None: + return {} + + result: dict[str, set[str]] = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.module and node.module.startswith("test."): + # from test.test_bar import helper -> test_bar: {helper} + parts = node.module.split(".") + if len(parts) >= 2: + submodule = parts[1] + if submodule not in ("support", "__init__"): + if submodule not in result: + result[submodule] = set() + for alias in node.names: + result[submodule].add(alias.name) + + return result + + def _build_test_import_graph(test_dir: pathlib.Path) -> dict[str, set[str]]: - """Build import graph for files within test directory. + """Build import graph for files within test directory (recursive). Args: test_dir: Path to Lib/test/ directory Returns: - Dict mapping filename (stem) -> set of test modules it imports + Dict mapping relative path (without .py) -> set of test modules it imports """ import_graph: dict[str, set[str]] = {} - for py_file in test_dir.glob("*.py"): + # Use **/*.py to recursively find all Python files + for py_file in test_dir.glob("**/*.py"): content = safe_read_text(py_file) if content is None: continue @@ -621,12 +654,30 @@ def _build_test_import_graph(test_dir: pathlib.Path) -> dict[str, set[str]]: imports.update(parse_test_imports(content)) # Also check direct imports of test modules all_imports = parse_lib_imports(content) - # Filter to only test modules that exist in test_dir + + # Check for files at same level or in test_dir for imp in all_imports: + # Check in same directory + if (py_file.parent / f"{imp}.py").exists(): + imports.add(imp) + # Check in test_dir root if (test_dir / f"{imp}.py").exists(): imports.add(imp) - import_graph[py_file.stem] = imports + # Handle "from test.X import Y" where Y is a file in test_dir/X/ + submodule_imports = _parse_test_submodule_imports(content) + for submodule, imported_names in submodule_imports.items(): + submodule_dir = test_dir / submodule + if submodule_dir.is_dir(): + for name in imported_names: + # Check if it's a file in the submodule directory + if (submodule_dir / f"{name}.py").exists(): + imports.add(name) + + # Use relative path from test_dir as key (without .py) + rel_path = py_file.relative_to(test_dir) + key = str(rel_path.with_suffix("")) + import_graph[key] = imports return import_graph @@ -667,15 +718,16 @@ def find_tests_importing_module( # Build test directory import graph for transitive analysis within test/ test_import_graph = _build_test_import_graph(test_dir) - # First pass: find all files (by stem) that directly import target modules + # First pass: find all files (by relative path) that directly import target modules directly_importing: set[str] = set() - for py_file in test_dir.glob("*.py"): + for py_file in test_dir.glob("**/*.py"): # Recursive glob content = safe_read_text(py_file) if content is None: continue imports = parse_lib_imports(content) - exclude_imports if imports & target_modules: - directly_importing.add(py_file.stem) + rel_path = py_file.relative_to(test_dir) + directly_importing.add(str(rel_path.with_suffix(""))) # Second pass: find files that transitively import via support files within test/ # BFS to find all files that import any file in all_importing @@ -683,15 +735,25 @@ def find_tests_importing_module( queue = list(directly_importing) while queue: current = queue.pop(0) - for file_stem, imports in test_import_graph.items(): - if current in imports and file_stem not in all_importing: - all_importing.add(file_stem) - queue.append(file_stem) + # Extract the filename (stem) from the relative path for matching + current_path = pathlib.Path(current) + current_stem = current_path.name + # For __init__.py, the import name is the parent directory name + # e.g., "test_json/__init__" -> can be imported as "test_json" + if current_stem == "__init__": + current_stem = current_path.parent.name + for file_key, imports in test_import_graph.items(): + if current_stem in imports and file_key not in all_importing: + all_importing.add(file_key) + queue.append(file_key) # Filter to only test_*.py files and build result paths result: set[pathlib.Path] = set() - for file_stem in all_importing: - if file_stem.startswith("test_"): - result.add(test_dir / f"{file_stem}.py") + for file_key in all_importing: + # file_key is like "test_foo" or "test_bar/test_sub" + path_parts = pathlib.Path(file_key) + filename = path_parts.name # Get just the filename part + if filename.startswith("test_"): + result.add(test_dir / f"{file_key}.py") return frozenset(result) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 5119b89445..7e55516a9a 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -208,7 +208,13 @@ def format_deps( if impacted_tests: lines.append(f"[+] impact: ({len(impacted_tests)} tests depend on {name})") # Sort tests and show with dependency info - for test_path in sorted(impacted_tests, key=lambda p: p.name): + test_dir = pathlib.Path(lib_prefix) / "test" + for test_path in sorted(impacted_tests, key=lambda p: str(p)): + # Get relative path from test directory for display + try: + display_name = str(test_path.relative_to(test_dir)) + except ValueError: + display_name = test_path.name # Determine if direct or via which module test_content = test_path.read_text(errors="ignore") from update_lib.deps import parse_lib_imports @@ -217,15 +223,15 @@ def format_deps( # Remove excluded modules from display (consistent with matching) test_imports = test_imports - exclude_imports if name in test_imports: - lines.append(f" - {test_path.name} (direct)") + lines.append(f" - {display_name} (direct)") else: # Find which transitive module is imported via_modules = test_imports & transitive_importers if via_modules: via_str = ", ".join(sorted(via_modules)) - lines.append(f" - {test_path.name} (via {via_str})") + lines.append(f" - {display_name} (via {via_str})") else: - lines.append(f" - {test_path.name}") + lines.append(f" - {display_name}") else: lines.append(f"[+] impact: (no tests depend on {name})") diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 70eeeeb7fa..1247c239a7 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -745,5 +745,101 @@ def test_chain_through_multiple_support_files(self): self.assertNotIn(test_dir / "helper_b.py", result) +class TestFindTestsInModuleDirectories(unittest.TestCase): + """Tests for finding tests inside test_*/ module directories.""" + + def test_finds_test_in_module_directory(self): + """Test files inside test_*/ directories should be found.""" + # Given: + # bar.py (target module in Lib/) + # test_bar/ + # __init__.py + # test_sub.py (imports bar) + # When: find_tests_importing_module("bar") + # Then: test_bar/test_sub.py IS included + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_bar_dir = test_dir / "test_bar" + test_bar_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_bar_dir / "__init__.py").write_text("") + (test_bar_dir / "test_sub.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # test_bar/test_sub.py should be in results + self.assertIn(test_bar_dir / "test_sub.py", result) + + def test_finds_nested_test_via_support_in_module_directory(self): + """Transitive deps through support files in module directories.""" + # Given: + # bar.py (target) + # test_bar/ + # __init__.py + # helper.py (imports bar) + # test_sub.py (imports helper via "from test.test_bar import helper") + # When: find_tests_importing_module("bar") + # Then: test_bar/test_sub.py IS included, helper.py is NOT + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_bar_dir = test_dir / "test_bar" + test_bar_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_bar_dir / "__init__.py").write_text("") + (test_bar_dir / "helper.py").write_text("import bar\n") + (test_bar_dir / "test_sub.py").write_text( + "from test.test_bar import helper\n" + ) + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # test_sub.py should be included (via helper) + self.assertIn(test_bar_dir / "test_sub.py", result) + # helper.py should NOT be in results (not a test file) + self.assertNotIn(test_bar_dir / "helper.py", result) + + def test_both_top_level_and_module_directory_tests_found(self): + """Both top-level test_*.py and test_*/test_*.py should be found.""" + # Given: + # bar.py (target) + # test_bar.py (top-level, imports bar) + # test_bar/ + # test_sub.py (imports bar) + # When: find_tests_importing_module("bar") + # Then: BOTH test_bar.py AND test_bar/test_sub.py are included + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_bar_dir = test_dir / "test_bar" + test_bar_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "test_bar.py").write_text("import bar\n") + (test_bar_dir / "__init__.py").write_text("") + (test_bar_dir / "test_sub.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # Both should be included + self.assertIn(test_dir / "test_bar.py", result) + self.assertIn(test_bar_dir / "test_sub.py", result) + + if __name__ == "__main__": unittest.main() From bb84973cd1ee1630cb80d24ef79f2e946f6b6fe6 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:18:58 +0900 Subject: [PATCH 05/10] Revert --exclude option of deps subcommand --- scripts/update_lib/deps.py | 4 +- scripts/update_lib/show_deps.py | 18 +---- scripts/update_lib/tests/test_deps.py | 101 -------------------------- 3 files changed, 4 insertions(+), 119 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 3d1d7bf267..c71c827b03 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -687,7 +687,6 @@ def find_tests_importing_module( module_name: str, lib_prefix: str = "Lib", include_transitive: bool = True, - exclude_imports: frozenset[str] = frozenset(), ) -> frozenset[pathlib.Path]: """Find all test files that import the given module (directly or transitively). @@ -698,7 +697,6 @@ def find_tests_importing_module( module_name: Module to search for (e.g., "datetime") lib_prefix: RustPython Lib directory (default: "Lib") include_transitive: Whether to include transitive dependencies - exclude_imports: Modules to exclude from test file imports when checking Returns: Frozenset of test_*.py file paths that depend on this module @@ -724,7 +722,7 @@ def find_tests_importing_module( content = safe_read_text(py_file) if content is None: continue - imports = parse_lib_imports(content) - exclude_imports + imports = parse_lib_imports(content) if imports & target_modules: rel_path = py_file.relative_to(test_dir) directly_importing.add(str(rel_path.with_suffix(""))) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 7e55516a9a..fe2c9fa805 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -146,7 +146,6 @@ def format_deps( max_depth: int = 10, _visited: set[str] | None = None, show_impact: bool = False, - exclude_imports: frozenset[str] = frozenset(), ) -> list[str]: """Format all dependency information for a module. @@ -157,7 +156,6 @@ def format_deps( max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules show_impact: Whether to show reverse dependencies (tests that import this module) - exclude_imports: Modules to exclude from impact analysis Returns: List of formatted lines @@ -202,7 +200,7 @@ def format_deps( # Show impact (reverse dependencies) if requested if show_impact: - impacted_tests = find_tests_importing_module(name, lib_prefix, exclude_imports=exclude_imports) + impacted_tests = find_tests_importing_module(name, lib_prefix) transitive_importers = get_transitive_imports(name, lib_prefix) if impacted_tests: @@ -220,8 +218,6 @@ def format_deps( from update_lib.deps import parse_lib_imports test_imports = parse_lib_imports(test_content) - # Remove excluded modules from display (consistent with matching) - test_imports = test_imports - exclude_imports if name in test_imports: lines.append(f" - {display_name} (direct)") else: @@ -244,7 +240,6 @@ def show_deps( lib_prefix: str = "Lib", max_depth: int = 10, show_impact: bool = False, - exclude_imports: frozenset[str] = frozenset(), ) -> None: """Show all dependency information for modules.""" # Expand "all" to all module names @@ -262,7 +257,7 @@ def show_deps( if i > 0: print() # blank line between modules for line in format_deps( - name, cpython_prefix, lib_prefix, max_depth, visited, show_impact, exclude_imports + name, cpython_prefix, lib_prefix, max_depth, visited, show_impact ): print(line) @@ -298,18 +293,11 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Show tests that import this module (reverse dependencies)", ) - parser.add_argument( - "--exclude", - action="append", - default=[], - help="Modules to exclude from impact analysis (can be repeated: --exclude unittest --exclude doctest)", - ) args = parser.parse_args(argv) try: - exclude_imports = frozenset(args.exclude) - show_deps(args.names, args.cpython, args.lib, args.depth, args.impact, exclude_imports) + show_deps(args.names, args.cpython, args.lib, args.depth, args.impact) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 1247c239a7..da1aef76cd 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -552,107 +552,6 @@ def test_empty_when_no_importers(self): self.assertEqual(result, frozenset()) -class TestFindTestsImportingModuleExclude(unittest.TestCase): - """Tests for exclude_imports parameter.""" - - def test_exclude_single_module(self): - """Test excluding a single module from import analysis.""" - # Given: - # bar.py (target module) - # unittest.py (module to exclude) - # test_foo.py imports: bar, unittest - # test_qux.py imports: unittest (only) - # When: find_tests_importing_module("bar", exclude_imports=frozenset({"unittest"})) - # Then: test_foo.py is included (bar matches), test_qux.py is excluded - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (lib_dir / "unittest.py").write_text("# unittest module") - - # test_foo imports both bar and unittest - (test_dir / "test_foo.py").write_text("import bar\nimport unittest\n") - # test_qux imports only unittest - (test_dir / "test_qux.py").write_text("import unittest\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module( - "bar", - lib_prefix=str(lib_dir), - exclude_imports=frozenset({"unittest"}) - ) - - # test_foo.py should be included (imports bar) - self.assertIn(test_dir / "test_foo.py", result) - # test_qux.py should be excluded (only imports unittest) - self.assertNotIn(test_dir / "test_qux.py", result) - - def test_exclude_transitive_via_excluded_module(self): - """Test that transitive dependencies via excluded modules are also excluded.""" - # Given: - # bar.py (target) - # baz.py imports bar - # unittest.py imports baz (so unittest transitively depends on bar) - # test_foo.py imports unittest (only) - # When: find_tests_importing_module("bar", exclude_imports=frozenset({"unittest"})) - # Then: test_foo.py is NOT included (unittest is excluded, no other path to bar) - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (lib_dir / "baz.py").write_text("import bar\n") - (lib_dir / "unittest.py").write_text("import baz\n") # unittest -> baz -> bar - - # test_foo imports only unittest - (test_dir / "test_foo.py").write_text("import unittest\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module( - "bar", - lib_prefix=str(lib_dir), - exclude_imports=frozenset({"unittest"}) - ) - - # test_foo.py should NOT be included - # (even though unittest -> baz -> bar, unittest is excluded) - self.assertNotIn(test_dir / "test_foo.py", result) - - def test_exclude_empty_set_same_as_default(self): - """Test that empty exclude set behaves same as no exclusion.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (test_dir / "test_foo.py").write_text("import bar\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - - result_default = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - - find_tests_importing_module.cache_clear() - result_empty = find_tests_importing_module( - "bar", - lib_prefix=str(lib_dir), - exclude_imports=frozenset() - ) - - self.assertEqual(result_default, result_empty) - - class TestFindTestsOnlyTestFiles(unittest.TestCase): """Tests for filtering to only test_*.py files in output.""" From b086002dd38b7825be4327050dfedea4073fdf82 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:36:38 +0900 Subject: [PATCH 06/10] Consolidate test module directories in deps impact output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add consolidate_test_paths() to group test_*/ directory contents into single entries (e.g., test_sqlite3/test_dbapi.py → test_sqlite3). Co-Authored-By: Claude Opus 4.5 --- scripts/update_lib/deps.py | 35 ++++++++++++++++ scripts/update_lib/show_deps.py | 36 ++++------------ scripts/update_lib/tests/test_deps.py | 59 +++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 28 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index c71c827b03..cf1545f0bd 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -755,3 +755,38 @@ def find_tests_importing_module( result.add(test_dir / f"{file_key}.py") return frozenset(result) + + +def consolidate_test_paths( + test_paths: frozenset[pathlib.Path], + test_dir: pathlib.Path, +) -> frozenset[str]: + """Consolidate test paths by grouping test_*/ directory contents into a single entry. + + Args: + test_paths: Frozenset of absolute paths to test files + test_dir: Path to the test directory (e.g., Lib/test) + + Returns: + Frozenset of consolidated test names: + - "test_foo" for Lib/test/test_foo.py + - "test_sqlite3" for any file in Lib/test/test_sqlite3/ + """ + consolidated: set[str] = set() + + for path in test_paths: + try: + rel_path = path.relative_to(test_dir) + parts = rel_path.parts + + if len(parts) == 1: + # test_foo.py -> test_foo + consolidated.add(rel_path.stem) + else: + # test_sqlite3/test_dbapi.py -> test_sqlite3 + consolidated.add(parts[0]) + except ValueError: + # Path not relative to test_dir, use stem + consolidated.add(path.stem) + + return frozenset(consolidated) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index fe2c9fa805..9e62dc1810 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -162,10 +162,10 @@ def format_deps( """ from update_lib.deps import ( DEPENDENCIES, + consolidate_test_paths, find_tests_importing_module, get_lib_paths, get_test_paths, - get_transitive_imports, ) if _visited is None: @@ -201,33 +201,13 @@ def format_deps( # Show impact (reverse dependencies) if requested if show_impact: impacted_tests = find_tests_importing_module(name, lib_prefix) - transitive_importers = get_transitive_imports(name, lib_prefix) - - if impacted_tests: - lines.append(f"[+] impact: ({len(impacted_tests)} tests depend on {name})") - # Sort tests and show with dependency info - test_dir = pathlib.Path(lib_prefix) / "test" - for test_path in sorted(impacted_tests, key=lambda p: str(p)): - # Get relative path from test directory for display - try: - display_name = str(test_path.relative_to(test_dir)) - except ValueError: - display_name = test_path.name - # Determine if direct or via which module - test_content = test_path.read_text(errors="ignore") - from update_lib.deps import parse_lib_imports - - test_imports = parse_lib_imports(test_content) - if name in test_imports: - lines.append(f" - {display_name} (direct)") - else: - # Find which transitive module is imported - via_modules = test_imports & transitive_importers - if via_modules: - via_str = ", ".join(sorted(via_modules)) - lines.append(f" - {display_name} (via {via_str})") - else: - lines.append(f" - {display_name}") + test_dir = pathlib.Path(lib_prefix) / "test" + consolidated = consolidate_test_paths(impacted_tests, test_dir) + + if consolidated: + lines.append(f"[+] impact: ({len(consolidated)} tests depend on {name})") + for test_name in sorted(consolidated): + lines.append(f" - {test_name}") else: lines.append(f"[+] impact: (no tests depend on {name})") diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index da1aef76cd..e91512088e 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -5,6 +5,7 @@ import unittest from update_lib.deps import ( + consolidate_test_paths, find_tests_importing_module, get_data_paths, get_lib_paths, @@ -740,5 +741,63 @@ def test_both_top_level_and_module_directory_tests_found(self): self.assertIn(test_bar_dir / "test_sub.py", result) +class TestConsolidateTestPaths(unittest.TestCase): + """Tests for consolidate_test_paths function.""" + + def test_top_level_test_file(self): + """Top-level test_*.py -> test_* (without .py).""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) + test_file = test_dir / "test_foo.py" + test_file.write_text("# test") + + result = consolidate_test_paths(frozenset({test_file}), test_dir) + self.assertEqual(result, frozenset({"test_foo"})) + + def test_module_directory_tests_consolidated(self): + """Multiple files in test_*/ directory -> single directory name.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) + module_dir = test_dir / "test_sqlite3" + module_dir.mkdir() + (module_dir / "test_dbapi.py").write_text("# test") + (module_dir / "test_backup.py").write_text("# test") + + result = consolidate_test_paths( + frozenset({module_dir / "test_dbapi.py", module_dir / "test_backup.py"}), + test_dir, + ) + self.assertEqual(result, frozenset({"test_sqlite3"})) + + def test_mixed_top_level_and_module_directory(self): + """Both top-level and module directory tests handled correctly.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) + # Top-level test + (test_dir / "test_foo.py").write_text("# test") + # Module directory tests + module_dir = test_dir / "test_sqlite3" + module_dir.mkdir() + (module_dir / "test_dbapi.py").write_text("# test") + (module_dir / "test_backup.py").write_text("# test") + + result = consolidate_test_paths( + frozenset({ + test_dir / "test_foo.py", + module_dir / "test_dbapi.py", + module_dir / "test_backup.py", + }), + test_dir, + ) + self.assertEqual(result, frozenset({"test_foo", "test_sqlite3"})) + + def test_empty_input(self): + """Empty input -> empty frozenset.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) + result = consolidate_test_paths(frozenset(), test_dir) + self.assertEqual(result, frozenset()) + + if __name__ == "__main__": unittest.main() From ef8c05a1cc92cfd6fb82b5ef65621d4d8200360a Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:49:01 +0900 Subject: [PATCH 07/10] Add --impact-only flag to deps subcommand for test runner integration Outputs space-separated test names for direct use with python3 -m test: python3 -m test $(python3 scripts/update_lib deps sqlite3 --impact-only) Co-Authored-By: Claude Opus 4.5 --- scripts/update_lib/show_deps.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 9e62dc1810..c89e15b12f 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -220,8 +220,14 @@ def show_deps( lib_prefix: str = "Lib", max_depth: int = 10, show_impact: bool = False, + impact_only: bool = False, ) -> None: """Show all dependency information for modules.""" + from update_lib.deps import ( + consolidate_test_paths, + find_tests_importing_module, + ) + # Expand "all" to all module names expanded_names = [] for name in names: @@ -230,6 +236,18 @@ def show_deps( else: expanded_names.append(name) + # Handle impact-only mode: output only space-separated test names + if impact_only: + all_tests: set[str] = set() + for name in expanded_names: + impacted = find_tests_importing_module(name, lib_prefix) + test_dir = pathlib.Path(lib_prefix) / "test" + consolidated = consolidate_test_paths(impacted, test_dir) + all_tests.update(consolidated) + if all_tests: + print(" ".join(sorted(all_tests))) + return + # Shared visited set across all modules visited: set[str] = set() @@ -273,11 +291,16 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Show tests that import this module (reverse dependencies)", ) + parser.add_argument( + "--impact-only", + action="store_true", + help="Output only impact test names, space-separated (for use with python3 -m test)", + ) args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth, args.impact) + show_deps(args.names, args.cpython, args.lib, args.depth, args.impact, args.impact_only) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) From c3b87b8d56769b3cdf4da0da1a0dae9733167f80 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:58:46 +0900 Subject: [PATCH 08/10] Rename impact to dependent tests and make it default in deps subcommand - Remove --impact flag, dependent tests are now shown by default - Rename --impact-only to --dependent-tests-only - Change output label from "[+] impact:" to "[+] dependent tests:" Co-Authored-By: Claude Opus 4.5 --- scripts/update_lib/show_deps.py | 43 +++++++++++++-------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index c89e15b12f..da47ccfca2 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -145,7 +145,6 @@ def format_deps( lib_prefix: str = "Lib", max_depth: int = 10, _visited: set[str] | None = None, - show_impact: bool = False, ) -> list[str]: """Format all dependency information for a module. @@ -155,7 +154,6 @@ def format_deps( lib_prefix: Local Lib directory prefix max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules - show_impact: Whether to show reverse dependencies (tests that import this module) Returns: List of formatted lines @@ -198,18 +196,17 @@ def format_deps( ) ) - # Show impact (reverse dependencies) if requested - if show_impact: - impacted_tests = find_tests_importing_module(name, lib_prefix) - test_dir = pathlib.Path(lib_prefix) / "test" - consolidated = consolidate_test_paths(impacted_tests, test_dir) + # Show dependent tests (reverse dependencies) + impacted_tests = find_tests_importing_module(name, lib_prefix) + test_dir = pathlib.Path(lib_prefix) / "test" + consolidated = consolidate_test_paths(impacted_tests, test_dir) - if consolidated: - lines.append(f"[+] impact: ({len(consolidated)} tests depend on {name})") - for test_name in sorted(consolidated): - lines.append(f" - {test_name}") - else: - lines.append(f"[+] impact: (no tests depend on {name})") + if consolidated: + lines.append(f"[+] dependent tests: ({len(consolidated)} tests depend on {name})") + for test_name in sorted(consolidated): + lines.append(f" - {test_name}") + else: + lines.append(f"[+] dependent tests: (no tests depend on {name})") return lines @@ -219,8 +216,7 @@ def show_deps( cpython_prefix: str = "cpython", lib_prefix: str = "Lib", max_depth: int = 10, - show_impact: bool = False, - impact_only: bool = False, + dependent_tests_only: bool = False, ) -> None: """Show all dependency information for modules.""" from update_lib.deps import ( @@ -236,8 +232,8 @@ def show_deps( else: expanded_names.append(name) - # Handle impact-only mode: output only space-separated test names - if impact_only: + # Handle dependent-tests-only mode: output only space-separated test names + if dependent_tests_only: all_tests: set[str] = set() for name in expanded_names: impacted = find_tests_importing_module(name, lib_prefix) @@ -255,7 +251,7 @@ def show_deps( if i > 0: print() # blank line between modules for line in format_deps( - name, cpython_prefix, lib_prefix, max_depth, visited, show_impact + name, cpython_prefix, lib_prefix, max_depth, visited ): print(line) @@ -287,20 +283,15 @@ def main(argv: list[str] | None = None) -> int: help="Maximum recursion depth for soft_deps tree (default: 10)", ) parser.add_argument( - "--impact", - action="store_true", - help="Show tests that import this module (reverse dependencies)", - ) - parser.add_argument( - "--impact-only", + "--dependent-tests-only", action="store_true", - help="Output only impact test names, space-separated (for use with python3 -m test)", + help="Output only dependent test names, space-separated (for use with python3 -m test)", ) args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth, args.impact, args.impact_only) + show_deps(args.names, args.cpython, args.lib, args.depth, args.dependent_tests_only) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) From 2f7ef1d2f6312a86b2c7465c4e9ce78e708e1b51 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 21 Jan 2026 15:00:23 +0000 Subject: [PATCH 09/10] Auto-format: ruff format --- scripts/update_lib/show_deps.py | 12 +++++++----- scripts/update_lib/tests/test_deps.py | 16 ++++++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index da47ccfca2..974cd9e052 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -202,7 +202,9 @@ def format_deps( consolidated = consolidate_test_paths(impacted_tests, test_dir) if consolidated: - lines.append(f"[+] dependent tests: ({len(consolidated)} tests depend on {name})") + lines.append( + f"[+] dependent tests: ({len(consolidated)} tests depend on {name})" + ) for test_name in sorted(consolidated): lines.append(f" - {test_name}") else: @@ -250,9 +252,7 @@ def show_deps( for i, name in enumerate(expanded_names): if i > 0: print() # blank line between modules - for line in format_deps( - name, cpython_prefix, lib_prefix, max_depth, visited - ): + for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited): print(line) @@ -291,7 +291,9 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth, args.dependent_tests_only) + show_deps( + args.names, args.cpython, args.lib, args.depth, args.dependent_tests_only + ) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index e91512088e..f06d6f2931 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -764,7 +764,9 @@ def test_module_directory_tests_consolidated(self): (module_dir / "test_backup.py").write_text("# test") result = consolidate_test_paths( - frozenset({module_dir / "test_dbapi.py", module_dir / "test_backup.py"}), + frozenset( + {module_dir / "test_dbapi.py", module_dir / "test_backup.py"} + ), test_dir, ) self.assertEqual(result, frozenset({"test_sqlite3"})) @@ -782,11 +784,13 @@ def test_mixed_top_level_and_module_directory(self): (module_dir / "test_backup.py").write_text("# test") result = consolidate_test_paths( - frozenset({ - test_dir / "test_foo.py", - module_dir / "test_dbapi.py", - module_dir / "test_backup.py", - }), + frozenset( + { + test_dir / "test_foo.py", + module_dir / "test_dbapi.py", + module_dir / "test_backup.py", + } + ), test_dir, ) self.assertEqual(result, frozenset({"test_foo", "test_sqlite3"})) From fac5e1d27cb84f8b282c8ea985724a0b0083a791 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Thu, 22 Jan 2026 00:51:10 +0900 Subject: [PATCH 10/10] Refactor --- scripts/update_lib/deps.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index cf1545f0bd..5c76fab8f5 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -7,8 +7,10 @@ - Test dependencies (auto-detected from 'from test import ...') """ +import ast import functools import pathlib +from collections import deque from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text from update_lib.path import construct_lib_path, resolve_module_path @@ -226,8 +228,6 @@ def parse_test_imports(content: str) -> set[str]: Returns: Set of module names imported from test package """ - import ast - tree = safe_parse_ast(content) if tree is None: return set() @@ -262,8 +262,6 @@ def parse_lib_imports(content: str) -> set[str]: Returns: Set of imported module names (top-level only) """ - import ast - tree = safe_parse_ast(content) if tree is None: return set() @@ -506,6 +504,7 @@ def resolve_all_paths( return result +@functools.cache def _build_import_graph(lib_prefix: str = "Lib") -> dict[str, set[str]]: """Build a graph of module imports from lib_prefix directory. @@ -585,10 +584,10 @@ def get_transitive_imports( # BFS from module_name following reverse edges visited: set[str] = set() - queue = list(reverse_graph.get(module_name, set())) + queue = deque(reverse_graph.get(module_name, set())) while queue: - current = queue.pop(0) + current = queue.popleft() if current in visited: continue visited.add(current) @@ -609,8 +608,6 @@ def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: Returns: Dict mapping submodule (e.g., "test_bar") -> set of imported names (e.g., {"helper"}) """ - import ast - tree = safe_parse_ast(content) if tree is None: return {} @@ -730,9 +727,9 @@ def find_tests_importing_module( # Second pass: find files that transitively import via support files within test/ # BFS to find all files that import any file in all_importing all_importing = set(directly_importing) - queue = list(directly_importing) + queue = deque(directly_importing) while queue: - current = queue.pop(0) + current = queue.popleft() # Extract the filename (stem) from the relative path for matching current_path = pathlib.Path(current) current_stem = current_path.name