diff --git a/CHANGELOG.md b/CHANGELOG.md index 886f817..3502cf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ +## v1.7.3 (2026-02-09) + +### Bug Fixes + +- Ensure the same version is installed in host and pyodide venv + ([#59](https://github.com/cloudflare/workers-py/pull/59), + [`f85a938`](https://github.com/cloudflare/workers-py/commit/f85a938758d4e6ecffa14c20bd772ec7539a4d24)) + + ## v1.7.2 (2026-02-05) ### Bug Fixes diff --git a/pyproject.toml b/pyproject.toml index 7bf16bf..aa94ec3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "workers-py" -version = "1.7.2" +version = "1.7.3" description = "A set of libraries and tools for Python Workers" readme = "README.md" requires-python = ">=3.12" diff --git a/src/pywrangler/sync.py b/src/pywrangler/sync.py index 8ac05ab..56e33b9 100644 --- a/src/pywrangler/sync.py +++ b/src/pywrangler/sync.py @@ -31,8 +31,12 @@ def get_venv_workers_token_path() -> Path: return get_venv_workers_path() / ".synced" +def get_vendor_modules_path() -> Path: + return get_project_root() / "python_modules" + + def get_vendor_token_path() -> Path: - return get_project_root() / "python_modules/.synced" + return get_vendor_modules_path() / ".synced" def get_pyodide_venv_path() -> Path: @@ -162,15 +166,20 @@ def temp_requirements_file(requirements: list[str]) -> Iterator[str]: yield temp_file.name -def _install_requirements_to_vendor(requirements: list[str]) -> None: - vendor_path = get_project_root() / "python_modules" +def _install_requirements_to_vendor(requirements: list[str]) -> str | None: + """Install packages to the Pyodide vendor directory. + + Returns: + Error message string if installation failed, None if successful. + """ + vendor_path = get_vendor_modules_path() logger.debug(f"Using vendor path: {vendor_path}") if len(requirements) == 0: logger.warning( f"Requirements list is empty. No dependencies to install in {vendor_path}." ) - return + return None # Install packages into vendor directory vendor_path.mkdir(parents=True, exist_ok=True) @@ -198,28 +207,7 @@ def _install_requirements_to_vendor(requirements: list[str]) -> None: env=os.environ | {"VIRTUAL_ENV": str(get_pyodide_venv_path())}, ) if result.returncode != 0: - logger.warning(result.stdout.strip()) - # Handle some common failures and give nicer error messages for them. - lowered_stdout = result.stdout.lower() - if "invalid peer certificate" in lowered_stdout: - logger.error( - "Installation failed because of an invalid peer certificate. Are your systems certificates correctly installed? Do you have an Enterprise VPN enabled?" - ) - elif "failed to fetch" in lowered_stdout: - logger.error( - "Installation failed because of a failed fetch. Is your network connection working?" - ) - elif "no solution found when resolving dependencies" in lowered_stdout: - logger.error( - "Installation failed because the packages you requested are not supported by Python Workers. See above for details." - ) - else: - logger.error( - "Installation of packages into the Python Worker failed. Possibly because these packages are not currently supported. See above for details." - ) - raise click.exceptions.Exit(code=result.returncode) - - _log_installed_packages(get_pyodide_venv_path()) + return result.stdout.strip() pyv = get_python_version() shutil.rmtree(vendor_path) @@ -237,24 +225,19 @@ def _install_requirements_to_vendor(requirements: list[str]) -> None: f"Packages installed in [bold]{relative_vendor_path}[/bold].", extra={"markup": True}, ) + return None -def _log_installed_packages(venv_path: Path) -> None: - result = run_command( - ["uv", "pip", "list", "--format=freeze"], - env=os.environ | {"VIRTUAL_ENV": str(venv_path)}, - capture_output=True, - check=False, - ) - if result.returncode == 0 and result.stdout.strip(): - logger.debug("Installed packages:") - for line in result.stdout.strip().split("\n"): - if line.strip(): - logger.debug(f" {line.strip()}") +def _install_requirements_to_venv(requirements: list[str]) -> str | None: + """Install packages to the native venv. + + Uses pinned versions from vendor directory if available to ensure host packages + accurately reflect what will run in production. + Returns: + Error message string if installation failed, None if successful. + """ -def _install_requirements_to_venv(requirements: list[str]) -> None: - # Create a requirements file for .venv-workers that includes pyodide-py venv_workers_path = get_venv_workers_path() project_root = get_project_root() relative_venv_workers_path = venv_workers_path.relative_to(project_root) @@ -265,25 +248,16 @@ def _install_requirements_to_venv(requirements: list[str]) -> None: f"Installing packages into [bold]{relative_venv_workers_path}[/bold]...", extra={"markup": True}, ) + with temp_requirements_file(requirements) as requirements_file: result = run_command( - [ - "uv", - "pip", - "install", - "-r", - requirements_file, - ], + ["uv", "pip", "install", "-r", requirements_file], check=False, env=os.environ | {"VIRTUAL_ENV": str(venv_workers_path)}, capture_output=True, ) if result.returncode != 0: - logger.warning(result.stdout.strip()) - logger.error( - "Failed to install the requirements defined in your pyproject.toml file. See above for details." - ) - raise click.exceptions.Exit(code=result.returncode) + return result.stdout.strip() get_venv_workers_token_path().touch() logger.info( @@ -291,16 +265,92 @@ def _install_requirements_to_venv(requirements: list[str]) -> None: extra={"markup": True}, ) + return None + + +def _log_installed_packages(venv_path: Path) -> None: + result = run_command( + ["uv", "pip", "list", "--format=freeze"], + env=os.environ | {"VIRTUAL_ENV": str(venv_path)}, + capture_output=True, + check=False, + ) + if result.returncode == 0 and result.stdout.strip(): + logger.debug("Installed packages:") + for line in result.stdout.strip().split("\n"): + if line.strip(): + logger.debug(f" {line.strip()}") + + +def _parse_pip_freeze(result: str) -> list[str]: + packages = [] + for line in result.strip().split("\n"): + # filter out empty lines and comments that we cannot handle just in case + line = line.strip() + if line and not line.startswith("#") and "==" in line: + packages.append(line) + return packages + + +def _get_vendor_package_versions() -> list[str]: + """Get pinned package versions from pyodide venv (e.g., ["shapely==2.0.7"]).""" + result = run_command( + ["uv", "pip", "freeze", "--path", str(get_vendor_modules_path())], + env=os.environ | {"VIRTUAL_ENV": str(get_pyodide_venv_path())}, + capture_output=True, + ) + if result.returncode != 0: + logger.warning("Failed to get package versions from pyodide venv") + return [] + + return _parse_pip_freeze(result.stdout) + def install_requirements(requirements: list[str]) -> None: - # Note: the order these are executed is important. - # We need to install to .venv-workers first, so that we can determine if the packages requested - # by the user are valid. - _install_requirements_to_venv(requirements) - # Then we install the same requirements to the vendor directory. If this installation - # fails while the above succeeded, it implies that Pyodide does not support these package - # requirements which allows us to give a nicer error message to the user. - _install_requirements_to_vendor(requirements) + # First, install to the Pyodide vendor directory. This determines the exact package + # versions that will run in production. + pyodide_error = _install_requirements_to_vendor(requirements) + + # Then install to .venv-workers using the pinned versions from vendor. + # This ensures host packages accurately reflect what will run in production. + # If the installation to the Pyodide vendor directory fails, use the original requirements + # to see if it fails in the native venv as well. + host_requirements = ( + requirements if pyodide_error else _get_vendor_package_versions() + ) + native_error = _install_requirements_to_venv(host_requirements) + + # Show the native error first (more likely to be actionable), then the Pyodide error. + if native_error: + logger.warning(native_error) + logger.error( + "Failed to install the requirements defined in your pyproject.toml file. See above for details." + ) + raise click.exceptions.Exit(code=1) + + if pyodide_error: + logger.warning(pyodide_error) + # Handle some common failures and give nicer error messages for them. + lowered_error = pyodide_error.lower() + if "invalid peer certificate" in lowered_error: + logger.error( + "Installation failed because of an invalid peer certificate. Are your systems certificates correctly installed? Do you have an Enterprise VPN enabled?" + ) + elif "failed to fetch" in lowered_error: + logger.error( + "Installation failed because of a failed fetch. Is your network connection working?" + ) + elif "no solution found when resolving dependencies" in lowered_error: + logger.error( + "Installation failed because the packages you requested are not supported by Python Workers. See above for details." + ) + else: + logger.error( + "Installation of packages into the Python Worker failed. Possibly because these packages are not currently supported. See above for details." + ) + raise click.exceptions.Exit(code=1) + + _log_installed_packages(get_venv_workers_path()) def _is_out_of_date(token: Path, time: float) -> bool: diff --git a/tests/test_cli.py b/tests/test_cli.py index 5260d3f..2c7d21a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -51,7 +51,6 @@ def test_dir(monkeypatch): monkeypatch.setattr( pywrangler_utils, "find_pyproject_toml", lambda: test_dir / "pyproject.toml" ) - try: yield test_dir.absolute() finally: @@ -147,7 +146,7 @@ def create_test_wrangler_toml( [], # Empty dependency list ], ) -def test_sync_command_integration(dependencies, test_dir): +def test_sync_command_integration(dependencies, test_dir): # noqa: C901 (test complexity) """Test the sync command with real commands running on the system.""" # Create a test pyproject.toml with dependencies test_deps = create_test_pyproject(test_dir, dependencies) @@ -225,6 +224,50 @@ def test_sync_command_integration(dependencies, test_dir): f"Package {dep} was not installed in .venv-workers" ) + if test_deps: + vendor_freeze_result = subprocess.run( + ["uv", "pip", "freeze", "--path", str(TEST_SRC_VENDOR)], + capture_output=True, + text=True, + cwd=test_dir, + check=True, + env=os.environ + | {"VIRTUAL_ENV": str(test_dir / ".venv-workers" / "pyodide-venv")}, + ) + vendor_packages = { + line.split("==")[0]: line.split("==")[1] + for line in vendor_freeze_result.stdout.strip().split("\n") + if line and "==" in line + } + + venv_freeze_result = subprocess.run( + ["uv", "pip", "freeze", "--path", str(site_packages_path)], + capture_output=True, + text=True, + cwd=test_dir, + check=True, + env=os.environ | {"VIRTUAL_ENV": str(TEST_VENV_WORKERS)}, + ) + venv_packages = { + line.split("==")[0]: line.split("==")[1] + for line in venv_freeze_result.stdout.strip().split("\n") + if line and "==" in line + } + + for pkg_name, vendor_version in vendor_packages.items(): + if pkg_name.lower().startswith("pyodide"): + continue + + assert pkg_name in venv_packages, ( + f"Package {pkg_name} found in vendor but not in venv" + ) + venv_version = venv_packages[pkg_name] + assert vendor_version == venv_version, ( + f"Version mismatch for {pkg_name}: " + f"vendor has {vendor_version}, " + f"venv has {venv_version}" + ) + def test_sync_command_handles_missing_pyproject(): """Test that the sync command correctly handles a missing pyproject.toml file.""" diff --git a/tests/test_version_sync.py b/tests/test_version_sync.py new file mode 100644 index 0000000..45b396f --- /dev/null +++ b/tests/test_version_sync.py @@ -0,0 +1,156 @@ +from unittest.mock import patch + +import pywrangler.sync as pywrangler_sync + + +def test_parse_pip_freeze(): + result = pywrangler_sync._parse_pip_freeze( + "shapely==2.0.7\nnumpy==1.26.4\nclick==8.1.7\n" + ) + + assert result == ["shapely==2.0.7", "numpy==1.26.4", "click==8.1.7"] + + result = pywrangler_sync._parse_pip_freeze( + "# Python 3.12.7\nshapely==2.0.7\n\n\nnumpy==1.26.4\n# Comment\n" + ) + + assert result == ["shapely==2.0.7", "numpy==1.26.4"] + + result = pywrangler_sync._parse_pip_freeze( + "shapely==2.0.7\nsome-package\nnumpy==1.26.4\n" + ) + + assert result == ["shapely==2.0.7", "numpy==1.26.4"] + + +class TestInstallRequirements: + @patch.object(pywrangler_sync, "_install_requirements_to_vendor") + @patch.object(pywrangler_sync, "_get_vendor_package_versions") + @patch.object(pywrangler_sync, "_install_requirements_to_venv") + def test_native_error_shown_before_pyodide_error( + self, mock_venv, mock_get_vendor, mock_vendor, caplog + ): + mocked_pyodide_error = "Pyodide install failed: no solution found" + mock_vendor.return_value = mocked_pyodide_error + mock_get_vendor.return_value = [] + mocked_native_error = "Native install failed: package not found" + mock_venv.return_value = mocked_native_error + + import click + import pytest + + with pytest.raises(click.exceptions.Exit): + pywrangler_sync.install_requirements(["nonexistent-package"]) + + assert mock_vendor.call_count == 1 + assert mock_venv.call_count == 1 + assert mock_get_vendor.call_count == 0 + + assert mock_vendor.call_args_list[0][0][0] == ["nonexistent-package"] + assert mock_venv.call_args_list[0][0][0] == ["nonexistent-package"] + + log_messages = [record.message for record in caplog.records] + native_idx = next( + i for i, msg in enumerate(log_messages) if mocked_native_error in msg + ) + pyodide_idx = next( + (i for i, msg in enumerate(log_messages) if mocked_pyodide_error in msg), + None, + ) + assert pyodide_idx is None, ( + "Pyodide error should not be shown when native error occurs" + ) + assert native_idx is not None + + @patch.object(pywrangler_sync, "_install_requirements_to_vendor") + @patch.object(pywrangler_sync, "_get_vendor_package_versions") + @patch.object(pywrangler_sync, "_install_requirements_to_venv") + def test_only_pyodide_error_shown_when_native_succeeds( + self, mock_venv, mock_get_vendor, mock_vendor, caplog + ): + mocked_pyodide_error = "Pyodide install failed: no solution found" + mock_vendor.return_value = mocked_pyodide_error + mock_get_vendor.return_value = [] + mock_venv.return_value = None + + import click + import pytest + + with pytest.raises(click.exceptions.Exit): + pywrangler_sync.install_requirements(["some-package"]) + + assert mock_vendor.call_count == 1 + assert mock_venv.call_count == 1 + # Pyodide installation failed, so _get_vendor_package_versions should not be called + assert mock_get_vendor.call_count == 0 + + assert mock_vendor.call_args_list[0][0][0] == ["some-package"] + + # native installation should be called with the original requirements + assert mock_venv.call_args_list[0][0][0] == ["some-package"] + + log_messages = [record.message for record in caplog.records] + assert any(mocked_pyodide_error in msg for msg in log_messages) + assert any( + "Installation of packages into the Python Worker failed. Possibly because these packages are not currently supported. See above for details." + in msg + for msg in log_messages + ) + + @patch.object(pywrangler_sync, "_install_requirements_to_vendor") + @patch.object(pywrangler_sync, "_get_vendor_package_versions") + @patch.object(pywrangler_sync, "_install_requirements_to_venv") + def test_pyodide_install_succeeds_but_native_installation_fail( + self, mock_venv, mock_get_vendor, mock_vendor, caplog + ): + mocked_native_error = "Native install failed: package not found" + mock_vendor.return_value = None + mock_get_vendor.return_value = ["some-package==1.0.0"] + mock_venv.return_value = mocked_native_error + + import click + import pytest + + with pytest.raises(click.exceptions.Exit): + pywrangler_sync.install_requirements(["some-package"]) + + assert mock_vendor.call_count == 1 + assert mock_venv.call_count == 1 + assert mock_get_vendor.call_count == 1 + + assert mock_vendor.call_args_list[0][0][0] == ["some-package"] + assert mock_venv.call_args_list[0][0][0] == ["some-package==1.0.0"] + + log_messages = [record.message for record in caplog.records] + assert any(mocked_native_error in msg for msg in log_messages) + assert any( + "Failed to install the requirements defined in your pyproject.toml file. See above for details." + in msg + for msg in log_messages + ) + + @patch.object(pywrangler_sync, "_install_requirements_to_vendor") + @patch.object(pywrangler_sync, "_get_vendor_package_versions") + @patch.object(pywrangler_sync, "_install_requirements_to_venv") + def test_known_pyodide_errors( + self, mock_venv, mock_get_vendor, mock_vendor, caplog + ): + common_errors = { + "invalid peer certificate": "Are your systems certificates correctly installed? Do you have an Enterprise VPN enabled?", + "failed to fetch": "Is your network connection working?", + "no solution found when resolving dependencies": "the packages you requested are not supported by Python Workers. See above for details.", + } + + for error, message in common_errors.items(): + mock_vendor.return_value = error + mock_get_vendor.return_value = [] + mock_venv.return_value = None + + import click + import pytest + + with pytest.raises(click.exceptions.Exit): + pywrangler_sync.install_requirements(["some-package"]) + + log_messages = [record.message for record in caplog.records] + assert any(message in msg for msg in log_messages) diff --git a/uv.lock b/uv.lock index 8c7b2f0..58bcbcf 100644 --- a/uv.lock +++ b/uv.lock @@ -500,7 +500,7 @@ wheels = [ [[package]] name = "workers-py" -version = "1.7.2" +version = "1.7.3" source = { editable = "." } dependencies = [ { name = "click" },