diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index ef19a9129ea6..7674081311de 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -6,7 +6,7 @@ on: - cron: "0 0 * * *" pull_request: paths: - - "requirements-tests.txt" + - "pyproject.toml" - ".github/workflows/daily.yml" # Please keep the permissions minimal, as stubtest runs arbitrary code from pypi. @@ -45,11 +45,11 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: requirements-tests.txt + cache-dependency-path: pyproject.toml allow-prereleases: true check-latest: true - name: Install dependencies - run: pip install -r requirements-tests.txt + run: pip install --group=dev - name: Run stubtest run: python tests/stubtest_stdlib.py @@ -69,10 +69,10 @@ jobs: python-version: "3.13" cache: pip cache-dependency-path: | - requirements-tests.txt + pyproject.toml stubs/**/METADATA.toml - name: Install dependencies - run: pip install -r requirements-tests.txt + run: pip install --group=dev - name: Run stubtest shell: bash run: | diff --git a/.github/workflows/meta_tests.yml b/.github/workflows/meta_tests.yml index dd368ed0cdde..6fa511f7eec9 100644 --- a/.github/workflows/meta_tests.yml +++ b/.github/workflows/meta_tests.yml @@ -12,7 +12,6 @@ on: - "tests/**" - "lib/**" - ".github/workflows/meta_tests.yml" - - "requirements-tests.txt" - "pyproject.toml" permissions: @@ -40,7 +39,7 @@ jobs: with: python-version: "3.13" - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - run: uv pip install -r requirements-tests.txt --system + - run: uv pip install --group=dev --system - run: python ./tests/typecheck_typeshed.py --platform=${{ matrix.platform }} pyright: name: Check scripts and tests with pyright @@ -57,7 +56,7 @@ jobs: # pytype_test.py imports pytype, we need to use Python 3.12 for now. python-version: "3.12" - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - run: uv pip install -r requirements-tests.txt --system + - run: uv pip install --group=dev --system - name: Run pyright on typeshed uses: jakebailey/pyright-action@v2 with: @@ -78,5 +77,5 @@ jobs: run: | git config --global user.name stubsabot git config --global user.email '<>' - - run: uv pip install -r requirements-tests.txt --system + - run: uv pip install --group=dev --system - run: python scripts/stubsabot.py --action-level local diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 6bddd58f8ded..0e9afc66f304 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -33,12 +33,12 @@ jobs: with: python-version: "3.13" - name: Install dependencies - run: pip install git+https://github.com/hauntsaninja/mypy_primer.git + run: pip install git+https://github.com/hauntsaninja/mypy_primer.git dependency-groups - name: Run mypy_primer shell: bash run: | cd typeshed_to_test - MYPY_VERSION=$(grep mypy== requirements-tests.txt | cut -d = -f 3) + MYPY_VERSION=$(python3 -m dependency_groups dev | grep mypy== | cut -d = -f3) echo "new commit" git rev-list --format=%s --max-count=1 $GITHUB_SHA git checkout -b upstream_main origin/main diff --git a/.github/workflows/stubsabot.yml b/.github/workflows/stubsabot.yml index 3648d5168aba..11409053d02c 100644 --- a/.github/workflows/stubsabot.yml +++ b/.github/workflows/stubsabot.yml @@ -33,7 +33,7 @@ jobs: git config --global user.name stubsabot git config --global user.email '<>' - name: Install dependencies - run: uv pip install -r requirements-tests.txt --system + run: uv pip install --group=dev --system - name: Run stubsabot run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python scripts/stubsabot.py --action-level everything diff --git a/.github/workflows/stubtest_stdlib.yml b/.github/workflows/stubtest_stdlib.yml index 2a574584ff08..42c95dd74317 100644 --- a/.github/workflows/stubtest_stdlib.yml +++ b/.github/workflows/stubtest_stdlib.yml @@ -41,10 +41,10 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: requirements-tests.txt + cache-dependency-path: pyproject.toml allow-prereleases: true check-latest: true - name: Install dependencies - run: pip install -r requirements-tests.txt + run: pip install --group=dev - name: Run stubtest run: python tests/stubtest_stdlib.py diff --git a/.github/workflows/stubtest_third_party.yml b/.github/workflows/stubtest_third_party.yml index 69ec7c6ac625..5dee7ed3cf18 100644 --- a/.github/workflows/stubtest_third_party.yml +++ b/.github/workflows/stubtest_third_party.yml @@ -44,10 +44,10 @@ jobs: python-version: "3.13" cache: pip cache-dependency-path: | - requirements-tests.txt + pyproject.toml stubs/**/METADATA.toml - name: Install dependencies - run: pip install -r requirements-tests.txt + run: pip install --group=dev - name: Run stubtest shell: bash run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 01a2af490699..91aeed59ae65 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,7 +31,7 @@ jobs: with: python-version: "3.13" - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - run: uv pip install -r requirements-tests.txt --system + - run: uv pip install --group=dev --system - run: python ./tests/check_typeshed_structure.py pytype: @@ -44,7 +44,7 @@ jobs: # Max supported Python version as of pytype 2024.10.11 python-version: "3.12" - uses: astral-sh/setup-uv@v6 - - run: uv pip install -r requirements-tests.txt --system + - run: uv pip install --group=dev --system - name: Install external dependencies for 3rd-party stubs run: | DEPENDENCIES=$( python tests/get_external_stub_requirements.py ) @@ -102,7 +102,7 @@ jobs: # TODO: figure out why that is (#11590) python-version: "3.11" - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - run: uv pip install -r requirements-tests.txt --system + - run: uv pip install --group=dev --system - run: python ./tests/regr_test.py --all --verbosity QUIET pyright: @@ -121,7 +121,7 @@ jobs: - uses: astral-sh/setup-uv@v6 - name: Install typeshed test-suite requirements # Install these so we can run `get_external_stub_requirements.py` - run: uv pip install -r requirements-tests.txt --system + run: uv pip install --group=dev --system - name: Install required APT packages run: | DEPENDENCIES=$( python tests/get_external_apt_dependencies.py ) @@ -187,5 +187,5 @@ jobs: - name: Run tests run: | cd stub_uploader - uv pip install -r requirements.txt --system + uv pip install --group=dev --system python -m pytest tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf2fbc3f7488..1b08e45c6bf7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,10 @@ repos: - id: mixed-line-ending args: [--fix=lf] - id: check-case-conflict + - repo: https://github.com/sirosen/dependency-groups + rev: 1.3.1 # must match pyproject.toml + hooks: + - id: lint-dependency-groups - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.4 # must match requirements-tests.txt hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 26a64dd85c11..d761ab4305f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,8 +63,8 @@ Note that some tests require extra setup steps to install the required dependenc ```bash $ python3 -m venv .venv $ source .venv/bin/activate - (.venv)$ pip install -U pip - (.venv)$ pip install -r requirements-tests.txt + (.venv)$ pip install -U "pip>=25.1" + (.venv)$ pip install --group=dev ``` @@ -79,8 +79,8 @@ Note that some tests require extra setup steps to install the required dependenc ```powershell > python -m venv .venv > .venv\Scripts\activate - (.venv) > pip install -U pip - (.venv) > pip install -r requirements-tests.txt + (.venv) > pip install -U "pip>=25.1" + (.venv) > pip install --group=dev ``` To be able to run pytype tests, you'll also need to install it manually @@ -100,8 +100,7 @@ as it's currently excluded from the requirements file: If you already have [uv](https://docs.astral.sh/uv/getting-started/installation/) installed, you can simply replace the commands above with: ```shell - uv venv - uv pip install -r requirements-tests.txt + uv pip install --group=dev ``` ```shell @@ -220,7 +219,7 @@ This has the following keys: in addition to the requirements in the `requires` field. * `apt_dependencies` (default: `[]`): A list of Ubuntu APT packages that need to be installed for stubtest to run successfully. -* `brew_dependencies` (default: `[]`): A list of MacOS Homebrew packages +* `brew_dependencies` (default: `[]`): A list of macOS Homebrew packages that need to be installed for stubtest to run successfully * `choco_dependencies` (default: `[]`): A list of Windows Chocolatey packages that need to be installed for stubtest to run successfully @@ -347,7 +346,7 @@ replacing `$INSERT_LIBRARY_NAME_HERE` with the name of the library: When the script has finished running, it will print instructions telling you what to do next. If it has been a while since you set up the virtualenv, make sure you have -the latest mypy (`pip install -r requirements-tests.txt`) before running the script. +the latest mypy (`pip install --group=dev`) before running the script. ### Supported type system features diff --git a/lib/ts_utils/paths.py b/lib/ts_utils/paths.py index 568a091a5ae6..91cc5476a9bc 100644 --- a/lib/ts_utils/paths.py +++ b/lib/ts_utils/paths.py @@ -10,7 +10,6 @@ STUBS_PATH: Final = TS_BASE_PATH / "stubs" PYPROJECT_PATH: Final = TS_BASE_PATH / "pyproject.toml" -REQUIREMENTS_PATH: Final = TS_BASE_PATH / "requirements-tests.txt" GITIGNORE_PATH: Final = TS_BASE_PATH / ".gitignore" TESTS_DIR: Final = "@tests" diff --git a/lib/ts_utils/utils.py b/lib/ts_utils/utils.py index 11d47acdfaff..f6262a614047 100644 --- a/lib/ts_utils/utils.py +++ b/lib/ts_utils/utils.py @@ -13,9 +13,11 @@ from typing_extensions import TypeAlias import pathspec +import tomllib +from dependency_groups import resolve from packaging.requirements import Requirement -from .paths import GITIGNORE_PATH, REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, allowlists_path, test_cases_path +from .paths import GITIGNORE_PATH, PYPROJECT_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, allowlists_path, test_cases_path if TYPE_CHECKING: from _typeshed import OpenTextMode @@ -95,12 +97,12 @@ def venv_python(venv_dir: Path) -> Path: @functools.cache def parse_requirements() -> Mapping[str, Requirement]: - """Return a dictionary of requirements from the requirements file.""" - with REQUIREMENTS_PATH.open(encoding="UTF-8") as requirements_file: - stripped_lines = map(strip_comments, requirements_file) - stripped_more = [li for li in stripped_lines if not li.startswith("-")] - requirements = map(Requirement, filter(None, stripped_more)) - return {requirement.name: requirement for requirement in requirements} + """Return a dictionary of requirements found in the dev dependency group.""" + with PYPROJECT_PATH.open("rb") as requirements_file: + pyproject = tomllib.load(requirements_file) + + requirements = [Requirement(requirement) for requirement in resolve(pyproject["dependency-groups"], "dev")] + return {requirement.name: requirement for requirement in requirements} def get_mypy_req() -> str: diff --git a/pyproject.toml b/pyproject.toml index b4a430f7510c..fb6f2c6a7bd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,41 @@ name = "typeshed" version = "0" requires-python = ">=3.9" # Minimum version to run tests, used by uv run +[dependency-groups] +dev = [ + # Type checkers that we test our stubs against. These should always + # be pinned to a specific version to make failure reproducible. + "mypy ==1.15.0", + "pyright ==1.1.400", + # pytype can be installed on Windows, but requires building wheels, let's not do that on the CI + "pytype ==2024.10.11; platform_system != 'Windows' and python_version >= '3.10' and python_version < '3.13'", + + # Libraries used by our various scripts. + # TODO (2025-05-09): Installing this on Python 3.14 on Windows fails at the moment. + "aiohttp ==3.11.15; python_version < '3.14'", + "dependency-groups ==1.3.1", # Must match .pre-commit-config.yaml. + # TODO (2025-05-09): Installing this on Python 3.14 on Windows fails at the moment. + "grpcio-tools >=1.66.2; python_version < '3.14'", # For grpc_tools.protoc" + "mypy-protobuf ==3.6.0", + "packaging ==24.2", + "pathspec >=0.11.1", + "pre-commit", + "ruff ==0.11.4", # Required by create_baseline_stubs.py. Must match .pre-commit-config.yaml. + # TODO (2025-05-07): Dependency libcst doesn't support Python 3.14 yet. + "stubdefaulter ==0.1.0; python_version < '3.14'", + "termcolor >=2.3", + "tomli ==2.2.1", + "tomlkit ==0.13.2", + "typing_extensions >=4.13.0rc1", + "uv ==0.7.4", + + # Utilities for typeshed infrastructure scripts. + "ts_utils @ file:lib", +] + +[tool.uv.sources] +ts_utils = { path = "lib", editable = true } + [tool.black] line-length = 130 target-version = ["py310"] diff --git a/requirements-tests.txt b/requirements-tests.txt deleted file mode 100644 index 1c0d6147c4ff..000000000000 --- a/requirements-tests.txt +++ /dev/null @@ -1,30 +0,0 @@ -# Type checkers that we test our stubs against. These should always -# be pinned to a specific version to make failure reproducible. -mypy==1.15.0 -pyright==1.1.400 -# pytype can be installed on Windows, but requires building wheels, let's not do that on the CI -pytype==2024.10.11; platform_system != "Windows" and python_version >= "3.10" and python_version < "3.13" - -# Libraries used by our various scripts. -# TODO (2025-05-09): Installing this on Python 3.14 on Windows fails at -# the moment. -aiohttp==3.11.15; python_version < "3.14" -# TODO (2025-05-09): No wheels exist for Python 3.14 yet, slowing down CI -# considerably and prone to fail. -grpcio-tools>=1.66.2; python_version < "3.14" # For grpc_tools.protoc -mypy-protobuf==3.6.0 -packaging==24.2 -pathspec>=0.11.1 -pre-commit -# Required by create_baseline_stubs.py. Must match .pre-commit-config.yaml. -ruff==0.11.4 -# TODO (2025-05-07): Dependency libcst doesn't support Python 3.14 yet. -stubdefaulter==0.1.0; python_version < "3.14" -termcolor>=2.3 -tomli==2.2.1 -tomlkit==0.13.2 -typing_extensions>=4.13.0rc1 -uv==0.7.4 - -# Utilities for typeshed infrastructure scripts. -ts_utils @ file:lib diff --git a/scripts/create_baseline_stubs.py b/scripts/create_baseline_stubs.py index db4fd9f3bf08..0c9dbb370722 100755 --- a/scripts/create_baseline_stubs.py +++ b/scripts/create_baseline_stubs.py @@ -84,8 +84,8 @@ async def get_project_urls_from_pypi(project: str, session: aiohttp.ClientSessio async def get_upstream_repo_url(project: str) -> str | None: - # aiohttp is overkill here, but it would also just be silly - # to have both requests and aiohttp in our requirements-tests.txt file. + # aiohttp is overkill here, but it would also just be silly to have + # both requests and aiohttp in our dev dependency-group in pyproject.toml. async with aiohttp.ClientSession() as session: project_urls = await get_project_urls_from_pypi(project, session) diff --git a/tests/check_typeshed_structure.py b/tests/check_typeshed_structure.py index 191173b4256b..cedf596570de 100755 --- a/tests/check_typeshed_structure.py +++ b/tests/check_typeshed_structure.py @@ -12,7 +12,7 @@ from pathlib import Path from ts_utils.metadata import read_metadata -from ts_utils.paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, TESTS_DIR, tests_path +from ts_utils.paths import PYPROJECT_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, TESTS_DIR, tests_path from ts_utils.utils import ( get_all_testcase_directories, get_gitignore_spec, @@ -166,10 +166,10 @@ def check_requirement_pins() -> None: """Check that type checkers and linters are pinned to an exact version.""" requirements = parse_requirements() for package in linters: - assert package in requirements, f"type checker/linter '{package}' not found in {REQUIREMENTS_PATH.name}" + assert package in requirements, f"type checker/linter '{package}' not found in {PYPROJECT_PATH.name}" spec = requirements[package].specifier - assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQUIREMENTS_PATH.name}" - msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQUIREMENTS_PATH.name}" + assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {PYPROJECT_PATH.name}" + msg = f"type checker/linter '{package}' is not pinned to an exact version in {PYPROJECT_PATH.name}" assert str(spec).startswith("=="), msg