Merge pull request #233 from aboutcode-org/netrc-format-fix-pluplus

Netrc format fix pluplus

Author: pombredanne

README.rst

@@ -2,10 +2,6 @@ python-inspector - inspect Python package dependencies and metadata
 =====================================================================
 
 
-Copyright (c) nexB Inc. and others.
-SPDX-License-Identifier: Apache-2.0
-Homepage: https://github.com/aboutcode-org/python-inspector and https://www.aboutcode.org/
-
 
 ``python-inspector`` is a collection of utilities to:
 
@@ -28,21 +24,21 @@ The goal of python-inspector is to be a comprehensive library
 that can handle every style of Python package layouts, manifests and lockfiles.
 
 
-Testing
---------
+SPDX-License-Identifier: Apache-2.0
 
-- Run the tests with::
+Copyright (c) AboutCode, nexB Inc. and others.
 
-    pytest -vvs
-
-- There are live tests to regenerate the tests with updated data run::
+Homepage: https://github.com/aboutcode-org/python-inspector and https://www.aboutcode.org/
 
-      PYINSP_REGEN_TEST_FIXTURES=yes pytest -vvs
 
 Usage
 --------
 
-- Install with pip::
+- Install the stable release with pip from PyPI::
+
+    pip install python-inspector
+
+- Or install the latest with pip::
 
     pip install git+https://github.com/aboutcode-org/python-inspector
 
@@ -51,8 +47,71 @@ Usage
     python-inspector --help
 
 
+Development
+--------------
+
+Run::
+
+    git clone https://github.com/aboutcode-org/python-inspector
+
+Create a virtual environment and install deps locally::
+
+    make dev
+    source venv/bin/activate
+
+
+When in the virtual environment, run python-inspector from that clone::
+
+    python-inspector --help
+
+
+Run tests::
+
+    make test
+
+Run code checks::
 
-Its companion libraries are:
+    make check
+
+Run code formatting::
+
+    make valie
+
+Check available make targets for further details
+
+
+
+More testing
+------------------
+
+- Run the tests with pytest::
+
+    pytest -vvs
+
+- Or run them faster using 12 cores ::
+
+    pytest -vvs --numprocesses=12
+
+
+Regenerate test files
+-----------------------------
+
+Some tests use live data from Pypi.org to run resolutions. When the package versions have
+changed, the resolution can change and some of the tests fail. We have an environment variable
+that regenerates the expected JSON result files when set.
+
+To regenerate expected test result files for the failed tests, use this command::
+
+    PYINSP_REGEN_TEST_FIXTURES=yes pytest -vvs --lf
+
+Then, carefully review the diff before committing the expected JSON test result files to validate
+that the changes are OK and mostly affect small changes in resolved package versions.
+
+
+Credits and dependencies
+---------------------------
+
+For info, python-inspector embeds or depends on these libraries:
 
 - ``pip-requirements-parser``, a mostly correct pip requirements parsing
   library extracted from pip.
@@ -72,6 +131,8 @@ Its companion libraries are:
 
 - ``packageurl-python`` to use Package URL to reference Python packages
 
+- ``scancode-toolkit`` for Python package manifest parsing.
+
 
 
 Acknowledgements, Funding, Support and Sponsoring

src/python_inspector/utils.py

@@ -16,20 +16,28 @@
 from typing import NamedTuple
 from typing import Optional
 
+from urllib.parse import urlparse
+
 import aiohttp
 import requests
 
 
 def get_netrc_auth(url, netrc):
     """
-    Return login and password if url is in netrc
+    Return login and password if either the hostname is in netrc or a default is set in netrc
     else return login and password as None
     """
+    hostname = urlparse(url).hostname
     hosts = netrc.hosts
-    if url in hosts:
-        url_auth = hosts.get(url)
+    if hostname in hosts:
+        url_auth = hosts.get(hostname)
         # netrc returns a tuple of (login, account, password)
         return (url_auth[0], url_auth[2])
+
+    if "default" in hosts:
+        default_auth = hosts.get("default")
+        return (default_auth[0], default_auth[2])
+
     return (None, None)
 
 

src/python_inspector/utils_pypi.py

@@ -1598,6 +1598,10 @@ async def fetch_links(
         name using the `index_url` of this repository.
         """
         package_url = f"{self.index_url}/{normalized_name}"
+
+        if not package_url.endswith("/"):
+            package_url += "/"
+
         text, _ = await CACHE.get(
             path_or_url=package_url,
             credentials=self.credentials,
@@ -1797,7 +1801,10 @@ async def get_remote_file_content(
 
     auth = None
     if credentials:
-        auth = (credentials.get("login"), credentials.get("password"))
+        login = credentials.get("login")
+        password = credentials.get("password")
+        if login and password:
+            auth = aiohttp.BasicAuth(login, password)
 
     async with aiohttp.ClientSession() as session:
         async with session.get(url, allow_redirects=True, headers=headers, auth=auth) as response:

tests/data/azure-devops.req-310-expected.json

@@ -2,7 +2,6 @@
   "headers": {
     "tool_name": "python-inspector",
     "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector",
-    "tool_version": "0.13.0",
     "options": [
       "--ignore-errors",
       "--index-url https://pypi.org/simple",
@@ -356,12 +355,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "pygments",
-      "version": "2.19.1",
+      "version": "2.19.2",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "Pygments\n~~~~~~~~\n\nPygments is a syntax highlighting package written in Python.\n\nIt is a generic syntax highlighter suitable for use in code hosting, forums,\nwikis or other applications that need to prettify source code.  Highlights\nare:\n\n* a wide range of over 500 languages and other text formats is supported\n* special attention is paid to details, increasing quality by a fair amount\n* support for new languages and formats are added easily\n* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image\n  formats that PIL supports and ANSI sequences\n* it is usable as a command-line tool and as a library\n\nCopyright 2006-2025 by the Pygments team, see ``AUTHORS``.\nLicensed under the BSD, see ``LICENSE`` for details.",
-      "release_date": "2025-01-06T17:26:25",
+      "release_date": "2025-06-21T13:39:07",
       "parties": [
         {
           "type": "person",
@@ -399,11 +398,11 @@
         "Topic :: Utilities"
       ],
       "homepage_url": null,
-      "download_url": "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl",
-      "size": 1225293,
+      "download_url": "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl",
+      "size": 1225217,
       "sha1": null,
-      "md5": "794747e68f6a2c85e86a8a49e4abb285",
-      "sha256": "9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c",
+      "md5": "c6f882e4a4ffd8543f245d4f7ce0d7fa",
+      "sha256": "86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b",
       "sha512": null,
       "bug_tracking_url": "https://github.com/pygments/pygments/issues",
       "code_view_url": "https://github.com/pygments/pygments",
@@ -423,20 +422,20 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/pygments/2.19.1/json",
+      "api_data_url": "https://pypi.org/pypi/pygments/2.19.2/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/[email protected].1"
+      "purl": "pkg:pypi/[email protected].2"
     },
     {
       "type": "pypi",
       "namespace": null,
       "name": "pytest",
-      "version": "8.4.0",
+      "version": "8.4.1",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "pytest: simple powerful testing with Python\n.. image:: https://github.com/pytest-dev/pytest/raw/main/doc/en/img/pytest_logo_curves.svg\n   :target: https://docs.pytest.org/en/stable/\n   :align: center\n   :height: 200\n   :alt: pytest\n\n\n------\n\n.. image:: https://img.shields.io/pypi/v/pytest.svg\n    :target: https://pypi.org/project/pytest/\n\n.. image:: https://img.shields.io/conda/vn/conda-forge/pytest.svg\n    :target: https://anaconda.org/conda-forge/pytest\n\n.. image:: https://img.shields.io/pypi/pyversions/pytest.svg\n    :target: https://pypi.org/project/pytest/\n\n.. image:: https://codecov.io/gh/pytest-dev/pytest/branch/main/graph/badge.svg\n    :target: https://codecov.io/gh/pytest-dev/pytest\n    :alt: Code coverage Status\n\n.. image:: https://github.com/pytest-dev/pytest/actions/workflows/test.yml/badge.svg\n    :target: https://github.com/pytest-dev/pytest/actions?query=workflow%3Atest\n\n.. image:: https://results.pre-commit.ci/badge/github/pytest-dev/pytest/main.svg\n   :target: https://results.pre-commit.ci/latest/github/pytest-dev/pytest/main\n   :alt: pre-commit.ci status\n\n.. image:: https://www.codetriage.com/pytest-dev/pytest/badges/users.svg\n    :target: https://www.codetriage.com/pytest-dev/pytest\n\n.. image:: https://readthedocs.org/projects/pytest/badge/?version=latest\n    :target: https://pytest.readthedocs.io/en/latest/?badge=latest\n    :alt: Documentation Status\n\n.. image:: https://img.shields.io/badge/Discord-pytest--dev-blue\n    :target: https://discord.com/invite/pytest-dev\n    :alt: Discord\n\n.. image:: https://img.shields.io/badge/Libera%20chat-%23pytest-orange\n    :target: https://web.libera.chat/#pytest\n    :alt: Libera chat\n\n\nThe ``pytest`` framework makes it easy to write small tests, yet\nscales to support complex functional testing for applications and libraries.\n\nAn example of a simple test:\n\n.. code-block:: python\n\n    # content of test_sample.py\n    def inc(x):\n        return x + 1\n\n\n    def test_answer():\n        assert inc(3) == 5\n\n\nTo execute it::\n\n    $ pytest\n    ============================= test session starts =============================\n    collected 1 items\n\n    test_sample.py F\n\n    ================================== FAILURES ===================================\n    _________________________________ test_answer _________________________________\n\n        def test_answer():\n    >       assert inc(3) == 5\n    E       assert 4 == 5\n    E        +  where 4 = inc(3)\n\n    test_sample.py:5: AssertionError\n    ========================== 1 failed in 0.04 seconds ===========================\n\n\nDue to ``pytest``'s detailed assertion introspection, only plain ``assert`` statements are used. See `getting-started <https://docs.pytest.org/en/stable/getting-started.html#our-first-test-run>`_ for more examples.\n\n\nFeatures\n--------\n\n- Detailed info on failing `assert statements <https://docs.pytest.org/en/stable/how-to/assert.html>`_ (no need to remember ``self.assert*`` names)\n\n- `Auto-discovery\n  <https://docs.pytest.org/en/stable/explanation/goodpractices.html#python-test-discovery>`_\n  of test modules and functions\n\n- `Modular fixtures <https://docs.pytest.org/en/stable/explanation/fixtures.html>`_ for\n  managing small or parametrized long-lived test resources\n\n- Can run `unittest <https://docs.pytest.org/en/stable/how-to/unittest.html>`_ (or trial)\n  test suites out of the box\n\n- Python 3.9+ or PyPy3\n\n- Rich plugin architecture, with over 1300+ `external plugins <https://docs.pytest.org/en/latest/reference/plugin_list.html>`_ and thriving community\n\n\nDocumentation\n-------------\n\nFor full documentation, including installation, tutorials and PDF documents, please see https://docs.pytest.org/en/stable/.\n\n\nBugs/Requests\n-------------\n\nPlease use the `GitHub issue tracker <https://github.com/pytest-dev/pytest/issues>`_ to submit bugs or request features.\n\n\nChangelog\n---------\n\nConsult the `Changelog <https://docs.pytest.org/en/stable/changelog.html>`__ page for fixes and enhancements of each version.\n\n\nSupport pytest\n--------------\n\n`Open Collective`_ is an online funding platform for open and transparent communities.\nIt provides tools to raise money and share your finances in full transparency.\n\nIt is the platform of choice for individuals and companies that want to make one-time or\nmonthly donations directly to the project.\n\nSee more details in the `pytest collective`_.\n\n.. _Open Collective: https://opencollective.com\n.. _pytest collective: https://opencollective.com/pytest\n\n\npytest for enterprise\n---------------------\n\nAvailable as part of the Tidelift Subscription.\n\nThe maintainers of pytest and thousands of other packages are working with Tidelift to deliver commercial support and\nmaintenance for the open source dependencies you use to build your applications.\nSave time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use.\n\n`Learn more. <https://tidelift.com/subscription/pkg/pypi-pytest?utm_source=pypi-pytest&utm_medium=referral&utm_campaign=enterprise&utm_term=repo>`_\n\nSecurity\n^^^^^^^^\n\npytest has never been associated with a security vulnerability, but in any case, to report a\nsecurity vulnerability please use the `Tidelift security contact <https://tidelift.com/security>`_.\nTidelift will coordinate the fix and disclosure.\n\n\nLicense\n-------\n\nCopyright Holger Krekel and others, 2004.\n\nDistributed under the terms of the `MIT`_ license, pytest is free and open source software.\n\n.. _`MIT`: https://github.com/pytest-dev/pytest/blob/main/LICENSE",
-      "release_date": "2025-06-02T17:36:27",
+      "release_date": "2025-06-18T05:48:03",
       "parties": [
         {
           "type": "person",
@@ -467,11 +466,11 @@
         "Topic :: Utilities"
       ],
       "homepage_url": null,
-      "download_url": "https://files.pythonhosted.org/packages/2f/de/afa024cbe022b1b318a3d224125aa24939e99b4ff6f22e0ba639a2eaee47/pytest-8.4.0-py3-none-any.whl",
-      "size": 363797,
+      "download_url": "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl",
+      "size": 365474,
       "sha1": null,
-      "md5": "e3ae70dc3edd90a392d9061ba2c8d6e6",
-      "sha256": "f40f825768ad76c0977cbacdf1fd37c6f7a468e460ea6a0636078f8972d4517e",
+      "md5": "6ad4ee79caee224776d07f155d91b7e7",
+      "sha256": "539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7",
       "sha512": null,
       "bug_tracking_url": "https://github.com/pytest-dev/pytest/issues",
       "code_view_url": "https://github.com/pytest-dev/pytest",
@@ -491,9 +490,9 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/pytest/8.4.0/json",
+      "api_data_url": "https://pypi.org/pypi/pytest/8.4.1/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/[email protected].0"
+      "purl": "pkg:pypi/[email protected].1"
     },
     {
       "type": "pypi",
@@ -649,17 +648,17 @@
       "dependencies": []
     },
     {
-      "package": "pkg:pypi/[email protected].1",
+      "package": "pkg:pypi/[email protected].2",
       "dependencies": []
     },
     {
-      "package": "pkg:pypi/[email protected].0",
+      "package": "pkg:pypi/[email protected].1",
       "dependencies": [
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/[email protected].1",
+        "pkg:pypi/[email protected].2",
         "pkg:pypi/[email protected]"
       ]
     },

tests/data/azure-devops.req-312-expected.json

@@ -6,7 +6,7 @@
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/txaio@23.1.1"
+        "pkg:pypi/txaio@23.6.1"
       ]
     },
     {
@@ -40,7 +40,7 @@
       "dependencies": []
     },
     {
-      "package": "pkg:pypi/txaio@23.1.1",
+      "package": "pkg:pypi/txaio@23.6.1",
       "dependencies": []
     }
   ],
@@ -52,6 +52,6 @@
     "pkg:pypi/[email protected]",
     "pkg:pypi/[email protected]",
     "pkg:pypi/[email protected]",
-    "pkg:pypi/txaio@23.1.1"
+    "pkg:pypi/txaio@23.6.1"
   ]
 ]

tests/data/azure-devops.req-313-expected.json

@@ -17,7 +17,7 @@
     {
       "package": "pkg:pypi/[email protected]",
       "dependencies": [
-        "pkg:pypi/zipp@3.22.0"
+        "pkg:pypi/zipp@3.23.0"
       ]
     },
     {
@@ -41,7 +41,7 @@
       ]
     },
     {
-      "package": "pkg:pypi/zipp@3.22.0",
+      "package": "pkg:pypi/zipp@3.23.0",
       "dependencies": []
     }
   ],
@@ -53,6 +53,6 @@
     "pkg:pypi/[email protected]",
     "pkg:pypi/[email protected]",
     "pkg:pypi/[email protected]",
-    "pkg:pypi/zipp@3.22.0"
+    "pkg:pypi/zipp@3.23.0"
   ]
 ]