diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e3df22f..1ce6f72 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,10 @@ Changelog ========= +Unreleased +---------- +- Include PyPI source distribution URLs in package repository_download_url (#134). + v0.8.5 -------- - Update pypa gh-action-pypi-publish tag to fixed commit SHA (https://github.com/aboutcode-org/fetchcode/pull/221) diff --git a/src/fetchcode/package.py b/src/fetchcode/package.py index e8c30a3..6071232 100644 --- a/src/fetchcode/package.py +++ b/src/fetchcode/package.py @@ -207,17 +207,21 @@ def get_pypi_data_from_purl(purl): for num in releases: version_purl = PackageURL(type=purl.type, name=name, version=num) - release = releases.get(num) or [{}] - release = release[0] - download_url = release.get("url") if purl.version and version_purl.version != purl.version: continue + release_files = releases.get(num) or [{}] + download_url = release_files[0].get("url") + repository_download_url = next( + (file.get("url") for file in release_files if file.get("packagetype") == "sdist"), + None, + ) yield Package( homepage_url=homepage_url, api_url=api_url, bug_tracking_url=bug_tracking_url, code_view_url=code_view_url, download_url=download_url, + repository_download_url=repository_download_url, declared_license=license, **version_purl.to_dict(), ) diff --git a/tests/data/pypi.json b/tests/data/pypi.json index 8be75ff..f37707f 100644 --- a/tests/data/pypi.json +++ b/tests/data/pypi.json @@ -32,7 +32,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/6e/49/43b514bfdaf4af12e6ef1f17aa25447157bcbb864c07775dacd72e8c8e02/Flask-0.1.tar.gz", "api_data_url": null }, { @@ -68,7 +68,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.10", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/f3/46/53d83cbdb79b27678c7b032d5deaa556655dd034cc747ee609b3e3cbf95b/Flask-0.10.tar.gz", "api_data_url": null }, { @@ -104,7 +104,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.10.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/db/9c/149ba60c47d107f85fe52564133348458f093dd5e6b57a5b60ab9ac517bb/Flask-0.10.1.tar.gz", "api_data_url": null }, { @@ -140,7 +140,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.11", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/dc/ca/c0ed9cc90c079085c698e284b672edbc1ffd6866b1830574095cbc5b7752/Flask-0.11.tar.gz", "api_data_url": null }, { @@ -176,7 +176,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.11.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/55/8a/78e165d30f0c8bb5d57c429a30ee5749825ed461ad6c959688872643ffb3/Flask-0.11.1.tar.gz", "api_data_url": null }, { @@ -212,7 +212,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.12", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/4b/3a/4c20183df155dd2e39168e35d53a388efb384a512ca6c73001d8292c094a/Flask-0.12.tar.gz", "api_data_url": null }, { @@ -248,7 +248,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.12.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/24/6e/11b9c57e46f276a8a8dfda85a2fa7ada62b0463b68693616c7ab5df356fa/Flask-0.12.1.tar.gz", "api_data_url": null }, { @@ -284,7 +284,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.12.2", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/eb/12/1c7bd06fcbd08ba544f25bf2c6612e305a70ea51ca0eda8007344ec3f123/Flask-0.12.2.tar.gz", "api_data_url": null }, { @@ -320,7 +320,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.12.3", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/80/84/ddf5d2141e84f71ba184ea58b3d9b9caaee9cc49ca0303051ac02381791c/Flask-0.12.3.tar.gz", "api_data_url": null }, { @@ -356,7 +356,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.12.4", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/1b/72/ffc594a6832337ace475f939e61c34a44cbb150cde9589f98c482b407dd8/Flask-0.12.4.tar.gz", "api_data_url": null }, { @@ -392,7 +392,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.12.5", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/32/57/3c33fe153ea008e9e0202eb028972178337c55777686aac03f41ade671f8/Flask-0.12.5.tar.gz", "api_data_url": null }, { @@ -428,7 +428,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.2", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/9a/db/245abc92428bcdfdc32d8017ddd1b079afffce9c74f94e34d1aa777bc771/Flask-0.2.tar.gz", "api_data_url": null }, { @@ -464,7 +464,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.3", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/8b/cb/706dbb37f4ef3a75366c9e715f41d22e73ca4594303f48d229d906c80632/Flask-0.3.tar.gz", "api_data_url": null }, { @@ -500,7 +500,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.3.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/e0/d7/4de91ad9fc1854e651cf03f87eff939a92cd06716645dee86b0382674ea3/Flask-0.3.1.tar.gz", "api_data_url": null }, { @@ -536,7 +536,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.4", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/a3/89/a4bf29e78a87e11f0f6fdd4d9e02a0aece1eecd38118496da58d4826d7e3/Flask-0.4.tar.gz", "api_data_url": null }, { @@ -572,7 +572,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.5", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/d4/6a/93500f2a7089b4e993fb095215979890b6204a5ba3f6b0f63dc6c3c6c827/Flask-0.5.tar.gz", "api_data_url": null }, { @@ -608,7 +608,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.5.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/16/a6/c458d3305e689d7e06a23eacee414ea10d870074a7673864ffea67109f9d/Flask-0.5.1.tar.gz", "api_data_url": null }, { @@ -644,7 +644,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.5.2", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/1c/b5/03c412ba48148e6c222e238201a0924360a85d755ce9597acbd99a1a6240/Flask-0.5.2.tar.gz", "api_data_url": null }, { @@ -680,7 +680,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.6", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/44/86/481371798994529e105633a50b2332638105a1e191053bc0f4bbc9b91791/Flask-0.6.tar.gz", "api_data_url": null }, { @@ -716,7 +716,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.6.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/8f/1c/453a427f55b91239b3368c8b975b55d089d5d79dc37545af41cd7157c187/Flask-0.6.1.tar.gz", "api_data_url": null }, { @@ -752,7 +752,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.7", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/43/08/e4907533c6ca0ebb1867182fa94b1ffa41fa3aba5f6cb4969e108262e92b/Flask-0.7.tar.gz", "api_data_url": null }, { @@ -788,7 +788,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.7.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/fe/3e/ad5eb51d4666e76f389cd4f9c6cc22e1544e0daf72419ccab8705e918911/Flask-0.7.1.tar.gz", "api_data_url": null }, { @@ -824,7 +824,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.7.2", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/1c/c7/a361d00f4c9ed3f1b7ab77976e820ca347f3b0aec4dee6c66fe5c5a2124d/Flask-0.7.2.tar.gz", "api_data_url": null }, { @@ -860,7 +860,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.8", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/f0/84/e3c207a6aad1acfdfe1eda20abeadff47035f24820f09ac6870f9c8a26a3/Flask-0.8.tar.gz", "api_data_url": null }, { @@ -896,7 +896,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.8.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/20/5d/f355d122c9d7a45d7846449f94b9f1d26df88556f705f14dd84a8fa264ea/Flask-0.8.1.tar.gz", "api_data_url": null }, { @@ -932,7 +932,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@0.9", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/49/0a/fe5021b35436202d3d4225a766f3bdc7fb51521ad89e73c5162db36cdbc7/Flask-0.9.tar.gz", "api_data_url": null }, { @@ -968,7 +968,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@1.0", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/99/ab/eedb921f26adf7057ade1291f9c1bfa35a506d64894f58546457ef658772/Flask-1.0.tar.gz", "api_data_url": null }, { @@ -1004,7 +1004,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@1.0.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/6d/2f/95a73db56fa2c2b3187bb69783cb2bea4327d1e7b2e0cf60e15df59502ee/Flask-1.0.1.tar.gz", "api_data_url": null }, { @@ -1040,7 +1040,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@1.0.2", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/4b/12/c1fbf4971fda0e4de05565694c9f0c92646223cff53f15b6eb248a310a62/Flask-1.0.2.tar.gz", "api_data_url": null }, { @@ -1076,7 +1076,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@1.0.3", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/e9/96/8f6d83828a77306a119e12b215a7b0637c955b408fb1c161311a6891b958/Flask-1.0.3.tar.gz", "api_data_url": null }, { @@ -1112,7 +1112,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@1.0.4", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/36/70/2234ee8842148cef44261c2cebca3a6384894bce6112b73b18693cdcc62f/Flask-1.0.4.tar.gz", "api_data_url": null }, { @@ -1148,7 +1148,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@1.1.0", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/1b/73/5133d483c4eac2c49f82a80bbb25c2d75e01177afe66f84ef8dc6d17c071/Flask-1.1.0.tar.gz", "api_data_url": null }, { @@ -1184,7 +1184,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@1.1.1", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/2e/80/3726a729de758513fd3dbc64e93098eb009c49305a97c6751de55b20b694/Flask-1.1.1.tar.gz", "api_data_url": null }, { @@ -1220,7 +1220,7 @@ "source_packages": [], "purl": "pkg:pypi/flask@1.1.2", "repository_homepage_url": null, - "repository_download_url": null, + "repository_download_url": "https://files.pythonhosted.org/packages/4e/0b/cb02268c90e67545a0e3a37ea1ca3d45de3aca43ceb7dbf1712fb5127d5d/Flask-1.1.2.tar.gz", "api_data_url": null } ] diff --git a/tests/test_package.py b/tests/test_package.py index e1e22b5..a345992 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -151,6 +151,82 @@ def test_pypi_packages(mock_get): check_packages(packages, expected_data) +@pytest.mark.parametrize( + "files, expected_download, expected_source", + [ + ( + [ + {"packagetype": "bdist_wheel", "url": "https://example.com/demo.whl"}, + {"packagetype": "sdist", "url": "https://example.com/demo.tar.gz"}, + ], + "https://example.com/demo.whl", + "https://example.com/demo.tar.gz", + ), + ( + [ + {"packagetype": "sdist", "url": "https://example.com/demo.zip"}, + {"packagetype": "bdist_wheel", "url": "https://example.com/demo.whl"}, + ], + "https://example.com/demo.zip", + "https://example.com/demo.zip", + ), + ( + [{"packagetype": "sdist", "url": "https://example.com/demo.tar.gz"}], + "https://example.com/demo.tar.gz", + "https://example.com/demo.tar.gz", + ), + ( + [{"packagetype": "bdist_wheel", "url": "https://example.com/demo.whl"}], + "https://example.com/demo.whl", + None, + ), + ([{"url": "https://example.com/demo.tar.gz"}], "https://example.com/demo.tar.gz", None), + ([{"packagetype": "sdist"}], None, None), + ([], None, None), + ], + ids=["wheel-first", "source-first", "source-only", "wheel-only", "untyped", "no-url", "empty"], +) +@mock.patch("fetchcode.package.get_response") +def test_pypi_source_distribution_url(mock_get, files, expected_download, expected_source): + mock_get.return_value = {"releases": {"1.0": files}} + + packages = list(info("pkg:pypi/demo")) + + mock_get.assert_called_once_with("https://pypi.org/pypi/demo/json") + assert len(packages) == 1 + package = packages[0] + assert package.purl == "pkg:pypi/demo@1.0" + assert package.download_url == expected_download + assert package.repository_download_url == expected_source + assert package.to_dict()["repository_download_url"] == expected_source + + +@pytest.mark.parametrize("version", [None, "2.0", "3.0"]) +@mock.patch("fetchcode.package.get_response") +def test_pypi_source_distribution_url_matches_release(mock_get, version): + mock_get.return_value = { + "releases": { + "1.0": [{"packagetype": "sdist", "url": "https://example.com/demo-1.0.tar.gz"}], + "2.0": [{"packagetype": "sdist", "url": "https://example.com/demo-2.0.tar.gz"}], + "3.0": [], + } + } + purl = "pkg:pypi/demo" + if version: + purl += f"@{version}" + + packages = list(info(purl)) + + expected = [ + ("1.0", "https://example.com/demo-1.0.tar.gz"), + ("2.0", "https://example.com/demo-2.0.tar.gz"), + ("3.0", None), + ] + if version: + expected = [item for item in expected if item[0] == version] + assert [(p.version, p.repository_download_url) for p in packages] == expected + + @mock.patch("fetchcode.package.get_response") def test_bitbucket_packages(mock_get): side_effect = [