Fix for issue declared license collection from nuspec (#2669)

src/packagedcode/nuget.py

@@ -18,6 +18,8 @@
 Handle NuGet packages and their manifests.
 """
 
+DEPRECATED_NUGET_LICENSE_URL = 'https://aka.ms/deprecateLicenseUrl'
+
 
 def get_dependencies(nuspec):
     """
@@ -79,6 +81,32 @@ def _get_dep_packs(deps, extra_data):
             extra_data=extra,
         )
 
+def get_declared_license(nuspec):
+    extracted_license_statement = None
+    license_file = None
+
+    license_data = nuspec.get('license')
+    if license_data:
+        if isinstance(license_data, dict):
+            license_type = license_data.get('@type')
+            license_text = license_data.get('#text')
+            if license_type == 'file':
+                license_file = license_text
+            else:
+                # "expression" (an SPDX license expression) or any other unspecified type: keep the text as is
+                extracted_license_statement = license_text
+        else:
+            # defensive: in case a plain string ever shows up here.
+            extracted_license_statement = license_data
+
+    license_url = nuspec.get('licenseUrl')
+    if license_url and license_url != DEPRECATED_NUGET_LICENSE_URL:
+        if extracted_license_statement:
+            extracted_license_statement = f'{extracted_license_statement}\n{license_url}'
+        else:
+            extracted_license_statement = license_url
+
+    return extracted_license_statement, license_file
 
 def get_urls(name, version, **kwargs):
     return dict(
@@ -155,14 +183,20 @@ def parse(cls, location, package_only=False):
 
         urls = get_urls(name, version)
 
-        extracted_license_statement = None
-        # See https://docs.microsoft.com/en-us/nuget/reference/nuspec#license
-        # This is a SPDX license expression
-        if 'license' in nuspec:
-            extracted_license_statement = nuspec.get('license')
-        # Deprecated and not a license expression, just a URL
-        elif 'licenseUrl' in nuspec:
-            extracted_license_statement = nuspec.get('licenseUrl')
+        # extracted_license_statement = None
+        # # See https://docs.microsoft.com/en-us/nuget/reference/nuspec#license
+        # # This is a SPDX license expression
+        # if 'license' in nuspec:
+        #     extracted_license_statement = nuspec.get('license')
+        # # Deprecated and not a license expression, just a URL
+        # elif 'licenseUrl' in nuspec:
+        #     extracted_license_statement = nuspec.get('licenseUrl')
+
+        extracted_license_statement, license_file = get_declared_license(nuspec)
+        extra_data = {}
+        if license_file:
+            extra_data['license_file'] = license_file
+
 
         package_data = dict(
             datasource_id=cls.datasource_id,
@@ -176,6 +210,7 @@ def parse(cls, location, package_only=False):
             extracted_license_statement=extracted_license_statement,
             copyright=nuspec.get('copyright') or None,
             vcs_url=vcs_url,
+            extra_data=extra_data,
             **urls,
         )
         yield models.PackageData.from_data(package_data, package_only)

tests/packagedcode/data/nuget/ExpressionLicense.nuspec

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+  <metadata>
+    <id>ExpressionLicenseExample</id>
+    <version>1.0.0</version>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license type="expression">MIT</license>
+  </metadata>
+</package>

tests/packagedcode/data/nuget/ExpressionLicense.nuspec.json.expected

@@ -0,0 +1,69 @@
+[
+  {
+    "type": "nuget",
+    "namespace": null,
+    "name": "ExpressionLicenseExample",
+    "version": "1.0.0",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": null,
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": "mit",
+    "declared_license_expression_spdx": "MIT",
+    "license_detections": [
+      {
+        "license_expression": "mit",
+        "license_expression_spdx": "MIT",
+        "matches": [
+          {
+            "license_expression": "mit",
+            "license_expression_spdx": "MIT",
+            "from_file": null,
+            "start_line": 1,
+            "end_line": 1,
+            "matcher": "1-spdx-id",
+            "score": 100.0,
+            "matched_length": 1,
+            "match_coverage": 100.0,
+            "rule_relevance": 100,
+            "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06",
+            "rule_url": null,
+            "matched_text": "MIT"
+          }
+        ],
+        "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf"
+      }
+    ],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": "MIT",
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {},
+    "dependencies": [],
+    "repository_homepage_url": "https://www.nuget.org/packages/ExpressionLicenseExample/1.0.0",
+    "repository_download_url": "https://www.nuget.org/api/v2/package/ExpressionLicenseExample/1.0.0",
+    "api_data_url": "https://api.nuget.org/v3/registration3/expressionlicenseexample/1.0.0.json",
+    "datasource_id": "nuget_nupsec",
+    "purl": "pkg:nuget/ExpressionLicenseExample@1.0.0"
+  }
+]

tests/packagedcode/data/nuget/LicenseURLOnly.nuspec

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+  <metadata>
+    <id>LicenseUrlOnlyExample</id>
+    <version>1.0.0</version>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <licenseUrl>https://opensource.org/licenses/MIT</licenseUrl>
+  </metadata>
+</package>

tests/packagedcode/data/nuget/LicenseURLOnly.nuspec.json.expected

@@ -0,0 +1,69 @@
+[
+  {
+    "type": "nuget",
+    "namespace": null,
+    "name": "LicenseUrlOnlyExample",
+    "version": "1.0.0",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": null,
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": "mit",
+    "declared_license_expression_spdx": "MIT",
+    "license_detections": [
+      {
+        "license_expression": "mit",
+        "license_expression_spdx": "MIT",
+        "matches": [
+          {
+            "license_expression": "mit",
+            "license_expression_spdx": "MIT",
+            "from_file": null,
+            "start_line": 1,
+            "end_line": 1,
+            "matcher": "1-hash",
+            "score": 100.0,
+            "matched_length": 5,
+            "match_coverage": 100.0,
+            "rule_relevance": 100,
+            "rule_identifier": "mit_65.RULE",
+            "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_65.RULE",
+            "matched_text": "https://opensource.org/licenses/MIT"
+          }
+        ],
+        "identifier": "mit-8c45a313-6646-a283-ebeb-697e96d45185"
+      }
+    ],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": "https://opensource.org/licenses/MIT",
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {},
+    "dependencies": [],
+    "repository_homepage_url": "https://www.nuget.org/packages/LicenseUrlOnlyExample/1.0.0",
+    "repository_download_url": "https://www.nuget.org/api/v2/package/LicenseUrlOnlyExample/1.0.0",
+    "api_data_url": "https://api.nuget.org/v3/registration3/licenseurlonlyexample/1.0.0.json",
+    "datasource_id": "nuget_nupsec",
+    "purl": "pkg:nuget/LicenseUrlOnlyExample@1.0.0"
+  }
+]

tests/packagedcode/data/nuget/NoLicenseInfo.nuspec

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+  <metadata>
+    <id>NoLicenseInfoExample</id>
+    <version>1.0.0</version>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+  </metadata>
+</package>

tests/packagedcode/data/nuget/NoLicenseInfo.nuspec.json.expected

@@ -0,0 +1,46 @@
+[
+  {
+    "type": "nuget",
+    "namespace": null,
+    "name": "NoLicenseInfoExample",
+    "version": "1.0.0",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": null,
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {},
+    "dependencies": [],
+    "repository_homepage_url": "https://www.nuget.org/packages/NoLicenseInfoExample/1.0.0",
+    "repository_download_url": "https://www.nuget.org/api/v2/package/NoLicenseInfoExample/1.0.0",
+    "api_data_url": "https://api.nuget.org/v3/registration3/nolicenseinfoexample/1.0.0.json",
+    "datasource_id": "nuget_nupsec",
+    "purl": "pkg:nuget/NoLicenseInfoExample@1.0.0"
+  }
+]

tests/packagedcode/data/nuget/PlainLicense.nuspec

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+  <metadata>
+    <id>PlainLicenseExample</id>
+    <version>1.0.0</version>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <license>MIT</license>
+  </metadata>
+</package>

tests/packagedcode/data/nuget/PlainLicense.nuspec.json.expected

@@ -0,0 +1,69 @@
+[
+  {
+    "type": "nuget",
+    "namespace": null,
+    "name": "PlainLicenseExample",
+    "version": "1.0.0",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": null,
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": "mit",
+    "declared_license_expression_spdx": "MIT",
+    "license_detections": [
+      {
+        "license_expression": "mit",
+        "license_expression_spdx": "MIT",
+        "matches": [
+          {
+            "license_expression": "mit",
+            "license_expression_spdx": "MIT",
+            "from_file": null,
+            "start_line": 1,
+            "end_line": 1,
+            "matcher": "1-spdx-id",
+            "score": 100.0,
+            "matched_length": 1,
+            "match_coverage": 100.0,
+            "rule_relevance": 100,
+            "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06",
+            "rule_url": null,
+            "matched_text": "MIT"
+          }
+        ],
+        "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf"
+      }
+    ],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": "MIT",
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {},
+    "dependencies": [],
+    "repository_homepage_url": "https://www.nuget.org/packages/PlainLicenseExample/1.0.0",
+    "repository_download_url": "https://www.nuget.org/api/v2/package/PlainLicenseExample/1.0.0",
+    "api_data_url": "https://api.nuget.org/v3/registration3/plainlicenseexample/1.0.0.json",
+    "datasource_id": "nuget_nupsec",
+    "purl": "pkg:nuget/PlainLicenseExample@1.0.0"
+  }
+]