Merge pull request #909 from nexB/prepare-release30

Prepare release 30

CHANGELOG.rst

@@ -11,14 +11,17 @@ Version v30.0.0
   are available.
   Because of these extensive changes, it is not possible to migrate existing imported
   data to the new schema. You will need instead to restart imports from an empty database
-  or request access to the new vulnerablecode.io live instance.
-  You can track the progress in this issue: https://github.com/nexB/vulnerablecode/issues/597 
+  or access the new public.vulnerablecode.io live instance. We also provide a database dump.
+
+- You can track the progress of this refactoring in this issue:
+  https://github.com/nexB/vulnerablecode/issues/597 
 
 - We added new data sources including PYSEC, GitHub and GitLab.
 
 - We improved the documentation including adding development examples for importers and improvers.
 
 - We removed the ability to edit relationships from the UI. The UI is now read-only.
+
 - We replace the web UI with a brand new UI based on the same overall look and feel as ScanCode.io.
 
 - We added support for NixOS as a Linux deployment target.
@@ -42,15 +45,17 @@ Version v30.0.0
     - Add new attribute `is_resolved`
     - Add namespace filter
 
-- We have provided backward compatibility for `url` and `unresolved_vulnerabilities` for now
+- We have provided backward compatibility for `url` and `unresolved_vulnerabilities` for now.
+  These will be removed in the next major version and should be considered as deprecated.
 
-- There is a new experimental cpe/ API endpoint to lookup for vulnerabilities by CPE and 
+- There is a new experimental `cpe/` API endpoint to lookup for vulnerabilities by CPE and 
   another aliases/ endpoint to lookup for vulnerabilities by aliases. These two endpoints will be
   replaced by query parameters on the main vulnerabilities/ endpoint when stabilized.
 
-- Added filters for vulnerabilities endpoint to get fixed packages in accordance to the details given in filters:
-  For example, when you call the endpoint this way ``/api/vulnerabilities?type=pypi&namespace=foo&name=bar``,
-  you will receive only fixed versioned purls of the type ``pypi``, namespace ``foo`` and name ``bar``.
+- Added filters for vulnerabilities endpoint to get fixed packages in accordance
+  to the details given in filters: For example, when you call the endpoint this way
+  ``/api/vulnerabilities?type=pypi&namespace=foo&name=bar``, you will receive only
+  fixed versioned purls of the type ``pypi``, namespace ``foo`` and name ``bar``.
 
 - Package endpoint will give fixed packages of only those that
   matches type, name, namespace, subpath and qualifiers of the package queried.
@@ -69,10 +74,14 @@ Version v30.0.0
   their API Key in the REST API.
   Users can be created using the Django "createsuperuser" management command.
 
+- The data license is now CC-BY-SA-4.0 as this is the highest common
+  denominator license among all the data sources we collect and aggregate. 
+
+
 Other:
 
-- we dropped calver to use a plain semver.
-- we adopted vers and the new univers library to handle version ranges.
+- We dropped calver to use a plain semver.
+- We adopted vers and the new univers library to handle version ranges.
 
 
 Version v20.10

etc/scripts/backup.sh

@@ -0,0 +1,6 @@
+
+# backup current Db
+
+DBDUMP=vcio-db-dump-$(date +"%Y-%m-%d_%H%M").dump
+echo "Backup vulnerablecode current DB to: $DBDUMP"
+sudo -u postgres pg_dump --format=c vulnerablecode > $DBDUMP

etc/scripts/restore.sh

@@ -0,0 +1,9 @@
+
+# backup current Db and then restore a dump
+
+./backup.sh
+echo "Restore vulnerablecode dump from: $1"
+sudo -u postgres dropdb vulnerablecode
+sudo -u postgres createdb --encoding=utf-8 --owner=vulnerablecode vulnerablecode
+sudo -u postgres pg_restore --verbose -d vulnerablecode $1
+

etc/thirdparty/virtualenv.pyz.ABOUT

@@ -1,7 +1,7 @@
 about_resource: virtualenv.pyz
 name: get-virtualenv
-version: 20.13.3
-download_url: https://github.com/pypa/get-virtualenv/raw/20.13.3/public/virtualenv.pyz
+version: 20.16.5
+download_url: https://github.com/pypa/get-virtualenv/raw/20.16.5/public/virtualenv.pyz
 description: virtualenv is a tool to create isolated Python environments.
 homepage_url: https://github.com/pypa/virtualenv
 license_expression: lgpl-2.1-plus AND (bsd-new OR apache-2.0) AND mit AND python AND bsd-new
@@ -10,4 +10,4 @@ copyright: Copyright (c) The Python Software Foundation and others
 redistribute: yes
 attribute: yes
 track_changes: yes
-package_url: pkg:github/pypa/get-virtualenv@20.13.1#public/virtualenv.pyz
+package_url: pkg:github/pypa/get-virtualenv@20.16.5#public/virtualenv.pyz

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 30.0.0rc3
+version = 30.0.0rc5
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

vulnerabilities/forms.py

@@ -9,29 +9,21 @@
 
 from django import forms
 
-from vulnerabilities.models import Package
 
+class PackageSearchForm(forms.Form):
 
-def get_known_package_types():
-    """
-    Return a list of known package types.
-    """
-    pkg_types = [(i.type, i.type) for i in Package.objects.distinct("type").all()]
-    pkg_types.append((None, "Any type"))
-    return pkg_types
-
-
-class PackageForm(forms.Form):
-
-    package_name = forms.CharField(
-        required=False, widget=forms.TextInput(attrs={"placeholder": "Package name or purl"})
+    search = forms.CharField(
+        required=True,
+        widget=forms.TextInput(
+            attrs={"placeholder": "Package name, purl or purl fragment"},
+        ),
     )
 
 
-class VulnerabilityForm(forms.Form):
+class VulnerabilitySearchForm(forms.Form):
 
-    vulnerability_id = forms.CharField(
-        required=False,
+    search = forms.CharField(
+        required=True,
         widget=forms.TextInput(
             attrs={"placeholder": "Vulnerability id or alias such as CVE or GHSA"}
         ),

vulnerabilities/improve_runner.py

@@ -11,14 +11,19 @@
 from datetime import datetime
 from datetime import timezone
 from typing import List
-from typing import Tuple
 
+from django.core.exceptions import ValidationError
 from django.db import transaction
 
-from vulnerabilities import models
-from vulnerabilities.importer import PackageURL
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
+from vulnerabilities.models import Alias
+from vulnerabilities.models import Package
+from vulnerabilities.models import PackageRelatedVulnerability
+from vulnerabilities.models import Vulnerability
+from vulnerabilities.models import VulnerabilityReference
+from vulnerabilities.models import VulnerabilityRelatedReference
+from vulnerabilities.models import VulnerabilitySeverity
 
 logger = logging.getLogger(__name__)
 
@@ -63,46 +68,59 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver
     logger.info(f"Improving advisory id: {advisory.id}")
 
     for inference in inferences:
-        vuln = get_or_create_vulnerability_and_aliases(
-            inference.vulnerability_id, inference.aliases, inference.summary
+        vulnerability = get_or_create_vulnerability_and_aliases(
+            vulnerability_id=inference.vulnerability_id,
+            alias_names=inference.aliases,
+            summary=inference.summary,
         )
-        if not vuln:
+
+        if not vulnerability:
             logger.warn(f"Unable to get vulnerability for inference: {inference!r}")
             continue
 
         for ref in inference.references:
-            reference, _ = models.VulnerabilityReference.objects.get_or_create(
-                reference_id=ref.reference_id, url=ref.url
+
+            reference = VulnerabilityReference.objects.get_or_none(
+                reference_id=ref.reference_id,
+                url=ref.url,
             )
 
-            models.VulnerabilityRelatedReference.objects.update_or_create(
-                reference=reference, vulnerability=vuln
+            if not reference:
+                reference = create_valid_vulnerability_reference(
+                    reference_id=ref.reference_id,
+                    url=ref.url,
+                )
+                if not reference:
+                    continue
+
+            VulnerabilityRelatedReference.objects.update_or_create(
+                reference=reference,
+                vulnerability=vulnerability,
             )
 
             for severity in ref.severities:
-                _vs, updated = models.VulnerabilitySeverity.objects.update_or_create(
+                _vs, updated = VulnerabilitySeverity.objects.update_or_create(
                     scoring_system=severity.system.identifier,
                     reference=reference,
                     defaults={"value": str(severity.value)},
                 )
                 if updated:
                     logger.info(f"Severity updated for reference {ref!r} to {severity.value!r}")
 
-        if inference.affected_purls:
-            for pkg in inference.affected_purls:
-                vulnerable_package, _ = _get_or_create_package(pkg)
-                models.PackageRelatedVulnerability(
-                    vulnerability=vuln,
-                    package=vulnerable_package,
-                    created_by=improver_name,
-                    confidence=inference.confidence,
-                    fix=False,
-                ).update_or_create()
+        for affected_purl in inference.affected_purls or []:
+            vulnerable_package = Package.objects.get_or_create_from_purl(purl=affected_purl)
+            PackageRelatedVulnerability(
+                vulnerability=vulnerability,
+                package=vulnerable_package,
+                created_by=improver_name,
+                confidence=inference.confidence,
+                fix=False,
+            ).update_or_create()
 
         if inference.fixed_purl:
-            fixed_package, _ = _get_or_create_package(inference.fixed_purl)
-            models.PackageRelatedVulnerability(
-                vulnerability=vuln,
+            fixed_package = Package.objects.get_or_create_from_purl(purl=inference.fixed_purl)
+            PackageRelatedVulnerability(
+                vulnerability=vulnerability,
                 package=fixed_package,
                 created_by=improver_name,
                 confidence=inference.confidence,
@@ -113,26 +131,25 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver
     advisory.save()
 
 
-def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]:
-    query_kwargs = {}
-    # TODO: this should be revisited as this should best be a model or manager method... and possibly streamlined
-    query_kwargs = dict(
-        type=p.type or "",
-        namespace=p.namespace or "",
-        name=p.name or "",
-        version=p.version or "",
-        qualifiers=p.qualifiers or {},
-        subpath=p.subpath or "",
+def create_valid_vulnerability_reference(url, reference_id=None):
+    """
+    Create and return a new validated VulnerabilityReference from a
+    ``url`` and ``reference_id``.
+    Return None and log a warning if this is not a valid reference.
+    """
+    reference = VulnerabilityReference(
+        reference_id=reference_id,
+        url=url,
     )
 
-    return models.Package.objects.get_or_create(**query_kwargs)
-
+    try:
+        reference.full_clean()
+    except ValidationError as e:
+        logger.warning(f"Invalid vulnerability reference: {reference!r}: {e}")
+        return
 
-def _package_url_to_package(purl: PackageURL) -> models.Package:
-    # FIXME: this is is likely creating a package from a purl?
-    p = models.Package()
-    p.set_package_url(purl)
-    return p
+    reference.save()
+    return reference
 
 
 def get_or_create_vulnerability_and_aliases(vulnerability_id, alias_names, summary):
@@ -145,9 +162,9 @@ def get_or_create_vulnerability_and_aliases(vulnerability_id, alias_names, summa
     new_alias_names = set()
     for alias_name in alias_names:
         try:
-            alias = models.Alias.objects.get(alias=alias_name)
+            alias = Alias.objects.get(alias=alias_name)
             existing_vulns.add(alias.vulnerability)
-        except models.Alias.DoesNotExist:
+        except Alias.DoesNotExist:
             new_alias_names.add(alias_name)
 
     # If given set of aliases point to different vulnerabilities in the
@@ -179,14 +196,14 @@ def get_or_create_vulnerability_and_aliases(vulnerability_id, alias_names, summa
         vulnerability = existing_alias_vuln
     elif vulnerability_id:
         try:
-            vulnerability = models.Vulnerability.objects.get(vulnerability_id=vulnerability_id)
-        except models.Vulnerability.DoesNotExist:
+            vulnerability = Vulnerability.objects.get(vulnerability_id=vulnerability_id)
+        except Vulnerability.DoesNotExist:
             logger.warn(
                 f"Given vulnerability_id: {vulnerability_id} does not exist in the database"
             )
             return
     else:
-        vulnerability = models.Vulnerability(summary=summary)
+        vulnerability = Vulnerability(summary=summary)
         vulnerability.save()
 
     if summary and summary != vulnerability.summary:
@@ -196,7 +213,7 @@ def get_or_create_vulnerability_and_aliases(vulnerability_id, alias_names, summa
         )
 
     for alias_name in new_alias_names:
-        alias = models.Alias(alias=alias_name, vulnerability=vulnerability)
+        alias = Alias(alias=alias_name, vulnerability=vulnerability)
         alias.save()
         logger.info(f"New alias for {vulnerability!r}: {alias_name}")
 

vulnerabilities/improvers/add_missing_refid.py

@@ -0,0 +1,46 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import re
+from typing import Iterable
+
+from django.db.models import Q
+from django.db.models.query import QuerySet
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.improver import Improver
+from vulnerabilities.improver import Inference
+from vulnerabilities.models import VulnerabilityReference
+
+"""
+Improver that looks for CVE References without an id and tries to set one.
+"""
+
+
+class CveIdImprover(Improver):
+    """
+    Add a CVE reference id when missing.
+    Note that we only look for uppercase CVE for now
+    """
+
+    @property
+    def interesting_advisories(self) -> QuerySet:
+        return VulnerabilityReference.objects.filter(
+            Q(reference_id__isnull=True) | Q(reference_id__exact=""),
+            url__contains="nvd.nist.gov/vuln/detail/CVE-",
+        )
+
+    def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
+        cve_pattern = re.compile(r"(CVE-\d{4}-\d{4,7})").search
+        for ref in self.interesting_advisories:
+            cve_match = cve_pattern(ref.url)
+            if cve_match:
+                cve = cve_match.group()
+                ref.reference_id = cve
+                ref.save()

vulnerabilities/migrations/0005_auto_20220329_0938.py

@@ -1,4 +1,3 @@
-# Generated by Django 4.0.2 on 2022-03-29 09:38
 
 import hashlib
 import json

vulnerabilities/migrations/0013_auto_20220503_0941.py

@@ -1,5 +1,3 @@
-# Generated by Django 4.0.4 on 2022-05-03 09:41
-
 from django.db import migrations
 
 from django.utils.http import int_to_base36