[PATCH 19 of 54] merged

Wald Commits scm-commit at wald.intevation.org
Wed Jan 7 10:56:36 CET 2015


# HG changeset patch
# User Benoît Allard <benoit.allard at greenbone.net>
# Date 1413298176 -7200
# Node ID 8a89b7a591e657c3daef81465d887251219693e4
# Parent  8e23ba7d4167b654b7e6fe756ec73dc1beac98f0
# Parent  90852c11fabd6c0b2addc042607bc864c8ebd7d8
merged

diff -r 8e23ba7d4167 -r 8a89b7a591e6 CHANGES
--- a/CHANGES	Wed Sep 24 17:47:14 2014 +0200
+++ b/CHANGES	Tue Oct 14 16:49:36 2014 +0200
@@ -1,3 +1,22 @@
+FarolLuz 0.1.1 (????-??-??)
+===========================
+
+This is the first patch release of FarolLuz 0.1
+
+This release add support for reading / writing incomplete CVRF documents.
+
+Main changes since FarolLuz 0.1:
+--------------------------------
+* Allow writing of incomplete CVRF documents.
+* Allow parsing of incomplete CVRF documents.
+* Add a method to extract a document ID.
+* Add methods to extract Product references in a Document.
+* Add method to get a Vulnerability Note per ordinal.
+* Fix issue where Acknowledgment could only have one Name and Organization.
+* Complete the CVRF template with missing elements
+* Improve validation.
+
+
 FarolLuz 0.1 (2014-09-23)
 =========================
 
diff -r 8e23ba7d4167 -r 8a89b7a591e6 farolluz/cvrf.py
--- a/farolluz/cvrf.py	Wed Sep 24 17:47:14 2014 +0200
+++ b/farolluz/cvrf.py	Tue Oct 14 16:49:36 2014 +0200
@@ -65,6 +65,9 @@
     def addAlias(self, alias):
         self._aliases.append(alias)
 
+    def getId(self):
+        return self._id
+
     def validate(self):
         if not self._id:
             raise ValidationError('Document ID cannot be left empty')
@@ -92,6 +95,9 @@
     def setGenerator(self, generator):
         self._generator = generator
 
+    def getId(self):
+        return self._identification.getId()
+
     def validate(self):
         if self._identification is None:
             raise ValidationError('Document Tracking needs to have an Identification')
@@ -225,18 +231,19 @@
 
 
 class CVRFAcknowledgment(object):
-    def __init__(self, name=None, organization=None, description=None,
+    def __init__(self, names=[], organizations=[], description=None,
                  url=None):
-        self._name = name
-        self._organization = organization
+        self._names = names
+        self._organizations = organizations
         self._description = description
         self._url = url
 
     def getTitle(self):
-        return "%s - %s" % (self._name, self._organization)
+        return "%s - %s" % (', '.join(self._names),
+                            ', '.join(self._organizations))
 
     def validate(self):
-        if (not self._name) and (not self._organization) and (not self._description):
+        if (not self._names) and (not self._organizations) and (not self._description):
             raise ValidationError('An Acknowledgment must have at least a Name, an Organization or a Description')
 
 
@@ -249,18 +256,9 @@
         self._products = []
         self._groups = []
 
-    def addBranch(self, branch):
-        parent = self.getBranch(branch.getParent().getPath())
-        if parent is self:
-            self._branches.append(branch)
-        else:
-            parent._childs.append(branch)
-
     def addProduct(self, product):
-        if product not in self._products:
-            self._products.append(product)
-        if product._parent is not self:
-            product._parent._product = product
+        """ Add to the product list """
+        self._products.append(product)
 
     def addRelationship(self, rel):
         self._relationships.append(rel)
@@ -396,9 +394,9 @@
     def __init__(self, _type, name, parentbranch):
         self._type = _type
         self._name = name
-        self._parentbranch = parentbranch
         self._childs = []
         self._product = None
+        self.link(parentbranch)
 
     def getParent(self):
         return self._parentbranch
@@ -455,6 +453,15 @@
             self.getParent()._childs.remove(self)
         self._parentbranch = None
 
+    def link(self, parent):
+        """ Actually, only set the parent """
+        self._parentbranch = parent
+        if self.isRoot():
+            parent._branches.append(self)
+        else:
+            parent._childs.append(self)
+
+
     def validate(self):
         if not self._type:
             raise ValidationError('A Branch must have a Type')
@@ -475,10 +482,10 @@
     def __init__(self, productid, name, parent, cpe=None):
         self._productid = productid
         self._name = name
+        self._cpe = cpe
         # Can be None (directly under the tree), a ProductBranch, or a
         # Relationship
-        self._parent = parent
-        self._cpe = cpe
+        self.link(parent)
 
     def isRoot(self):
         return isinstance(self._parent, CVRFProductTree)
@@ -502,13 +509,18 @@
         return None
 
     def unlink(self):
-        """ Unset our _parent, and remove us from the _parent._childs """
-        if self.isRoot():
-            self._parent._products.remove(self)
-        else:
+        """ Unset our _parent, and remove us from the _parent._childs
+        We are still in the product list.
+        """
+        if not self.isRoot():
             self._parent._product = None
         self._parent = None
 
+    def link(self, parent):
+        self._parent = parent
+        if not self.isRoot():
+            parent._product = self
+
     def validate(self):
         if not self._productid:
             raise ValidationError('A Product must have a ProductID')
@@ -655,6 +667,46 @@
             return self._id._value
         return "#%d" % self._ordinal
 
+    def getNote(self, ordinal):
+        for note in self._notes:
+            if note._ordinal == ordinal:
+                return note
+        return None
+
+    def mentionsProdId(self, productid):
+        """ Returns in which sub element, self is mentioning the productid """
+        for category in (self._productstatuses, self._threats, self._cvsss, self._remediations):
+            for subelem in category:
+                if productid in subelem._productids:
+                    yield subelem
+
+    def isMentioningProdId(self, productid):
+        """ Returns if self is mentioning the productid """
+        for e in self.mentionsProdId(productid):
+            # We only need to know if the generator yield at least one elem.
+            return True
+        return False
+
+    def mentionsGroupId(self, groupid):
+        for category in (self._threats, self._remediations):
+            for subelem in category:
+                if groupid in subelem._groupids:
+                    yield subelem
+
+    def isMentioningGroupId(self, groupids):
+        """ Make sure you call this with a list (not a generator or a tuple)
+        when wished """
+        if not isinstance(groupids, list):
+            groupids = [groupids]
+        for groupid in groupids:
+            print "testing GroupId: ", groupid
+            for _ in self.mentionsGroupId(groupid):
+                # We only need to know if the generator yield at least one elem.
+                print 'True'
+                return True
+        print 'False'
+        return False
+
     def validate(self, productids, groupids):
         if not self._ordinal:
             raise ValidationError('A Vulnerability must have an ordinal')
@@ -672,10 +724,22 @@
             cwe.validate()
         for status in self._productstatuses:
             status.validate(productids)
+        pids = set()
+        for status in self._productstatuses:
+            for pid in status._productids:
+                if pid in pids:
+                    raise ValidationError('ProductID %s mentionned in two different ProductStatuses for Vulnerability %d' % (pid, self._ordinal))
+                pids.add(pid)
         for threat in self._threats:
             threat.validate(productids, groupids)
         for cvss in self._cvsss:
             cvss.validate(productids)
+        pids = set()
+        for cvss in self._cvsss:
+            for pid in (cvss._productids or productids):
+                if pid in pids:
+                    raise ValidationError('ProductID %s mentionned in two different CVSS Score Sets for Vulnerability %d' % (pid, self._ordinal))
+                pids.add(pid)
         for remediation in self._remediations:
             remediation.validate(productids, groupids)
         for reference in self._references:
@@ -684,7 +748,6 @@
             acknowledgment.validate()
 
 
-
 class CVRFInvolvement(object):
     PARTIES = CVRFPublisher.TYPES
     STATUSES = ('Open', 'Disputed', 'In Progress', 'Completed',
@@ -726,6 +789,7 @@
 class CVRFProductStatus(object):
     TYPES = ('First Affected', 'Known Affected', 'Known Not Affected',
              'First Fixed', 'Fixed', 'Recommended', 'Last Affected')
+    NAME = "Product Status"
     def __init__(self, _type):
         self._type = _type
         self._productids = []
@@ -750,6 +814,7 @@
 
 class CVRFThreat(object):
     TYPES = ('Impact', 'Exploit Status', 'Target Set')
+    NAME = "Threat"
     def __init__(self, _type, description):
         self._type = _type
         self._description = description
@@ -792,6 +857,7 @@
               'C': {'N':0.0, 'P':0.275, 'C':0.66},
               'I': {'N':0.0, 'P':0.275, 'C':0.66},
               'A': {'N':0.0, 'P':0.275, 'C':0.66}}
+    NAME = "CVSS Score Set"
     def __init__(self, basescore):
         self._basescore = basescore
         self._temporalscore = None
@@ -844,6 +910,7 @@
 class CVRFRemediation(object):
     TYPES = ('Workaround', 'Mitigation', 'Vendor Fix', 'None Available',
              'Will Not Fix')
+    NAME = "Remediation"
     def __init__(self, _type, description):
         self._type = _type
         self._description = description
@@ -972,12 +1039,39 @@
                         products.add(productid)
         return set(self.getProductForID(p) for p in products)
 
+    def isProductOrphan(self, productid):
+        """ Returns if a productid is mentionned nowhere in the document """
+        # We first look at the ProductTree
+        ptree = self._producttree
+        for relation in ptree._relationships:
+            if productid == relation._productreference:
+                return False
+            if productid == relation._relatestoproductreference:
+                return False
+        groupids = [g._groupid for g in ptree._groups if productid in g._productids]
+        if len(groupids) > 0:
+            return False
+        # Go through all the Vulnerabilities
+        for vulnerability in self._vulnerabilities:
+            if vulnerability.isMentioningProdId(productid):
+                return False
+            for groupid in groupids:
+                if vulnerability.isMentioningGroupId(groupid):
+                    return False
+        return True
+
     def getNote(self, ordinal):
         for note in self._notes:
             if note._ordinal == ordinal:
                 return note
         return None
 
+    def getDocId(self):
+        if self._tracking is not None:
+            return self._tracking.getId()
+        # Make up something ...
+        return self._title.lower()
+
     def validate(self):
         if not self._title:
             raise ValidationError('Document Title cannot be empty')
diff -r 8e23ba7d4167 -r 8a89b7a591e6 farolluz/parsers/cvrf.py
--- a/farolluz/parsers/cvrf.py	Wed Sep 24 17:47:14 2014 +0200
+++ b/farolluz/parsers/cvrf.py	Tue Oct 14 16:49:36 2014 +0200
@@ -94,9 +94,14 @@
 
 
 def parseAcknowledgment(elem, ns='cvrf'):
+    names = []
+    for cvrfname in elem.findall(UN(ns, 'Name')):
+        names.append(cvrfname.text.strip())
+    orgs = []
+    for cvrforg in elem.findall(UN(ns, 'Organization')):
+        orgs.append(cvrforg.text.strip())
     return CVRFAcknowledgment(
-        elem.findtext(UN(ns, 'Name')),
-        elem.findtext(UN(ns, 'Organization')),
+        names, orgs,
         elem.findtext(UN(ns, 'Description')),
         elem.findtext(UN(ns, 'URL')),
     )
@@ -247,47 +252,51 @@
         cvrfdoc.findtext(UN('cvrf', 'DocumentTitle')).strip(),
         cvrfdoc.findtext(UN('cvrf', 'DocumentType')).strip()
     )
+
     cvrfpub = cvrfdoc.find(UN('cvrf', 'DocumentPublisher'))
-    pub = CVRFPublisher(cvrfpub.attrib['Type'], cvrfpub.attrib.get('VendorID'))
-    doc.setPublisher(pub)
-    contact = cvrfpub.find(UN('cvrf', 'ContactDetails'))
-    if contact is not None:
-        pub.setContact(contact.text.strip())
-    authority = cvrfpub.find(UN('cvrf', 'IssuingAuthority'))
-    if authority is not None:
-        pub.setAuthority(authority.text.strip())
+    if cvrfpub is not None:
+        pub = CVRFPublisher(cvrfpub.attrib['Type'], cvrfpub.attrib.get('VendorID'))
+        doc.setPublisher(pub)
+        contact = cvrfpub.find(UN('cvrf', 'ContactDetails'))
+        if contact is not None:
+            pub.setContact(contact.text.strip())
+        authority = cvrfpub.find(UN('cvrf', 'IssuingAuthority'))
+        if authority is not None:
+            pub.setAuthority(authority.text.strip())
+
     cvrftracking = cvrfdoc.find(UN('cvrf', 'DocumentTracking'))
-    identification = CVRFTrackingID(
-        cvrftracking.findtext('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'ID')])).strip()
-    )
-    for cvrfalias in cvrftracking.findall('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'Alias')])):
-        identification.addAlias(cvrfalias.text.strip())
-    tracking = CVRFTracking(
-        identification,
-        cvrftracking.findtext(UN('cvrf', 'Status')).strip(),
-        parseVersion(cvrftracking.findtext(UN('cvrf', 'Version')).strip()),
-        parseDate(cvrftracking.findtext(UN('cvrf', 'InitialReleaseDate')).strip()),
-        parseDate(cvrftracking.findtext(UN('cvrf', 'CurrentReleaseDate')).strip())
-    )
-    doc.setTracking(tracking)
-    for cvrfrev in cvrftracking.findall('/'.join([UN('cvrf', 'RevisionHistory'), UN('cvrf', 'Revision')])):
-        rev = CVRFRevision(
-            parseVersion(cvrfrev.findtext(UN('cvrf', 'Number')).strip()),
-            parseDate(cvrfrev.findtext(UN('cvrf', 'Date')).strip()),
-            cvrfrev.findtext(UN('cvrf', 'Description')).strip(),
+    if cvrftracking is not None:
+        identification = CVRFTrackingID(
+            cvrftracking.findtext('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'ID')])).strip()
         )
-        tracking.addRevision(rev)
+        for cvrfalias in cvrftracking.findall('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'Alias')])):
+            identification.addAlias(cvrfalias.text.strip())
+        tracking = CVRFTracking(
+            identification,
+            cvrftracking.findtext(UN('cvrf', 'Status')).strip(),
+            parseVersion(cvrftracking.findtext(UN('cvrf', 'Version')).strip()),
+            parseDate(cvrftracking.findtext(UN('cvrf', 'InitialReleaseDate')).strip()),
+            parseDate(cvrftracking.findtext(UN('cvrf', 'CurrentReleaseDate')).strip())
+        )
+        doc.setTracking(tracking)
+        for cvrfrev in cvrftracking.findall('/'.join([UN('cvrf', 'RevisionHistory'), UN('cvrf', 'Revision')])):
+            rev = CVRFRevision(
+                parseVersion(cvrfrev.findtext(UN('cvrf', 'Number')).strip()),
+                parseDate(cvrfrev.findtext(UN('cvrf', 'Date')).strip()),
+                cvrfrev.findtext(UN('cvrf', 'Description')).strip(),
+            )
+            tracking.addRevision(rev)
 
-    xmlgenerator = cvrftracking.find(UN('cvrf', 'Generator'))
-    if xmlgenerator is not None:
-        generator = CVRFGenerator()
-        xmlengine = xmlgenerator.findtext(UN('cvrf', 'Engine'))
-        if xmlengine is not None:
-            generator.setEngine(xmlengine.strip())
-        xmldate = xmlgenerator.findtext(UN('cvrf', 'Date'))
-        if xmldate is not None:
-            generator.setDate(parseDate(xmldate.strip()))
-        tracking.setGenerator(generator)
+        xmlgenerator = cvrftracking.find(UN('cvrf', 'Generator'))
+        if xmlgenerator is not None:
+            generator = CVRFGenerator()
+            xmlengine = xmlgenerator.findtext(UN('cvrf', 'Engine'))
+            if xmlengine is not None:
+                generator.setEngine(xmlengine.strip())
+            xmldate = xmlgenerator.findtext(UN('cvrf', 'Date'))
+            if xmldate is not None:
+                generator.setDate(parseDate(xmldate.strip()))
+            tracking.setGenerator(generator)
 
     for cvrfnote in cvrfdoc.findall('/'.join([UN('cvrf', 'DocumentNotes'), UN('cvrf', 'Note')])):
         doc.addNote(parseNote(cvrfnote))
@@ -315,8 +324,8 @@
     cvrfptree = cvrfdoc.find(UN('prod', 'ProductTree'))
     if cvrfptree is not None:
         producttree = doc.createProductTree()
-        for branch in parseProdBranch(cvrfptree, producttree):
-            producttree.addBranch(branch)
+        # We need to exhaust our generator ...
+        for _ in parseProdBranch(cvrfptree, producttree): pass
 
         for product in cvrfptree.findall(UN('prod', 'FullProductName')):
             producttree.addProduct(parseFullProductName(product, producttree))
diff -r 8e23ba7d4167 -r 8a89b7a591e6 farolluz/templates/cvrf.j2
--- a/farolluz/templates/cvrf.j2	Wed Sep 24 17:47:14 2014 +0200
+++ b/farolluz/templates/cvrf.j2	Tue Oct 14 16:49:36 2014 +0200
@@ -25,7 +25,7 @@
 
 <?xml version="1.0" encoding="utf-8"?>
 
-{#- Some macros for producttree generation #}
+{#- A macro for producttree generation #}
 {%- macro FullProductNames(producttree, parent) %}
   {%- for product in producttree._products %}
     {%- if product._parent is sameas parent %}
@@ -36,15 +36,41 @@
   {%- endfor %}
 {%- endmacro %}
 
+{#- Some macros about more generic types #}
 {%- macro Note(note) -%}
   <Note{{ {'Type': note._type, 'Ordinal': note._ordinal, 'Title': note._title, 'Audience': note._audience} | xmlattr }}>
     {{- note._note | escape -}}
   </Note>
-{%- endmacro %}
+{%- endmacro -%}
+
+{%- macro Reference(reference) -%}
+  <Reference{{ {'Type': reference._type} | xmlattr }}>
+    <URL>{{ reference._url }}</URL>
+    <Description>{{ reference._description }}</Description>
+  </Reference>
+{%- endmacro -%}
+
+{%- macro Acknowledgment(acknowledgment) -%}
+  <Acknowledgment>
+    {%- for name in acknowledgment._names %}
+    <Name>{{ name }}</Name>
+    {%- endfor %}
+    {%- for organization in acknowledgment._organizations %}
+    <Organization>{{ organization }}</Organization>
+    {%- endfor %}
+    {%- if acknowledgment._description %}
+    <Description>{{ acknowledgment._description }}</Description>
+    {%- endif %}
+    {%- if acknowledgment._url %}
+    <URL>{{ acknowledgment._url }}</URL>
+    {%- endif %}
+  </Acknowledgment>
+{%- endmacro -%}
+
 <cvrfdoc xmlns="http://www.icasi.org/CVRF/schema/cvrf/1.1">
   <DocumentTitle>{{ cvrf._title }}</DocumentTitle>
   <DocumentType>{{ cvrf._type }}</DocumentType>
-  {%- with publisher = cvrf._publisher %}
+  {%- with publisher = cvrf._publisher %}{% if publisher %}
   <DocumentPublisher{{ {'Type': publisher._type, 'VendorID': publisher._vendorid} | xmlattr }}>
     {%- if publisher._contact %}
     <ContactDetails>{{ publisher._contact }}</ContactDetails>
@@ -53,8 +79,8 @@
     <IssuingAuthority>{{ publisher._authority }}</IssuingAuthority>
     {%- endif %}
   </DocumentPublisher>
-  {%- endwith %}
-  {%- with tracking = cvrf._tracking %}
+  {%- endif %}{% endwith %}
+  {%- with tracking = cvrf._tracking %}{% if tracking %}
   <DocumentTracking>
     <Identification>
       <ID>{{ tracking._identification._id }}</ID>
@@ -88,7 +114,7 @@
     </Generator>
     {%- endif %}
   </DocumentTracking>
-  {%- endwith %}
+  {%- endif %}{% endwith %}
   {%- if cvrf._notes %}
   <DocumentNotes>
     {%- for note in cvrf._notes %}
@@ -107,30 +133,14 @@
   {%- if cvrf._references %}
   <DocumentReferences>
     {%- for reference in cvrf._references %}
-    <Reference{{ {'Type': reference._type} | xmlattr }}>
-      <URL>{{ reference._url }}</URL>
-      <Description>{{ reference._description }}</Description>
-    </Reference>
+    {{ Reference(reference) }}
     {%- endfor %}
   </DocumentReferences>
   {%- endif %}
   {%- if cvrf._acknowledgments %}
   <Acknowledgments>
     {%- for acknowledgment in cvrf._acknowledgments %}
-    <Acknowledgment>
-      {%- if acknowledgment._name %}
-      <Name>{{ acknowledgment._name }}</Name>
-      {%- endif %}
-      {%- if acknowledgment._organization %}
-      <Organization>{{ acknowledgment._organization }}</Organization>
-      {%- endif %}
-      {%- if acknowledgment._description %}
-      <Description>{{ acknowledgment._description }}</Description>
-      {%- endif %}
-      {%- if acknowledgment._url %}
-      <URL>{{ acknowledgment._url }}</URL>
-      {%- endif %}
-    </Acknowledgment>
+    {{ Acknowledgment(acknowledgment) }}
     {%- endfor %}
   </Acknowledgments>
   {%- endif %}
@@ -143,8 +153,8 @@
       {{- FullProductNames(producttree, branch) }}
     </Branch>
     {%- endfor %}
-    {{ FullProductNames(producttree, producttree) }}
-    {%- for relationship in producttree._relationships -%}
+    {{- FullProductNames(producttree, producttree) }}
+    {%- for relationship in producttree._relationships %}
     <Relationship{{ {'ProductReference': relationship._productreference, 'RelationType': relationship._relationtype, 'RelatesToProductReference': relationship._relatestoproductreference} | xmlattr }}>
       {{- FullProductNames(producttree, relationship) }}
     </Relationship>
@@ -152,7 +162,7 @@
     {%- if producttree._groups %}
     <ProductGroups>
       {%- for group in producttree._groups %}
-      <Group{{ {'GroupID': group._id} | xmlattr }}>
+      <Group{{ {'GroupID': group._groupid} | xmlattr }}>
         {%- if group._description %}
         <Description>{{ group._description }}</Description>
         {%- endif %}
@@ -215,6 +225,21 @@
       {%- endfor %}
     </ProductStatuses>
     {%- endif %}
+    {%- if vulnerability._threats %}
+    <Threats>
+      {%- for threat in vulnerability._threats %}
+      <Threat Type="{{ threat._type }}"{{ ' Date="%s"' % threat._date.isoformat() if threat._date }}>
+        <Description>{{ threat._description }}</Description>
+        {%- for productid in threat._productids %}
+        <ProductID>{{ productid }}</ProductID>
+        {%- endfor %}
+        {%- for groupid in threat._groupids %}
+        <GroupID>{{ groupid }}</GroupID>
+        {%- endfor %}
+      </Threat>
+      {%- endfor %}
+    </Threats>
+    {%- endif %}
     {%- if vulnerability._cvsss %}
     <CVSSScoreSets>
       {%- for cvss in vulnerability._cvsss %}
@@ -230,7 +255,7 @@
         <Vector>{{ cvss._vector }}</Vector>
         {%- endif %}
         {%- for productid in cvss._productids %}
-        <ProductID>{{productid}}</ProductID>
+        <ProductID>{{ productid }}</ProductID>
         {%- endfor %}
       </ScoreSet>
       {%- endfor %}
@@ -257,6 +282,20 @@
       {%- endfor %}
     </Remediations>
     {%- endif %}
+    {%- if vulnerability._references %}
+    <References>
+      {%- for reference in vulnerability._references %}
+      {{ Reference(reference) }}
+      {%- endfor %}
+    </References>
+    {%- endif %}
+    {%- if vulnerability._acknowledgments %}
+    <Acknowledgments>
+      {%- for acknowledgment in vulnerability._acknowledgments %}
+      {{ Acknowledgment(acknowledgment) }}
+      {%- endfor %}
+    </Acknowledgments>
+    {%- endif %}
   </Vulnerability>
   {%- endfor %}
 </cvrfdoc>


More information about the Farol-commits mailing list