Add support for parsing complex license expressions

.circleci/config.yml

@@ -1,82 +1,86 @@
 version: 2
 
 jobs:
-  python_2_7_15:
+  python_3_6_5:
     shell: /bin/bash --login
     macos:
-      xcode: '9.0'
+      xcode: '9.4.1'
     steps:
     - checkout
     - run: |
         brew update
         python --version
         sudo -H pip install --upgrade virtualenv
         brew install pyenv
+        echo 'eval "$(pyenv init --path)"' >> ~/.bash_profile
         echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
         source ~/.bash_profile
-        pyenv install 2.7.15
+        pyenv install 3.6.5
         pyenv versions
-        pyenv global 2.7.15
+        pyenv global 3.6.5
         python --version
     - run: python setup.py install
     - run: python setup.py test
 
-  python_3_4_8:
+  python_3_7_10:
     shell: /bin/bash --login
     macos:
-      xcode: '9.0'
+      xcode: '9.4.1'
     steps:
     - checkout
     - run: |
         brew update
         python --version
         sudo -H pip install --upgrade virtualenv
         brew install pyenv
+        echo 'eval "$(pyenv init --path)"' >> ~/.bash_profile
         echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
         source ~/.bash_profile
-        pyenv install 3.4.8
+        pyenv install 3.7.10
         pyenv versions
-        pyenv global 3.4.8
+        pyenv global 3.7.10
         python --version
     - run: python setup.py install
     - run: python setup.py test
 
-  python_3_5_5:
+  python_3_8_10:
     shell: /bin/bash --login
     macos:
-      xcode: '9.0'
+      xcode: '9.4.1'
     steps:
     - checkout
     - run: |
         brew update
         python --version
         sudo -H pip install --upgrade virtualenv
         brew install pyenv
+        echo 'eval "$(pyenv init --path)"' >> ~/.bash_profile
         echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
         source ~/.bash_profile
-        pyenv install 3.5.5
+        pyenv install 3.8.10
         pyenv versions
-        pyenv global 3.5.5
+        pyenv global 3.8.10
         python --version
     - run: python setup.py install
     - run: python setup.py test
 
-  python_3_6_5:
+  python_3_9_5:
     shell: /bin/bash --login
     macos:
-      xcode: '9.0'
+      xcode: '9.4.1'
     steps:
     - checkout
     - run: |
         brew update
         python --version
         sudo -H pip install --upgrade virtualenv
         brew install pyenv
+        echo 'eval "$(pyenv init --path)"' >> ~/.bash_profile
         echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
         source ~/.bash_profile
-        pyenv install 3.6.5
+        pyenv install 3.9.5
         pyenv versions
-        pyenv global 3.6.5
+        pyenv global 3.9.5
         python --version
     - run: python setup.py install
     - run: python setup.py test
@@ -85,7 +89,7 @@ workflows:
   version: 2
   python_matrix_build:
     jobs:
-      - python_2_7_15
-      - python_3_4_8
-      - python_3_5_5
       - python_3_6_5
+      - python_3_7_10
+      - python_3_8_10
+      - python_3_9_5

.travis.yml

@@ -1,9 +1,10 @@
 language: python
 
 python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
+  - "3.6"
+  - "3.7"
+  - "3.8"
+  - "3.9"
 
 install:
   - ./setup.py install

README.md

@@ -11,6 +11,9 @@
 [5]: https://ci.appveyor.com/api/projects/status/0bf9glha2yg9x8ef/branch/master?svg=true
 [6]: https://ci.appveyor.com/project/spdx/tools-python/branch/master
 
+
+# Information
+
 This library implements an SPDX tag/value and RDF parser, validator and handler in Python.
 This is the result of an initial GSoC contribution by @[ah450](https://github.com/ah450) (or https://github.com/a-h-i) and 
 is maintained by a community of SPDX adopters and enthusiasts.
@@ -41,37 +44,35 @@ Pypi: https://pypi.python.org/pypi/spdx-tools
 
 # How to use
 
-Example tag/value parsing usage:
-```Python
-    from spdx.parsers.tagvalue import Parser
-    from spdx.parsers.tagvaluebuilders import Builder
-    from spdx.parsers.loggers import StandardLogger
-    p = Parser(Builder(), StandardLogger())
-    p.build()
-    # data is a string containing the SPDX file.
-    document, error = p.parse(data)
+## Command-line usage:
+
+1. **PARSER** (for parsing any format):
+* Use   `parser --file <filename>`   where  `<filename>`  is the location of the file.              
+Try running :   `parser --file data/SPDXRdfExample.rdf`.
+
+* Or you can use  `parser`  only and then it will automatically prompt/ask for  `filename`. 
 
-```
+* for help - use `parser --help` 
 
-The `examples` directory contains several code samples. Here some of them:
 
-* `parse_tv.py` is an example tag/value parsing usage.
-  Try running `python parse_tv.py ../data/SPDXSimpleTag.tag `
+2. **CONVERTOR** (for converting one format to another):
+* If I/O formats are known:
+
+    * Use `convertor --infile/-i <input_file> --outfile/-o <output_file>` where `<input_file>` is the location of the file to be converted 
+    (Note: only RDF and Tag formated supported) and `<output_file>` is the location of the output file.  
+    Try running : `convertor --infile data/SPDXRdfExample.rdf --outfile output.json` 
 
-* `write_tv.py` provides an example of writing tag/value files.
-  Run `python write_tv.py sample.tag` to test it.
+* If I/O formats are not known:
 
-* `pp_tv.py` demonstrates how to pretty-print a tag/value file.
-   To test it run `python pp_tv.py ../data/SPDXTagExample.tag pretty.tag`.
+    * Use `convertor --from/-f <input_format> <input_file> --to/-t <output_format> <output_file>` where `<input_format>` is the manually enterred format of the input file (can be either rdf or tag)
+    and `<out_format>` (can be tag, rdf, json, yaml, xml) is the manually enterred format of the output file. 
+    Try running : `convertor --from tag data/SPDXTagExample.in --to yaml output.out` 
 
-* `parse_rdf.py` demonstrates how to parse an RDF file and print out document 
-   information. To test it run `python parse_rdf.py ../data/SPDXRdfExample.rdf`
+* If anyone format is known and other is not, you can use the mixture of the above two points.      
+Ex. : `convertor -f rdf data/SPDXRdfExample.xyz -o output.xml`
 
-* `rdf_to_tv.py` demonstrates how to convert an RDF file to a tag/value one.
-   To test it run `python rdf_to_tv.py ../data/SPDXRdfExample.rdf converted.tag`
+* for help - use `convertor --help`
 
-* `pp_rdf.py` demonstrates how to pretty-print an RDF file, to test it run 
-  `python pp_rdf.py ../data/SPDXRdfExample.rdf pretty.rdf`
 
 
 # Installation

appveyor.yml

@@ -2,12 +2,10 @@ version: "{build}"
 
 environment:
   matrix:
-    - PYTHON_EXE: "C:\\Python27\\python.exe"
-    - PYTHON_EXE: "C:\\Python27-x64\\python.exe"
-    - PYTHON_EXE: "C:\\Python36\\python.exe"
     - PYTHON_EXE: "C:\\Python36-x64\\python.exe"
-    - PYTHON_EXE: "C:\\Python37\\python.exe"
     - PYTHON_EXE: "C:\\Python37-x64\\python.exe"
+    - PYTHON_EXE: "C:\\Python38-x64\\python.exe"
+    - PYTHON_EXE: "C:\\Python39-x64\\python.exe"
 
 install:
   - "%PYTHON_EXE% --version"

data/SPDXJsonExample.json

@@ -5,7 +5,7 @@
         "documentDescribes": [
             {
                 "Package": {
-                    "id": "SPDXRef-Package", 
+                    "SPDXID": "SPDXRef-Package", 
                     "originator": "Organization: SPDX", 
                     "files": [
                         {
@@ -28,14 +28,14 @@
                                 "licenseComments": "This license is used by Jena", 
                                 "checksums": [
                                     {
-                                        "value": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125", 
+                                        "checksumValue": "3ab4e1c67a2d28fced849ee1bb76e7391b93f125", 
                                         "algorithm": "checksumAlgorithm_sha1"
                                     }
                                 ], 
                                 "fileTypes": [
                                     "fileType_archive"
                                 ], 
-                                "id": "SPDXRef-File1"
+                                "SPDXID": "SPDXRef-File1"
                             }
                         }, 
                         {
@@ -49,14 +49,14 @@
                                 "licenseConcluded": "Apache-2.0", 
                                 "checksums": [
                                     {
-                                        "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", 
+                                        "checksumValue": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", 
                                         "algorithm": "checksumAlgorithm_sha1"
                                     }
                                 ], 
                                 "fileTypes": [
                                     "fileType_source"
                                 ], 
-                                "id": "SPDXRef-File2"
+                                "SPDXID": "SPDXRef-File2"
                             }
                         }
                     ], 
@@ -77,17 +77,18 @@
                     "sourceInfo": "Version 1.0 of the SPDX Translator application", 
                     "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", 
                     "packageVerificationCode": {
-                        "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", 
-                        "excludedFilesNames": [
+                        "packageVerificationCodeValue": "4e3211c67a2d28fced849ee1bb76e7391b93feba", 
+                        "packageVerificationCodeExcludedFiles": [
                             "SpdxTranslatorSpdx.rdf", 
                             "SpdxTranslatorSpdx.txt"
                         ]
                     }, 
                     "licenseConcluded": "(Apache-1.0 AND LicenseRef-2 AND MPL-1.1 AND LicenseRef-3 AND LicenseRef-4 AND Apache-2.0 AND LicenseRef-1)", 
-                    "supplier": "Organization: Linux Foundation", 
+                    "supplier": "Organization: Linux Foundation",
+                    "attributionTexts" : [ "The GNU C Library is free software.  See the file COPYING.LIB for copying conditions, and LICENSES for notices about a few contributions that require these additional notices to be distributed.  License copyright years may be listed using range notation, e.g., 1996-2015, indicating that every year in the range, inclusive, is a copyrightable year that would otherwise be listed individually." ], 
                     "checksums": [
                         {
-                            "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", 
+                            "checksumValue": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", 
                             "algorithm": "checksumAlgorithm_sha1"
                         }
                     ], 
@@ -111,23 +112,36 @@
         "externalDocumentRefs": [
             {
                 "checksum": {
-                    "value": "d6a770ba38583ed4bb4525bd96e50461655d2759", 
+                    "checksumValue": "d6a770ba38583ed4bb4525bd96e50461655d2759", 
                     "algorithm": "checksumAlgorithm_sha1"
                 }, 
-                "spdxDocumentNamespace": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", 
+                "spdxDocument": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", 
                 "externalDocumentId": "DocumentRef-spdx-tool-2.1"
             }
         ], 
-        "namespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", 
+        "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", 
         "annotations": [
             {
                 "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", 
                 "annotationType": "REVIEW", 
-                "id": "SPDXRef-45", 
+                "SPDXID": "SPDXRef-45", 
                 "annotationDate": "2012-06-13T00:00:00Z", 
                 "annotator": "Person: Jim Reviewer"
             }
         ], 
+        "relationships" : [ {
+    "spdxElementId" : "SPDXRef-DOCUMENT",
+    "relatedSpdxElement" : "SPDXRef-Package",
+    "relationshipType" : "CONTAINS"
+  }, {
+    "spdxElementId" : "SPDXRef-DOCUMENT",
+    "relatedSpdxElement" : "SPDXRef-File",
+    "relationshipType" : "DESCRIBES"
+  }, {
+    "spdxElementId" : "SPDXRef-DOCUMENT",
+    "relatedSpdxElement" : "SPDXRef-Package",
+    "relationshipType" : "DESCRIBES"
+  } ],
         "dataLicense": "CC0-1.0", 
         "reviewers": [
             {
@@ -141,7 +155,7 @@
                 "reviewDate": "2011-03-13T00:00:00Z"
             }
         ], 
-        "extractedLicenseInfos": [
+        "hasExtractedLicensingInfos": [
             {
                 "extractedText": "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n\u00a9 Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: \n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. \nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. \nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. \nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", 
                 "licenseId": "LicenseRef-2"
@@ -165,8 +179,8 @@
                 "licenseId": "LicenseRef-1"
             }
         ], 
-        "specVersion": "SPDX-2.1", 
-        "id": "SPDXRef-DOCUMENT", 
+        "spdxVersion": "SPDX-2.1", 
+        "SPDXID": "SPDXRef-DOCUMENT", 
         "snippets": [
             {
                 "comment": "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later.", 
@@ -177,7 +191,7 @@
                     "Apache-2.0"
                 ], 
                 "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", 
-                "id": "SPDXRef-Snippet", 
+                "SPDXID": "SPDXRef-Snippet", 
                 "fileId": "SPDXRef-DoapSource"
             }
         ]

data/SPDXRdfExample.rdf

@@ -39,7 +39,7 @@
       </ExternalDocumentRef>
     </externalDocumentRef>
     <referencesFile>
-      <File rdf:about="https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File1">>
+      <File rdf:about="https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File1">
         <licenseConcluded>
           <ExtractedLicensingInfo rdf:nodeID="A1">
             <extractedText>/*
@@ -78,6 +78,7 @@
           </j.0:Project>
         </artifactOf>
         <licenseComments>This license is used by Jena</licenseComments>
+        <attributionText>The GNU C Library is free software.  See the file COPYING.LIB for copying conditions, and LICENSES for notices about a few contributions that require these additional notices to be distributed.  License copyright years may be listed using range notation, e.g., 1996-2015, indicating that every year in the range, inclusive, is a copyrightable year that would otherwise be listed individually.</attributionText>
         <fileName>Jenna-2.6.3/jena-2.6.3-sources.jar</fileName>
         <fileType rdf:resource="http://spdx.org/rdf/terms#fileType_archive"/>
         <copyrightText>(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP</copyrightText>
@@ -115,6 +116,12 @@
         <annotator>Person: Jim Reviewer</annotator>
       </Annotation>
     </annotation>
+    <relationship>
+      <Relationship>
+        <relationshipType rdf:resource="http://spdx.org/rdf/terms#relationshipType_describes"/>
+        <relatedSpdxElement rdf:resource="http://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-Package"/>
+      </Relationship>
+    </relationship>
     <referencesFile>
       <File rdf:about="https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-File2">
         <copyrightText>Copyright 2010, 2011 Source Auditor Inc.</copyrightText>

data/SPDXSimpleTag.tag

@@ -34,9 +34,11 @@ PackageSourceInfo: <text>Version 1.0 of test</text>
 PackageFileName: test-1.0.zip
 PackageSupplier: Organization:ACME
 PackageOriginator: Organization:ACME
+PackageAttributionText: <text>The GNU C Library is free software.  See the file COPYING.LIB for copying conditions, and LICENSES for notices about a few contributions that require these additional notices to be distributed.  License copyright years may be listed using range notation, e.g., 1996-2015, indicating that every year in the range, inclusive, is a copyrightable year that would otherwise be listed individually.</text>
 PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12
 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (something.rdf, something.txt)
 PackageDescription: <text>A package.</text>
+PackageAttributionText: <text>The GNU C Library is free software.  See the file COPYING.LIB for copying conditions, and LICENSES for notices about a few contributions that require these additional notices to be distributed.  License copyright years may be listed using range notation, e.g., 1996-2015, indicating that every year in the range, inclusive, is a copyrightable year that would otherwise be listed individually.</text>
 PackageCopyrightText: <text> Copyright 2010, 2011 Acme Inc.</text>
 PackageLicenseDeclared: Apache-2.0
 PackageLicenseConcluded: (LicenseRef-2.0 and Apache-2.0)