diff --git a/spdx/document.py b/spdx/document.py index e0232d7cc..1396b1732 100644 --- a/spdx/document.py +++ b/spdx/document.py @@ -279,6 +279,7 @@ class Document(object): - documentNamespace: SPDX document specific namespace. Mandatory, one. Type: str - creation_info: SPDX file creation info. Mandatory, one. Type: CreationInfo - package: Package described by this document. Mandatory, one. Type: Package + - unpackaged_files: Files not part of any package. Optional zero or more. Type: File. - extracted_licenses: List of licenses extracted that are not part of the SPDX license list. Optional, many. Type: ExtractedLicense. - reviews: SPDX document review information, Optional zero or more. @@ -316,6 +317,7 @@ def __init__( self.packages = [] if package is not None: self.packages.append(package) + self.unpackaged_files = [] self.extracted_licenses = [] self.reviews = [] self.annotations = [] @@ -331,6 +333,9 @@ def add_annotation(self, annotation): def add_relationships(self, relationship): self.relationships.append(relationship) + def add_file(self, value): + self.unpackaged_files.append(value) + def add_extr_lic(self, lic): self.extracted_licenses.append(lic) @@ -370,10 +375,16 @@ def package(self, value): @property def files(self): + warnings.warn('document.package and document.files are deprecated; ' + 'use document.packages instead', + DeprecationWarning) return self.package.files @files.setter def files(self, value): + warnings.warn('document.package and document.files are deprecated; ' + 'use document.packages instead', + DeprecationWarning) self.package.files = value @property @@ -400,6 +411,7 @@ def validate(self, messages=None): self.validate_creation_info(messages) self.validate_packages(messages) self.validate_extracted_licenses(messages) + self.validate_unpackaged_files(messages) self.validate_reviews(messages) self.validate_snippet(messages) self.validate_annotations(messages) @@ -486,4 +498,8 @@ def validate_extracted_licenses(self, messages): messages.append( "Document extracted licenses must be of type " "spdx.document.ExtractedLicense and not " + type(lic) - ) \ No newline at end of file + ) + + def validate_unpackaged_files(self, messages): + for spdx_file in self.unpackaged_files: + messages = spdx_file.validate(messages) diff --git a/spdx/parsers/rdf.py b/spdx/parsers/rdf.py index c014c3d85..575b69321 100644 --- a/spdx/parsers/rdf.py +++ b/spdx/parsers/rdf.py @@ -17,6 +17,7 @@ from rdflib import Namespace from rdflib import RDF from rdflib import RDFS +from rdflib.term import URIRef from spdx import document from spdx import utils @@ -365,6 +366,7 @@ def parse_package(self, p_term): self.p_pkg_summary(p_term, self.spdx_namespace["summary"]) self.p_pkg_descr(p_term, self.spdx_namespace["description"]) self.p_pkg_comment(p_term, self.spdx_namespace["comment"]) + self.p_pkg_has_file(p_term, self.spdx_namespace["hasFile"]) def p_pkg_cr_text(self, p_term, predicate): try: @@ -396,6 +398,14 @@ def p_pkg_comment(self, p_term, predicate): except CardinalityError: self.more_than_one_error("package comment") + def p_pkg_has_file(self, p_term, predicate): + for _, _, has_file_object in self.graph.triples((p_term, predicate, None)): + for index, spdx_file in enumerate(self.doc.unpackaged_files): + object_spdx_id = str(has_file_object) + if spdx_file.spdx_id == object_spdx_id: + self.doc.packages[-1].files.append(spdx_file) + del self.doc.unpackaged_files[index] + def p_pkg_attribution_text(self, p_term, predicate): try: for _, _, attribute_text in self.graph.triples((p_term, predicate, None)): @@ -1299,6 +1309,11 @@ def parse(self, fil): ): self.handle_extracted_license(s) + for s, _p, o in self.graph.triples( + (URIRef(self.doc.spdx_id), self.spdx_namespace["referencesFile"], None) + ): + self.parse_file(o) # Document-level ref file (unpackaged) + for s, _p, o in self.graph.triples( (None, RDF.type, self.spdx_namespace["Package"]) ): @@ -1312,7 +1327,9 @@ def parse(self, fil): for s, _p, o in self.graph.triples( (None, self.spdx_namespace["referencesFile"], None) ): - self.parse_file(o) + if not str(s) == self.doc.spdx_id: + # self.parse_file(o) # TODO: Package child ref file? + raise NotImplementedError(f"Processing {s} referencesFile") for s, _p, o in self.graph.triples( (None, RDF.type, self.spdx_namespace["Snippet"]) diff --git a/spdx/parsers/rdfbuilders.py b/spdx/parsers/rdfbuilders.py index 3affb7329..7f3356fc2 100644 --- a/spdx/parsers/rdfbuilders.py +++ b/spdx/parsers/rdfbuilders.py @@ -386,9 +386,9 @@ def set_file_chksum(self, doc, chk_sum): Set the file check sum, if not already set. chk_sum - A string Raise CardinalityError if already defined. - Raise OrderError if no package previously defined. + Raise OrderError if no package, package file, or unpackaged file is defined. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_chksum_set: self.file_chksum_set = True self.file(doc).chk_sum = checksum.Algorithm("SHA1", chk_sum) @@ -403,7 +403,7 @@ def set_file_license_comment(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one per file. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_license_comment_set: self.file_license_comment_set = True self.file(doc).license_comment = text @@ -417,7 +417,7 @@ def set_file_attribution_text(self, doc, text): """ Set the file's attribution text. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): self.assert_package_exists() self.file(doc).attribution_text = text return True @@ -427,7 +427,7 @@ def set_file_copyright(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_copytext_set: self.file_copytext_set = True self.file(doc).copyright = text @@ -442,7 +442,7 @@ def set_file_comment(self, doc, text): Raise OrderError if no package or no file defined. Raise CardinalityError if more than one comment set. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_comment_set: self.file_comment_set = True self.file(doc).comment = text @@ -457,7 +457,7 @@ def set_file_notice(self, doc, text): Raise OrderError if no package or file defined. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_notice_set: self.file_notice_set = True self.file(doc).notice = tagvaluebuilders.str_from_text(text) diff --git a/spdx/parsers/tagvalue.py b/spdx/parsers/tagvalue.py index eda58fa70..4b645f385 100644 --- a/spdx/parsers/tagvalue.py +++ b/spdx/parsers/tagvalue.py @@ -564,7 +564,8 @@ def p_spdx_id(self, p): value = p[2] if not self.builder.doc_spdx_id_set: self.builder.set_doc_spdx_id(self.document, value) - elif not self.builder.package_spdx_id_set: + elif (not self.builder.package_spdx_id_set + and self.builder.package_set): self.builder.set_pkg_spdx_id(self.document, value) else: self.builder.set_file_spdx_id(self.document, value) diff --git a/spdx/parsers/tagvaluebuilders.py b/spdx/parsers/tagvaluebuilders.py index caf7f2ff9..7c1ceccde 100644 --- a/spdx/parsers/tagvaluebuilders.py +++ b/spdx/parsers/tagvaluebuilders.py @@ -1031,28 +1031,39 @@ def __init__(self): # FIXME: this state does not make sense self.reset_file_stat() + def _build_file_valid(self, doc): + """ + Return true if building a File for referenced document is valid. + """ + if self.has_package(doc) and self.has_file(doc): + return True + if not self.package_set and self.has_unpackaged_file(doc): + return True + return False + def set_file_name(self, doc, name): """ - Raise OrderError if no package defined. + Set the file name. """ if self.has_package(doc): doc.packages[-1].files.append(file.File(name)) - # A file name marks the start of a new file instance. - # The builder must be reset - # FIXME: this state does not make sense - self.reset_file_stat() - return True else: - raise OrderError("File::Name") + # Starting with SPDX 2.0, file entries may proceed any package information section + doc.unpackaged_files.append(file.File(name)) + # A file name marks the start of a new file instance. + # The builder must be reset + # FIXME: this state does not make sense + self.reset_file_stat() + return True def set_file_spdx_id(self, doc, spdx_id): """ Set the file SPDX Identifier. - Raise OrderError if no package or no file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if malformed value. Raise CardinalityError if more than one spdx_id set. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_spdx_id_set: self.file_spdx_id_set = True if validations.validate_file_spdx_id(spdx_id): @@ -1067,11 +1078,11 @@ def set_file_spdx_id(self, doc, spdx_id): def set_file_comment(self, doc, text): """ - Raise OrderError if no package or no file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise CardinalityError if more than one comment set. Raise SPDXValueError if text is not free form text. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_comment_set: self.file_comment_set = True if validations.validate_file_comment(text): @@ -1089,7 +1100,7 @@ def set_file_attribution_text(self, doc, text): Set the file's attribution text . Raise SPDXValueError if text is not free form text. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if validations.validate_file_attribution_text(text): self.file(doc).comment = str_from_text(text) return True @@ -1098,7 +1109,7 @@ def set_file_attribution_text(self, doc, text): def set_file_type(self, doc, type_value): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise CardinalityError if more than one type set. Raise SPDXValueError if type is unknown. """ @@ -1108,7 +1119,7 @@ def set_file_type(self, doc, type_value): "ARCHIVE": file.FileType.ARCHIVE, "OTHER": file.FileType.OTHER, } - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_type_set: self.file_type_set = True if type_value in type_dict.keys(): @@ -1123,10 +1134,10 @@ def set_file_type(self, doc, type_value): def set_file_chksum(self, doc, chksum): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise CardinalityError if more than one chksum set. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_chksum_set: self.file_chksum_set = True self.file(doc).chk_sum = checksum_from_sha1(chksum) @@ -1138,11 +1149,11 @@ def set_file_chksum(self, doc, chksum): def set_concluded_license(self, doc, lic): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise CardinalityError if already set. Raise SPDXValueError if malformed. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_conc_lics_set: self.file_conc_lics_set = True if validations.validate_lics_conc(lic): @@ -1157,10 +1168,10 @@ def set_concluded_license(self, doc, lic): def set_file_license_in_file(self, doc, lic): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if malformed value. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if validations.validate_file_lics_in_file(lic): self.file(doc).add_lics(lic) return True @@ -1171,11 +1182,11 @@ def set_file_license_in_file(self, doc, lic): def set_file_license_comment(self, doc, text): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if text is not free form text. Raise CardinalityError if more than one per file. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_license_comment_set: self.file_license_comment_set = True if validations.validate_file_lics_comment(text): @@ -1189,11 +1200,11 @@ def set_file_license_comment(self, doc, text): def set_file_copyright(self, doc, text): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if not free form text or NONE or NO_ASSERT. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_copytext_set: self.file_copytext_set = True if validations.validate_file_cpyright(text): @@ -1211,11 +1222,11 @@ def set_file_copyright(self, doc, text): def set_file_notice(self, doc, text): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. Raise SPDXValueError if not free form text. Raise CardinalityError if more than one. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): if not self.file_notice_set: self.file_notice_set = True if validations.validate_file_notice(text): @@ -1229,18 +1240,18 @@ def set_file_notice(self, doc, text): def add_file_contribution(self, doc, value): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): self.file(doc).add_contrib(value) else: raise OrderError("File::Contributor") def add_file_dep(self, doc, value): """ - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): self.file(doc).add_depend(value) else: raise OrderError("File::Dependency") @@ -1248,17 +1259,20 @@ def add_file_dep(self, doc, value): def set_file_atrificat_of_project(self, doc, symbol, value): """ Set a file name, uri or home artificat. - Raise OrderError if no package or file defined. + Raise OrderError if no package, package file, or unpackaged file is defined. """ - if self.has_package(doc) and self.has_file(doc): + if (self._build_file_valid(doc)): self.file(doc).add_artifact(symbol, value) else: raise OrderError("File::Artificat") def file(self, doc): """ - Return the last file in the document's package's file list. + Return the last file in document's package's file list (or + unpackaged list if no package has been set). """ + if not self.package_set: + return doc.unpackaged_files[-1] return doc.packages[-1].files[-1] def has_file(self, doc): @@ -1268,6 +1282,12 @@ def has_file(self, doc): """ return len(doc.packages[-1].files) != 0 + def has_unpackaged_file(self, doc): + """ + Return true if the document has at least one file not part of any package. + """ + return len(doc.unpackaged_files) != 0 + def has_package(self, doc): """ Return true if the document has a package. diff --git a/spdx/writers/jsonyamlxml.py b/spdx/writers/jsonyamlxml.py index faac513dd..7bc49b31a 100644 --- a/spdx/writers/jsonyamlxml.py +++ b/spdx/writers/jsonyamlxml.py @@ -182,7 +182,10 @@ def create_artifact_info(self, file): return artifact_of_objects - def create_file_info(self, package): + def create_pkg_file_info(self, package): + return self.create_file_info(package.files) + + def create_file_info(self, files): file_types = { 1: "fileType_source", 2: "fileType_binary", @@ -190,7 +193,6 @@ def create_file_info(self, package): 4: "fileType_other", } file_objects = [] - files = package.files for file in files: file_object = dict() @@ -471,10 +473,13 @@ def create_document(self): self.document_object["SPDXID"] = self.spdx_id(self.document.spdx_id) self.document_object["name"] = self.document.name + if self.document.unpackaged_files: + self.document_object["files"] = self.create_file_info(self.document.unpackaged_files) + package_objects = [] for package in self.document.packages: package_info_object = self.create_package_info(package) - package_info_object["files"] = self.create_file_info(package) + package_info_object["files"] = self.create_pkg_file_info(package) package_objects.append({"Package": package_info_object}) self.document_object["documentDescribes"] = package_objects diff --git a/spdx/writers/tagvalue.py b/spdx/writers/tagvalue.py index 7ffd80d4b..092754ac7 100644 --- a/spdx/writers/tagvalue.py +++ b/spdx/writers/tagvalue.py @@ -358,6 +358,11 @@ def write_document(document, out, validate=True): write_relationship(relationship, out) write_separators(out) + # Write out unpackaged file info + for spdx_file in document.unpackaged_files: + write_file(spdx_file, out) + write_separators(out) + # Write out package info for package in document.packages: write_package(package, out) diff --git a/tests/data/doc_parse/SBOMexpected.json b/tests/data/doc_parse/SBOMexpected.json index 848d19ab0..436ff7815 100644 --- a/tests/data/doc_parse/SBOMexpected.json +++ b/tests/data/doc_parse/SBOMexpected.json @@ -30,6 +30,7 @@ ], "created": "2020-07-23T18:30:22Z", "creatorComment": null, + "files": [], "packages": [ { "id": "SPDXRef-Package-xyz", diff --git a/tests/data/doc_parse/expected.json b/tests/data/doc_parse/expected.json index 346de7167..c09768a87 100644 --- a/tests/data/doc_parse/expected.json +++ b/tests/data/doc_parse/expected.json @@ -34,6 +34,7 @@ ], "created": "2010-02-03T00:00:00Z", "creatorComment": "This is an example of an SPDX spreadsheet format", + "files": [], "packages": [ { "id": "SPDXRef-Package", diff --git a/tests/data/doc_parse/spdx-expected-unpackaged-files.json b/tests/data/doc_parse/spdx-expected-unpackaged-files.json new file mode 100644 index 000000000..f47714a67 --- /dev/null +++ b/tests/data/doc_parse/spdx-expected-unpackaged-files.json @@ -0,0 +1,296 @@ +{ + "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-DOCUMENT", + "specVersion": { + "major": 2, + "minor": 1 + }, + "documentNamespace": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206", + "name": "Sample_Document-V2.1", + "comment": "This is a sample spreadsheet", + "dataLicense": { + "type": "Single", + "identifier": "CC0-1.0", + "name": "Creative Commons Zero v1.0 Universal" + }, + "licenseListVersion": { + "major": 3, + "minor": 6 + }, + "creators": [ + { + "name": "A. Developer", + "email": null, + "type": "Person" + }, + { + "name": "Sample Group Inc.", + "email": null, + "type": "Organization" + }, + { + "name": "SomeExampleTool-V1.0", + "type": "Tool" + } + ], + "created": "2022-05-31T00:00:00Z", + "creatorComment": "This is an example of an SPDX spreadsheet format", + "files": [ + { + "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-File1", + "name": "file.bin", + "type": 4, + "comment": null, + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "copyrightText": "Copyright 2022 Example Client Inc.", + "licenseComment": null, + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "1b30f385d353620c845594d7ef96b3dd6d17a69a" + }, + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [], + "artifactOfProjectHome": [], + "artifactOfProjectURI": [] + } + ], + "packages": [ + { + "id": "SPDXRef-Package", + "name": "SPDX Translator", + "packageFileName": "spdxtranslator-1.0.zip", + "summary": "SPDX Translator utility", + "description": "This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document.", + "versionInfo": "Version 0.9.2", + "sourceInfo": "Version 1.0 of the SPDX Translator application", + "downloadLocation": "http://www.spdx.org/tools", + "homepage": null, + "originator": { + "name": "SPDX", + "email": null, + "type": "Organization" + }, + "supplier": { + "name": "Linux Foundation", + "email": null, + "type": "Organization" + }, + "licenseConcluded": { + "type": "Conjunction", + "identifier": [ + "Apache-1.0", + "Apache-2.0", + "LicenseRef-2", + "LicenseRef-3", + "LicenseRef-4", + "MPL-1.1" + ], + "name": [ + "Apache License 1.0", + "Apache License 2.0", + "CyberNeko License", + "LicenseRef-2", + "LicenseRef-4", + "Mozilla Public License 1.1" + ] + }, + "licenseDeclared": { + "type": "Conjunction", + "identifier": [ + "Apache-2.0", + "LicenseRef-2", + "LicenseRef-3", + "LicenseRef-4", + "MPL-1.1" + ], + "name": [ + "Apache License 2.0", + "CyberNeko License", + "LicenseRef-2", + "LicenseRef-4", + "Mozilla Public License 1.1" + ] + }, + "copyrightText": " Copyright 2010, 2011 Source Auditor Inc.", + "licenseComment": "The declared license information can be found in the NOTICE file at the root of the archive file", + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "files": [ + { + "id": "https://spdx.org/spdxdocs/spdx-example-F16320D4-36AB-476D-9691-59E1BAD31206#SPDXRef-File2", + "name": "etc/message.dat", + "type": 4, + "comment": null, + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "copyrightText": "Copyright 2022 Example Client Inc.", + "licenseComment": null, + "notice": null, + "checksum": { + "identifier": "SHA1", + "value": "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + }, + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ], + "contributors": [], + "dependencies": [], + "artifactOfProjectName": [], + "artifactOfProjectHome": [], + "artifactOfProjectURI": [] + } + ], + "licenseInfoFromFiles": [ + { + "type": "Single", + "identifier": "Apache-1.0", + "name": "Apache License 1.0" + }, + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + { + "type": "Single", + "identifier": "LicenseRef-2", + "name": "LicenseRef-2" + }, + { + "type": "Single", + "identifier": "LicenseRef-3", + "name": "CyberNeko License" + }, + { + "type": "Single", + "identifier": "LicenseRef-4", + "name": "LicenseRef-4" + }, + { + "type": "Single", + "identifier": "MPL-1.1", + "name": "Mozilla Public License 1.1" + } + ], + "verificationCode": { + "value": "4e3211c67a2d28fced849ee1bb76e7391b93feba", + "excludedFilesNames": [ + "SpdxTranslatorSpdx.rdf", + "SpdxTranslatorSpdx.txt" + ] + } + } + ], + "externalDocumentRefs": [ + { + "externalDocumentId": "DocumentRef-spdx-tool-2.1", + "spdxDocument": "https://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301", + "checksum": { + "identifier": "SHA1", + "value": "d6a770ba38583ed4bb4525bd96e50461655d2759" + } + } + ], + "extractedLicenses": [ + { + "name": "LicenseRef-2", + "identifier": "LicenseRef-2", + "text": "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n\u00a9 Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: \n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. \nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. \nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. \nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", + "comment": null, + "cross_refs": [] + }, + { + "name": "CyberNeko License", + "identifier": "LicenseRef-3", + "text": "The CyberNeko Software License, Version 1.0\n\n \n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n \nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer. \n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment: \n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior \n written permission. For written permission, please contact \n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, \nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT \nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR \nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, \nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE \nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, \nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "comment": "This is tye CyperNeko License", + "cross_refs": [ + "http://justasample.url.com", + "http://people.apache.org/~andyc/neko/LICENSE" + ] + }, + { + "name": "LicenseRef-4", + "identifier": "LicenseRef-4", + "text": "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */ ", + "comment": null, + "cross_refs": [] + } + ], + "annotations": [ + { + "id": "https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#SPDXRef-45", + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "type": "REVIEW", + "annotator": { + "name": "Jim Reviewer", + "email": null, + "type": "Person" + }, + "date": "2012-06-13T00:00:00Z" + } + ], + "reviews": [ + { + "comment": "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses", + "reviewer": { + "name": "Joe Reviewer", + "email": null, + "type": "Person" + }, + "date": "2010-02-10T00:00:00Z" + }, + { + "comment": "Another example reviewer.", + "reviewer": { + "name": "Suzanne Reviewer", + "email": null, + "type": "Person" + }, + "date": "2011-03-13T00:00:00Z" + } + ], + "snippets": [ + { + "id": "SPDXRef-Snippet", + "name": "from linux kernel", + "comment": "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later.", + "copyrightText": "Copyright 2008-2010 John Smith", + "licenseComments": "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", + "fileId": "SPDXRef-DoapSource", + "licenseConcluded": { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + }, + "licenseInfoFromSnippet": [ + { + "type": "Single", + "identifier": "Apache-2.0", + "name": "Apache License 2.0" + } + ] + } + ] +} diff --git a/tests/data/doc_parse/spdx-expected.json b/tests/data/doc_parse/spdx-expected.json index 3514bfcc6..d57354f1d 100644 --- a/tests/data/doc_parse/spdx-expected.json +++ b/tests/data/doc_parse/spdx-expected.json @@ -34,6 +34,7 @@ ], "created": "2010-02-03T00:00:00Z", "creatorComment": "This is an example of an SPDX spreadsheet format", + "files": [], "packages": [ { "id": "SPDXRef-Package", diff --git a/tests/data/formats/SPDXRdfExample.rdf b/tests/data/formats/SPDXRdfExample.rdf index 4637b4ca9..4d5a778d2 100644 --- a/tests/data/formats/SPDXRdfExample.rdf +++ b/tests/data/formats/SPDXRdfExample.rdf @@ -39,7 +39,7 @@ - > + /* @@ -142,7 +142,7 @@ http://www.spdx.org/tools true Organization:Linux Foundation - + @@ -269,7 +269,7 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SPDX Translator Version 0.9.2 - + Copyright 2010, 2011 Source Auditor Inc. diff --git a/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf new file mode 100644 index 000000000..d5443ddfc --- /dev/null +++ b/tests/data/formats/SPDXRdfUnpackagedFileExample.rdf @@ -0,0 +1,263 @@ + + + + + + from linux kernel + Copyright 2008-2010 John Smith + The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz. + This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0-or-later. + + + + + Sample_Document-V2.1 + + + 2022-05-31T00:00:00Z + This is an example of an SPDX spreadsheet format + Tool: SomeExampleTool-V1.0 + Organization: Sample Group Inc. + Person: A. Developer + + + SPDX-2.1 + + + DocumentRef-spdx-tool-2.1 + + + + d6a770ba38583ed4bb4525bd96e50461655d2759 + + + + + + + + Copyright 2022 Example Client Inc. + + + + + + 1b30f385d353620c845594d7ef96b3dd6d17a69a + + + + file.bin + + + + + This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses + 2010-02-10T00:00:00Z + Person: Joe Reviewer + + + + + + + Another example reviewer. + 2011-03-13T00:00:00Z + Person: Suzanne Reviewer + + + + + + This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses + 2012-06-13T00:00:00Z + Person: Jim Reviewer + + + + + + + + + + + Copyright 2022 Example Client Inc. + + + + + + 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 + + + + etc/message.dat + + + + + + http://www.spdx.org/tools + true + Organization:Linux Foundation + + + + + 4e3211c67a2d28fced849ee1bb76e7391b93feba + SpdxTranslatorSpdx.txt + SpdxTranslatorSpdx.rdf + + + + + + + This package includes the GRDDL parser developed by Hewlett Packard under the following license: +© Copyright 2007 Hewlett-Packard Development Company, LP + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + LicenseRef-2 + + + + + + + + /* + * (c) Copyright 2009 University of Bristol + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + LicenseRef-4 + + + + + http://justasample.url.com + http://people.apache.org/~andyc/neko/LICENSE + CyberNeko License + This is tye CyperNeko License + The CyberNeko Software License, Version 1.0 + + +(C) Copyright 2002-2005, Andy Clark. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +3. The end-user documentation included with the redistribution, + if any, must include the following acknowledgment: + "This product includes software developed by Andy Clark." + Alternately, this acknowledgment may appear in the software itself, + if and wherever such third-party acknowledgments normally appear. + +4. The names "CyberNeko" and "NekoHTML" must not be used to endorse + or promote products derived from this software without prior + written permission. For written permission, please contact + andyc@cyberneko.net. + +5. Products derived from this software may not be called "CyberNeko", + nor may "CyberNeko" appear in their name, without prior written + permission of the author. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + LicenseRef-3 + + + + + Version 1.0 of the SPDX Translator application + + + 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 + + + + spdxtranslator-1.0.zip + This utility translates and SPDX RDF XML document to a spreadsheet, translates a spreadsheet to an SPDX RDF XML document and translates an SPDX RDFa document to an SPDX RDF XML document. + + SPDX Translator + Version 0.9.2 + + Copyright 2010, 2011 Source Auditor Inc. + + + + + + + + + + + Organization:SPDX + The declared license information can be found in the NOTICE file at the root of the archive file + SPDX Translator utility + + + + + + org.apache.commons:commons-lang:3.2.1 + NIST National Vulnerability Database (NVD) describes security vulnerabilities (CVEs) which affect Vendor Product Version acmecorp:acmenator:6.6.6 + + + + + This is a sample spreadsheet + + + + + diff --git a/tests/test_rdf_parser.py b/tests/test_rdf_parser.py index 8f52f58b7..13800fa4b 100644 --- a/tests/test_rdf_parser.py +++ b/tests/test_rdf_parser.py @@ -45,3 +45,12 @@ def check_document(self, document, expected_loc, regen=False): expected = json.load(ex) assert result == expected + + def test_rdf_parser_unpacked_files(self, regen=False): + parser = rdf.Parser(RDFBuilder(), StandardLogger()) + test_file = utils_test.get_test_loc('formats/SPDXRdfUnpackagedFileExample.rdf', test_data_dir=utils_test.test_data_dir) + with io.open(test_file, 'rb') as f: + document, _ = parser.parse(f) + expected_loc = utils_test.get_test_loc('doc_parse/spdx-expected-unpackaged-files.json', test_data_dir=utils_test.test_data_dir) + self.check_document(document, expected_loc, regen=regen) + diff --git a/tests/test_tag_value_parser.py b/tests/test_tag_value_parser.py index 54dde115e..9c5bdc1e1 100644 --- a/tests/test_tag_value_parser.py +++ b/tests/test_tag_value_parser.py @@ -207,6 +207,17 @@ class TestParser(TestCase): 'ReviewComment: Alice was also here.' ]) + unpackaged_file_str = '\n'.join([ + 'FileName: testfile.text-info', + 'SPDXID: SPDXRef-File', + 'FileType: OTHER', + 'FileChecksum: SHA1: c940141b1f4d098f12812675e9cfa5fe72e07bab', + 'LicenseConcluded: Apache-2.0', + 'LicenseInfoInFile: Apache-2.0', + 'FileCopyrightText: Copyright 2022 Acme Inc.', + 'FileComment: Very long file' + ]) + package_str = '\n'.join([ 'PackageName: Test', 'SPDXID: SPDXRef-Package', @@ -259,7 +270,7 @@ class TestParser(TestCase): 'LicenseInfoInSnippet: Apache-2.0', ]) - complete_str = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format(document_str, creation_str, review_str, package_str, file_str, snippet_str) + complete_str = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}\n{6}'.format(document_str, creation_str, review_str, unpackaged_file_str, package_str, file_str, snippet_str) def setUp(self): self.p = Parser(Builder(), StandardLogger()) @@ -319,6 +330,16 @@ def test_file(self): assert len(spdx_file.artifact_of_project_home) == 1 assert len(spdx_file.artifact_of_project_uri) == 1 + def test_unpackaged_file(self): + document, error = self.p.parse(self.complete_str) + assert document is not None + assert not error + assert len(document.files) == 1 + spdx_file = document.unpackaged_files[0] + assert spdx_file.name == 'testfile.text-info' + assert spdx_file.spdx_id == 'SPDXRef-File' + assert spdx_file.type == spdx.file.FileType.OTHER + def test_unknown_tag(self): try: diff --git a/tests/test_write_anything.py b/tests/test_write_anything.py index 9dceaffcb..a997c88f2 100644 --- a/tests/test_write_anything.py +++ b/tests/test_write_anything.py @@ -35,7 +35,12 @@ "SPDXRdfExample.rdf-yaml", "SPDXRdfExample.rdf-xml", "SPDXRdfExample.rdf-json", - "SPDXRdfExample.rdf-tag" + "SPDXRdfExample.rdf-tag", + "SPDXRdfUnpackagedFileExample.rdf-rdf", + "SPDXRdfUnpackagedFileExample.rdf-yaml", + "SPDXRdfUnpackagedFileExample.rdf-xml", + "SPDXRdfUnpackagedFileExample.rdf-json", + "SPDXRdfUnpackagedFileExample.rdf-tag" } @pytest.mark.parametrize("out_format", ['rdf', 'yaml', 'xml', 'json', 'tag']) diff --git a/tests/utils_test.py b/tests/utils_test.py index 92840f108..6e7ced09d 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -510,6 +510,7 @@ def to_dict(cls, doc): ('creators', [cls.entity_to_dict(creator) for creator in creators]), ('created', utils.datetime_iso_format(doc.creation_info.created)), ('creatorComment', doc.creation_info.comment), + ('files', cls.files_to_list(doc.unpackaged_files)), ('packages', [cls.package_to_dict(p) for p in doc.packages]), ('externalDocumentRefs', cls.ext_document_references_to_list(sorted(doc.ext_document_references))), ('extractedLicenses', cls.extracted_licenses_to_list(sorted(doc.extracted_licenses))),