From 8050fd9c41a92c75ec2ba9eb10ed9a919c375fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Thu, 9 Nov 2023 16:28:20 +0100 Subject: [PATCH 01/41] [issue-771] fix license expression error handling in tag-value parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/spdx_tools/spdx/parser/tagvalue/parser.py | 10 ++++- .../parser/tagvalue/test_tag_value_parser.py | 38 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/spdx_tools/spdx/parser/tagvalue/parser.py b/src/spdx_tools/spdx/parser/tagvalue/parser.py index a11f08679..32be1143e 100644 --- a/src/spdx_tools/spdx/parser/tagvalue/parser.py +++ b/src/spdx_tools/spdx/parser/tagvalue/parser.py @@ -14,7 +14,7 @@ import re from beartype.typing import Any, Dict, List -from license_expression import get_spdx_licensing +from license_expression import ExpressionError, get_spdx_licensing from ply import yacc from ply.yacc import LRParser @@ -233,7 +233,13 @@ def p_none(self, p): @grammar_rule("license_or_no_assertion_or_none : LINE") def p_license(self, p): - p[0] = get_spdx_licensing().parse(p[1]) + try: + p[0] = get_spdx_licensing().parse(p[1]) + except ExpressionError as err: + error_message = f"Error while parsing license expression: {p[1]}" + if err.args: + error_message += f": {err.args[0]}" + self.current_element["logger"].append(error_message) @grammar_rule("actor_or_no_assertion : PERSON_VALUE\n | ORGANIZATION_VALUE") def p_actor_values(self, p): diff --git a/tests/spdx/parser/tagvalue/test_tag_value_parser.py b/tests/spdx/parser/tagvalue/test_tag_value_parser.py index 33defcb9d..9f347fc08 100644 --- a/tests/spdx/parser/tagvalue/test_tag_value_parser.py +++ b/tests/spdx/parser/tagvalue/test_tag_value_parser.py @@ -98,3 +98,41 @@ def test_document_with_mixed_values(): "Element Package is not the current element in scope, probably the expected " "tag to start the element (PackageName) is missing. Line: 4" ] + + +def test_faulty_license_expression(): + parser = Parser() + document_str = "\n".join( + [ + f"SPDXID:{DOCUMENT_SPDX_ID}", + "FileName: File with faulty license expression", + "SPDXID: SPDXRef-File", + "FileChecksum: SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759", + "LicenseConcluded: LicenseRef-foo/bar", + "PackageName: Package with faulty license expression", + "SPDXID: SPDXRef-Package", + "PackageDownloadLocation: www.download.com", + "PackageLicenseConcluded: LicenseRef-bar/foo", + "SnippetSPDXID: SPDXRef-Snippet", + "SnippetName: Snippet with faulty license expression", + "SnippetLicenseConcluded: LicenseRef-foo/foo", + ] + ) + + with pytest.raises(SPDXParsingError) as err: + parser.parse(document_str) + + assert err.value.get_messages() == [ + 'Error while parsing File: ["Error while parsing license expression: ' + "LicenseRef-foo/bar: Invalid license key: the valid characters are: letters " + "and numbers, underscore, dot, colon or hyphen signs and spaces: " + "'LicenseRef-foo/bar'\"]", + 'Error while parsing Package: ["Error while parsing license expression: ' + "LicenseRef-bar/foo: Invalid license key: the valid characters are: letters " + "and numbers, underscore, dot, colon or hyphen signs and spaces: " + "'LicenseRef-bar/foo'\"]", + 'Error while parsing Snippet: ["Error while parsing license expression: ' + "LicenseRef-foo/foo: Invalid license key: the valid characters are: letters " + "and numbers, underscore, dot, colon or hyphen signs and spaces: " + "'LicenseRef-foo/foo'\"]", + ] From a4d063b7014383365c739f5b7fb3c53a0e0a2344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Thu, 7 Dec 2023 15:35:07 +0100 Subject: [PATCH 02/41] [issue-773] shorten output of FileNotFoundError when called via the CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/spdx_tools/spdx/clitools/pyspdxtools.py | 4 ++++ tests/spdx/test_cli.py | 1 + 2 files changed, 5 insertions(+) diff --git a/src/spdx_tools/spdx/clitools/pyspdxtools.py b/src/spdx_tools/spdx/clitools/pyspdxtools.py index 8603b8401..6a46b0d84 100644 --- a/src/spdx_tools/spdx/clitools/pyspdxtools.py +++ b/src/spdx_tools/spdx/clitools/pyspdxtools.py @@ -113,6 +113,10 @@ def main(infile: str, outfile: str, version: str, novalidation: bool, graph: boo logging.error(log_string) sys.exit(1) + except FileNotFoundError as err: + logging.error(f"{err.strerror}: {err.filename}") + sys.exit(1) + if __name__ == "__main__": main() diff --git a/tests/spdx/test_cli.py b/tests/spdx/test_cli.py index 6fcbe3733..0019cf3a3 100644 --- a/tests/spdx/test_cli.py +++ b/tests/spdx/test_cli.py @@ -42,6 +42,7 @@ def test_cli_with_system_exit_code_0(options): "data/invalid/spdx-trivy-vmware_log-intelligence-fluentd-sha256_086af034f561f343f633be9d9f9e95f65ae6c61b8ddb2c6755ef5bb25b40f53a.json", # noqa: E501 ), ), + ("-i", "non_existent_file.spdx"), ], ) def test_cli_with_system_exit_code_1(options): From 4432e66d46e42513f26d808cb3778ba71bd450bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Thu, 7 Dec 2023 16:19:31 +0100 Subject: [PATCH 03/41] [issue-774] fix tag-value output when related_spdx_element_id is "NONE" or "NOASSERTION" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- .../tagvalue_writer_helper_functions.py | 4 +++- .../test_tagvalue_writer_helper_functions.py | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/spdx_tools/spdx/writer/tagvalue/tagvalue_writer_helper_functions.py b/src/spdx_tools/spdx/writer/tagvalue/tagvalue_writer_helper_functions.py index 907c155b7..98f670252 100644 --- a/src/spdx_tools/spdx/writer/tagvalue/tagvalue_writer_helper_functions.py +++ b/src/spdx_tools/spdx/writer/tagvalue/tagvalue_writer_helper_functions.py @@ -83,7 +83,9 @@ def scan_relationships( files_by_spdx_id = {file.spdx_id: file for file in files} packages_spdx_ids = [package.spdx_id for package in packages] for relationship in relationships: - if ( + if relationship.related_spdx_element_id in [SpdxNoAssertion(), SpdxNone()]: + relationships_to_write.append(relationship) + elif ( relationship.relationship_type == RelationshipType.CONTAINS and relationship.spdx_element_id in packages_spdx_ids and relationship.related_spdx_element_id in files_by_spdx_id.keys() diff --git a/tests/spdx/writer/tagvalue/test_tagvalue_writer_helper_functions.py b/tests/spdx/writer/tagvalue/test_tagvalue_writer_helper_functions.py index d101fd37c..4949b434b 100644 --- a/tests/spdx/writer/tagvalue/test_tagvalue_writer_helper_functions.py +++ b/tests/spdx/writer/tagvalue/test_tagvalue_writer_helper_functions.py @@ -5,7 +5,7 @@ import pytest -from spdx_tools.spdx.model import ActorType, RelationshipType, SpdxNoAssertion +from spdx_tools.spdx.model import ActorType, RelationshipType, SpdxNoAssertion, SpdxNone from spdx_tools.spdx.writer.tagvalue.tagvalue_writer_helper_functions import scan_relationships, write_actor from tests.spdx.fixtures import actor_fixture, file_fixture, package_fixture, relationship_fixture @@ -16,6 +16,18 @@ def test_scan_relationships(): packages = [package_fixture(spdx_id=first_package_spdx_id), package_fixture(spdx_id=second_package_spdx_id)] file_spdx_id = "SPDXRef-File" files = [file_fixture(spdx_id=file_spdx_id)] + no_assertion_relationship = relationship_fixture( + spdx_element_id=second_package_spdx_id, + relationship_type=RelationshipType.CONTAINS, + related_spdx_element_id=SpdxNoAssertion(), + comment=None, + ) + none_relationship = relationship_fixture( + spdx_element_id=second_package_spdx_id, + relationship_type=RelationshipType.CONTAINS, + related_spdx_element_id=SpdxNone(), + comment=None, + ) relationships = [ relationship_fixture( spdx_element_id=first_package_spdx_id, @@ -29,11 +41,13 @@ def test_scan_relationships(): related_spdx_element_id=file_spdx_id, comment=None, ), + no_assertion_relationship, + none_relationship, ] relationships_to_write, contained_files_by_package_id = scan_relationships(relationships, packages, files) - assert relationships_to_write == [] + assert relationships_to_write == [no_assertion_relationship, none_relationship] assert contained_files_by_package_id == {first_package_spdx_id: files, second_package_spdx_id: files} From 5916d43da25612c3e3fd5cf1d339a4bf001d2ff4 Mon Sep 17 00:00:00 2001 From: Stanislav Pankevich Date: Tue, 9 Jan 2024 21:40:28 +0100 Subject: [PATCH 04/41] spdx3: software_purpose: add REQUIREMENT type Signed-off-by: Stanislav Pankevich --- src/spdx_tools/spdx3/model/software/software_purpose.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spdx_tools/spdx3/model/software/software_purpose.py b/src/spdx_tools/spdx3/model/software/software_purpose.py index 4e071181f..e3a4d48cf 100644 --- a/src/spdx_tools/spdx3/model/software/software_purpose.py +++ b/src/spdx_tools/spdx3/model/software/software_purpose.py @@ -24,5 +24,6 @@ class SoftwarePurpose(Enum): OPERATING_SYSTEM = auto() OTHER = auto() PATCH = auto() + REQUIREMENT = auto() SOURCE = auto() TEST = auto() From 0e1c9eed5d22b2fb6c73e2588d9e21052f213283 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Thu, 22 Feb 2024 17:52:15 +0100 Subject: [PATCH 05/41] [issue-798] fix tag-value parser: parse Tool or Organization as annotator Signed-off-by: Meret Behrens --- src/spdx_tools/spdx/parser/tagvalue/parser.py | 2 +- .../parser/tagvalue/test_annotation_parser.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/spdx_tools/spdx/parser/tagvalue/parser.py b/src/spdx_tools/spdx/parser/tagvalue/parser.py index 32be1143e..a190dd7d7 100644 --- a/src/spdx_tools/spdx/parser/tagvalue/parser.py +++ b/src/spdx_tools/spdx/parser/tagvalue/parser.py @@ -488,7 +488,7 @@ def p_snippet_range(self, p): # parsing methods for annotation - @grammar_rule("annotator : ANNOTATOR PERSON_VALUE\n| TOOL_VALUE\n| ORGANIZATION_VALUE") + @grammar_rule("annotator : ANNOTATOR PERSON_VALUE\n| ANNOTATOR TOOL_VALUE\n| ANNOTATOR ORGANIZATION_VALUE") def p_annotator(self, p): self.initialize_new_current_element(Annotation) set_value(p, self.current_element, method_to_apply=ActorParser.parse_actor) diff --git a/tests/spdx/parser/tagvalue/test_annotation_parser.py b/tests/spdx/parser/tagvalue/test_annotation_parser.py index 629fe72e9..f10ebe8c6 100644 --- a/tests/spdx/parser/tagvalue/test_annotation_parser.py +++ b/tests/spdx/parser/tagvalue/test_annotation_parser.py @@ -34,6 +34,28 @@ def test_parse_annotation(): assert annotation.spdx_id == DOCUMENT_SPDX_ID +def test_parse_annotation_with_organization_as_annotator(): + parser = Parser() + annotation_str = "\n".join( + [ + "Annotator: Organization: some-organization", + "AnnotationDate: 2010-01-29T18:30:22Z", + "AnnotationComment: Document level annotation", + "AnnotationType: OTHER", + f"SPDXREF: {DOCUMENT_SPDX_ID}", + ] + ) + document = parser.parse("\n".join([DOCUMENT_STR, annotation_str])) + assert document is not None + assert len(document.annotations) == 1 + annotation = document.annotations[0] + assert annotation.annotator.name == "some-organization" + assert annotation.annotation_date == datetime(2010, 1, 29, 18, 30, 22) + assert annotation.annotation_comment == "Document level annotation" + assert annotation.annotation_type == AnnotationType.OTHER + assert annotation.spdx_id == DOCUMENT_SPDX_ID + + @pytest.mark.parametrize( "annotation_str, expected_message", [ From a1ceee6e20163f31d6c555d6a58dfce7440f2d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Thu, 7 Dec 2023 17:24:22 +0100 Subject: [PATCH 06/41] [issue-775] catch decoding errors while parsing using the cli tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/spdx_tools/spdx/clitools/pyspdxtools.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/spdx_tools/spdx/clitools/pyspdxtools.py b/src/spdx_tools/spdx/clitools/pyspdxtools.py index 6a46b0d84..4219c6b81 100644 --- a/src/spdx_tools/spdx/clitools/pyspdxtools.py +++ b/src/spdx_tools/spdx/clitools/pyspdxtools.py @@ -14,9 +14,13 @@ # limitations under the License. import logging import sys +from json import JSONDecodeError +from xml.parsers.expat import ExpatError +from xml.sax import SAXParseException import click from beartype.typing import List +from yaml.scanner import ScannerError from spdx_tools.spdx.graph_generation import export_graph_from_document from spdx_tools.spdx.model import Document @@ -113,6 +117,22 @@ def main(infile: str, outfile: str, version: str, novalidation: bool, graph: boo logging.error(log_string) sys.exit(1) + except JSONDecodeError as err: + logging.error(f"Invalid JSON provided: {err.args[0]}") + sys.exit(1) + + except ScannerError as err: + logging.error("Invalid YAML provided: " + "\n".join([str(arg) for arg in err.args])) + sys.exit(1) + + except ExpatError as err: + logging.error(f"Invalid XML provided: {err.args[0]}") + sys.exit(1) + + except SAXParseException as err: + logging.error(f"Invalid RDF-XML provided: {str(err)}") + sys.exit(1) + except FileNotFoundError as err: logging.error(f"{err.strerror}: {err.filename}") sys.exit(1) From 590bbfa8601a2d4590c6579e20e6fc249c523636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 26 Jul 2024 12:25:58 +0200 Subject: [PATCH 07/41] [issue-806] replace Licensing() with spdx_licensing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- .../jsonlikedict/license_expression_parser.py | 5 +++-- tests/spdx/jsonschema/test_file_converter.py | 10 +++++----- tests/spdx/jsonschema/test_package_converter.py | 14 +++++++------- tests/spdx/jsonschema/test_snippet_converter.py | 10 +++++----- tests/spdx/model/test_package.py | 7 ++++--- tests/spdx/parser/jsonlikedict/test_file_parser.py | 6 +++--- .../jsonlikedict/test_license_expression_parser.py | 5 ++++- .../parser/jsonlikedict/test_package_parser.py | 12 ++++++------ .../parser/jsonlikedict/test_snippet_parser.py | 6 +++--- tests/spdx/validation/test_package_validator.py | 4 ++-- 10 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/spdx_tools/spdx/parser/jsonlikedict/license_expression_parser.py b/src/spdx_tools/spdx/parser/jsonlikedict/license_expression_parser.py index 29c0ebb99..5a3b545f6 100644 --- a/src/spdx_tools/spdx/parser/jsonlikedict/license_expression_parser.py +++ b/src/spdx_tools/spdx/parser/jsonlikedict/license_expression_parser.py @@ -2,8 +2,9 @@ # # SPDX-License-Identifier: Apache-2.0 from beartype.typing import Union -from license_expression import ExpressionError, LicenseExpression, Licensing +from license_expression import ExpressionError, LicenseExpression +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import SpdxNoAssertion, SpdxNone from spdx_tools.spdx.parser.error import SPDXParsingError @@ -18,7 +19,7 @@ def parse_license_expression(license_expression_str: str) -> Union[LicenseExpres return SpdxNone() try: - license_expression = Licensing().parse(license_expression_str) + license_expression = spdx_licensing.parse(license_expression_str) except ExpressionError as err: err_msg = f'Error parsing LicenseExpression: "{license_expression_str}"' if err.args: diff --git a/tests/spdx/jsonschema/test_file_converter.py b/tests/spdx/jsonschema/test_file_converter.py index 8e6c14049..560ecb1a4 100644 --- a/tests/spdx/jsonschema/test_file_converter.py +++ b/tests/spdx/jsonschema/test_file_converter.py @@ -7,8 +7,8 @@ from unittest.mock import MagicMock, NonCallableMagicMock import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.jsonschema.annotation_converter import AnnotationConverter from spdx_tools.spdx.jsonschema.file_converter import FileConverter from spdx_tools.spdx.jsonschema.file_properties import FileProperty @@ -81,8 +81,8 @@ def test_successful_conversion(converter: FileConverter): spdx_id="spdxId", checksums=[Checksum(ChecksumAlgorithm.SHA224, "sha224"), Checksum(ChecksumAlgorithm.MD2, "md2")], file_types=[FileType.SPDX, FileType.OTHER], - license_concluded=Licensing().parse("MIT and GPL-2.0"), - license_info_in_file=[Licensing().parse("MIT"), Licensing().parse("GPL-2.0"), SpdxNoAssertion()], + license_concluded=spdx_licensing.parse("MIT and GPL-2.0"), + license_info_in_file=[spdx_licensing.parse("MIT"), spdx_licensing.parse("GPL-2.0"), SpdxNoAssertion()], license_comment="licenseComment", copyright_text="copyrightText", comment="comment", @@ -115,8 +115,8 @@ def test_successful_conversion(converter: FileConverter): converter.json_property_name(FileProperty.FILE_NAME): "name", converter.json_property_name(FileProperty.FILE_TYPES): ["SPDX", "OTHER"], converter.json_property_name(FileProperty.LICENSE_COMMENTS): "licenseComment", - converter.json_property_name(FileProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0", - converter.json_property_name(FileProperty.LICENSE_INFO_IN_FILES): ["MIT", "GPL-2.0", "NOASSERTION"], + converter.json_property_name(FileProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0-only", + converter.json_property_name(FileProperty.LICENSE_INFO_IN_FILES): ["MIT", "GPL-2.0-only", "NOASSERTION"], converter.json_property_name(FileProperty.NOTICE_TEXT): "notice", } diff --git a/tests/spdx/jsonschema/test_package_converter.py b/tests/spdx/jsonschema/test_package_converter.py index 9365c8214..c6e63d9f8 100644 --- a/tests/spdx/jsonschema/test_package_converter.py +++ b/tests/spdx/jsonschema/test_package_converter.py @@ -7,8 +7,8 @@ from unittest.mock import MagicMock, NonCallableMagicMock import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.jsonschema.annotation_converter import AnnotationConverter from spdx_tools.spdx.jsonschema.package_converter import PackageConverter from spdx_tools.spdx.jsonschema.package_properties import PackageProperty @@ -123,9 +123,9 @@ def test_successful_conversion(converter: PackageConverter): checksums=[Checksum(ChecksumAlgorithm.SHA1, "sha1"), Checksum(ChecksumAlgorithm.BLAKE2B_256, "blake")], homepage="homepage", source_info="sourceInfo", - license_concluded=Licensing().parse("MIT and GPL-2.0"), - license_info_from_files=[Licensing().parse("MIT"), Licensing().parse("GPL-2.0")], - license_declared=Licensing().parse("MIT or GPL-2.0 "), + license_concluded=spdx_licensing.parse("MIT and GPL-2.0"), + license_info_from_files=[spdx_licensing.parse("MIT"), spdx_licensing.parse("GPL-2.0")], + license_declared=spdx_licensing.parse("MIT or GPL-2.0 "), license_comment="licenseComment", copyright_text="copyrightText", summary="summary", @@ -168,9 +168,9 @@ def test_successful_conversion(converter: PackageConverter): ], converter.json_property_name(PackageProperty.HOMEPAGE): "homepage", converter.json_property_name(PackageProperty.SOURCE_INFO): "sourceInfo", - converter.json_property_name(PackageProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0", - converter.json_property_name(PackageProperty.LICENSE_INFO_FROM_FILES): ["MIT", "GPL-2.0"], - converter.json_property_name(PackageProperty.LICENSE_DECLARED): "MIT OR GPL-2.0", + converter.json_property_name(PackageProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0-only", + converter.json_property_name(PackageProperty.LICENSE_INFO_FROM_FILES): ["MIT", "GPL-2.0-only"], + converter.json_property_name(PackageProperty.LICENSE_DECLARED): "MIT OR GPL-2.0-only", converter.json_property_name(PackageProperty.LICENSE_COMMENTS): "licenseComment", converter.json_property_name(PackageProperty.COPYRIGHT_TEXT): "copyrightText", converter.json_property_name(PackageProperty.SUMMARY): "summary", diff --git a/tests/spdx/jsonschema/test_snippet_converter.py b/tests/spdx/jsonschema/test_snippet_converter.py index a677343b0..36eecd4ac 100644 --- a/tests/spdx/jsonschema/test_snippet_converter.py +++ b/tests/spdx/jsonschema/test_snippet_converter.py @@ -7,8 +7,8 @@ from unittest.mock import MagicMock, NonCallableMagicMock import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.jsonschema.annotation_converter import AnnotationConverter from spdx_tools.spdx.jsonschema.snippet_converter import SnippetConverter from spdx_tools.spdx.jsonschema.snippet_properties import SnippetProperty @@ -72,8 +72,8 @@ def test_successful_conversion(converter: SnippetConverter): file_spdx_id=file_spdx_id, byte_range=(1, 2), line_range=(3, 4), - license_concluded=Licensing().parse("MIT and GPL-2.0"), - license_info_in_snippet=[Licensing().parse("MIT"), Licensing().parse("GPL-2.0")], + license_concluded=spdx_licensing.parse("MIT and GPL-2.0"), + license_info_in_snippet=[spdx_licensing.parse("MIT"), spdx_licensing.parse("GPL-2.0")], license_comment="licenseComment", copyright_text="copyrightText", comment="comment", @@ -98,8 +98,8 @@ def test_successful_conversion(converter: SnippetConverter): converter.json_property_name(SnippetProperty.COMMENT): "comment", converter.json_property_name(SnippetProperty.COPYRIGHT_TEXT): "copyrightText", converter.json_property_name(SnippetProperty.LICENSE_COMMENTS): "licenseComment", - converter.json_property_name(SnippetProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0", - converter.json_property_name(SnippetProperty.LICENSE_INFO_IN_SNIPPETS): ["MIT", "GPL-2.0"], + converter.json_property_name(SnippetProperty.LICENSE_CONCLUDED): "MIT AND GPL-2.0-only", + converter.json_property_name(SnippetProperty.LICENSE_INFO_IN_SNIPPETS): ["MIT", "GPL-2.0-only"], converter.json_property_name(SnippetProperty.NAME): "name", converter.json_property_name(SnippetProperty.RANGES): [ { diff --git a/tests/spdx/model/test_package.py b/tests/spdx/model/test_package.py index c533b2812..2f015f1e0 100644 --- a/tests/spdx/model/test_package.py +++ b/tests/spdx/model/test_package.py @@ -6,8 +6,9 @@ from unittest import mock import pytest -from license_expression import LicenseExpression, Licensing +from license_expression import LicenseExpression +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import Checksum, ChecksumAlgorithm, Package, PackagePurpose, SpdxNoAssertion, SpdxNone @@ -30,7 +31,7 @@ def test_correct_initialization(actor, verif_code, checksum, ext_ref): "homepage", "source_info", None, - [Licensing().parse("license and expression"), SpdxNoAssertion()], + [spdx_licensing.parse("license and expression"), SpdxNoAssertion()], SpdxNone(), "comment on license", "copyright", @@ -57,7 +58,7 @@ def test_correct_initialization(actor, verif_code, checksum, ext_ref): assert package.homepage == "homepage" assert package.source_info == "source_info" assert package.license_concluded is None - assert package.license_info_from_files == [Licensing().parse("license and expression"), SpdxNoAssertion()] + assert package.license_info_from_files == [spdx_licensing.parse("license and expression"), SpdxNoAssertion()] assert package.license_declared == SpdxNone() assert package.license_comment == "comment on license" assert package.copyright_text == "copyright" diff --git a/tests/spdx/parser/jsonlikedict/test_file_parser.py b/tests/spdx/parser/jsonlikedict/test_file_parser.py index e1c6b7a5e..bbe5b70ca 100644 --- a/tests/spdx/parser/jsonlikedict/test_file_parser.py +++ b/tests/spdx/parser/jsonlikedict/test_file_parser.py @@ -4,8 +4,8 @@ from unittest import TestCase import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import Checksum, ChecksumAlgorithm, FileType, SpdxNoAssertion, SpdxNone from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.dict_parsing_functions import parse_list_of_elements @@ -82,10 +82,10 @@ def test_parse_file(copyright_text, expected_copyright_text): "IBM Corporation", ], ) - assert file.license_concluded == Licensing().parse("(LGPL-2.0-only OR LicenseRef-2)") + assert file.license_concluded == spdx_licensing.parse("(LGPL-2.0-only OR LicenseRef-2)") TestCase().assertCountEqual( file.license_info_in_file, - [Licensing().parse("GPL-2.0-only"), Licensing().parse("LicenseRef-2"), SpdxNoAssertion()], + [spdx_licensing.parse("GPL-2.0-only"), spdx_licensing.parse("LicenseRef-2"), SpdxNoAssertion()], ) assert ( file.license_comment == "The concluded license was taken from the package level that the file was included in." diff --git a/tests/spdx/parser/jsonlikedict/test_license_expression_parser.py b/tests/spdx/parser/jsonlikedict/test_license_expression_parser.py index fca36b837..d78a88af3 100644 --- a/tests/spdx/parser/jsonlikedict/test_license_expression_parser.py +++ b/tests/spdx/parser/jsonlikedict/test_license_expression_parser.py @@ -16,6 +16,8 @@ [ ("First License", spdx_licensing.parse("First License")), ("Second License", spdx_licensing.parse("Second License")), + ("Apache-1.1", spdx_licensing.parse("Apache-1.1")), + ("Zlib", spdx_licensing.parse("zlib")), ("NOASSERTION", SpdxNoAssertion()), ("NONE", SpdxNone()), ], @@ -34,7 +36,8 @@ def test_parse_license_expression(license_expression_str, expected_license): ( "LGPL-2.1, GPL-2.0, GPL-3.0", [ - "Error parsing LicenseExpression: \"LGPL-2.1, GPL-2.0, GPL-3.0\": Invalid license key: the valid characters are: letters and numbers, underscore, dot, colon or hyphen signs and spaces: 'LGPL-2.1, GPL-2.0, GPL-3.0'" # noqa: E501 + # the error message we receive from the license_expression library somehow cuts off the last license + "Error parsing LicenseExpression: \"LGPL-2.1, GPL-2.0, GPL-3.0\": Invalid license key: the valid characters are: letters and numbers, underscore, dot, colon or hyphen signs and spaces: 'LGPL-2.1, GPL-2.0,'" # noqa: E501 ], ), ("Apache License (2.0)", ['Error parsing LicenseExpression: "Apache License (2.0)"']), diff --git a/tests/spdx/parser/jsonlikedict/test_package_parser.py b/tests/spdx/parser/jsonlikedict/test_package_parser.py index 5b933df8b..209cf5c0c 100644 --- a/tests/spdx/parser/jsonlikedict/test_package_parser.py +++ b/tests/spdx/parser/jsonlikedict/test_package_parser.py @@ -5,8 +5,8 @@ from unittest import TestCase import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import ( Actor, ActorType, @@ -173,17 +173,17 @@ def test_parse_package( ) assert package.homepage == expected_homepage assert package.source_info == "uses glibc-2_11-branch from git://sourceware.org/git/glibc.git." - assert package.license_concluded == Licensing().parse("(LGPL-2.0-only OR LicenseRef-3)") + assert package.license_concluded == spdx_licensing.parse("(LGPL-2.0-only OR LicenseRef-3)") TestCase().assertCountEqual( package.license_info_from_files, [ - Licensing().parse("GPL-2.0-only"), - Licensing().parse("LicenseRef-2"), - Licensing().parse("LicenseRef-1"), + spdx_licensing.parse("GPL-2.0-only"), + spdx_licensing.parse("LicenseRef-2"), + spdx_licensing.parse("LicenseRef-1"), SpdxNoAssertion(), ], ) - assert package.license_declared == Licensing().parse("(LGPL-2.0-only AND LicenseRef-3)") + assert package.license_declared == spdx_licensing.parse("(LGPL-2.0-only AND LicenseRef-3)") assert ( package.license_comment == "The license for this project changed with the release of version x.y. The version of the project included" diff --git a/tests/spdx/parser/jsonlikedict/test_snippet_parser.py b/tests/spdx/parser/jsonlikedict/test_snippet_parser.py index 257a00e3a..3b9b62a19 100644 --- a/tests/spdx/parser/jsonlikedict/test_snippet_parser.py +++ b/tests/spdx/parser/jsonlikedict/test_snippet_parser.py @@ -4,8 +4,8 @@ from unittest import TestCase import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.model import SpdxNoAssertion, SpdxNone from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.snippet_parser import SnippetParser @@ -65,8 +65,8 @@ def test_parse_snippet(copyright_text, expected_copyright_text): assert snippet.byte_range == (310, 420) assert snippet.line_range == (5, 23) assert snippet.file_spdx_id == "SPDXRef-DoapSource" - assert snippet.license_info_in_snippet == [Licensing().parse("GPL-2.0-only"), SpdxNoAssertion()] - assert snippet.license_concluded == Licensing().parse("GPL-2.0-only") + assert snippet.license_info_in_snippet == [spdx_licensing.parse("GPL-2.0-only"), SpdxNoAssertion()] + assert snippet.license_concluded == spdx_licensing.parse("GPL-2.0-only") assert snippet.attribution_texts == ["Some example attibution text."] diff --git a/tests/spdx/validation/test_package_validator.py b/tests/spdx/validation/test_package_validator.py index a6ef976ef..d0516fff2 100644 --- a/tests/spdx/validation/test_package_validator.py +++ b/tests/spdx/validation/test_package_validator.py @@ -6,8 +6,8 @@ from unittest import TestCase import pytest -from license_expression import Licensing +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.constants import DOCUMENT_SPDX_ID from spdx_tools.spdx.model import Relationship, RelationshipType, SpdxNoAssertion, SpdxNone from spdx_tools.spdx.validation.package_validator import validate_package, validate_package_within_document @@ -45,7 +45,7 @@ def test_valid_package(): ( package_fixture( files_analyzed=False, - license_info_from_files=[Licensing().parse("some_license")], + license_info_from_files=[spdx_licensing.parse("some_license")], verification_code=None, ), "license_info_from_files must be None if files_analyzed is False, but is: [LicenseSymbol('some_license', " From eded3dbf029d4bda3ac9bfbdfca98ddc75e08b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 26 Jul 2024 12:44:18 +0200 Subject: [PATCH 08/41] [issue-815] fix CI for Python 3.7 on MacOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.7 had to be removed completely as even the runner on older macos did not start. Signed-off-by: Armin Tänzer --- .github/workflows/check_codestyle.yml | 3 +++ .github/workflows/install_and_test.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/check_codestyle.yml b/.github/workflows/check_codestyle.yml index 15dfd17f9..92860a527 100644 --- a/.github/workflows/check_codestyle.yml +++ b/.github/workflows/check_codestyle.yml @@ -21,6 +21,9 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + exclude: # see https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 + - python-version: "3.7" + os: macos-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml index 4884b9493..038eb4717 100644 --- a/.github/workflows/install_and_test.yml +++ b/.github/workflows/install_and_test.yml @@ -18,6 +18,9 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + exclude: # see https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 + - python-version: "3.7" + os: macos-latest steps: - uses: actions/checkout@v3 From 40331ac14c3433b2ff81b58b3ad87929db86082e Mon Sep 17 00:00:00 2001 From: Stanislav Pankevich Date: Tue, 9 Jan 2024 20:57:37 +0100 Subject: [PATCH 09/41] spdx3: element_writer: unindent creation information Signed-off-by: Stanislav Pankevich --- .../spdx3/writer/console/console.py | 2 +- .../writer/tag_value/test_write_document.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/spdx3/writer/tag_value/test_write_document.py diff --git a/src/spdx_tools/spdx3/writer/console/console.py b/src/spdx_tools/spdx3/writer/console/console.py index a2ec81ed5..28b5f9cfa 100644 --- a/src/spdx_tools/spdx3/writer/console/console.py +++ b/src/spdx_tools/spdx3/writer/console/console.py @@ -22,6 +22,6 @@ def write_value(tag: str, value: Optional[Union[bool, str, dict, list, Enum]], o def write_and_possibly_indent(text: str, indent: bool, out: TextIO): if indent: - out.write(f"\t{text}\n") + out.write(f" {text}\n") else: out.write(f"{text}\n") diff --git a/tests/spdx3/writer/tag_value/test_write_document.py b/tests/spdx3/writer/tag_value/test_write_document.py new file mode 100644 index 000000000..580c61b81 --- /dev/null +++ b/tests/spdx3/writer/tag_value/test_write_document.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2024 spdx contributors +# +# SPDX-License-Identifier: Apache-2.0 +import io +from datetime import datetime + +from semantic_version import Version + +from spdx_tools.spdx3.model import CreationInfo, ProfileIdentifierType, SpdxDocument +from spdx_tools.spdx3.writer.console.spdx_document_writer import write_spdx_document + + +def test_render_creation_info(): + fake_datetime = datetime(year=2024, month=1, day=1) + spec_version = Version("3.0.0") + creation_info = CreationInfo( + spec_version=spec_version, + created=fake_datetime, + created_by=[], + profile=[ProfileIdentifierType.SOFTWARE], + ) + spdx_document = SpdxDocument( + spdx_id="SPDXRef-FOO", + name="BAR", + element=[], + root_element=[], + creation_info=creation_info, + ) + output_str = io.StringIO() + write_spdx_document(spdx_document, text_output=output_str) + + assert ( + output_str.getvalue() + == """\ +## SPDX Document +SPDXID: SPDXRef-FOO +name: BAR +# Creation Information + specVersion: 3.0.0 + created: 2024-01-01T00:00:00Z + profile: SOFTWARE + data license: CC0-1.0 +elements: +""" # noqa: W291 # elements: are printed with a space + ) From 321788330ce119eccfa1220c21036d8e45eec445 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Fri, 2 Aug 2024 15:43:09 +0200 Subject: [PATCH 10/41] [issue-788] fix tag-value parser to allow NONE and NOASSERTION for package source info as they are valid strings Signed-off-by: Meret Behrens # fixes 788 --- src/spdx_tools/spdx/parser/tagvalue/parser.py | 8 ++++++-- .../parser/tagvalue/test_package_parser.py | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/spdx_tools/spdx/parser/tagvalue/parser.py b/src/spdx_tools/spdx/parser/tagvalue/parser.py index a190dd7d7..ffa0dbd56 100644 --- a/src/spdx_tools/spdx/parser/tagvalue/parser.py +++ b/src/spdx_tools/spdx/parser/tagvalue/parser.py @@ -180,7 +180,7 @@ def p_current_element_error(self, p): "file_comment : FILE_COMMENT text_or_line\n " "file_license_concluded : FILE_LICENSE_CONCLUDED license_or_no_assertion_or_none\n " "package_name : PKG_NAME LINE\n description : PKG_DESCRIPTION text_or_line\n " - "summary : PKG_SUMMARY text_or_line\n source_info : PKG_SOURCE_INFO text_or_line\n " + "summary : PKG_SUMMARY text_or_line\n source_info : PKG_SOURCE_INFO text_or_line_including_no_assertion\n " "homepage : PKG_HOMEPAGE line_or_no_assertion_or_none\n " "download_location : PKG_DOWNLOAD_LOCATION line_or_no_assertion_or_none\n " "originator : PKG_ORIGINATOR actor_or_no_assertion\n supplier : PKG_SUPPLIER actor_or_no_assertion\n " @@ -216,7 +216,11 @@ def p_unknown_tag(self, p): def p_text(self, p): p[0] = str_from_text(p[1]) - @grammar_rule("text_or_line : LINE\n line_or_no_assertion : LINE\nline_or_no_assertion_or_none : text_or_line") + @grammar_rule( + "text_or_line : LINE\n line_or_no_assertion : LINE\nline_or_no_assertion_or_none : text_or_line\n" + "text_or_line_including_no_assertion : text_or_line\ntext_or_line_including_no_assertion : NO_ASSERTION\n" + "text_or_line_including_no_assertion : NONE" + ) def p_line(self, p): p[0] = p[1] diff --git a/tests/spdx/parser/tagvalue/test_package_parser.py b/tests/spdx/parser/tagvalue/test_package_parser.py index 470f1e2f7..18524262c 100644 --- a/tests/spdx/parser/tagvalue/test_package_parser.py +++ b/tests/spdx/parser/tagvalue/test_package_parser.py @@ -83,6 +83,26 @@ def test_parse_package(): assert package.valid_until_date == datetime(2022, 1, 1, 12) +def test_parse_package_with_no_assertion_as_source_info(): + parser = Parser() + package_str = "\n".join( + [ + "PackageName: Test", + "SPDXID: SPDXRef-Package", + "PackageDownloadLocation: http://example.com/test", + "FilesAnalyzed: true", + "PackageSummary: Test package", + "PackageSourceInfo: NOASSERTION", + ] + ) + document = parser.parse("\n".join([DOCUMENT_STR, package_str])) + assert document is not None + package = document.packages[0] + assert package.name == "Test" + assert package.spdx_id == "SPDXRef-Package" + assert package.source_info == "NOASSERTION" + + @pytest.mark.parametrize( "package_str, expected_message", [ From f6fcf20770de2acdeef68ba74362ce45bc9111ab Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Fri, 2 Aug 2024 16:27:08 +0200 Subject: [PATCH 11/41] [issue-815] Add legacy os macos-13 to CI Signed-off-by: Meret Behrens --- .github/workflows/check_codestyle.yml | 3 +++ .github/workflows/install_and_test.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/check_codestyle.yml b/.github/workflows/check_codestyle.yml index 92860a527..4cc16b9c1 100644 --- a/.github/workflows/check_codestyle.yml +++ b/.github/workflows/check_codestyle.yml @@ -24,6 +24,9 @@ jobs: exclude: # see https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 - python-version: "3.7" os: macos-latest + include: + - python-version: "3.7" + os: macos-13 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml index 038eb4717..0f66cc73f 100644 --- a/.github/workflows/install_and_test.yml +++ b/.github/workflows/install_and_test.yml @@ -21,6 +21,9 @@ jobs: exclude: # see https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 - python-version: "3.7" os: macos-latest + include: + - python-version: "3.7" + os: macos-13 steps: - uses: actions/checkout@v3 From 279f27165b4d83263a09a0a950ddb78b3eacfaee Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Fri, 2 Aug 2024 16:39:15 +0200 Subject: [PATCH 12/41] [issue-788] fix tag-value parser to allow NONE and NOASSERTION for package source info as they are valid strings Signed-off-by: Meret Behrens --- src/spdx_tools/spdx/parser/tagvalue/parser.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/spdx_tools/spdx/parser/tagvalue/parser.py b/src/spdx_tools/spdx/parser/tagvalue/parser.py index ffa0dbd56..50096bda2 100644 --- a/src/spdx_tools/spdx/parser/tagvalue/parser.py +++ b/src/spdx_tools/spdx/parser/tagvalue/parser.py @@ -180,7 +180,7 @@ def p_current_element_error(self, p): "file_comment : FILE_COMMENT text_or_line\n " "file_license_concluded : FILE_LICENSE_CONCLUDED license_or_no_assertion_or_none\n " "package_name : PKG_NAME LINE\n description : PKG_DESCRIPTION text_or_line\n " - "summary : PKG_SUMMARY text_or_line\n source_info : PKG_SOURCE_INFO text_or_line_including_no_assertion\n " + "summary : PKG_SUMMARY text_or_line\n source_info : PKG_SOURCE_INFO text_or_line\n " "homepage : PKG_HOMEPAGE line_or_no_assertion_or_none\n " "download_location : PKG_DOWNLOAD_LOCATION line_or_no_assertion_or_none\n " "originator : PKG_ORIGINATOR actor_or_no_assertion\n supplier : PKG_SUPPLIER actor_or_no_assertion\n " @@ -212,14 +212,13 @@ def p_generic_value(self, p): def p_unknown_tag(self, p): self.logger.append(f"Unknown tag provided in line {p.lineno(1)}") - @grammar_rule("text_or_line : TEXT") + @grammar_rule("text_or_line : TEXT\n line_or_no_assertion_or_none : TEXT") def p_text(self, p): p[0] = str_from_text(p[1]) @grammar_rule( - "text_or_line : LINE\n line_or_no_assertion : LINE\nline_or_no_assertion_or_none : text_or_line\n" - "text_or_line_including_no_assertion : text_or_line\ntext_or_line_including_no_assertion : NO_ASSERTION\n" - "text_or_line_including_no_assertion : NONE" + "text_or_line : LINE\n line_or_no_assertion : LINE\nline_or_no_assertion_or_none : LINE\n" + "text_or_line : NO_ASSERTION\n text_or_line : NONE" ) def p_line(self, p): p[0] = p[1] From 1973ce1cc74c1da65e2b4d2841ae17f84c2e5bd0 Mon Sep 17 00:00:00 2001 From: Meret Behrens Date: Fri, 16 Aug 2024 08:49:32 +0200 Subject: [PATCH 13/41] [issue-788] Improve test Signed-off-by: Meret Behrens --- .../parser/tagvalue/test_package_parser.py | 20 ------------- .../parser/tagvalue/test_tag_value_parser.py | 28 ++++++++++++++++++- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/tests/spdx/parser/tagvalue/test_package_parser.py b/tests/spdx/parser/tagvalue/test_package_parser.py index 18524262c..470f1e2f7 100644 --- a/tests/spdx/parser/tagvalue/test_package_parser.py +++ b/tests/spdx/parser/tagvalue/test_package_parser.py @@ -83,26 +83,6 @@ def test_parse_package(): assert package.valid_until_date == datetime(2022, 1, 1, 12) -def test_parse_package_with_no_assertion_as_source_info(): - parser = Parser() - package_str = "\n".join( - [ - "PackageName: Test", - "SPDXID: SPDXRef-Package", - "PackageDownloadLocation: http://example.com/test", - "FilesAnalyzed: true", - "PackageSummary: Test package", - "PackageSourceInfo: NOASSERTION", - ] - ) - document = parser.parse("\n".join([DOCUMENT_STR, package_str])) - assert document is not None - package = document.packages[0] - assert package.name == "Test" - assert package.spdx_id == "SPDXRef-Package" - assert package.source_info == "NOASSERTION" - - @pytest.mark.parametrize( "package_str, expected_message", [ diff --git a/tests/spdx/parser/tagvalue/test_tag_value_parser.py b/tests/spdx/parser/tagvalue/test_tag_value_parser.py index 9f347fc08..c866e2726 100644 --- a/tests/spdx/parser/tagvalue/test_tag_value_parser.py +++ b/tests/spdx/parser/tagvalue/test_tag_value_parser.py @@ -6,7 +6,7 @@ import pytest from spdx_tools.spdx.constants import DOCUMENT_SPDX_ID -from spdx_tools.spdx.model import Relationship, RelationshipType +from spdx_tools.spdx.model import Relationship, RelationshipType, SpdxNoAssertion, SpdxNone from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.tagvalue.parser import Parser from tests.spdx.parser.tagvalue.test_creation_info_parser import DOCUMENT_STR @@ -136,3 +136,29 @@ def test_faulty_license_expression(): "and numbers, underscore, dot, colon or hyphen signs and spaces: " "'LicenseRef-foo/foo'\"]", ] + + +def test_parse_none_or_no_assertion_as_text(): + parser = Parser() + document_str = "\n".join( + [ + DOCUMENT_STR, + "PackageName: Test", + "SPDXID: SPDXRef-Package", + "PackageDownloadLocation: http://example.com/test", + "FilesAnalyzed: true", + "PackageSummary: NONE", + "PackageSourceInfo: NOASSERTION", + "PackageLicenseConcluded: NONE", + "PackageLicenseDeclared: NOASSERTION", + ] + ) + document = parser.parse(document_str) + assert document is not None + package = document.packages[0] + assert package.name == "Test" + assert package.spdx_id == "SPDXRef-Package" + assert package.source_info == "NOASSERTION" + assert package.summary == "NONE" + assert package.license_concluded == SpdxNone() + assert package.license_declared == SpdxNoAssertion() From a25937fa18ef3cfc57192502924a6ae723e7057d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 27 Sep 2024 09:29:31 +0200 Subject: [PATCH 14/41] update changelog for 0.8.3 release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e55dfaa1d..f17ee022c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## v0.8.3 (2024-09-27) + +### New features and changes + +* tag-value parser: + * fixed license expression error handling + * fixed parsing Tool or Organization as annotator + * allow "NONE" and "NOASSERTION" as strings for fields requiring text +* CLI tool: + * shortened the output of the FileNotFoundError + * now catches decoding errors while parsing +* fixed tag-value output when the ID of a related SPDX element is "NONE" or "NOASSERTION" +* spdx3: + * added REQUIREMENT type to software_purpose + * unindent creation information in element_writer +* internal: + * replaced remaining occurrences of Licensing() with spdx_licensing + * fixed CI not working for Python 3.7 on latest MacOS + +### Contributors + +This release was made possible by the following contributors. Thank you very much! + +* Stanislav Pankevich @stanislaw +* Meret Behrens @meretp +* Maximilian Huber @maxhbr +* Armin Tänzer @armintaenzertng + ## v0.8.2 (2023-10-12) ### New features and changes From 0677ae7d31a0361d49df1a69dfd83c9fb27a39f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 11 Apr 2025 14:17:46 +0200 Subject: [PATCH 15/41] [issue-839] Drop support for Python 3.7 (we are long past its EOL) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- .github/workflows/check_codestyle.yml | 8 +------- .github/workflows/install_and_test.yml | 8 +------- .github/workflows/prepare_release.yml | 2 +- pyproject.toml | 3 +-- 4 files changed, 4 insertions(+), 17 deletions(-) diff --git a/.github/workflows/check_codestyle.yml b/.github/workflows/check_codestyle.yml index 4cc16b9c1..08b5e125f 100644 --- a/.github/workflows/check_codestyle.yml +++ b/.github/workflows/check_codestyle.yml @@ -20,13 +20,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] - exclude: # see https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 - - python-version: "3.7" - os: macos-latest - include: - - python-version: "3.7" - os: macos-13 + python-version: [ "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml index 0f66cc73f..2149dd267 100644 --- a/.github/workflows/install_and_test.yml +++ b/.github/workflows/install_and_test.yml @@ -17,13 +17,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] - exclude: # see https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 - - python-version: "3.7" - os: macos-latest - include: - - python-version: "3.7" - os: macos-13 + python-version: [ "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml index 1197a3d1d..cb2f5a67f 100644 --- a/.github/workflows/prepare_release.yml +++ b/.github/workflows/prepare_release.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: '3.8' - name: Set up dependencies run: | python -m pip install --upgrade pip diff --git a/pyproject.toml b/pyproject.toml index aff57e36b..bab596e20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,14 +16,13 @@ classifiers = [ "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] urls = { Homepage = "https://github.com/spdx/tools-python" } -requires-python = ">=3.7" +requires-python = ">=3.8" dependencies = ["click", "pyyaml", "xmltodict", "rdflib", "beartype", "uritools", "license_expression", "ply", "semantic_version"] dynamic = ["version"] From f19674c7a6af2c687b5d4435334b9780089aac19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 11 Apr 2025 15:10:11 +0200 Subject: [PATCH 16/41] Add pyshacl as development dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise, pytest will complain. Signed-off-by: Armin Tänzer --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bab596e20..004e67347 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dynamic = ["version"] test = ["pytest", "pyshacl", "tzdata"] code_style = ["isort", "black", "flake8"] graph_generation = ["pygraphviz", "networkx"] -development = ["black", "flake8", "isort", "networkx", "pytest"] +development = ["black", "flake8", "isort", "networkx", "pytest", "pyshacl"] [project.scripts] pyspdxtools = "spdx_tools.spdx.clitools.pyspdxtools:main" From 6817ca7056cadfd71c5619b7d75452461e4bbb18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 11 Apr 2025 15:12:56 +0200 Subject: [PATCH 17/41] [issue-844] Update excepted typecheck error after beartype update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/spdx_tools/common/typing/dataclass_with_properties.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spdx_tools/common/typing/dataclass_with_properties.py b/src/spdx_tools/common/typing/dataclass_with_properties.py index 3f13950d5..e0b5c9614 100644 --- a/src/spdx_tools/common/typing/dataclass_with_properties.py +++ b/src/spdx_tools/common/typing/dataclass_with_properties.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from beartype import beartype -from beartype.roar import BeartypeCallHintException +from beartype.roar import BeartypeCallHintParamViolation def dataclass_with_properties(cls): @@ -30,7 +30,7 @@ def set_field(self, value: field_type): def set_field_with_error_conversion(self, value: field_type): try: set_field(self, value) - except BeartypeCallHintException as err: + except BeartypeCallHintParamViolation as err: error_message: str = f"SetterError {self.__class__.__name__}: {err}" # As setters are created dynamically, their argument name is always "value". We replace it by the From 8dc336f783e993d7e347d20b8ecd50b8808abf70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 11 Apr 2025 15:29:36 +0200 Subject: [PATCH 18/41] Small formatting fixes after black update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/spdx_tools/spdx/parser/jsonlikedict/snippet_parser.py | 2 +- src/spdx_tools/spdx3/bump_from_spdx2/relationship.py | 2 +- src/spdx_tools/spdx3/writer/console/__init__.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/spdx_tools/spdx/parser/jsonlikedict/snippet_parser.py b/src/spdx_tools/spdx/parser/jsonlikedict/snippet_parser.py index f00779407..3914e686b 100644 --- a/src/spdx_tools/spdx/parser/jsonlikedict/snippet_parser.py +++ b/src/spdx_tools/spdx/parser/jsonlikedict/snippet_parser.py @@ -127,7 +127,7 @@ def validate_pointer_and_get_type(pointer: Dict) -> RangeType: @staticmethod def convert_range_from_str( - _range: Tuple[Union[int, str], Union[int, str]] + _range: Tuple[Union[int, str], Union[int, str]], ) -> Tuple[Union[int, str], Union[int, str]]: # XML does not support integers, so we have to convert from string (if possible) if not _range: diff --git a/src/spdx_tools/spdx3/bump_from_spdx2/relationship.py b/src/spdx_tools/spdx3/bump_from_spdx2/relationship.py index 918cbdafb..dd6909c49 100644 --- a/src/spdx_tools/spdx3/bump_from_spdx2/relationship.py +++ b/src/spdx_tools/spdx3/bump_from_spdx2/relationship.py @@ -218,7 +218,7 @@ def bump_relationship( def determine_completeness_and_to( - related_spdx_element_id: Union[str, SpdxNone, SpdxNoAssertion] + related_spdx_element_id: Union[str, SpdxNone, SpdxNoAssertion], ) -> Tuple[Optional[RelationshipCompleteness], List[str]]: if isinstance(related_spdx_element_id, SpdxNoAssertion): completeness = RelationshipCompleteness.NOASSERTION diff --git a/src/spdx_tools/spdx3/writer/console/__init__.py b/src/spdx_tools/spdx3/writer/console/__init__.py index 39bbda884..46eabf875 100644 --- a/src/spdx_tools/spdx3/writer/console/__init__.py +++ b/src/spdx_tools/spdx3/writer/console/__init__.py @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: 2023 spdx contributors # # SPDX-License-Identifier: Apache-2.0 -""" This is a temporary package to write the implemented model of spdx_tools.spdx3.0 to console. As soon as - serialization formats are properly defined this package can be deleted.""" +"""This is a temporary package to write the implemented model of spdx_tools.spdx3.0 to console. As soon as +serialization formats are properly defined this package can be deleted.""" From aa225fbc5b599a52880708a0c3a0c48e34fc2263 Mon Sep 17 00:00:00 2001 From: Zalan Blenessy Date: Tue, 1 Jul 2025 11:20:55 +0200 Subject: [PATCH 19/41] [issue-854] Remove some control characters from JSON SPDX Signed-off-by: Zalan Blenessy --- .../spdx/parser/json/json_parser.py | 23 +++++++++- tests/spdx/data/ControlCharacters.spdx.json | 46 +++++++++++++++++++ .../parser/jsonlikedict/test_json_parser.py | 11 +++++ 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 tests/spdx/data/ControlCharacters.spdx.json create mode 100644 tests/spdx/parser/jsonlikedict/test_json_parser.py diff --git a/src/spdx_tools/spdx/parser/json/json_parser.py b/src/spdx_tools/spdx/parser/json/json_parser.py index 219ccfed2..675ed2560 100644 --- a/src/spdx_tools/spdx/parser/json/json_parser.py +++ b/src/spdx_tools/spdx/parser/json/json_parser.py @@ -3,14 +3,33 @@ # SPDX-License-Identifier: Apache-2.0 import json -from beartype.typing import Dict +from beartype.typing import Any, Dict from spdx_tools.spdx.model import Document from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser +# chars we don't want to see in SBOMs +CONTROL_CHARS_MAP = { + 8: None, # ASCII/UTF-8: backspace + 12: None, # ASCII/UTF-8: formfeed +} + + +def remove_control_chars_from_value(value: Any) -> Any: + if isinstance(value, str): + return value.translate(CONTROL_CHARS_MAP) + elif isinstance(value, list): + for i in range(len(value)): + value[i] = remove_control_chars_from_value(value[i]) + return value + + +def remove_json_control_chars_hook(pairs: list) -> dict: + return {k: remove_control_chars_from_value(v) for k, v in pairs} + def parse_from_file(file_name: str, encoding: str = "utf-8") -> Document: with open(file_name, encoding=encoding) as file: - input_doc_as_dict: Dict = json.load(file) + input_doc_as_dict: Dict = json.load(file, object_pairs_hook=remove_json_control_chars_hook) return JsonLikeDictParser().parse(input_doc_as_dict) diff --git a/tests/spdx/data/ControlCharacters.spdx.json b/tests/spdx/data/ControlCharacters.spdx.json new file mode 100644 index 000000000..d79ee2e92 --- /dev/null +++ b/tests/spdx/data/ControlCharacters.spdx.json @@ -0,0 +1,46 @@ +{ + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "creationInfo": { + "created": "2020-11-24T01:12:27Z", + "creators": ["Person: Nisha \b\f K (nishak@vmware.com)"] + }, + "name": "golang-dist", + "documentNamespace": "https://swinslow.net/spdx-examples/example7/golang-dist-492dfde4-318b-49f7-b48c-934bfafbde48", + "documentDescribes": ["SPDXRef-golang-dist"], + "packages": [ + { + "name": "go1.16.4.linux-amd64", + "SPDXID": "SPDXRef-golang-dist", + "downloadLocation": "https://golang.org/dl/go1.16.4.linux-amd64.tar.gz", + "versionInfo": "1.16.4", + "filesAnalyzed": false, + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "7154e88f5a8047aad4b80ebace58a059e36e7e2e4eb3b383127a28c711b4ff59" + } + ], + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "LicenseRef-Golang-BSD-plus-Patents", + "copyrightText": "Copyright (c) 2009 The Go Authors. \b All rights reserved." + }, + { + "name": "go", + "SPDXID": "SPDXRef-go-compiler", + "downloadLocation": "https://golang.org/dl/go1.16.4.linux-amd64.tar.gz", + "versionInfo": "1.16.4", + "filesAnalyzed": false, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ], + "hasExtractedLicensingInfos": [ + { + "licenseId": "LicenseRef-Golang-BSD-plus-Patents", + "extractedText": "Golang BSD plus Patents \"\\\/\b\f\n\r\t" + } + ] +} diff --git a/tests/spdx/parser/jsonlikedict/test_json_parser.py b/tests/spdx/parser/jsonlikedict/test_json_parser.py new file mode 100644 index 000000000..ab3249d63 --- /dev/null +++ b/tests/spdx/parser/jsonlikedict/test_json_parser.py @@ -0,0 +1,11 @@ +import os + +from spdx_tools.spdx.parser.json import json_parser + + +def test_parse_control_characters(): + doc = json_parser.parse_from_file( + os.path.join(os.path.dirname(__file__), "../../data/ControlCharacters.spdx.json") + ) + assert doc.creation_info.creators[0].name == "Nisha K" + assert doc.extracted_licensing_info[0].extracted_text == 'Golang BSD plus Patents "\\/\n\r\t' From 47751b680e4cea0f9ad035d37619f598c34af736 Mon Sep 17 00:00:00 2001 From: konsulten Date: Mon, 30 Sep 2024 13:56:44 +0200 Subject: [PATCH 20/41] downloadLocation URIs not case sensitive Signed-off-by: Claes Nordmark --- src/spdx_tools/spdx/validation/uri_validators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spdx_tools/spdx/validation/uri_validators.py b/src/spdx_tools/spdx/validation/uri_validators.py index c14d196f4..b2bc91692 100644 --- a/src/spdx_tools/spdx/validation/uri_validators.py +++ b/src/spdx_tools/spdx/validation/uri_validators.py @@ -18,6 +18,7 @@ download_location_pattern = ( "^(((" + supported_download_repos + "\\+)?" + url_pattern + ")|" + git_pattern + "|" + bazaar_pattern + ")$" ) +compiled_pattern = re.compile(download_location_pattern, re.IGNORECASE) def validate_url(url: str) -> List[str]: @@ -28,7 +29,7 @@ def validate_url(url: str) -> List[str]: def validate_download_location(location: str) -> List[str]: - if not (validate_url(location) == [] or re.match(download_location_pattern, location)): + if not (validate_url(location) == [] or compiled_pattern.match(location)): return [f"must be a valid URL or download location according to the specification, but is: {location}"] return [] From 0b5210acc5a60c44a857259b6356973196feefbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 8 Jul 2025 10:11:33 +0200 Subject: [PATCH 21/41] [issue-825] Allow capital letters in validate_url() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/spdx_tools/spdx/validation/uri_validators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/spdx_tools/spdx/validation/uri_validators.py b/src/spdx_tools/spdx/validation/uri_validators.py index b2bc91692..7720c3fb5 100644 --- a/src/spdx_tools/spdx/validation/uri_validators.py +++ b/src/spdx_tools/spdx/validation/uri_validators.py @@ -12,6 +12,8 @@ "\\/\\/|ftp:\\/\\/)?([\\w\\-.!~*'()%;:&=+$,]+@)?[a-z0-9]+([\\-\\.]{1}[a-z0-9]+){0,100}\\.[a-z]{2,5}" "(:[0-9]{1,5})?(\\/.*)?" ) +url_pattern_ignore_case = re.compile(url_pattern, re.IGNORECASE) + supported_download_repos: str = "(git|hg|svn|bzr)" git_pattern = "(git\\+git@[a-zA-Z0-9\\.\\-]+:[a-zA-Z0-9/\\\\.@\\-]+)" bazaar_pattern = "(bzr\\+lp:[a-zA-Z0-9\\.\\-]+)" @@ -22,7 +24,7 @@ def validate_url(url: str) -> List[str]: - if not re.match(url_pattern, url): + if not url_pattern_ignore_case.match(url): return [f"must be a valid URL, but is: {url}"] return [] From b7f9a3defe8b0af1414cd75f1447ec242428f1d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 8 Jul 2025 10:12:29 +0200 Subject: [PATCH 22/41] [issue-825] Add tests for allowing capital letters in URL/URIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- tests/spdx/validation/test_uri_validators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/spdx/validation/test_uri_validators.py b/tests/spdx/validation/test_uri_validators.py index 2d374ee8e..0fb4ec7cb 100644 --- a/tests/spdx/validation/test_uri_validators.py +++ b/tests/spdx/validation/test_uri_validators.py @@ -14,6 +14,7 @@ "https://spdx.org/spdxdocs/spdx-tools-v1.2-3F2504E0-4F89-41D3-9A0C-0305E82...", "http://some.url", "http://ftp.gnu.org/gnu/glibc/glibc-ports-2.15.tar.gz", + "HTTP://SOME.URL", ], ) def test_valid_url(input_value): @@ -79,6 +80,7 @@ def test_invalid_url(input_value): "bzr+https://bzr.myproject.org/MyProject/trunk@2019", "bzr+http://bzr.myproject.org/MyProject/trunk@v1.0", "bzr+https://bzr.myproject.org/MyProject/trunk@2019#src/somefile.c", + "BZR+HTTPS://BZR.MYPROJECT.ORG/MYPROJECT/TRUNK@2019#SRC/SOMEFILE.C", ], ) def test_valid_package_download_location(input_value): @@ -106,6 +108,7 @@ def test_invalid_package_download_location(input_value): "https://spdx.org/spdxdocs/spdx-tools-v1.2-3F2504E0-4F89-41D3-9A0C-0305E82...", "h://someweirdtest^?", "https://some.uri that goes on!?", + "HTtPS://SOME.URI With CAPITALS", ], ) def test_valid_uri(input_value): From a0016e2d7fe8b2a72126a10eb36b470bfbf074da Mon Sep 17 00:00:00 2001 From: Olivier Benjamin Date: Fri, 5 Sep 2025 14:55:05 +0200 Subject: [PATCH 23/41] README.md: State that the experimental SPDX3 support is write-only. As described in issue #760, the tools do not currently support reading SPDX3.0 documents. This makes it clearer in the README. Signed-off-by: Olivier Benjamin (Schneider Electric) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 99112ea02..e089bd138 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ The main features of v0.8 are: stable at this point, as the spec is still evolving. SPDX3-related code is contained in a separate subpackage "spdx3" and its use is optional. We do not recommend using it in production code yet. +Note that v0.8 only supports **writing**, not **reading** SPDX 3.0 documents. +See [#760](https://github.com/spdx/tools-python/issues/760) for details. # Information From 92a62cf78db49a30a3b3b84a8c70058d0448502b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 9 Jan 2026 14:54:10 +0100 Subject: [PATCH 24/41] update GitHub Action: upload pages artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * v1 uses the deprecated version v3 of upload artifact Signed-off-by: Armin Tänzer --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 095f9ec7a..95530f802 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,7 +28,7 @@ jobs: - name: Generate docs run: pdoc spdx_tools -o docs/ - name: Upload docs as artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@v3 with: path: docs/ From 8a4c3ecc704f972e4c8af81c7ffae67e2329ce63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 9 Jan 2026 15:03:05 +0100 Subject: [PATCH 25/41] update GitHub workflows to Python 3.14, drop support for 3.8 and 3.9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Python 3.9 reached end of life in 2025 Signed-off-by: Armin Tänzer --- .github/workflows/check_codestyle.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/install_and_test.yml | 2 +- .github/workflows/integration_test.yml | 4 ++-- .github/workflows/prepare_release.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check_codestyle.yml b/.github/workflows/check_codestyle.yml index 08b5e125f..327f2f54a 100644 --- a/.github/workflows/check_codestyle.yml +++ b/.github/workflows/check_codestyle.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.14" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 95530f802..be0d749b1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -19,7 +19,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.14' - name: Install dependencies run: | sudo apt-get install graphviz-dev diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml index 2149dd267..d661745be 100644 --- a/.github/workflows/install_and_test.yml +++ b/.github/workflows/install_and_test.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index 2a37fa7a6..3a749a3f0 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -13,10 +13,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.11 + - name: Set up Python 3.14 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.14 - name: Installation run: | python -m pip install --upgrade pip diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml index cb2f5a67f..15439f41f 100644 --- a/.github/workflows/prepare_release.yml +++ b/.github/workflows/prepare_release.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.14' - name: Set up dependencies run: | python -m pip install --upgrade pip From e9019e039eae296ee99b506597a97fdafdb88d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 9 Jan 2026 15:08:28 +0100 Subject: [PATCH 26/41] [issue-866] fix annotation access from class instance to class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was a bug in the old code that only now throws an error in Python 3.14. Signed-off-by: Armin Tänzer --- src/spdx_tools/spdx3/writer/console/relationship_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spdx_tools/spdx3/writer/console/relationship_writer.py b/src/spdx_tools/spdx3/writer/console/relationship_writer.py index 1a8b16a3e..9f61e5214 100644 --- a/src/spdx_tools/spdx3/writer/console/relationship_writer.py +++ b/src/spdx_tools/spdx3/writer/console/relationship_writer.py @@ -12,5 +12,5 @@ def write_relationship(relationship: Relationship, text_output: TextIO, heading: if heading: text_output.write("## Relationship\n") write_element_properties(relationship, text_output) - for property_name in relationship.__annotations__.keys(): + for property_name in Relationship.__annotations__.keys(): write_value(property_name, getattr(relationship, property_name), text_output) From ed5d515a47370eb98c2941d2ca4a04f2093e5704 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 26 Nov 2024 13:00:07 +0000 Subject: [PATCH 27/41] Add return type Document for parse_file Signed-off-by: Arthit Suriyawongkul --- src/spdx_tools/spdx/parser/parse_anything.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spdx_tools/spdx/parser/parse_anything.py b/src/spdx_tools/spdx/parser/parse_anything.py index ff41e2b77..b47dc50dc 100644 --- a/src/spdx_tools/spdx/parser/parse_anything.py +++ b/src/spdx_tools/spdx/parser/parse_anything.py @@ -12,6 +12,7 @@ import logging from spdx_tools.spdx.formats import FileFormat, file_name_to_format +from spdx_tools.spdx.model import Document from spdx_tools.spdx.parser.json import json_parser from spdx_tools.spdx.parser.rdf import rdf_parser from spdx_tools.spdx.parser.tagvalue import tagvalue_parser @@ -19,7 +20,7 @@ from spdx_tools.spdx.parser.yaml import yaml_parser -def parse_file(file_name: str, encoding: str = "utf-8"): +def parse_file(file_name: str, encoding: str = "utf-8") -> Document: if encoding != "utf-8": logging.warning( "It's recommended to use the UTF-8 encoding for any SPDX file. Consider changing the encoding of the file." From 7636b1b686f5b7541cfa7d67b27461c7b23e0a1e Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 23 Apr 2025 20:37:36 +0100 Subject: [PATCH 28/41] Use import.resources instead of __file__ Potentially fix #257 and allow zip-safe = true in pyproject.toml Signed-off-by: Arthit Suriyawongkul --- .../spdx3/writer/json_ld/json_ld_writer.py | 12 +++++++---- .../spdx3/writer/json_ld/owl_to_context.py | 9 +++++--- tests/spdx/test_cli.py | 21 ++++++++++--------- .../writer/json_ld/test_json_ld_writer.py | 5 +++-- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/spdx_tools/spdx3/writer/json_ld/json_ld_writer.py b/src/spdx_tools/spdx3/writer/json_ld/json_ld_writer.py index 69a4d763c..3c3e0819a 100644 --- a/src/spdx_tools/spdx3/writer/json_ld/json_ld_writer.py +++ b/src/spdx_tools/spdx3/writer/json_ld/json_ld_writer.py @@ -2,20 +2,24 @@ # # SPDX-License-Identifier: Apache-2.0 import json -import os +from importlib import resources from spdx_tools.spdx3.payload import Payload -from spdx_tools.spdx3.writer.json_ld.json_ld_converter import convert_payload_to_json_ld_list_of_elements +from spdx_tools.spdx3.writer.json_ld.json_ld_converter import ( + convert_payload_to_json_ld_list_of_elements, +) def write_payload(payload: Payload, file_name: str): element_list = convert_payload_to_json_ld_list_of_elements(payload) # this will be obsolete as soon as the context is publicly available under some URI - with open(os.path.join(os.path.dirname(__file__), "context.json"), "r") as infile: + # Note: 3.0.1 context is now available at + # https://spdx.org/rdf/3.0.1/spdx-context.jsonld + with resources.files("spdx_tools.spdx3.writer.json_ld").joinpath("context.json").open("r") as infile: context = json.load(infile) complete_dict = {"@context": context, "@graph": element_list} - with open(file_name + ".jsonld", "w") as out: + with open(file_name + ".jsonld", "w", encoding="utf-8") as out: json.dump(complete_dict, out, indent=2) diff --git a/src/spdx_tools/spdx3/writer/json_ld/owl_to_context.py b/src/spdx_tools/spdx3/writer/json_ld/owl_to_context.py index f6bc7c35a..cdeb3144e 100644 --- a/src/spdx_tools/spdx3/writer/json_ld/owl_to_context.py +++ b/src/spdx_tools/spdx3/writer/json_ld/owl_to_context.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 import json -import os.path +from importlib import resources # current workflow: markdown files + spec_parser -> model.ttl -> convert to json_ld: SPDX_OWL.json -> # use the function below to generate context.json @@ -108,8 +108,11 @@ def convert_spdx_owl_to_jsonld_context(spdx_owl: str = "SPDX_OWL.json"): else: print(f"unknown node_type: {node_type}") - with open(os.path.join(os.path.dirname(__file__), "context.json"), "w") as infile: - json.dump(context_dict, infile) + with resources.as_file( + resources.files("spdx_tools.spdx3.writer.json_ld").joinpath("context.json") + ) as context_path: + with open(context_path, "w", encoding="utf-8") as outfile: + json.dump(context_dict, outfile) if __name__ == "__main__": diff --git a/tests/spdx/test_cli.py b/tests/spdx/test_cli.py index 0019cf3a3..52ea85cf5 100644 --- a/tests/spdx/test_cli.py +++ b/tests/spdx/test_cli.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023 spdx contributors # # SPDX-License-Identifier: Apache-2.0 -import os +from importlib import resources import pytest from click.testing import CliRunner @@ -12,16 +12,16 @@ @pytest.mark.parametrize( "options", [ - ("--infile", os.path.join(os.path.dirname(__file__), "data/SPDXJSONExample-v2.3.spdx.json")), - ("-i", os.path.join(os.path.dirname(__file__), "data/SPDXJSONExample-v2.3.spdx.json"), "--novalidation"), + ("--infile", str(resources.files("tests.spdx.data").joinpath("SPDXJSONExample-v2.3.spdx.json"))), + ("-i", str(resources.files("tests.spdx.data").joinpath("SPDXJSONExample-v2.3.spdx.json")), "--novalidation"), ( "-i", - os.path.join(os.path.dirname(__file__), "data/SPDXJSONExample-v2.3.spdx.json"), + str(resources.files("tests.spdx.data").joinpath("SPDXJSONExample-v2.3.spdx.json")), "--novalidation", "--version", "SPDX-2.3", ), - ("-i", os.path.join(os.path.dirname(__file__), "data/SPDXJSONExample-v2.3.spdx.json"), "-o", "-"), + ("-i", str(resources.files("tests.spdx.data").joinpath("SPDXJSONExample-v2.3.spdx.json")), "-o", "-"), ], ) def test_cli_with_system_exit_code_0(options): @@ -37,9 +37,10 @@ def test_cli_with_system_exit_code_0(options): [ ( "-i", - os.path.join( - os.path.dirname(__file__), - "data/invalid/spdx-trivy-vmware_log-intelligence-fluentd-sha256_086af034f561f343f633be9d9f9e95f65ae6c61b8ddb2c6755ef5bb25b40f53a.json", # noqa: E501 + str( + resources.files("tests.spdx.data.invalid").joinpath( + "spdx-trivy-vmware_log-intelligence-fluentd-sha256_086af034f561f343f633be9d9f9e95f65ae6c61b8ddb2c6755ef5bb25b40f53a.json" + ) ), ), ("-i", "non_existent_file.spdx"), @@ -57,8 +58,8 @@ def test_cli_with_system_exit_code_1(options): "options", [ (), - ("-i", os.path.join(os.path.dirname(__file__), "data/SPDXJSONExample-v2.3.spdx.json"), "--version"), - ("-i", os.path.join(os.path.dirname(__file__), "data/SPDXJSONExample-v2.3.spdx.json"), "-o"), + ("-i", str(resources.files("tests.spdx.data").joinpath("SPDXJSONExample-v2.3.spdx.json")), "--version"), + ("-i", str(resources.files("tests.spdx.data").joinpath("SPDXJSONExample-v2.3.spdx.json")), "-o"), ], ) def test_cli_with_system_exit_code_2(options): diff --git a/tests/spdx3/writer/json_ld/test_json_ld_writer.py b/tests/spdx3/writer/json_ld/test_json_ld_writer.py index f46469b1f..4718f3e6e 100644 --- a/tests/spdx3/writer/json_ld/test_json_ld_writer.py +++ b/tests/spdx3/writer/json_ld/test_json_ld_writer.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023 spdx contributors # # SPDX-License-Identifier: Apache-2.0 -import os +from importlib import resources from spdx_tools.spdx3.bump_from_spdx2.spdx_document import bump_spdx_document from spdx_tools.spdx3.payload import Payload @@ -15,4 +15,5 @@ def test_json_writer(): payload: Payload = bump_spdx_document(spdx2_document) # this currently generates an actual file to look at, this should be changed to a temp file later - write_payload(payload, os.path.join(os.path.dirname(__file__), "../../../SPDX3_jsonld_test")) + with resources.as_file(resources.files("tests.spdx3.writer.json_ld").joinpath("SPDX3_jsonld_test")) as output_file: + write_payload(payload, str(output_file)) From 0ce225c41bd89063b75f22bb32f30a7fdb4d6b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 9 Jan 2026 16:16:18 +0100 Subject: [PATCH 29/41] linter fix: reduce line length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- tests/spdx/test_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/spdx/test_cli.py b/tests/spdx/test_cli.py index 52ea85cf5..d3b0bd4f3 100644 --- a/tests/spdx/test_cli.py +++ b/tests/spdx/test_cli.py @@ -39,7 +39,8 @@ def test_cli_with_system_exit_code_0(options): "-i", str( resources.files("tests.spdx.data.invalid").joinpath( - "spdx-trivy-vmware_log-intelligence-fluentd-sha256_086af034f561f343f633be9d9f9e95f65ae6c61b8ddb2c6755ef5bb25b40f53a.json" + "spdx-trivy-vmware_log-intelligence-fluentd-" + "sha256_086af034f561f343f633be9d9f9e95f65ae6c61b8ddb2c6755ef5bb25b40f53a.json" ) ), ), From f37a321c7cddd190fdb89486fd0f96499eeca0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 9 Jan 2026 16:43:56 +0100 Subject: [PATCH 30/41] update actions/deploy-pages to v4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * This hopefully fixes the GitHub actions workflow that currently doesn't find the correct url. Signed-off-by: Armin Tänzer --- .github/workflows/docs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index be0d749b1..46dc4157a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,6 +7,7 @@ on: push: branches: - main + workflow_dispatch: permissions: contents: read @@ -44,4 +45,4 @@ jobs: steps: - id: deployment name: Deploy docs to GitHub pages - uses: actions/deploy-pages@v2 + uses: actions/deploy-pages@v4 From fa5a3348e0ca2da3a5bd84a8d30fa77226886c4e Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 8 Oct 2024 13:43:46 +0200 Subject: [PATCH 31/41] Fix Markdown warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove trailing whitespaces - Ensure one blank line before and after headings/lists - Enforce one H1 heading in a document, the rest will be H2, H3, etc. - Standardize the use of bullet markup (using all `-`, instead of mixing `-` and `*`) Signed-off-by: Arthit Suriyawongkul Signed-off-by: Armin Tänzer --- CONTRIBUTING.md | 31 +++++--- DOCUMENTATION.md | 22 +++++- README.md | 196 +++++++++++++++++++++++++---------------------- 3 files changed, 143 insertions(+), 106 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c4a04341..88638e70e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ intention prior to creating a patch. ## Development process -We use the GitHub flow that is described here: https://guides.github.com/introduction/flow/ +We use the GitHub flow that is described here: Here's the process to make changes to the codebase: @@ -30,15 +30,19 @@ Here's the process to make changes to the codebase: and optionally follow the further steps described to sync your fork and the original repository. 4. Create a new branch in your fork and set up environment: + ```sh git checkout -b fix-or-improve-something python -m venv ./venv ./venv/bin/activate pip install -e ".[development]" ``` - Note: By using the group `[development]` for the installation, all dependencies (including optional ones) will be - installed. This way we make sure that all tests are executed. + + Note: By using the group `[development]` for the installation, all dependencies (including optional ones) will be + installed. This way we make sure that all tests are executed. + 5. Make some changes and commit them to the branch: + ```sh git commit --signoff -m 'description of my changes' ``` @@ -49,27 +53,33 @@ Here's the process to make changes to the codebase: of [the Developer Certificate of Origin](https://developercertificate.org/). Git has utilities for signing off on commits: `git commit -s` or `--signoff` signs a current commit, and `git rebase --signoff ` retroactively signs a range of past commits. + 6. Test your changes: + ```sh pytest -vvs # in the repo root ``` -7. Check your code style. When opening a pull request, your changes will automatically be checked with `isort`, `black` - and `flake8` to make sure your changes fit with the rest of the code style. +7. Check your code style. When opening a pull request, your changes will automatically be checked with `isort`, `black` + and `flake8` to make sure your changes fit with the rest of the code style. + ```sh # run the following commands in the repo root - isort src tests + isort src tests black src tests - flake8 src tests + flake8 src tests ``` - `black` and `isort` will automatically format the code and sort the imports. The configuration for these linters + + `black` and `isort` will automatically format the code and sort the imports. The configuration for these linters can be found in the `pyproject.toml`. `flake8` lists all problems found which then need to be resolved manually. The configuration for the linter can be found in the `.flake8` file. 8. Push the branch to your fork on GitHub: + ```sh git push origin fix-or-improve-something ``` + 9. Make a pull request on GitHub. 10. Continue making more changes and commits on the branch, with `git commit --signoff` and `git push`. 11. When done, write a comment on the PR asking for a code review. @@ -77,6 +87,7 @@ Here's the process to make changes to the codebase: possible, or with `squash`. 13. The temporary branch on GitHub should be deleted (there is a button for deleting it). 14. Delete the local branch as well: + ```sh git checkout master git pull -p @@ -84,11 +95,11 @@ Here's the process to make changes to the codebase: git branch -d fix-or-improve-something ``` -# How to run tests +## How to run tests The tests framework is using pytest: -``` +```sh pip install pytest pytest -vvs ``` diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index f8c7e274b..5119dd467 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1,12 +1,15 @@ # Code architecture documentation ## Package Overview + Beneath the top-level package `spdx_tools` you will find three sub-packages: + - `spdx`, which contains the code to create, parse, write and validate SPDX documents of versions 2.2 and 2.3 - `spdx3`, which will contain the same feature set for versions 3.x once they are released - `common`, which contains code that is shared between the different versions, such as type-checking and `spdx_licensing`. ## `spdx` + The `spdx` package contains the code dealing with SPDX-2 documents. The subpackages serve the purpose to divide the code into logically independent chunks. Shared code can be found in the top-level modules here. `model`, `parser`, `validation` and `writer` constitute the four main components of this library and are further described below. @@ -14,9 +17,11 @@ The subpackages serve the purpose to divide the code into logically independent `jsonschema` and `rdfschema` contain code specific to the corresponding serialization format. ### `model` + The internal data model closely follows the [official SPDX-2.3 specification](https://spdx.github.io/spdx-spec/v2.3/). Entrypoint to the model is the `Document` class, which has the following attributes: + - `creation_info`: a single instance of the `CreationInfo` class - `packages`: a list of `Package` objects - `files`: a list of `File` objects @@ -35,6 +40,7 @@ A custom extension of the `@dataclass` annotation is used that is called `@datac Apart from all the usual `dataclass` functionality, this implements fields of a class as properties with their own getter and setter methods. This is used in particular to implement type checking when properties are set. Source of truth for these checks are the attribute definitions at the start of the respective class that must specify the correct type hint. + The `beartype` library is used to check type conformity (`typeguard` was used in the past but has been replaced since due to performance issues). In case of a type mismatch a `TypeError` is raised. To ensure that all possible type errors are found during the construction of an object, a custom `__init__()` that calls `check_types_and_set_values()` is part of every class. @@ -43,26 +49,31 @@ This function tries to set all values provided by the constructor and collects a For the SPDX values `NONE` and `NOASSERTION` the classes `SpdxNone` and `SpdxNoAssertion` are used, respectively. Both can be instantiated without any arguments. ### `parser` + The parsing and writing modules are split into subpackages according to the serialization formats: `json`, `yaml`, `xml`, `tagvalue` and `rdf`. As the first three share the same tree structure that can be parsed into a dictionary, their shared logic is contained in the `jsonlikedict` package. One overarching concept of all parsers is the goal of dealing with parsing errors (like faulty types or missing mandatory fields) as long as possible before failing. Thus, the `SPDXParsingError` that is finally raised collects as much information as possible about all parsing errors that occurred. #### `tagvalue` + Since Tag-Value is an SPDX-specific format, there exist no readily available parsers for it. -This library implements its own deserialization code using the `ply` library's `lex` module for lexing and the `yacc` module for parsing. +This library implements its own deserialization code using the `ply` library's `lex` module for lexing and the `yacc` module for parsing. #### `rdf` + The `rdflib` library is used to deserialize RDF graphs from XML format. -The graph is then being parsed and translated into the internal data model. +The graph is then being parsed and translated into the internal data model. #### `json`, `yaml`, `xml` + In a first step, all three of JSON, YAML and XML formats are deserialized into a dictionary representing their tree structure. This is achieved via the `json`, `yaml` and `xmltodict` packages, respectively. Special note has to be taken in the XML case which does not support lists and numbers. The logic concerning the translation from these dicts to the internal data model can be found in the `jsonlikedict` package. ### `writer` + For serialization purposes, only non-null fields are written out. All writers expect a valid SPDX document from the internal model as input. To ensure this is actually the case, the standard behaviour of every writer function is to call validation before the writing process. @@ -71,18 +82,21 @@ Also by default, all list properties in the model are scanned for duplicates whi This can be disabled by setting the `drop_duplicates` boolean to false. #### `tagvalue` + The ordering of the tags follows the [example in the official specification](https://github.com/spdx/spdx-spec/blob/development/v2.3.1/examples/SPDXTagExample-v2.3.spdx). #### `rdf` + The RDF graph is constructed from the internal data model and serialized to XML format afterward, using the `rdflib` library. #### `json`, `yaml`, `xml` + As all three of JSON, YAML and XML formats share the same tree structure, the first step is to generate the dictionary representing that tree. This is achieved by the `DocumentConverter` class in the `jsonschema` package. Subsequently, the dictionary is serialized using the `json`, `yaml` and `xmltodict` packages, respectively. - ### `validation` + The `validation` package takes care of all nonconformities with the SPDX specification that are not due to incorrect typing. This mainly includes checks for correctly formatted strings or the actual existence of references SPDXIDs. Entrypoint is the `document_validator` module with the `validate_full_spdx_document()` function. @@ -93,6 +107,7 @@ Validation and reference checking of SPDXIDs (and possibly external document ref For the validation of license expressions we utilise the `license-expression` library's `validate` and `parse` functions, which take care of checking license symbols against the [SPDX license list](https://spdx.org/licenses/). Invalidities are captured in instances of a custom `ValidationMessage` class. This has two attributes: + - `validation_message` is a string that describes the actual problem - `validation_context` is a `ValidationContext` object that helps to pinpoint the source of the problem by providing the faulty element's SPDXID (if it has one), the parent SPDXID (if that is known), the element's type and finally the full element itself. It is left open to the implementer which of this information to use in the following evaluation of the validation process. @@ -101,6 +116,7 @@ Every validation function returns a list of `ValidationMessage` objects, which a That is, if an empty list is returned, the document is valid. ## `spdx3` + Due to the SPDX-3 model still being in development, this package is still a work in progress. However, as the basic building blocks of parsing, writing, creation and validation are still important in the new version, the `spdx3` package is planned to be structured similarly to the `spdx` package. diff --git a/README.md b/README.md index e089bd138..0dbe4514c 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,9 @@ CI status (Linux, macOS and Windows): [![Install and Test][1]][2] [1]: https://github.com/spdx/tools-python/actions/workflows/install_and_test.yml/badge.svg - [2]: https://github.com/spdx/tools-python/actions/workflows/install_and_test.yml - -# Breaking changes v0.7 -> v0.8 +## Breaking changes v0.7 -> v0.8 Please be aware that the upcoming 0.8 release has undergone a significant refactoring in preparation for the upcoming SPDX v3.0 release, leading to breaking changes in the API. @@ -15,126 +13,137 @@ Please refer to the [migration guide](https://github.com/spdx/tools-python/wiki/ to update your existing code. The main features of v0.8 are: + - full validation of SPDX documents against the v2.2 and v2.3 specification - support for SPDX's RDF format with all v2.3 features -- experimental support for the upcoming SPDX v3 specification. Note, however, that support is neither complete nor - stable at this point, as the spec is still evolving. SPDX3-related code is contained in a separate subpackage "spdx3" +- experimental support for the upcoming SPDX v3 specification. Note, however, that support is neither complete nor + stable at this point, as the spec is still evolving. SPDX3-related code is contained in a separate subpackage "spdx3" and its use is optional. We do not recommend using it in production code yet. Note that v0.8 only supports **writing**, not **reading** SPDX 3.0 documents. See [#760](https://github.com/spdx/tools-python/issues/760) for details. -# Information +## Information This library implements SPDX parsers, convertors, validators and handlers in Python. -- Home: https://github.com/spdx/tools-python -- Issues: https://github.com/spdx/tools-python/issues -- PyPI: https://pypi.python.org/pypi/spdx-tools -- Browse the API: https://spdx.github.io/tools-python +- Home: +- Issues: +- PyPI: +- Browse the API: -Important updates regarding this library are shared via the SPDX tech mailing list: https://lists.spdx.org/g/Spdx-tech. +Important updates regarding this library are shared via the SPDX tech mailing list: . - -# License +## License [Apache-2.0](LICENSE) -# Features +## Features -* API to create and manipulate SPDX v2.2 and v2.3 documents -* Parse, convert, create and validate SPDX files -* supported formats: Tag/Value, RDF, JSON, YAML, XML -* visualize the structure of a SPDX document by creating an `AGraph`. Note: This is an optional feature and requires +- API to create and manipulate SPDX v2.2 and v2.3 documents +- Parse, convert, create and validate SPDX files +- supported formats: Tag/Value, RDF, JSON, YAML, XML +- visualize the structure of a SPDX document by creating an `AGraph`. Note: This is an optional feature and requires additional installation of optional dependencies ## Experimental support for SPDX 3.0 -* Create v3.0 elements and payloads -* Convert v2.2/v2.3 documents to v3.0 -* Serialize to JSON-LD -See [Quickstart to SPDX 3.0](#quickstart-to-spdx-30) below. -The implementation is based on the descriptive markdown files in the repository https://github.com/spdx/spdx-3-model (latest commit: a5372a3c145dbdfc1381fc1f791c68889aafc7ff). +- Create v3.0 elements and payloads +- Convert v2.2/v2.3 documents to v3.0 +- Serialize to JSON-LD +See [Quickstart to SPDX 3.0](#quickstart-to-spdx-30) below. +The implementation is based on the descriptive Markdown files in the repository (latest commit: a5372a3c145dbdfc1381fc1f791c68889aafc7ff). -# Installation +## Installation As always you should work in a virtualenv (venv). You can install a local clone -of this repo with `yourenv/bin/pip install .` or install it from PyPI +of this repo with `yourenv/bin/pip install .` or install it from PyPI (check for the [newest release](https://pypi.org/project/spdx-tools/#history) and install it like `yourenv/bin/pip install spdx-tools==0.8.0a2`). Note that on Windows it would be `Scripts` instead of `bin`. -# How to use +## How to use -## Command-line usage +### Command-line usage 1. **PARSING/VALIDATING** (for parsing any format): -* Use `pyspdxtools -i ` where `` is the location of the file. The input format is inferred automatically from the file ending. + - Use `pyspdxtools -i ` where `` is the location of the file. The input format is inferred automatically from the file ending. -* If you are using a source distribution, try running: - `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json` + - If you are using a source distribution, try running: + `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json` 2. **CONVERTING** (for converting one format to another): -* Use `pyspdxtools -i -o ` where `` is the location of the file to be converted - and `` is the location of the output file. The input and output formats are inferred automatically from the file endings. + - Use `pyspdxtools -i -o ` where `` is the location of the file to be converted + and `` is the location of the output file. The input and output formats are inferred automatically from the file endings. -* If you are using a source distribution, try running: - `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json -o output.tag` + - If you are using a source distribution, try running: + `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json -o output.tag` -* If you want to skip the validation process, provide the `--novalidation` flag, like so: - `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json -o output.tag --novalidation` + - If you want to skip the validation process, provide the `--novalidation` flag, like so: + `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json -o output.tag --novalidation` (use this with caution: note that undetected invalid documents may lead to unexpected behavior of the tool) - -* For help use `pyspdxtools --help` + + - For help use `pyspdxtools --help` 3. **GRAPH GENERATION** (optional feature) -* This feature generates a graph representing all elements in the SPDX document and their connections based on the provided - relationships. The graph can be rendered to a picture. Below is an example for the file `tests/data/SPDXJSONExample-v2.3.spdx.json`: -![SPDXJSONExample-v2.3.spdx.png](assets/SPDXJSONExample-v2.3.spdx.png) -* Make sure you install the optional dependencies `networkx` and `pygraphviz`. To do so run `pip install ".[graph_generation]"`. -* Use `pyspdxtools -i --graph -o ` where `` is an output file name with valid format for `pygraphviz` (check - the documentation [here](https://pygraphviz.github.io/documentation/stable/reference/agraph.html#pygraphviz.AGraph.draw)). -* If you are using a source distribution, try running - `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json --graph -o SPDXJSONExample-v2.3.spdx.png` to generate - a png with an overview of the structure of the example file. - -## Library usage + - This feature generates a graph representing all elements in the SPDX document and their connections based on the provided + relationships. The graph can be rendered to a picture. Below is an example for the file `tests/data/SPDXJSONExample-v2.3.spdx.json`: + ![SPDXJSONExample-v2.3.spdx.png](assets/SPDXJSONExample-v2.3.spdx.png) + + - Make sure you install the optional dependencies `networkx` and `pygraphviz`. To do so run `pip install ".[graph_generation]"`. + - Use `pyspdxtools -i --graph -o ` where `` is an output file name with valid format for `pygraphviz` (check + the documentation [here](https://pygraphviz.github.io/documentation/stable/reference/agraph.html#pygraphviz.AGraph.draw)). + - If you are using a source distribution, try running + `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json --graph -o SPDXJSONExample-v2.3.spdx.png` to generate + a png with an overview of the structure of the example file. + +### Library usage + 1. **DATA MODEL** - * The `spdx_tools.spdx.model` package constitutes the internal SPDX v2.3 data model (v2.2 is simply a subset of this). All relevant classes for SPDX document creation are exposed in the `__init__.py` found [here](src%2Fspdx_tools%2Fspdx%2Fmodel%2F__init__.py). - * SPDX objects are implemented via `@dataclass_with_properties`, a custom extension of `@dataclass`. - * Each class starts with a list of its properties and their possible types. When no default value is provided, the property is mandatory and must be set during initialization. - * Using the type hints, type checking is enforced when initializing a new instance or setting/getting a property on an instance + + - The `spdx_tools.spdx.model` package constitutes the internal SPDX v2.3 data model (v2.2 is simply a subset of this). All relevant classes for SPDX document creation are exposed in the `__init__.py` found [here](src%2Fspdx_tools%2Fspdx%2Fmodel%2F__init__.py). + - SPDX objects are implemented via `@dataclass_with_properties`, a custom extension of `@dataclass`. + - Each class starts with a list of its properties and their possible types. When no default value is provided, the property is mandatory and must be set during initialization. + - Using the type hints, type checking is enforced when initializing a new instance or setting/getting a property on an instance (wrong types will raise `ConstructorTypeError` or `TypeError`, respectively). This makes it easy to catch invalid properties early and only construct valid documents. - * Note: in-place manipulations like `list.append(item)` will circumvent the type checking (a `TypeError` will still be raised when reading `list` again). We recommend using `list = list + [item]` instead. - * The main entry point of an SPDX document is the `Document` class from the [document.py](src%2Fspdx_tools%2Fspdx%2Fmodel%2Fdocument.py) module, which links to all other classes. - * For license handling, the [license_expression](https://github.com/nexB/license-expression) library is used. - * Note on `documentDescribes` and `hasFiles`: These fields will be converted to relationships in the internal data model. As they are deprecated, these fields will not be written in the output. + - Note: in-place manipulations like `list.append(item)` will circumvent the type checking (a `TypeError` will still be raised when reading `list` again). We recommend using `list = list + [item]` instead. + - The main entry point of an SPDX document is the `Document` class from the [document.py](src%2Fspdx_tools%2Fspdx%2Fmodel%2Fdocument.py) module, which links to all other classes. + - For license handling, the [license_expression](https://github.com/nexB/license-expression) library is used. + - Note on `documentDescribes` and `hasFiles`: These fields will be converted to relationships in the internal data model. As they are deprecated, these fields will not be written in the output. + 2. **PARSING** - * Use `parse_file(file_name)` from the `parse_anything.py` module to parse an arbitrary file with one of the supported file endings. - * Successful parsing will return a `Document` instance. Unsuccessful parsing will raise `SPDXParsingError` with a list of all encountered problems. + + - Use `parse_file(file_name)` from the `parse_anything.py` module to parse an arbitrary file with one of the supported file endings. + - Successful parsing will return a `Document` instance. Unsuccessful parsing will raise `SPDXParsingError` with a list of all encountered problems. + 3. **VALIDATING** - * Use `validate_full_spdx_document(document)` to validate an instance of the `Document` class. - * This will return a list of `ValidationMessage` objects, each consisting of a String describing the invalidity and a `ValidationContext` to pinpoint the source of the validation error. - * Validation depends on the SPDX version of the document. Note that only versions `SPDX-2.2` and `SPDX-2.3` are supported by this tool. + + - Use `validate_full_spdx_document(document)` to validate an instance of the `Document` class. + - This will return a list of `ValidationMessage` objects, each consisting of a String describing the invalidity and a `ValidationContext` to pinpoint the source of the validation error. + - Validation depends on the SPDX version of the document. Note that only versions `SPDX-2.2` and `SPDX-2.3` are supported by this tool. + 4. **WRITING** - * Use `write_file(document, file_name)` from the `write_anything.py` module to write a `Document` instance to the specified file. + + - Use `write_file(document, file_name)` from the `write_anything.py` module to write a `Document` instance to the specified file. The serialization format is determined from the filename ending. - * Validation is performed per default prior to the writing process, which is cancelled if the document is invalid. You can skip the validation via `write_file(document, file_name, validate=False)`. + - Validation is performed per default prior to the writing process, which is cancelled if the document is invalid. You can skip the validation via `write_file(document, file_name, validate=False)`. Caution: Only valid documents can be serialized reliably; serialization of invalid documents is not supported. -## Example +### Example + Here are some examples of possible use cases to quickly get you started with the spdx-tools. If you want more examples, like how to create an SPDX document from scratch, have a look [at the examples folder](examples). + ```python import logging from license_expression import get_spdx_licensing -from spdx_tools.spdx.model import (Checksum, ChecksumAlgorithm, File, +from spdx_tools.spdx.model import (Checksum, ChecksumAlgorithm, File, FileType, Relationship, RelationshipType) from spdx_tools.spdx.parser.parse_anything import parse_file from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document @@ -149,14 +158,14 @@ document.creation_info.name = "new document name" # define a file and a DESCRIBES relationship between the file and the document checksum = Checksum(ChecksumAlgorithm.SHA1, "71c4025dd9897b364f3ebbb42c484ff43d00791c") -file = File(name="./fileName.py", spdx_id="SPDXRef-File", checksums=[checksum], - file_types=[FileType.TEXT], +file = File(name="./fileName.py", spdx_id="SPDXRef-File", checksums=[checksum], + file_types=[FileType.TEXT], license_concluded=get_spdx_licensing().parse("MIT and GPL-2.0"), license_comment="licenseComment", copyright_text="copyrightText") relationship = Relationship("SPDXRef-DOCUMENT", RelationshipType.DESCRIBES, "SPDXRef-File") -# add the file and the relationship to the document +# add the file and the relationship to the document # (note that we do not use "document.files.append(file)" as that would circumvent the type checking) document.files = document.files + [file] document.relationships = document.relationships + [relationship] @@ -167,51 +176,52 @@ validation_messages = validate_full_spdx_document(document) for validation_message in validation_messages: logging.warning(validation_message.validation_message) -# if there are no validation messages, the document is valid +# if there are no validation messages, the document is valid # and we can safely serialize it without validating again if not validation_messages: write_file(document, "new_spdx_document.rdf", validate=False) ``` -# Quickstart to SPDX 3.0 +## Quickstart to SPDX 3.0 + In contrast to SPDX v2, all elements are now subclasses of the central `Element` class. -This includes packages, files, snippets, relationships, annotations, but also SBOMs, SpdxDocuments, and more. +This includes packages, files, snippets, relationships, annotations, but also SBOMs, SpdxDocuments, and more. For serialization purposes, all Elements that are to be serialized into the same file are collected in a `Payload`. This is just a dictionary that maps each Element's SpdxId to itself. Use the `write_payload()` functions to serialize a payload. -There currently are two options: -* The `spdx_tools.spdx3.writer.json_ld.json_ld_writer` module generates a JSON-LD file of the payload. -* The `spdx_tools.spdx3.writer.console.payload_writer` module prints a debug output to console. Note that this is not an official part of the SPDX specification and will probably be dropped as soon as a better standard emerges. +There currently are two options: + +- The `spdx_tools.spdx3.writer.json_ld.json_ld_writer` module generates a JSON-LD file of the payload. +- The `spdx_tools.spdx3.writer.console.payload_writer` module prints a debug output to console. Note that this is not an official part of the SPDX specification and will probably be dropped as soon as a better standard emerges. You can convert an SPDX v2 document to v3 via the `spdx_tools.spdx3.bump_from_spdx2.spdx_document` module. The `bump_spdx_document()` function will return a payload containing an `SpdxDocument` Element and one Element for each package, file, snippet, relationship, or annotation contained in the v2 document. +## Dependencies -# Dependencies - -* PyYAML: https://pypi.org/project/PyYAML/ for handling YAML. -* xmltodict: https://pypi.org/project/xmltodict/ for handling XML. -* rdflib: https://pypi.python.org/pypi/rdflib/ for handling RDF. -* ply: https://pypi.org/project/ply/ for handling tag-value. -* click: https://pypi.org/project/click/ for creating the CLI interface. -* beartype: https://pypi.org/project/beartype/ for type checking. -* uritools: https://pypi.org/project/uritools/ for validation of URIs. -* license-expression: https://pypi.org/project/license-expression/ for handling SPDX license expressions. +- PyYAML: for handling YAML. +- xmltodict: for handling XML. +- rdflib: for handling RDF. +- ply: for handling tag-value. +- click: for creating the CLI interface. +- beartype: for type checking. +- uritools: for validation of URIs. +- license-expression: for handling SPDX license expressions. -# Support +## Support -* Submit issues, questions or feedback at https://github.com/spdx/tools-python/issues -* Join the chat at https://gitter.im/spdx-org/Lobby -* Join the discussion on https://lists.spdx.org/g/spdx-tech and - https://spdx.dev/participate/tech/ +- Submit issues, questions or feedback at +- Join the chat at +- Join the discussion on and + -# Contributing +## Contributing Contributions are very welcome! See [CONTRIBUTING.md](./CONTRIBUTING.md) for instructions on how to contribute to the codebase. -# History +## History This is the result of an initial GSoC contribution by @[ah450](https://github.com/ah450) -(or https://github.com/a-h-i) and is maintained by a community of SPDX adopters and enthusiasts. +(or ) and is maintained by a community of SPDX adopters and enthusiasts. In order to prepare for the release of SPDX v3.0, the repository has undergone a major refactoring during the time from 11/2022 to 07/2023. From 26fca3c6d78f52c7a5d2a1287e5ec8a235ae3b74 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 8 Oct 2024 19:44:31 +0700 Subject: [PATCH 32/41] Fix process.md Signed-off-by: Arthit Suriyawongkul --- .../spdx3/writer/json_ld/process.md | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/spdx_tools/spdx3/writer/json_ld/process.md b/src/spdx_tools/spdx3/writer/json_ld/process.md index 7d04d5ccd..1968e6e48 100644 --- a/src/spdx_tools/spdx3/writer/json_ld/process.md +++ b/src/spdx_tools/spdx3/writer/json_ld/process.md @@ -1,27 +1,29 @@ -### Workflow +# Workflow Process to produce context file and a serialization example: -1. Run -``` -spec-parser --gen-md --gen-refs --gen-rdf ../spdx-3-model/model -``` -- spdx-3-model (commit: 6cb4316, last commit where spec-parser is able to run)
-- spec-parser (main with commits from PR 44, 45) +1. Run -2. Convert the generated `spec-parser/md_generated/model.ttl` to a json-ld file using https://frogcat.github.io/ttl2jsonld/demo/. + ```sh + spec-parser --gen-md --gen-refs --gen-rdf ../spdx-3-model/model + ``` + + - spdx-3-model (commit: 6cb4316, last commit where spec-parser is able to run) + - spec-parser (main with commits from PR 44, 45) + +2. Convert the generated `spec-parser/md_generated/model.ttl` to a json-ld file using . 3. Convert owl to context using `convert_spdx_owl_to_jsonld_context("SPDX_OWL.json")`. 4. Place the generated `context.json` in `spdx_tools/spdx3/writer/jsonld/`. 5. To generate the jsonld from the testfile run -``` -pyspdxtools3 -i ./tests/spdx/data/SPDXJSONExample-v2.3.spdx.json -o example_with_context -``` + ```sh + pyspdxtools3 -i ./tests/spdx/data/SPDXJSONExample-v2.3.spdx.json -o example_with_context + ``` -### Manually +## Manually +## Known limitations -### Known limitations - Validation of enums does not work - Additional keys seem to be ignored in validation -- inherited properties aren't validated +- Inherited properties aren't validated From 661f2220fef28272e88e85e40829ac954eb4fbe2 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Tue, 8 Oct 2024 21:47:00 +0700 Subject: [PATCH 33/41] Fix tests/spdx/data path Signed-off-by: Arthit Suriyawongkul --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0dbe4514c..776135c98 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The implementation is based on the descriptive Markdown files in the repository As always you should work in a virtualenv (venv). You can install a local clone of this repo with `yourenv/bin/pip install .` or install it from PyPI (check for the [newest release](https://pypi.org/project/spdx-tools/#history) and install it like -`yourenv/bin/pip install spdx-tools==0.8.0a2`). Note that on Windows it would be `Scripts` +`yourenv/bin/pip install spdx-tools==0.8.3`). Note that on Windows it would be `Scripts` instead of `bin`. ## How to use @@ -72,7 +72,7 @@ instead of `bin`. - Use `pyspdxtools -i ` where `` is the location of the file. The input format is inferred automatically from the file ending. - If you are using a source distribution, try running: - `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json` + `pyspdxtools -i tests/spdx/data/SPDXJSONExample-v2.3.spdx.json` 2. **CONVERTING** (for converting one format to another): @@ -80,10 +80,10 @@ instead of `bin`. and `` is the location of the output file. The input and output formats are inferred automatically from the file endings. - If you are using a source distribution, try running: - `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json -o output.tag` + `pyspdxtools -i tests/spdx/data/SPDXJSONExample-v2.3.spdx.json -o output.tag` - If you want to skip the validation process, provide the `--novalidation` flag, like so: - `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json -o output.tag --novalidation` + `pyspdxtools -i tests/spdx/data/SPDXJSONExample-v2.3.spdx.json -o output.tag --novalidation` (use this with caution: note that undetected invalid documents may lead to unexpected behavior of the tool) - For help use `pyspdxtools --help` @@ -91,14 +91,14 @@ instead of `bin`. 3. **GRAPH GENERATION** (optional feature) - This feature generates a graph representing all elements in the SPDX document and their connections based on the provided - relationships. The graph can be rendered to a picture. Below is an example for the file `tests/data/SPDXJSONExample-v2.3.spdx.json`: + relationships. The graph can be rendered to a picture. Below is an example for the file `tests/spdx/data/SPDXJSONExample-v2.3.spdx.json`: ![SPDXJSONExample-v2.3.spdx.png](assets/SPDXJSONExample-v2.3.spdx.png) - Make sure you install the optional dependencies `networkx` and `pygraphviz`. To do so run `pip install ".[graph_generation]"`. - Use `pyspdxtools -i --graph -o ` where `` is an output file name with valid format for `pygraphviz` (check the documentation [here](https://pygraphviz.github.io/documentation/stable/reference/agraph.html#pygraphviz.AGraph.draw)). - If you are using a source distribution, try running - `pyspdxtools -i tests/data/SPDXJSONExample-v2.3.spdx.json --graph -o SPDXJSONExample-v2.3.spdx.png` to generate + `pyspdxtools -i tests/spdx/data/SPDXJSONExample-v2.3.spdx.json --graph -o SPDXJSONExample-v2.3.spdx.png` to generate a png with an overview of the structure of the example file. ### Library usage From 61f771356572de7c058b025440544bcf9b27bdc8 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 10 Oct 2024 15:13:44 +0700 Subject: [PATCH 34/41] Update spec-parser parameters Signed-off-by: Arthit Suriyawongkul --- src/spdx_tools/spdx3/writer/json_ld/process.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/spdx_tools/spdx3/writer/json_ld/process.md b/src/spdx_tools/spdx3/writer/json_ld/process.md index 1968e6e48..485df8805 100644 --- a/src/spdx_tools/spdx3/writer/json_ld/process.md +++ b/src/spdx_tools/spdx3/writer/json_ld/process.md @@ -5,16 +5,17 @@ Process to produce context file and a serialization example: 1. Run ```sh - spec-parser --gen-md --gen-refs --gen-rdf ../spdx-3-model/model + python spec-parser/main.py spdx-3-model/model parser_output ``` - - spdx-3-model (commit: 6cb4316, last commit where spec-parser is able to run) - - spec-parser (main with commits from PR 44, 45) + - spdx-3-model (main; where v3.0.1 development happens) + - spec-parser (main) -2. Convert the generated `spec-parser/md_generated/model.ttl` to a json-ld file using . -3. Convert owl to context using `convert_spdx_owl_to_jsonld_context("SPDX_OWL.json")`. -4. Place the generated `context.json` in `spdx_tools/spdx3/writer/jsonld/`. -5. To generate the jsonld from the testfile run +2. Convert the generated `parser_output/rdf/spdx-model.ttl` to a JSON-LD file + using . +3. Convert OWL to context using `owl_to_context.py`. +4. Place the generated `context.json` in `src/spdx_tools/spdx3/writer/json_ld/`. +5. To generate the JSON-LD from the test file, run: ```sh pyspdxtools3 -i ./tests/spdx/data/SPDXJSONExample-v2.3.spdx.json -o example_with_context From 93a5232c8dc80c52fc5c754521567e46f3c0ca43 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 10 Oct 2024 19:03:39 +0700 Subject: [PATCH 35/41] Add link to official serialization doc Signed-off-by: Arthit Suriyawongkul --- src/spdx_tools/spdx3/writer/json_ld/process.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/spdx_tools/spdx3/writer/json_ld/process.md b/src/spdx_tools/spdx3/writer/json_ld/process.md index 485df8805..02dc7b715 100644 --- a/src/spdx_tools/spdx3/writer/json_ld/process.md +++ b/src/spdx_tools/spdx3/writer/json_ld/process.md @@ -1,5 +1,10 @@ # Workflow +Official SPDX v3.0 serialization documentation and context file +are available at: + +## Manually generate context file + Process to produce context file and a serialization example: 1. Run @@ -21,8 +26,6 @@ Process to produce context file and a serialization example: pyspdxtools3 -i ./tests/spdx/data/SPDXJSONExample-v2.3.spdx.json -o example_with_context ``` -## Manually - ## Known limitations - Validation of enums does not work From 11db2cf71ae4e02ae932b03bdfac328aecae499d Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 10 Oct 2024 19:16:19 +0700 Subject: [PATCH 36/41] Add link to SPDX 3.0 model Signed-off-by: Arthit Suriyawongkul --- CHANGELOG.md | 4 ++-- README.md | 16 +++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f17ee022c..56126e898 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -141,12 +141,12 @@ Starting a Changelog. * Dropped Python 2 support. Python >= 3.6 is now required. * Added `pyspdxtools_convertor` and `pyspdxtools_parser` CLI scripts. See [the readme](README.md) for usage instructions. -* Updated the tools to support SPDX versions up to 2.3 and to conform with the specification. Apart from many bugfixes +* Updated the tools to support SPDX versions up to 2.3 and to conform with the specification. Apart from many bugfixes and new properties, some of the more significant changes include: * Support for multiple packages per document * Support for multiple checksums for packages and files * Support for files outside a package -* **Note**: Validation was updated to follow the 2.3 specification. Since there is currently no support for +* **Note**: Validation was updated to follow the 2.3 specification. Since there is currently no support for version-specific handling, some details may be handled incorrectly for documents using lower versions. The changes are mostly restricted to properties becoming optional and new property values becoming available, and should be of limited impact. See https://spdx.github.io/spdx-spec/v2.3/diffs-from-previous-editions/ diff --git a/README.md b/README.md index 776135c98..a8e5cafd8 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ This library implements SPDX parsers, convertors, validators and handlers in Pyt - PyPI: - Browse the API: -Important updates regarding this library are shared via the SPDX tech mailing list: . +Important updates regarding this library are shared via +the SPDX tech mailing list: . ## License @@ -42,9 +43,10 @@ Important updates regarding this library are shared via the SPDX tech mailing li - API to create and manipulate SPDX v2.2 and v2.3 documents - Parse, convert, create and validate SPDX files -- supported formats: Tag/Value, RDF, JSON, YAML, XML -- visualize the structure of a SPDX document by creating an `AGraph`. Note: This is an optional feature and requires -additional installation of optional dependencies +- Supported formats: Tag/Value, RDF, JSON, YAML, XML +- Visualize the structure of a SPDX document by creating an `AGraph`. + Note: This is an optional feature and requires + additional installation of optional dependencies ## Experimental support for SPDX 3.0 @@ -53,7 +55,11 @@ additional installation of optional dependencies - Serialize to JSON-LD See [Quickstart to SPDX 3.0](#quickstart-to-spdx-30) below. -The implementation is based on the descriptive Markdown files in the repository (latest commit: a5372a3c145dbdfc1381fc1f791c68889aafc7ff). +The implementation is based on the descriptive Markdown files in the repository + +(commit: a5372a3c145dbdfc1381fc1f791c68889aafc7ff). +The latest SPDX 3.0 model is available at +. ## Installation From a02a3c4c3d2d01140134ab166d7514c3a78cd3b5 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Thu, 10 Oct 2024 22:12:22 +0700 Subject: [PATCH 37/41] Remove mentions that SPDX 3 is not released yet Signed-off-by: Arthit Suriyawongkul --- CHANGELOG.md | 6 +++--- DOCUMENTATION.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56126e898..e44b33018 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -143,9 +143,9 @@ Starting a Changelog. * Added `pyspdxtools_convertor` and `pyspdxtools_parser` CLI scripts. See [the readme](README.md) for usage instructions. * Updated the tools to support SPDX versions up to 2.3 and to conform with the specification. Apart from many bugfixes and new properties, some of the more significant changes include: - * Support for multiple packages per document - * Support for multiple checksums for packages and files - * Support for files outside a package + * Support for multiple packages per document + * Support for multiple checksums for packages and files + * Support for files outside a package * **Note**: Validation was updated to follow the 2.3 specification. Since there is currently no support for version-specific handling, some details may be handled incorrectly for documents using lower versions. The changes are mostly restricted to properties becoming optional and new property values becoming diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 5119dd467..6a9bcb2a5 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -5,7 +5,7 @@ Beneath the top-level package `spdx_tools` you will find three sub-packages: - `spdx`, which contains the code to create, parse, write and validate SPDX documents of versions 2.2 and 2.3 -- `spdx3`, which will contain the same feature set for versions 3.x once they are released +- `spdx3`, which will contain the same feature set for versions 3.x - `common`, which contains code that is shared between the different versions, such as type-checking and `spdx_licensing`. ## `spdx` @@ -117,7 +117,7 @@ That is, if an empty list is returned, the document is valid. ## `spdx3` -Due to the SPDX-3 model still being in development, this package is still a work in progress. +This package is still a work in progress. However, as the basic building blocks of parsing, writing, creation and validation are still important in the new version, the `spdx3` package is planned to be structured similarly to the `spdx` package. From 6405c49a9e189bbb2c1b1eb2892033d902ac3779 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 9 Jan 2026 18:12:23 +0000 Subject: [PATCH 38/41] pyproject.toml: Update Python versions + project URLs Also remove `zip-safe` setuptools config (issue #257 got fixed; the config is also deprecated) Signed-off-by: Arthit Suriyawongkul --- pyproject.toml | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 004e67347..798c95c41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,35 +9,51 @@ maintainers = [ { name = "Philippe Ombredanne", email = "pombredanne@gmail.com" }, { name = "SPDX group at the Linux Foundation and others" }, ] -license = { text = "Apache-2.0" } +license = "Apache-2.0" description = "SPDX parser and tools." readme = "README.md" classifiers = [ "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +requires-python = ">=3.10" +dependencies = [ + "beartype", + "click", + "license_expression", + "ply", + "pyyaml", + "rdflib", + "semantic_version", + "uritools", + "xmltodict", ] -urls = { Homepage = "https://github.com/spdx/tools-python" } -requires-python = ">=3.8" -dependencies = ["click", "pyyaml", "xmltodict", "rdflib", "beartype", "uritools", "license_expression", "ply", "semantic_version"] dynamic = ["version"] [project.optional-dependencies] -test = ["pytest", "pyshacl", "tzdata"] -code_style = ["isort", "black", "flake8"] -graph_generation = ["pygraphviz", "networkx"] -development = ["black", "flake8", "isort", "networkx", "pytest", "pyshacl"] +test = ["pyshacl", "pytest", "tzdata"] +code_style = ["black", "flake8", "isort"] +graph_generation = ["networkx", "pygraphviz"] +development = ["black", "flake8", "isort", "networkx", "pyshacl", "pytest"] [project.scripts] pyspdxtools = "spdx_tools.spdx.clitools.pyspdxtools:main" pyspdxtools3 = "spdx_tools.spdx3.clitools.pyspdxtools3:main" +[project.urls] +Homepage = "https://github.com/spdx/tools-python" +Documentation = "https://spdx.github.io/tools-python/" +Repository = "https://github.com/spdx/tools-python.git" +Issues = "https://github.com/spdx/tools-python/issues" +Changelog = "https://github.com/spdx/tools-python/blob/main/CHANGELOG.md" + [tool.setuptools] -zip-safe = false # because of the uses of __file__: https://github.com/spdx/tools-python/issues/257 include-package-data = true [tool.setuptools.packages.find] From a0794d9f81c45dd57923ffe7252f41df3898d041 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 9 Jan 2026 18:19:43 +0000 Subject: [PATCH 39/41] Remove License from classifiers Signed-off-by: Arthit Suriyawongkul --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 798c95c41..abfec60c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,12 +15,14 @@ readme = "README.md" classifiers = [ "Intended Audience :: Developers", "Intended Audience :: System Administrators", - "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Operating System :: OS Independent", ] requires-python = ">=3.10" dependencies = [ From ae2fe8bac644ca77d86891e2fdfe356ee49543a4 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 9 Jan 2026 18:30:25 +0000 Subject: [PATCH 40/41] Add development status Signed-off-by: Arthit Suriyawongkul --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index abfec60c9..e773d841c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,10 @@ license = "Apache-2.0" description = "SPDX parser and tools." readme = "README.md" classifiers = [ + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: System Administrators", + "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", @@ -22,7 +24,9 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", - "Operating System :: OS Independent", + "Topic :: Software Development :: Documentation", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Utilities", ] requires-python = ">=3.10" dependencies = [ From 4c9e2a63f95bf3a17b7e48ed8b5a192cb4816ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Fri, 16 Jan 2026 09:00:59 +0100 Subject: [PATCH 41/41] add changelog for upcoming release 0.8.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e44b33018..808f5e134 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## v0.8.4 (2026-01-16) + +### New features and changes + +* change supported Python versions: + * add support for Python 3.14 + * dropped support for Python 3.7, 3.8, 3.9 (which all have reached their end-of-life) +* fixes + * download location URIs are not case sensitive anymore + * removed some control characters from JSON SPDX + * remove usages of `__file__` to make the project zip-safe +* fixes to `README.md` and `process.md` + +### Contributors + +This release was made possible by the following contributors. Thank you very much! + +* @zbleness +* @clabbenius +* Oliver Benjamin @oli-ben +* Arthit Suriyawongkul @bact +* Armin Tänzer @armintaenzertng + ## v0.8.3 (2024-09-27) ### New features and changes