From 484374c62f70ff1bc02abbe73cc16edd1996ac66 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Tue, 7 Mar 2023 11:54:24 +0100 Subject: [PATCH 001/115] tox: fix errors with tox 4 commands[0]> sh -ec 'python3 -m isort $(git ls-files "*.py")' failed with sh is not allowed, use allowlist_externals to allow it install_deps> tox-install.sh .tox black flake8 isort pylint lint: failed with tox-install.sh (resolves to .../tox-install.sh) is not allowed, use allowlist_externals to allow it Signed-off-by: Robin Jarry --- tox.ini | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index c3186ae7..16e883b8 100644 --- a/tox.ini +++ b/tox.ini @@ -11,12 +11,16 @@ basepython = python3 description = Compile extension and run tests against {envname}. changedir = tests/ install_command = {toxinidir}/tox-install.sh {envdir} {opts} {packages} +allowlist_externals = + {toxinidir}/tox-install.sh commands = python -Wd -m unittest discover -c [testenv:coverage] changedir = . deps = coverage install_command = {toxinidir}/tox-install.sh {envdir} {opts} {packages} +allowlist_externals = + {toxinidir}/tox-install.sh commands = python -Wd -m coverage run -m unittest discover -c tests/ python -m coverage report @@ -32,7 +36,7 @@ deps = isort skip_install = true install_command = python3 -m pip install {opts} {packages} -whitelist_externals = +allowlist_externals = /bin/sh /usr/bin/sh commands = @@ -48,9 +52,10 @@ deps = flake8 isort pylint -whitelist_externals = +allowlist_externals = /bin/sh /usr/bin/sh + {toxinidir}/tox-install.sh commands = sh -ec 'python3 -m black -t py36 --diff --check $(git ls-files "*.py")' sh -ec 'python3 -m flake8 $(git ls-files "*.py")' From 6cd3b21b214e05f7e2d65fc0a81a2b790bee3e75 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Tue, 7 Mar 2023 11:57:36 +0100 Subject: [PATCH 002/115] black: update format with 23.1.0 Fix the version of these crappy dependencies once again. Signed-off-by: Robin Jarry --- libyang/context.py | 1 - libyang/data.py | 2 -- libyang/diff.py | 4 ---- libyang/schema.py | 22 ++-------------------- pylintrc | 2 +- tests/test_diff.py | 1 - tests/test_schema.py | 1 - tox.ini | 12 ++++++------ 8 files changed, 9 insertions(+), 36 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index d5f797d9..5b89e50a 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -21,7 +21,6 @@ # ------------------------------------------------------------------------------------- class Context: - __slots__ = ("cdata",) def __init__( diff --git a/libyang/data.py b/libyang/data.py index 357339ee..92d8c511 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -296,7 +296,6 @@ def new_path( opt_bin_value: bool = False, opt_canon_value: bool = False, ): - opt = 0 if opt_update: opt |= lib.LYD_NEW_PATH_UPDATE @@ -944,7 +943,6 @@ def value(self) -> Any: @staticmethod def cdata_leaf_value(cdata, context: "libyang.Context" = None) -> Any: - val = lib.lyd_get_value(cdata) if val == ffi.NULL: return None diff --git a/libyang/diff.py b/libyang/diff.py index 57099518..c3a2a4a9 100644 --- a/libyang/diff.py +++ b/libyang/diff.py @@ -78,7 +78,6 @@ class SNodeDiff: # ------------------------------------------------------------------------------------- class SNodeRemoved(SNodeDiff): - __slots__ = ("node",) def __init__(self, node: SNode): @@ -94,7 +93,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class SNodeAdded(SNodeDiff): - __slots__ = ("node",) def __init__(self, node: SNode): @@ -106,7 +104,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class SNodeAttributeChanged(SNodeDiff): - __slots__ = ("old", "new", "value") def __init__(self, old: SNode, new: SNode, value: Any = None): @@ -289,7 +286,6 @@ class UnitsAdded(SNodeAttributeChanged): # ------------------------------------------------------------------------------------- def snode_changes(old: SNode, new: SNode) -> Iterator[SNodeDiff]: - if old.nodetype() != new.nodetype(): yield NodeTypeRemoved(old, new, old.keyword()) yield NodeTypeAdded(old, new, new.keyword()) diff --git a/libyang/schema.py b/libyang/schema.py index bf9c1647..47731402 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -6,7 +6,7 @@ from typing import IO, Any, Dict, Iterator, Optional, Tuple, Union from _libyang import ffi, lib -from .util import IOType, c2str, init_output, ly_array_iter, str2c +from .util import IOType, LibyangError, c2str, init_output, ly_array_iter, str2c # ------------------------------------------------------------------------------------- @@ -44,7 +44,6 @@ def printer_flags( # ------------------------------------------------------------------------------------- class Module: - __slots__ = ("context", "cdata") def __init__(self, context: "libyang.Context", cdata): @@ -228,7 +227,6 @@ def parse_data_dict( # ------------------------------------------------------------------------------------- class Revision: - __slots__ = ("context", "cdata", "module") def __init__(self, context: "libyang.Context", cdata, module): @@ -272,7 +270,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class Extension: - __slots__ = ("context", "cdata") def __init__(self, context: "libyang.Context", cdata, module_parent: Module = None): @@ -295,7 +292,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class ExtensionParsed(Extension): - __slots__ = ("module_parent",) def __init__(self, context: "libyang.Context", cdata, module_parent: Module = None): @@ -318,7 +314,6 @@ def module(self) -> Module: # ------------------------------------------------------------------------------------- class ExtensionCompiled(Extension): - __slots__ = ("cdata_def",) def __init__(self, context: "libyang.Context", cdata): @@ -336,7 +331,6 @@ def module(self) -> Module: # ------------------------------------------------------------------------------------- class _EnumBit: - __slots__ = ("context", "cdata") def __init__(self, context: "libyang.Context", cdata): @@ -388,7 +382,6 @@ class Bit(_EnumBit): # ------------------------------------------------------------------------------------- class Type: - __slots__ = ("context", "cdata", "cdata_parsed") UNKNOWN = lib.LY_TYPE_UNKNOWN @@ -609,7 +602,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class Feature: - __slots__ = ("context", "cdata") def __init__(self, context: "libyang.Context", cdata): @@ -653,7 +645,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class IfFeatureExpr: - __slots__ = ("context", "cdata", "module_features", "compiled") def __init__(self, context: "libyang.Context", cdata, module_features=None): @@ -692,7 +683,7 @@ def get_feature(name): for feature in self.module_features: if feature.name() == name: return feature.cdata - raise Exception("No feature %s in module" % name) + raise LibyangError("No feature %s in module" % name) def parse_iffeature(tokens): def oper2(op): @@ -789,7 +780,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class IfFeature(IfFeatureExprTree): - __slots__ = ("context", "cdata") def __init__(self, context: "libyang.Context", cdata): @@ -812,7 +802,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class IfNotFeature(IfFeatureExprTree): - __slots__ = ("context", "child") def __init__(self, context: "libyang.Context", child: IfFeatureExprTree): @@ -831,7 +820,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class IfAndFeatures(IfFeatureExprTree): - __slots__ = ("context", "a", "b") def __init__( @@ -856,7 +844,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class IfOrFeatures(IfFeatureExprTree): - __slots__ = ("context", "a", "b") def __init__( @@ -881,7 +868,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- class SNode: - __slots__ = ("context", "cdata", "cdata_parsed") CONTAINER = lib.LYS_CONTAINER @@ -1076,7 +1062,6 @@ def new(context: "libyang.Context", cdata) -> "SNode": # ------------------------------------------------------------------------------------- @SNode.register(SNode.LEAF) class SLeaf(SNode): - __slots__ = ("cdata_leaf", "cdata_leaf_parsed") def __init__(self, context: "libyang.Context", cdata): @@ -1123,7 +1108,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- @SNode.register(SNode.LEAFLIST) class SLeafList(SNode): - __slots__ = ("cdata_leaflist", "cdata_leaflist_parsed") def __init__(self, context: "libyang.Context", cdata): @@ -1176,7 +1160,6 @@ def __str__(self): # ------------------------------------------------------------------------------------- @SNode.register(SNode.CONTAINER) class SContainer(SNode): - __slots__ = ("cdata_container", "cdata_container_parsed") def __init__(self, context: "libyang.Context", cdata): @@ -1209,7 +1192,6 @@ def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: # ------------------------------------------------------------------------------------- @SNode.register(SNode.LIST) class SList(SNode): - __slots__ = ("cdata_list", "cdata_list_parsed") def __init__(self, context: "libyang.Context", cdata): diff --git a/pylintrc b/pylintrc index 658c45ef..97a7cec1 100644 --- a/pylintrc +++ b/pylintrc @@ -525,4 +525,4 @@ min-public-methods=0 # Exceptions that will emit a warning when being caught. Defaults to # "Exception". -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception diff --git a/tests/test_diff.py b/tests/test_diff.py index 7155b3e0..e586413d 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -28,7 +28,6 @@ # ------------------------------------------------------------------------------------- class DiffTest(unittest.TestCase): - expected_diffs = frozenset( ( (BaseTypeAdded, "/yolo-system:conf/speed"), diff --git a/tests/test_schema.py b/tests/test_schema.py index 3904fbef..a326d2b7 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -276,7 +276,6 @@ def test_iter_tree(self): # ------------------------------------------------------------------------------------- class ListTest(unittest.TestCase): - SCHEMA_PATH = "/yolo-system:conf/url" DATA_PATH = "/yolo-system:conf/url[host='%s'][proto='%s']" diff --git a/tox.ini b/tox.ini index 16e883b8..8ccfa441 100644 --- a/tox.ini +++ b/tox.ini @@ -32,8 +32,8 @@ basepython = python3 description = Format python code using isort and black. changedir = . deps = - black==22.10.0 - isort + black~=23.1.0 + isort~=5.12.0 skip_install = true install_command = python3 -m pip install {opts} {packages} allowlist_externals = @@ -48,10 +48,10 @@ basepython = python3 description = Run coding style checks. changedir = . deps = - black==22.10.0 - flake8 - isort - pylint + black~=23.1.0 + flake8~=6.0.0 + isort~=5.12.0 + pylint~=2.16.2 allowlist_externals = /bin/sh /usr/bin/sh From 5b10c4368bdfc4e7b92fc567b8433246be00caa6 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Tue, 7 Mar 2023 16:17:04 +0100 Subject: [PATCH 003/115] tox: test with libyang master by default Add special case for the devel branch. Currently, the devel branch fails to parse one of our module... Signed-off-by: Robin Jarry --- .github/workflows/ci.yml | 18 ++++++++++++++++++ Makefile | 4 ++-- tox-install.sh | 2 +- tox.ini | 6 +++++- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed936040..9cfbb845 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,8 @@ jobs: toxenv: py39 - python: "3.10" toxenv: py310 + - python: "3.11" + toxenv: py311 - python: pypy3.9 toxenv: pypy3 steps: @@ -53,6 +55,22 @@ jobs: - run: python -m pip install --upgrade tox - run: python -m tox -e ${{ matrix.toxenv }} + libyang_devel: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: pip + restore-keys: pip + - run: python -m pip install --upgrade pip setuptools wheel + - run: python -m pip install --upgrade tox + - run: python -m tox -e lydevel + coverage: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index fcc52d83..90147aaa 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2020 Robin Jarry +# Copyright (c) 2018-2023 Robin Jarry # SPDX-License-Identifier: MIT all: lint tests @@ -7,7 +7,7 @@ lint: tox -e lint tests: - tox -e py37 + tox -e py3 format: tox -e format diff --git a/tox-install.sh b/tox-install.sh index 428659d9..56e6e2a7 100755 --- a/tox-install.sh +++ b/tox-install.sh @@ -32,7 +32,7 @@ download() # build and install libyang into the virtualenv src="${LIBYANG_SRC:-$venv/.src}" if ! [ -d "$src" ]; then - libyang_branch="${LIBYANG_BRANCH:-devel}" + libyang_branch="${LIBYANG_BRANCH:-master}" download "https://github.com/CESNET/libyang" "$libyang_branch" "$src" fi diff --git a/tox.ini b/tox.ini index 8ccfa441..f8e3c972 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = format,lint,py{36,37,38,39,310,py3},coverage +envlist = format,lint,py{36,37,38,39,310,311,py3,3},lydevel,coverage skip_missing_interpreters = true isolated_build = true distdir = {toxinidir}/dist @@ -15,6 +15,10 @@ allowlist_externals = {toxinidir}/tox-install.sh commands = python -Wd -m unittest discover -c +[testenv:lydevel] +setenv = + LIBYANG_BRANCH=devel + [testenv:coverage] changedir = . deps = coverage From 56ca55aa3238ceb5a91411fe04ba37cf3defe6d8 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Tue, 7 Mar 2023 11:45:04 +0100 Subject: [PATCH 004/115] data: document differences between the native dict format and json The native dict format is not a 1:1 translation of the JSON format. This was a deliberate design choice to make it easier to work with data nodes in python code. Document the differences. Fixes: https://github.com/CESNET/libyang-python/issues/60 Signed-off-by: Robin Jarry --- libyang/data.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libyang/data.py b/libyang/data.py index 92d8c511..3323b0ec 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -710,6 +710,17 @@ def print_dict( """ Convert a DNode object to a python dictionary. + The format is inspired by the YANG JSON format described in :rfc:`7951` but has + some differences: + + * ``int64`` and ``uint64`` values are represented by python ``int`` values + instead of string values. + * ``decimal64`` values are represented by python ``float`` values instead of + string values. + * ``empty`` values are represented by python ``None`` values instead of + ``[None]`` list instances. To check if an ``empty`` leaf is set in a + container, you should use the idiomatic ``if "foo" in container:`` construct. + :arg bool strip_prefixes: If True (the default), module prefixes are stripped from dictionary keys. If False, dictionary keys are in the form ``:``. From d5c5cb7e4a6cd07945c81045f3622a31bcae1286 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Mon, 13 Mar 2023 17:46:27 +0100 Subject: [PATCH 005/115] diff: add an option to use data_path When comparing two models including choices/cases, schema_diff will find differences, because the schema are actually different. Add an option to use the node data_paths, which will ignore choice/case. Signed-off-by: Samuel Gauthier --- libyang/diff.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/libyang/diff.py b/libyang/diff.py index c3a2a4a9..4caaf34b 100644 --- a/libyang/diff.py +++ b/libyang/diff.py @@ -12,6 +12,7 @@ def schema_diff( ctx_old: Context, ctx_new: Context, exclude_node_cb: Optional[Callable[[SNode], bool]] = None, + use_data_path: bool = False, ) -> Iterator["SNodeDiff"]: """ Compare two libyang Context objects, for a given set of paths and return all @@ -25,6 +26,9 @@ def schema_diff( Optionnal user callback that will be called with each node that is found in each context. If the callback returns a "trueish" value, the node will be excluded from the diff (as well as all its children). + :arg use_data_path: + Use data path instead of schema path to compare the nodes. Using data path + ignores choices and cases. :return: An iterator that yield `SNodeDiff` objects. @@ -40,7 +44,11 @@ def flatten(node, dic): """ if exclude_node_cb(node): return - dic[node.schema_path()] = node + if use_data_path: + path = node.data_path() + else: + path = node.schema_path() + dic[path] = node if isinstance(node, (SContainer, SList, SNotif, SRpc, SRpcInOut)): for child in node: flatten(child, dic) From d5957da13ec67cf2996ba86d66412392156c55d5 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Wed, 24 May 2023 14:53:52 +0200 Subject: [PATCH 006/115] cffi: fix compilation with latest version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove field that causes build errors with the latest libyang versions: build/temp.linux-x86_64-cpython-37/_libyang.c:8143:32: error: initialization of ‘struct hash_table **’ from incompatible pointer type ‘struct ly_ht **’ [-Werror=incompatible-pointer-types] children_ht is not used anywhere, remove it. Signed-off-by: Robin Jarry --- cffi/cdefs.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 0ccb9772..0cef63a8 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -719,7 +719,6 @@ struct lyd_node_inner { }; }; struct lyd_node *child; - struct hash_table *children_ht; ...; }; From 93676bc9a829d5c94ba645db5bfbd90a13aab387 Mon Sep 17 00:00:00 2001 From: focksor surooi Date: Thu, 6 Apr 2023 17:36:31 +0800 Subject: [PATCH 007/115] readme: fix example code Remove deprecated config argument in module.parse_data_dict. Signed-off-by: focksor surooi --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8caf69e5..0387cf57 100644 --- a/README.rst +++ b/README.rst @@ -185,7 +185,7 @@ Data Tree ... {'name': 'lo', 'address': '127.0.0.1'}, ... ], ... }, - ... }, config=True) + ... }) >>> print(node.print_mem('xml', pretty=True)) From bfe8a0c9aece794982cace3a592e834f38d499ac Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Fri, 7 Jul 2023 18:08:53 +0200 Subject: [PATCH 008/115] schema: add parsed schema for union types When available, let's add the parsed schema for union types, they will be more complete, adding names, extensions, ranges, units, patterns, etc. Signed-off-by: Samuel Gauthier Acked-by: Robin Jarry --- libyang/schema.py | 11 +++++++++-- tests/test_schema.py | 6 ++++++ tests/yang/wtf/wtf-types.yang | 25 +++++++++++++++++++++---- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 47731402..75934824 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -479,9 +479,16 @@ def leafref_path(self) -> Optional["str"]: def union_types(self) -> Iterator["Type"]: if self.cdata.basetype != self.UNION: return + t = ffi.cast("struct lysc_type_union *", self.cdata) - for union_type in ly_array_iter(t.types): - yield Type(self.context, union_type, None) + if self.cdata_parsed and self.cdata_parsed.types != ffi.NULL: + for union_type, union_type_parsed in zip( + ly_array_iter(t.types), ly_array_iter(self.cdata_parsed.types) + ): + yield Type(self.context, union_type, union_type_parsed) + else: + for union_type in ly_array_iter(t.types): + yield Type(self.context, union_type, None) def enums(self) -> Iterator[Enum]: if self.cdata.basetype != self.ENUM: diff --git a/tests/test_schema.py b/tests/test_schema.py index a326d2b7..995b6406 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -406,6 +406,12 @@ def test_leaf_type_union(self): self.assertEqual(t.base(), Type.UNION) types = set(u.name() for u in t.union_types()) self.assertEqual(types, set(["int16", "int32", "uint16", "uint32"])) + for u in t.union_types(): + ext = u.get_extension( + "type-desc", prefix="omg-extensions", arg_value=f"<{u.name()}>" + ) + self.assertIsInstance(ext, Extension) + self.assertEqual(len(list(u.extensions())), 2) bases = set(t.basenames()) self.assertEqual(bases, set(["int16", "int32", "uint16", "uint32"])) diff --git a/tests/yang/wtf/wtf-types.yang b/tests/yang/wtf/wtf-types.yang index 6d712854..fc1fe8eb 100644 --- a/tests/yang/wtf/wtf-types.yang +++ b/tests/yang/wtf/wtf-types.yang @@ -2,6 +2,8 @@ module wtf-types { namespace "urn:yang:wtf:types"; prefix t; + import omg-extensions { prefix ext; } + typedef str { type string; } @@ -12,17 +14,32 @@ module wtf-types { } } + extension signed; + extension unsigned; + typedef unsigned { type union { - type uint16; - type uint32; + type uint16 { + t:unsigned; + ext:type-desc ""; + } + type uint32 { + ext:type-desc ""; + t:unsigned; + } } } typedef signed { type union { - type int16; - type int32; + type int16 { + ext:type-desc ""; + t:signed; + } + type int32 { + ext:type-desc ""; + t:signed; + } } } From 30120a39aeda73bd3fb29b913e7d4e9ab6b10d6c Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Fri, 7 Jul 2023 18:03:48 +0200 Subject: [PATCH 009/115] all: allow to add fields to classes It can be handy to add custom fields to classes, let's add support for it. As a side-node, this does not degrade performance for access / write for the fields that are in the slots. Signed-off-by: Samuel Gauthier Acked-by: Robin Jarry --- libyang/context.py | 2 +- libyang/data.py | 2 +- libyang/diff.py | 2 +- libyang/schema.py | 16 ++++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index 5b89e50a..e64112e0 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -21,7 +21,7 @@ # ------------------------------------------------------------------------------------- class Context: - __slots__ = ("cdata",) + __slots__ = ("cdata", "__dict__") def __init__( self, diff --git a/libyang/data.py b/libyang/data.py index 3323b0ec..a8c0ed92 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -193,7 +193,7 @@ class DNode: Data tree node. """ - __slots__ = ("context", "cdata", "free_func") + __slots__ = ("context", "cdata", "free_func", "__dict__") def __init__(self, context: "libyang.Context", cdata): """ diff --git a/libyang/diff.py b/libyang/diff.py index 4caaf34b..b2a15118 100644 --- a/libyang/diff.py +++ b/libyang/diff.py @@ -81,7 +81,7 @@ def flatten(node, dic): # ------------------------------------------------------------------------------------- class SNodeDiff: - pass + __slots__ = ("__dict__",) # ------------------------------------------------------------------------------------- diff --git a/libyang/schema.py b/libyang/schema.py index 75934824..4b870116 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -44,7 +44,7 @@ def printer_flags( # ------------------------------------------------------------------------------------- class Module: - __slots__ = ("context", "cdata") + __slots__ = ("context", "cdata", "__dict__") def __init__(self, context: "libyang.Context", cdata): self.context = context @@ -227,7 +227,7 @@ def parse_data_dict( # ------------------------------------------------------------------------------------- class Revision: - __slots__ = ("context", "cdata", "module") + __slots__ = ("context", "cdata", "module", "__dict__") def __init__(self, context: "libyang.Context", cdata, module): self.context = context @@ -270,7 +270,7 @@ def __str__(self): # ------------------------------------------------------------------------------------- class Extension: - __slots__ = ("context", "cdata") + __slots__ = ("context", "cdata", "__dict__") def __init__(self, context: "libyang.Context", cdata, module_parent: Module = None): self.context = context @@ -331,7 +331,7 @@ def module(self) -> Module: # ------------------------------------------------------------------------------------- class _EnumBit: - __slots__ = ("context", "cdata") + __slots__ = ("context", "cdata", "__dict__") def __init__(self, context: "libyang.Context", cdata): self.context = context @@ -382,7 +382,7 @@ class Bit(_EnumBit): # ------------------------------------------------------------------------------------- class Type: - __slots__ = ("context", "cdata", "cdata_parsed") + __slots__ = ("context", "cdata", "cdata_parsed", "__dict__") UNKNOWN = lib.LY_TYPE_UNKNOWN BINARY = lib.LY_TYPE_BINARY @@ -609,7 +609,7 @@ def __str__(self): # ------------------------------------------------------------------------------------- class Feature: - __slots__ = ("context", "cdata") + __slots__ = ("context", "cdata", "__dict__") def __init__(self, context: "libyang.Context", cdata): self.context = context @@ -652,7 +652,7 @@ def __str__(self): # ------------------------------------------------------------------------------------- class IfFeatureExpr: - __slots__ = ("context", "cdata", "module_features", "compiled") + __slots__ = ("context", "cdata", "module_features", "compiled", "__dict__") def __init__(self, context: "libyang.Context", cdata, module_features=None): """ @@ -875,7 +875,7 @@ def __str__(self): # ------------------------------------------------------------------------------------- class SNode: - __slots__ = ("context", "cdata", "cdata_parsed") + __slots__ = ("context", "cdata", "cdata_parsed", "__dict__") CONTAINER = lib.LYS_CONTAINER LEAF = lib.LYS_LEAF From 7aaea74971a3915150d63f918eae083d0c9d5508 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Tue, 11 Jul 2023 15:18:59 +0200 Subject: [PATCH 010/115] schema: add basic support for choice and case Let's add enough to parse a model with choices and cases, so that iter_tree is working. Before this patch, the unit test added by this patch would fail with: > ====================================================================== > ERROR: test_iter_tree (test_schema.ContainerTest) > ---------------------------------------------------------------------- > Traceback (most recent call last): > File "(...)/libyang-python/tests/test_schema.py", line 278, in test_iter_tree > tree = list(self.container.iter_tree()) > File "(...)/libyang-python/.tox/py3/lib/python3.9/site-packages/libyang/schema.py", line 1148, in iter_tree > yield self.new(self.context, n) > File "(...)/libyang-python/.tox/py3/lib/python3.9/site-packages/libyang/schema.py", line 1192, in new > raise TypeError("node type %s not implemented" % cdata.nodetype) > TypeError: node type 2 not implemented Signed-off-by: Samuel Gauthier Acked-by: Robin Jarry --- libyang/schema.py | 22 ++++++++++++++++++++++ tests/test_diff.py | 6 ++++++ tests/test_schema.py | 12 ++++++++---- tests/yang/yolo/yolo-system.yang | 13 +++++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 4b870116..3e8f1c53 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -878,6 +878,8 @@ class SNode: __slots__ = ("context", "cdata", "cdata_parsed", "__dict__") CONTAINER = lib.LYS_CONTAINER + CHOICE = lib.LYS_CHOICE + CASE = lib.LYS_CASE LEAF = lib.LYS_LEAF LEAFLIST = lib.LYS_LEAFLIST LIST = lib.LYS_LIST @@ -1196,6 +1198,26 @@ def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: return iter_children(self.context, self.cdata, types=types) +# ------------------------------------------------------------------------------------- +@SNode.register(SNode.CHOICE) +class SChoice(SNode): + def __iter__(self) -> Iterator[SNode]: + return self.children() + + def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: + return iter_children(self.context, self.cdata, types=types) + + +# ------------------------------------------------------------------------------------- +@SNode.register(SNode.CASE) +class SCase(SNode): + def __iter__(self) -> Iterator[SNode]: + return self.children() + + def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: + return iter_children(self.context, self.cdata, types=types) + + # ------------------------------------------------------------------------------------- @SNode.register(SNode.LIST) class SList(SNode): diff --git a/tests/test_diff.py b/tests/test_diff.py index e586413d..d3edadc9 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -81,6 +81,12 @@ class DiffTest(unittest.TestCase): (EnumStatusAdded, "/yolo-system:state/url/proto"), (EnumStatusRemoved, "/yolo-system:conf/url/proto"), (EnumStatusRemoved, "/yolo-system:state/url/proto"), + (SNodeAdded, "/yolo-system:conf/pill/red/out"), + (SNodeAdded, "/yolo-system:state/pill/red/out"), + (SNodeAdded, "/yolo-system:conf/pill/blue/in"), + (SNodeAdded, "/yolo-system:state/pill/blue/in"), + (SNodeAdded, "/yolo-system:alarm-triggered/severity"), + (SNodeAdded, "/yolo-system:alarm-triggered/description"), ) ) diff --git a/tests/test_schema.py b/tests/test_schema.py index 995b6406..e0aae835 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -258,20 +258,20 @@ def test_cont_attrs(self): def test_cont_iter(self): children = list(iter(self.container)) - self.assertEqual(len(children), 9) + self.assertEqual(len(children), 11) def test_cont_children_leafs(self): leafs = list(self.container.children(types=(SNode.LEAF,))) - self.assertEqual(len(leafs), 7) + self.assertEqual(len(leafs), 9) def test_cont_parent(self): self.assertIsNone(self.container.parent()) def test_iter_tree(self): tree = list(self.container.iter_tree()) - self.assertEqual(len(tree), 15) - tree = list(self.container.iter_tree(full=True)) self.assertEqual(len(tree), 20) + tree = list(self.container.iter_tree(full=True)) + self.assertEqual(len(tree), 25) # ------------------------------------------------------------------------------------- @@ -455,3 +455,7 @@ def test_leaf_parent(self): self.assertIsNotNone(parent) self.assertIsInstance(parent, SList) self.assertEqual(parent.name(), "url") + + def test_iter_tree(self): + leaf = next(self.ctx.find_path("/yolo-system:conf")) + self.assertEqual(len(list(leaf.iter_tree(full=True))), 23) diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index 9ee49f04..44196b47 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -60,6 +60,19 @@ module yolo-system { type string; } + choice pill { + case red { + leaf out { + type boolean; + } + } + case blue { + leaf in { + type boolean; + } + } + } + list url { description "An URL."; From 7dea0b291b639d2d66f05a6fb1e69ba8502ac000 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Tue, 11 Jul 2023 16:04:47 +0200 Subject: [PATCH 011/115] schema: add module import Parse the import section of a module in an Import class. Signed-off-by: Samuel Gauthier Acked-by: Robin Jarry --- libyang/schema.py | 50 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_schema.py | 6 ++++++ 2 files changed, 56 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index 3e8f1c53..1b669330 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -112,6 +112,10 @@ def revisions(self) -> Iterator["Revision"]: for revision in ly_array_iter(self.cdata.parsed.revs): yield Revision(self.context, revision, self) + def imports(self) -> Iterator["Import"]: + for i in ly_array_iter(self.cdata.parsed.imports): + yield Import(self.context, i, self) + def __iter__(self) -> Iterator["SNode"]: return self.children() @@ -268,6 +272,52 @@ def __str__(self): return self.date() +# ------------------------------------------------------------------------------------- +class Import: + __slots__ = ("context", "cdata", "module", "__dict__") + + def __init__(self, context: "libyang.Context", cdata, module): + self.context = context + self.cdata = cdata # C type: "struct lysp_revision *" + self.module = module + + def name(self) -> str: + return c2str(self.cdata.name) + + def prefix(self) -> Optional[str]: + return c2str(self.cdata.prefix) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + def get_extension( + self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None + ) -> Optional["ExtensionParsed"]: + for ext in self.extensions(): + if ext.name() != name: + continue + if prefix is not None and ext.module().name() != prefix: + continue + if arg_value is not None and ext.argument() != arg_value: + continue + return ext + return None + + def __repr__(self): + cls = self.__class__ + return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) + + def __str__(self): + return self.name() + + # ------------------------------------------------------------------------------------- class Extension: __slots__ = ("context", "cdata", "__dict__") diff --git a/tests/test_schema.py b/tests/test_schema.py index e0aae835..b111d12c 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -73,6 +73,12 @@ def test_mod_enable_features(self): self.assertTrue(self.module.feature_state("turbo-boost")) self.module.feature_disable_all() + def test_mod_imports(self): + imports = list(self.module.imports()) + self.assertEqual(imports[0].name(), "omg-extensions") + self.assertEqual(imports[1].name(), "wtf-types") + self.assertEqual(len(imports), 2) + def test_mod_features(self): features = list(self.module.features()) self.assertEqual(len(features), 2) From fdfabf9287f54ea53492016f9c90ac994e65859f Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Tue, 11 Jul 2023 16:06:33 +0200 Subject: [PATCH 012/115] schema: fix type module method The module function of the Type class was not working. Fix it, and restore the unit test. Note that it returns the module where the type is defined, not the typedef. Signed-off-by: Samuel Gauthier Acked-by: Robin Jarry --- libyang/schema.py | 6 ++---- tests/test_schema.py | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 1b669330..9ea306c4 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -626,11 +626,9 @@ def all_patterns(self) -> Iterator[Tuple[str, bool]]: yield from self.patterns() def module(self) -> Module: - # TODO: pointer to the parsed module wehere is the type defined is in self.cdata_parsed.pmod - # however there is no way how to get name of the module from lysp_module - if not self.cdata.der.module: + if not self.cdata_parsed: return None - return Module(self.context, self.cdata.der.module) + return Module(self.context, self.cdata_parsed.pmod.mod) def extensions(self) -> Iterator[ExtensionCompiled]: for extension in ly_array_iter(self.cdata.exts): diff --git a/tests/test_schema.py b/tests/test_schema.py index b111d12c..a7cf7054 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -376,9 +376,9 @@ def test_leaf_type_derived(self): self.assertIsInstance(t, Type) self.assertEqual(t.name(), "types:host") self.assertEqual(t.base(), Type.STRING) - # mod = t.module() - # self.assertIsNot(mod, None) - # self.assertEqual(mod.name(), "wtf-types") + mod = t.module() + self.assertIsNot(mod, None) + self.assertEqual(mod.name(), "yolo-system") def test_leaf_type_status(self): leaf = next(self.ctx.find_path("/yolo-system:conf/yolo-system:hostname")) From fd7a990e96a904386ec1a32af59ce527b0277d5d Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Tue, 11 Jul 2023 16:05:43 +0200 Subject: [PATCH 013/115] schema: add get_module_from_prefix to module class This new method from the Module class gives a module given the prefix defined in the import section. Signed-off-by: Samuel Gauthier Acked-by: Robin Jarry --- libyang/schema.py | 7 +++++++ tests/test_schema.py | 1 + 2 files changed, 8 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index 9ea306c4..cced2dfb 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -116,6 +116,13 @@ def imports(self) -> Iterator["Import"]: for i in ly_array_iter(self.cdata.parsed.imports): yield Import(self.context, i, self) + def get_module_from_prefix(self, prefix: str) -> Optional["Module"]: + for i in self.imports(): + if i.prefix() != prefix: + continue + return self.context.get_module(i.name()) + return None + def __iter__(self) -> Iterator["SNode"]: return self.children() diff --git a/tests/test_schema.py b/tests/test_schema.py index a7cf7054..d704999d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -379,6 +379,7 @@ def test_leaf_type_derived(self): mod = t.module() self.assertIsNot(mod, None) self.assertEqual(mod.name(), "yolo-system") + self.assertEqual(mod.get_module_from_prefix("types").name(), "wtf-types") def test_leaf_type_status(self): leaf = next(self.ctx.find_path("/yolo-system:conf/yolo-system:hostname")) From aaca4df3a2701936c6a938dd632d30395ce62006 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Tue, 11 Jul 2023 16:07:19 +0200 Subject: [PATCH 014/115] schema: add typedef support Parse the typedef section in a Typedef class, add typedefs / get_typedef methods to the Module class, and add a method to get the Typedef object associated to a Type. Signed-off-by: Samuel Gauthier Acked-by: Robin Jarry --- cffi/cdefs.h | 11 ++++++ libyang/__init__.py | 1 + libyang/schema.py | 72 +++++++++++++++++++++++++++++++++++ tests/test_schema.py | 2 + tests/yang/wtf/wtf-types.yang | 2 + 5 files changed, 88 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 0cef63a8..c1eaee39 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -978,6 +978,17 @@ LY_ERR lyd_dup_siblings(const struct lyd_node *, struct lyd_node_inner *, uint32 LY_ERR lyd_dup_single(const struct lyd_node *, struct lyd_node_inner *, uint32_t, struct lyd_node **); void lyd_free_meta_single(struct lyd_meta *); +struct lysp_tpdf { + const char *name; + const char *units; + struct lysp_qname dflt; + const char *dsc; + const char *ref; + struct lysp_ext_instance *exts; + struct lysp_type type; + uint16_t flags; +}; + struct lysc_when { struct lyxp_expr *cond; struct lysc_node *context; diff --git a/libyang/__init__.py b/libyang/__init__.py index 1207a6b3..aa9dcca9 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -83,6 +83,7 @@ SRpc, SRpcInOut, Type, + Typedef, ) from .util import DataType, IOType, LibyangError from .xpath import ( diff --git a/libyang/schema.py b/libyang/schema.py index cced2dfb..83bb4793 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -112,6 +112,17 @@ def revisions(self) -> Iterator["Revision"]: for revision in ly_array_iter(self.cdata.parsed.revs): yield Revision(self.context, revision, self) + def typedefs(self) -> Iterator["Typedef"]: + for typedef in ly_array_iter(self.cdata.parsed.typedefs): + yield Typedef(self.context, typedef) + + def get_typedef(self, name: str) -> Optional["Typedef"]: + for typedef in self.typedefs(): + if typedef.name() != name: + continue + return typedef + return None + def imports(self) -> Iterator["Import"]: for i in ly_array_iter(self.cdata.parsed.imports): yield Import(self.context, i, self) @@ -533,6 +544,14 @@ def leafref_path(self) -> Optional["str"]: lr = ffi.cast("struct lysc_type_leafref *", self.cdata) return c2str(lib.lyxp_get_expr(lr.path)) + def typedef(self) -> "Typedef": + if ":" in self.name(): + module_prefix, type_name = self.name().split(":") + import_module = self.module().get_module_from_prefix(module_prefix) + if import_module: + return import_module.get_typedef(type_name) + return None + def union_types(self) -> Iterator["Type"]: if self.cdata.basetype != self.UNION: return @@ -662,6 +681,59 @@ def __str__(self): return self.name() +# ------------------------------------------------------------------------------------- +class Typedef: + __slots__ = ("context", "cdata", "__dict__") + + def __init__(self, context: "libyang.Context", cdata): + self.context = context + self.cdata = cdata # C type: "struct lysp_tpdf *" + + def name(self) -> str: + return c2str(self.cdata.name) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def units(self) -> Optional[str]: + return c2str(self.cdata.units) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def extensions(self) -> Iterator[ExtensionCompiled]: + ext = ffi.cast("struct lysc_ext_instance *", self.cdata.exts) + if ext == ffi.NULL: + return + for extension in ly_array_iter(ext): + yield ExtensionCompiled(self.context, extension) + + def get_extension( + self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None + ) -> Optional[ExtensionCompiled]: + for ext in self.extensions(): + if ext.name() != name: + continue + if prefix is not None and ext.module().name() != prefix: + continue + if arg_value is not None and ext.argument() != arg_value: + continue + return ext + return None + + def deprecated(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_DEPRC) + + def obsolete(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_OBSLT) + + def module(self) -> Module: + return Module(self.context, self.cdata.module) + + def __str__(self): + return self.name() + + # ------------------------------------------------------------------------------------- class Feature: __slots__ = ("context", "cdata", "__dict__") diff --git a/tests/test_schema.py b/tests/test_schema.py index d704999d..bb3a1325 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -380,6 +380,8 @@ def test_leaf_type_derived(self): self.assertIsNot(mod, None) self.assertEqual(mod.name(), "yolo-system") self.assertEqual(mod.get_module_from_prefix("types").name(), "wtf-types") + self.assertEqual(t.typedef().name(), "host") + self.assertEqual(t.typedef().description(), "my host type.") def test_leaf_type_status(self): leaf = next(self.ctx.find_path("/yolo-system:conf/yolo-system:hostname")) diff --git a/tests/yang/wtf/wtf-types.yang b/tests/yang/wtf/wtf-types.yang index fc1fe8eb..75a0a778 100644 --- a/tests/yang/wtf/wtf-types.yang +++ b/tests/yang/wtf/wtf-types.yang @@ -12,6 +12,8 @@ module wtf-types { type str { pattern "[a-z]+"; } + description + "my host type."; } extension signed; From ef885cc949774f8cfc622072376fdb8f651cbe4f Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Wed, 12 Jul 2023 10:55:07 +0200 Subject: [PATCH 015/115] schema: use typedef description for type When available, use the typedef's description for type. Signed-off-by: Samuel Gauthier Acked-by: Robin Jarry --- libyang/schema.py | 3 +++ tests/test_schema.py | 1 + 2 files changed, 4 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index 83bb4793..c6d822c2 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -516,6 +516,9 @@ def name(self) -> str: return self.basename() def description(self) -> Optional[str]: + typedef = self.typedef() + if typedef: + return typedef.description() return None def base(self) -> int: diff --git a/tests/test_schema.py b/tests/test_schema.py index bb3a1325..b88f0092 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -382,6 +382,7 @@ def test_leaf_type_derived(self): self.assertEqual(mod.get_module_from_prefix("types").name(), "wtf-types") self.assertEqual(t.typedef().name(), "host") self.assertEqual(t.typedef().description(), "my host type.") + self.assertEqual(t.description(), "my host type.") def test_leaf_type_status(self): leaf = next(self.ctx.find_path("/yolo-system:conf/yolo-system:hostname")) From bff1462c0b67f94207f2cf5897b3776bad121de4 Mon Sep 17 00:00:00 2001 From: Bill Stephens Date: Wed, 2 Aug 2023 09:15:48 -0500 Subject: [PATCH 016/115] data: fix merge operation lyd_merge_* functions may modify the passed dnode pointer. Make sure to update the self.cdata pointer after the operation. Fixes: #49 Signed-off-by: Bill Stephens --- libyang/data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libyang/data.py b/libyang/data.py index a8c0ed92..69649eba 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -552,6 +552,8 @@ def merge( if ret != lib.LY_SUCCESS: raise self.context.error("merge failed") + self.cdata = node_p[0] + def iter_tree(self) -> Iterator["DNode"]: n = next_n = self.cdata while n != ffi.NULL: From d8549c8a900a6a5ef3bb86f8d7859849e7964b5a Mon Sep 17 00:00:00 2001 From: Adam Allen Date: Mon, 18 Sep 2023 15:30:32 +0100 Subject: [PATCH 017/115] Instantiate `ly_ctx_new` with `LY_CTX_NO_YANG_LIB` When instantiating a Context without a yang library loading data will trigger the following validation error *unless* the context is instantiated correctly *or* the validation flags `LYD_VALIDATE_NO_STATE` or `LYD_VALIDATE_PRESENT` are used. ``` 'Data location "/ietf-yang-library:yang-library".: Mandatory node "content-id" instance does not exist.' ``` --- libyang/context.py | 1 + tests/test_data.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/libyang/context.py b/libyang/context.py index e64112e0..57432941 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -61,6 +61,7 @@ def __init__( search_path = ":".join(search_paths) if search_paths else None if yanglib_path is None: + options |= lib.LY_CTX_NO_YANGLIBRARY if lib.ly_ctx_new(str2c(search_path), options, ctx) != lib.LY_SUCCESS: raise self.error("cannot create context") else: diff --git a/tests/test_data.py b/tests/test_data.py index b217b97a..becb5d0c 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -78,6 +78,46 @@ def test_data_parse_config_json(self): finally: dnode.free() + JSON_CONFIG_WITH_STATE = """{ + "yolo-system:state": { + "speed": 4321 + }, + "yolo-system:conf": { + "hostname": "foo", + "url": [ + { + "proto": "https", + "host": "github.com", + "path": "/CESNET/libyang-python", + "enabled": false + }, + { + "proto": "http", + "host": "foobar.com", + "port": 8080, + "path": "/index.html", + "enabled": true + } + ], + "number": [ + 1000, + 2000, + 3000 + ], + "speed": 1234 + } +} +""" + + def test_data_parse_config_json_without_yang_lib(self): + dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json") + self.assertIsInstance(dnode, DContainer) + try: + j = dnode.print_mem("json", with_siblings=True) + self.assertEqual(j, self.JSON_CONFIG_WITH_STATE) + finally: + dnode.free() + JSON_CONFIG_ADD_LIST_ITEM = """{ "yolo-system:conf": { "hostname": "foo", From a2159c801c2706b6dbf01f73be9bc043f3ec9707 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Wed, 24 Jan 2024 13:50:41 +0100 Subject: [PATCH 018/115] tox: fix lint with python 3.12 Fix the following error when running pylint with python 3.12: Exception on node ImportFrom in file 'cffi/build.py' Traceback (most recent call last): File ".../python3.12/site-packages/pylint/checkers/imports.py", line 846, in _get_imported_module return importnode.do_import_module(modname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/nodes/_base_nodes.py", line 146, in do_import_module return mymodule.import_module( ^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/nodes/scoped_nodes/scoped_nodes.py", line 530, in import_module return AstroidManager().ast_from_module_name( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/manager.py", line 246, in ast_from_module_name return self.ast_from_file(found_spec.location, modname, fallback=False) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/manager.py", line 138, in ast_from_file return AstroidBuilder(self).file_build(filepath, modname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/builder.py", line 144, in file_build module, builder = self._data_build(data, modname, path) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/builder.py", line 204, in _data_build module = builder.visit_module(node, modname, node_file, package) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/rebuilder.py", line 254, in visit_module [self.visit(child, newnode) for child in node.body], ^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../python3.12/site-packages/astroid/rebuilder.py", line 609, in visit visit_method = getattr(self, visit_name) ^^^^^^^^^^^^^^^^^^^^^^^^^ AttributeError: 'TreeRebuilder' object has no attribute 'visit_typealias' Update all lint and format dependencies to their latest stable releases. Ignore the new use-implicit-booleaness-not-comparison-to-string and use-implicit-booleaness-not-comparison-to-zero pylint rules. Use the correct Hashable reference from collections.abc instead of the deprecated typing alias. Signed-off-by: Robin Jarry --- .github/workflows/ci.yml | 2 ++ libyang/keyed_list.py | 3 ++- pylintrc | 2 ++ tox.ini | 18 +++++++++++------- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cfbb845..4c308738 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,8 @@ jobs: toxenv: py310 - python: "3.11" toxenv: py311 + - python: "3.12" + toxenv: py312 - python: pypy3.9 toxenv: pypy3 steps: diff --git a/libyang/keyed_list.py b/libyang/keyed_list.py index 02b030fa..1a98af32 100644 --- a/libyang/keyed_list.py +++ b/libyang/keyed_list.py @@ -1,8 +1,9 @@ # Copyright (c) 2020 6WIND S.A. # SPDX-License-Identifier: MIT +from collections.abc import Hashable import copy -from typing import Any, Hashable, Iterable, Optional, Tuple, Union +from typing import Any, Iterable, Optional, Tuple, Union # ------------------------------------------------------------------------------------- diff --git a/pylintrc b/pylintrc index 97a7cec1..16a9f0ae 100644 --- a/pylintrc +++ b/pylintrc @@ -77,6 +77,8 @@ disable= too-many-return-statements, too-many-statements, unused-argument, + use-implicit-booleaness-not-comparison-to-string, + use-implicit-booleaness-not-comparison-to-zero, wrong-import-order, [REPORTS] diff --git a/tox.ini b/tox.ini index f8e3c972..9fd15d15 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = format,lint,py{36,37,38,39,310,311,py3,3},lydevel,coverage +envlist = format,lint,py{36,37,38,39,310,311,312,py3,3},lydevel,coverage skip_missing_interpreters = true isolated_build = true distdir = {toxinidir}/dist @@ -36,8 +36,8 @@ basepython = python3 description = Format python code using isort and black. changedir = . deps = - black~=23.1.0 - isort~=5.12.0 + black~=23.12.1 + isort~=5.13.2 skip_install = true install_command = python3 -m pip install {opts} {packages} allowlist_externals = @@ -52,10 +52,14 @@ basepython = python3 description = Run coding style checks. changedir = . deps = - black~=23.1.0 - flake8~=6.0.0 - isort~=5.12.0 - pylint~=2.16.2 + astroid~=3.0.2 + black~=23.12.1 + flake8~=7.0.0 + isort~=5.13.2 + pycodestyle~=2.11.1 + pyflakes~=3.2.0 + pylint~=3.0.3 + setuptools~=69.0.3 allowlist_externals = /bin/sh /usr/bin/sh From 097412cdb8ff40993755661f9760dadb8a5c9bbc Mon Sep 17 00:00:00 2001 From: Matthieu Ternisien d'Ouville Date: Wed, 17 Jan 2024 08:33:29 +0100 Subject: [PATCH 019/115] schema: enable getting node data path without list key There is no way to get a node data path (path without choice/case) without list keys. Adds the path_type parameter to the SNode.schema_path() method. This parameter can takes 3 values: - SNode.PATH_LOG: returns the path with schema-only nodes (choice, case) included, the default - SNode.PATH_DATA: returns the path without schema-only nodes - SNode.PATH_DATA_PATTERN: similar to PATH_DATA with list keys added (the one used by data_path()) The SNode.PATH_LOG is set by default to not change the original behavior. The SNode.data_path() method now calls SNode.schema_path() with self.PATH_DATA_PATTERN instead of lib.lysc_path(). Here is an example of the output difference between schema_path(), data_path(), and schema_path(path_type=SNode.PATH_DATA) with a node included in a choice and a list: node.schema_path(): /ietf-keystore:keystore/asymmetric-keys/asymmetric-key/private-key-type/private-key/private-key node.data_path() or node.schema_path(SNode.PATH_DATA_PATTERN): /ietf-keystore:keystore/asymmetric-keys/asymmetric-key[name='%s']/private-key node.schema_path(SNode.PATH_DATA): /ietf-keystore:keystore/asymmetric-keys/asymmetric-key/private-key Tests have been updated accordingly. Signed-off-by: Matthieu Ternisien d'Ouville --- libyang/schema.py | 21 +++++++++++---------- tests/test_schema.py | 15 ++++++++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index c6d822c2..6b0c803b 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1034,6 +1034,10 @@ class SNode: ANYDATA: "anydata", } + PATH_LOG = lib.LYSC_PATH_LOG + PATH_DATA = lib.LYSC_PATH_DATA + PATH_DATA_PATTERN = lib.LYSC_PATH_DATA_PATTERN + def __init__(self, context: "libyang.Context", cdata): self.context = context self.cdata = cdata # C type: "struct lysc_node *" @@ -1079,22 +1083,19 @@ def status(self) -> str: def module(self) -> Module: return Module(self.context, self.cdata.module) - def schema_path(self) -> str: + def schema_path(self, path_type: int = PATH_LOG) -> str: try: - s = lib.lysc_path(self.cdata, lib.LYSC_PATH_LOG, ffi.NULL, 0) + s = lib.lysc_path(self.cdata, path_type, ffi.NULL, 0) return c2str(s) finally: lib.free(s) def data_path(self, key_placeholder: str = "'%s'") -> str: - try: - s = lib.lysc_path(self.cdata, lib.LYSC_PATH_DATA_PATTERN, ffi.NULL, 0) - val = c2str(s) - if key_placeholder != "'%s'": - val = val.replace("'%s'", key_placeholder) - return val - finally: - lib.free(s) + val = self.schema_path(self.PATH_DATA_PATTERN) + + if key_placeholder != "'%s'": + val = val.replace("'%s'", key_placeholder) + return val def extensions(self) -> Iterator[ExtensionCompiled]: ext = ffi.cast("struct lysc_ext_instance *", self.cdata.exts) diff --git a/tests/test_schema.py b/tests/test_schema.py index b88f0092..f493ba14 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -282,13 +282,16 @@ def test_iter_tree(self): # ------------------------------------------------------------------------------------- class ListTest(unittest.TestCase): - SCHEMA_PATH = "/yolo-system:conf/url" - DATA_PATH = "/yolo-system:conf/url[host='%s'][proto='%s']" + PATH = { + "LOG": "/yolo-system:conf/url", + "DATA": "/yolo-system:conf/url", + "DATA_PATTERN": "/yolo-system:conf/url[host='%s'][proto='%s']", + } def setUp(self): self.ctx = Context(YANG_DIR) self.ctx.load_module("yolo-system") - self.list = next(self.ctx.find_path(self.SCHEMA_PATH)) + self.list = next(self.ctx.find_path(self.PATH["LOG"])) def tearDown(self): self.list = None @@ -300,9 +303,11 @@ def test_list_attrs(self): self.assertEqual(self.list.nodetype(), SNode.LIST) self.assertEqual(self.list.keyword(), "list") - self.assertEqual(self.list.schema_path(), self.SCHEMA_PATH) + self.assertEqual(self.list.schema_path(), self.PATH["LOG"]) - self.assertEqual(self.list.data_path(), self.DATA_PATH) + self.assertEqual(self.list.schema_path(SNode.PATH_DATA), self.PATH["DATA"]) + + self.assertEqual(self.list.data_path(), self.PATH["DATA_PATTERN"]) self.assertFalse(self.list.ordered()) def test_list_keys(self): From cccb9d6b90d955a8a7c678a878b5eb2b5ff9a0c1 Mon Sep 17 00:00:00 2001 From: Wataru Ishida Date: Sat, 29 Jan 2022 06:57:20 +0000 Subject: [PATCH 020/115] test: add keyless list test Keyless list is allowed for operational data. (see RFC7950 7.8.2.) It is used in ietf-netconf-notifications:netconf-config-change, for example. It is already supported, but untested, add a test case. Signed-off-by: Wataru Ishida Signed-off-by: Samuel Gauthier --- tests/test_data.py | 16 ++++++++++++++++ tests/test_diff.py | 3 +++ tests/test_schema.py | 2 +- tests/yang/yolo/yolo-system.yang | 8 ++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/test_data.py b/tests/test_data.py index becb5d0c..32c605fe 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -704,6 +704,22 @@ def test_notification_from_dict_module(self): dnotif.free() self.assertEqual(json.loads(j), json.loads(self.JSON_NOTIF)) + DICT_NOTIF_KEYLESS_LIST = { + "config-change": {"edit": [{"target": "a"}, {"target": "b"}]}, + } + + def test_data_to_dict_keyless_list(self): + module = self.ctx.get_module("yolo-system") + dnotif = module.parse_data_dict( + self.DICT_NOTIF_KEYLESS_LIST, strict=True, notification=True + ) + self.assertIsInstance(dnotif, DNotif) + try: + dic = dnotif.print_dict() + finally: + dnotif.free() + self.assertEqual(dic, self.DICT_NOTIF_KEYLESS_LIST) + XML_DIFF_STATE1 = """ foo 1234 diff --git a/tests/test_diff.py b/tests/test_diff.py index d3edadc9..49bf77a2 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -75,6 +75,9 @@ class DiffTest(unittest.TestCase): (SNodeAdded, "/yolo-system:alarm-triggered"), (SNodeAdded, "/yolo-system:alarm-triggered/severity"), (SNodeAdded, "/yolo-system:alarm-triggered/description"), + (SNodeAdded, "/yolo-system:config-change"), + (SNodeAdded, "/yolo-system:config-change/edit"), + (SNodeAdded, "/yolo-system:config-change/edit/target"), (EnumRemoved, "/yolo-system:conf/url/proto"), (EnumRemoved, "/yolo-system:state/url/proto"), (EnumStatusAdded, "/yolo-system:conf/url/proto"), diff --git a/tests/test_schema.py b/tests/test_schema.py index f493ba14..770a476c 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -56,7 +56,7 @@ def test_mod_filepath(self): def test_mod_iter(self): children = list(iter(self.module)) - self.assertEqual(len(children), 5) + self.assertEqual(len(children), 6) def test_mod_children_rpcs(self): rpcs = list(self.module.children(types=(SNode.RPC,))) diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index 44196b47..a34c421c 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -204,4 +204,12 @@ module yolo-system { type uint32; } } + + notification config-change { + list edit { + leaf target { + type string; + } + } + } } From 3b4d5099a04c16a93037cc49ca66a9f2ec14aa60 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Fri, 26 Jan 2024 00:23:52 +0100 Subject: [PATCH 021/115] schema: fix leaf-list defaults Instanciate a Type module to be able to compare with Type enum. The next commit will add unit tests. Fixes: 806be4c3bc4f ("Port to libyang 2") Fixes: #80 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index 6b0c803b..3aef4176 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1279,7 +1279,7 @@ def defaults(self) -> Iterator[str]: if not val: yield None ret = c2str(val) - val_type = self.cdata_leaflist.dflts[i].realtype + val_type = Type(self.context, self.cdata_leaflist.dflts[i].realtype, None) if val_type == Type.BOOL: ret = val == "true" elif val_type in Type.NUM_TYPES: From 3428ea055cd47d7f33e061c0d1ea8116af21b3d1 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Tue, 28 Nov 2023 14:59:21 +0100 Subject: [PATCH 022/115] schema: return float for decimal64 in default function This patch changes the output for leaf/leaf-list nodes using decimal64 type, and adds new unit test for this purpose. The leaf-list defaults function is reworked to be similar to the leaf default function. A new yang schema is added for clarity, and to avoid modifying all the tests using yolo-system.yang. Fixes: #80 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 17 +++++++++------ tests/test_schema.py | 31 ++++++++++++++++++++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 32 +++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 tests/yang/yolo/yolo-nodetypes.yang diff --git a/libyang/schema.py b/libyang/schema.py index 3aef4176..d85912d1 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1209,7 +1209,7 @@ def __init__(self, context: "libyang.Context", cdata): self.cdata_leaf = ffi.cast("struct lysc_node_leaf *", cdata) self.cdata_leaf_parsed = ffi.cast("struct lysp_node_leaf *", self.cdata_parsed) - def default(self) -> Union[None, bool, int, str]: + def default(self) -> Union[None, bool, int, str, float]: if not self.cdata_leaf.dflt: return None val = lib.lyd_value_get_canonical(self.context.cdata, self.cdata_leaf.dflt) @@ -1221,6 +1221,8 @@ def default(self) -> Union[None, bool, int, str]: return val == "true" if val_type.base() in Type.NUM_TYPES: return int(val) + if val_type.base() == Type.DEC64: + return float(val) return val def units(self) -> Optional[str]: @@ -1268,7 +1270,7 @@ def type(self) -> Type: self.context, self.cdata_leaflist.type, self.cdata_leaflist_parsed.type ) - def defaults(self) -> Iterator[str]: + def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: if self.cdata_leaflist.dflts == ffi.NULL: return arr_length = ffi.cast("uint64_t *", self.cdata_leaflist.dflts)[-1] @@ -1278,13 +1280,16 @@ def defaults(self) -> Iterator[str]: ) if not val: yield None - ret = c2str(val) + val = c2str(val) val_type = Type(self.context, self.cdata_leaflist.dflts[i].realtype, None) if val_type == Type.BOOL: - ret = val == "true" + yield val == "true" elif val_type in Type.NUM_TYPES: - ret = int(val) - yield ret + yield int(val) + elif val_type.base() == Type.DEC64: + yield float(val) + else: + yield val def must_conditions(self) -> Iterator[str]: pdata = self.cdata_leaflist_parsed diff --git a/tests/test_schema.py b/tests/test_schema.py index 770a476c..434618bf 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -474,3 +474,34 @@ def test_leaf_parent(self): def test_iter_tree(self): leaf = next(self.ctx.find_path("/yolo-system:conf")) self.assertEqual(len(list(leaf.iter_tree(full=True))), 23) + + +# ------------------------------------------------------------------------------------- +class LeafTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_leaf_default(self): + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) + self.assertIsInstance(leaf.default(), float) + + +# ------------------------------------------------------------------------------------- +class LeafListTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_leaflist_defaults(self): + leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/ratios")) + for d in leaflist.defaults(): + self.assertIsInstance(d, float) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang new file mode 100644 index 00000000..3dd76832 --- /dev/null +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -0,0 +1,32 @@ +module yolo-nodetypes { + yang-version 1.1; + namespace "urn:yang:yolo:nodetypes"; + prefix sys; + + description + "YOLO Nodetypes."; + + revision 2024-01-25 { + description + "Initial version."; + } + + container conf { + description + "Configuration."; + leaf percentage { + type decimal64 { + fraction-digits 2; + } + default 10.2; + } + + leaf-list ratios { + type decimal64 { + fraction-digits 2; + } + default 2.5; + default 2.6; + } + } +} From 6502d47d62154da80472ae95f0b36fa77efb13f3 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:42:12 +0100 Subject: [PATCH 023/115] context: add root_node to find_path This patch enhances the context find_path function to allow using relative paths as well, by adding an optional root_node attribute, from which the relative path is being evaluated. Fixes: #82 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/context.py | 14 ++++++++++++-- tests/test_context.py | 8 +++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index 57432941..0d8d0a67 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -175,17 +175,27 @@ def get_module(self, name: str) -> Module: return Module(self, mod) - def find_path(self, path: str, output: bool = False) -> Iterator[SNode]: + def find_path( + self, + path: str, + output: bool = False, + root_node: Optional["libyang.SNode"] = None, + ) -> Iterator[SNode]: if self.cdata is None: raise RuntimeError("context already destroyed") + if root_node is not None: + ctx_node = root_node.cdata + else: + ctx_node = ffi.NULL + flags = 0 if output: flags |= lib.LYS_FIND_XP_OUTPUT node_set = ffi.new("struct ly_set **") if ( - lib.lys_find_xpath(self.cdata, ffi.NULL, str2c(path), 0, node_set) + lib.lys_find_xpath(self.cdata, ctx_node, str2c(path), 0, node_set) != lib.LY_SUCCESS ): raise self.error("cannot find path") diff --git a/tests/test_context.py b/tests/test_context.py index 6e88a261..5650ecde 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -4,7 +4,7 @@ import os import unittest -from libyang import Context, LibyangError, Module, SRpc +from libyang import Context, LibyangError, Module, SLeaf, SLeafList YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") @@ -82,8 +82,10 @@ def test_ctx_load_invalid_module(self): def test_ctx_find_path(self): with Context(YANG_DIR) as ctx: ctx.load_module("yolo-system") - node = next(ctx.find_path("/yolo-system:format-disk")) - self.assertIsInstance(node, SRpc) + node = next(ctx.find_path("/yolo-system:conf/offline")) + self.assertIsInstance(node, SLeaf) + node2 = next(ctx.find_path("../number", root_node=node)) + self.assertIsInstance(node2, SLeafList) def test_ctx_iter_modules(self): with Context(YANG_DIR) as ctx: From c6caf46e08c084bc558fc271d1a6a93fd8430d21 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:48:04 +0100 Subject: [PATCH 024/115] schema: add a with_typedefs option to union_types This patch adds a new with_typedefs option to the union_types function that will return the typedefs instead of the resolved types when available. A unit test is added as well. Fixes: #83 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 13 ++++++++++++- tests/test_schema.py | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index d85912d1..7e72e809 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -555,16 +555,27 @@ def typedef(self) -> "Typedef": return import_module.get_typedef(type_name) return None - def union_types(self) -> Iterator["Type"]: + def union_types(self, with_typedefs: bool = False) -> Iterator["Type"]: if self.cdata.basetype != self.UNION: return + typedef = self.typedef() t = ffi.cast("struct lysc_type_union *", self.cdata) if self.cdata_parsed and self.cdata_parsed.types != ffi.NULL: for union_type, union_type_parsed in zip( ly_array_iter(t.types), ly_array_iter(self.cdata_parsed.types) ): yield Type(self.context, union_type, union_type_parsed) + elif ( + with_typedefs + and typedef + and typedef.cdata + and typedef.cdata.type.types != ffi.NULL + ): + for union_type, union_type_parsed in zip( + ly_array_iter(t.types), ly_array_iter(typedef.cdata.type.types) + ): + yield Type(self.context, union_type, union_type_parsed) else: for union_type in ly_array_iter(t.types): yield Type(self.context, union_type, None) diff --git a/tests/test_schema.py b/tests/test_schema.py index 434618bf..5bf5f888 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -420,7 +420,9 @@ def test_leaf_type_union(self): self.assertEqual(t.name(), "types:number") self.assertEqual(t.base(), Type.UNION) types = set(u.name() for u in t.union_types()) + types2 = set(u.name() for u in t.union_types(with_typedefs=True)) self.assertEqual(types, set(["int16", "int32", "uint16", "uint32"])) + self.assertEqual(types2, set(["signed", "unsigned"])) for u in t.union_types(): ext = u.get_extension( "type-desc", prefix="omg-extensions", arg_value=f"<{u.name()}>" From 6e42fb3dd9e448ac6e8e3c6599d4e864372dfebe Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Thu, 25 Jan 2024 23:15:15 +0100 Subject: [PATCH 025/115] data: fix DNode double free Calling free twice on a dnode is failing with libyang-python, when it is supported with libyang. That is because cdata is set to None in the free function. Set it to ffi.NULL so that libyang can see the NULL pointer. Fixes: #84 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/data.py | 2 +- tests/test_data.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/libyang/data.py b/libyang/data.py index 69649eba..c253c57e 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -881,7 +881,7 @@ def free(self, with_siblings: bool = True) -> None: else: self.free_internal(with_siblings) finally: - self.cdata = None + self.cdata = ffi.NULL def __repr__(self): cls = self.__class__ diff --git a/tests/test_data.py b/tests/test_data.py index 32c605fe..bf1a44fb 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -876,3 +876,8 @@ def test_add_defaults(self): node = dnode.find_path("/yolo-system:conf/speed") self.assertIsInstance(node, DLeaf) self.assertEqual(node.value(), 4321) + + def test_dnode_double_free(self): + dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", validate_present=True) + dnode.free() + dnode.free() From 6423babd1a5b26af3ce297477cdad6c90362e15f Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:10:04 +0100 Subject: [PATCH 026/115] data: add unlink function This patch adds unlink function from libyang to allow proper node cleanup procedure. Fixes: #84 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 2 ++ libyang/data.py | 6 ++++++ tests/test_data.py | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index c1eaee39..23edb262 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -267,6 +267,8 @@ LY_ERR lys_set_implemented(struct lys_module *, const char **); #define LYD_NEW_PATH_CANON_VALUE ... LY_ERR lyd_new_path(struct lyd_node *, const struct ly_ctx *, const char *, const char *, uint32_t, struct lyd_node **); LY_ERR lyd_find_xpath(const struct lyd_node *, const char *, struct ly_set **); +void lyd_unlink_siblings(struct lyd_node *node); +void lyd_unlink_tree(struct lyd_node *node); void lyd_free_all(struct lyd_node *node); void lyd_free_tree(struct lyd_node *node); diff --git a/libyang/data.py b/libyang/data.py index c253c57e..0ed18750 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -868,6 +868,12 @@ def merge_data_dict( rpcreply=rpcreply, ) + def unlink(self, with_siblings: bool = False) -> None: + if with_siblings: + lib.lyd_unlink_siblings(self.cdata) + else: + lib.lyd_unlink_tree(self.cdata) + def free_internal(self, with_siblings: bool = True) -> None: if with_siblings: lib.lyd_free_all(self.cdata) diff --git a/tests/test_data.py b/tests/test_data.py index bf1a44fb..84e68f82 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -881,3 +881,19 @@ def test_dnode_double_free(self): dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", validate_present=True) dnode.free() dnode.free() + + def test_dnode_unlink(self): + dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", validate_present=True) + self.assertIsInstance(dnode, DContainer) + try: + child = dnode.find_one("hostname") + self.assertIsInstance(child, DNode) + child.unlink(with_siblings=False) + self.assertIsNone(dnode.find_one("hostname")) + child = next(dnode.children(), None) + self.assertIsNot(child, None) + child.unlink(with_siblings=True) + child = next(dnode.children(), None) + self.assertIsNone(child, None) + finally: + dnode.free() From d5f48d6f4f7767148bb2e217aec414766fd643f6 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:11:28 +0100 Subject: [PATCH 027/115] context: add LY_CTX_LEAFREF_EXTENDED option This patch adds the ability to create context with an additional option for extended leafrefs, which can use deref() functions. Fixes: #85 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + cffi/source.c | 4 +-- libyang/context.py | 3 +++ tests/test_context.py | 5 ++++ tests/yang/yolo/yolo-leafref-extended.yang | 31 ++++++++++++++++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 tests/yang/yolo/yolo-leafref-extended.yang diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 23edb262..0e4bd97f 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -14,6 +14,7 @@ struct ly_ctx; #define LY_CTX_EXPLICIT_COMPILE ... #define LY_CTX_REF_IMPLEMENTED ... #define LY_CTX_SET_PRIV_PARSED ... +#define LY_CTX_LEAFREF_EXTENDED ... typedef enum { diff --git a/cffi/source.c b/cffi/source.c index f7fe18a2..2682dd88 100644 --- a/cffi/source.c +++ b/cffi/source.c @@ -9,6 +9,6 @@ #if (LY_VERSION_MAJOR != 2) #error "This version of libyang bindings only works with libyang 2.x" #endif -#if (LY_VERSION_MINOR < 25) -#error "Need at least libyang 2.25" +#if (LY_VERSION_MINOR < 37) +#error "Need at least libyang 2.37" #endif diff --git a/libyang/context.py b/libyang/context.py index 0d8d0a67..dc3e0052 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -28,6 +28,7 @@ def __init__( search_path: Optional[str] = None, disable_searchdir_cwd: bool = True, explicit_compile: Optional[bool] = False, + leafref_extended: bool = False, yanglib_path: Optional[str] = None, yanglib_fmt: str = "json", cdata=None, # C type: "struct ly_ctx *" @@ -41,6 +42,8 @@ def __init__( options |= lib.LY_CTX_DISABLE_SEARCHDIR_CWD if explicit_compile: options |= lib.LY_CTX_EXPLICIT_COMPILE + if leafref_extended: + options |= lib.LY_CTX_LEAFREF_EXTENDED # force priv parsed options |= lib.LY_CTX_SET_PRIV_PARSED diff --git a/tests/test_context.py b/tests/test_context.py index 5650ecde..59839284 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -106,3 +106,8 @@ def test_ctx_parse_module(self): with Context(YANG_DIR) as ctx: mod = ctx.parse_module_file(f, features=["turbo-boost", "networking"]) self.assertIsInstance(mod, Module) + + def test_ctx_leafref_extended(self): + with Context(YANG_DIR, leafref_extended=True) as ctx: + mod = ctx.load_module("yolo-leafref-extended") + self.assertIsInstance(mod, Module) diff --git a/tests/yang/yolo/yolo-leafref-extended.yang b/tests/yang/yolo/yolo-leafref-extended.yang new file mode 100644 index 00000000..0aa8bd2f --- /dev/null +++ b/tests/yang/yolo/yolo-leafref-extended.yang @@ -0,0 +1,31 @@ +module yolo-leafref-extended { + yang-version 1.1; + namespace "urn:yang:yolo:leafref-extended"; + prefix leafref-ext; + + revision 2025-01-25 { + description + "Initial version."; + } + + list list1 { + key leaf1; + leaf leaf1 { + type string; + } + leaf-list leaflist2 { + type string; + } + } + + leaf ref1 { + type leafref { + path "../list1/leaf1"; + } + } + leaf ref2 { + type leafref { + path "deref(../ref1)/../leaflist2"; + } + } +} From b62fb0d52498d9749918f1609b44b75b98c376c8 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:51:41 +0100 Subject: [PATCH 028/115] schema: add fraction_digits support This patch introduces fraction_digits() and all_fraction_digits() functions for SLeaf and SLeafList. Fixes: #86 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 16 ++++++++++++++++ tests/test_schema.py | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index 7e72e809..bd315596 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -622,6 +622,22 @@ def all_ranges(self) -> Iterator[str]: if rng is not None: yield rng + def fraction_digits(self) -> Optional[int]: + if not self.cdata_parsed: + return None + if self.cdata.basetype != self.DEC64: + return None + return self.cdata_parsed.fraction_digits + + def all_fraction_digits(self) -> Iterator[int]: + if self.cdata.basetype == lib.LY_TYPE_UNION: + for t in self.union_types(): + yield from t.all_fraction_digits() + else: + fd = self.fraction_digits() + if fd is not None: + yield fd + STR_TYPES = frozenset((STRING, BINARY, ENUM, IDENT, BITS)) def length(self) -> Optional[str]: diff --git a/tests/test_schema.py b/tests/test_schema.py index 5bf5f888..1287012a 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -477,6 +477,14 @@ def test_iter_tree(self): leaf = next(self.ctx.find_path("/yolo-system:conf")) self.assertEqual(len(list(leaf.iter_tree(full=True))), 23) + def test_leaf_type_fraction_digits(self): + self.ctx.load_module("yolo-nodetypes") + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) + self.assertIsInstance(leaf, SLeaf) + t = leaf.type() + self.assertIsInstance(t, Type) + self.assertEqual(next(t.all_fraction_digits(), None), 2) + # ------------------------------------------------------------------------------------- class LeafTest(unittest.TestCase): From 3e3af68211a1087df778b9d6dfe6a978c7edbcc0 Mon Sep 17 00:00:00 2001 From: Sergei Markov Date: Thu, 25 Jan 2024 13:57:28 +0800 Subject: [PATCH 029/115] context: add support for LYD_VALIDATE_MULTI_ERROR Thanks to the LYD_VALIDATE_MULTI_ERROR, the validation does not stop at the first error, but generates all the detected errors. Add a dedicated test. Fixes: #79 Signed-off-by: Sergei Markov Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + libyang/context.py | 9 ++++++++- libyang/data.py | 3 +++ tests/test_data.py | 25 +++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 0e4bd97f..ba6bd5d4 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -314,6 +314,7 @@ LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint3 #define LYD_VALIDATE_NO_STATE ... #define LYD_VALIDATE_PRESENT ... #define LYD_VALIDATE_OPTS_MASK ... +#define LYD_VALIDATE_MULTI_ERROR ... LY_ERR lyd_parse_data_mem(const struct ly_ctx *, const char *, LYD_FORMAT, uint32_t, uint32_t, struct lyd_node **); diff --git a/libyang/context.py b/libyang/context.py index dc3e0052..eefc4e05 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -339,6 +339,7 @@ def parse_data( ordered: bool = False, strict: bool = False, validate_present: bool = False, + validate_multi_error: bool = False, ) -> Optional[DNode]: if self.cdata is None: raise RuntimeError("context already destroyed") @@ -351,7 +352,9 @@ def parse_data( strict=strict, ) validation_flgs = validation_flags( - no_state=no_state, validate_present=validate_present + no_state=no_state, + validate_present=validate_present, + validate_multi_error=validate_multi_error, ) fmt = data_format(fmt) encode = True @@ -403,6 +406,7 @@ def parse_data_mem( ordered: bool = False, strict: bool = False, validate_present: bool = False, + validate_multi_error: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -416,6 +420,7 @@ def parse_data_mem( ordered=ordered, strict=strict, validate_present=validate_present, + validate_multi_error=validate_multi_error, ) def parse_data_file( @@ -430,6 +435,7 @@ def parse_data_file( ordered: bool = False, strict: bool = False, validate_present: bool = False, + validate_multi_error: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -443,6 +449,7 @@ def parse_data_file( ordered=ordered, strict=strict, validate_present=validate_present, + validate_multi_error=validate_multi_error, ) def __iter__(self) -> Iterator[Module]: diff --git a/libyang/data.py b/libyang/data.py index 0ed18750..c2bf95f5 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -171,12 +171,15 @@ def data_type(dtype): def validation_flags( no_state: bool = False, validate_present: bool = False, + validate_multi_error: bool = False, ) -> int: flags = 0 if no_state: flags |= lib.LYD_VALIDATE_NO_STATE if validate_present: flags |= lib.LYD_VALIDATE_PRESENT + if validate_multi_error: + flags |= lib.LYD_VALIDATE_MULTI_ERROR return flags diff --git a/tests/test_data.py b/tests/test_data.py index 84e68f82..25bec67d 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -256,6 +256,31 @@ def test_data_parse_config_xml(self): finally: dnode.free() + XML_CONFIG_MULTI_ERROR = """ + foo + + https + /CESNET/libyang-python + abcd + + 2000 + +""" + + def test_data_parse_config_xml_multi_error(self): + with self.assertRaises(Exception) as cm: + self.ctx.parse_data_mem( + self.XML_CONFIG_MULTI_ERROR, + "xml", + validate_present=True, + validate_multi_error=True, + ) + self.assertEqual( + str(cm.exception), + 'failed to parse data tree: Invalid boolean value "abcd".: ' + 'List instance is missing its key "host".', + ) + XML_STATE = """ foo From 95012d95d0aa8996fbf14a8099deaf84ec87e41c Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Fri, 26 Jan 2024 09:29:00 +0100 Subject: [PATCH 030/115] schema: add uniques function for SList This patch add ability get a list of SList unique definitions. Fixes: #88 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 9 ++++++++- tests/test_schema.py | 16 ++++++++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 21 +++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index bd315596..b1fd416c 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT from contextlib import suppress -from typing import IO, Any, Dict, Iterator, Optional, Tuple, Union +from typing import IO, Any, Dict, Iterator, List, Optional, Tuple, Union from _libyang import ffi, lib from .util import IOType, LibyangError, c2str, init_output, ly_array_iter, str2c @@ -1416,6 +1416,13 @@ def must_conditions(self) -> Iterator[str]: for must in ly_array_iter(pdata.musts): yield c2str(must.arg.str) + def uniques(self) -> Iterator[List[SNode]]: + for unique in ly_array_iter(self.cdata_list.uniques): + nodes = [] + for node in ly_array_iter(unique): + nodes.append(SNode.new(self.context, node)) + yield nodes + def __str__(self): return "%s [%s]" % (self.name(), ", ".join(k.name() for k in self.keys())) diff --git a/tests/test_schema.py b/tests/test_schema.py index 1287012a..4524a0a6 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -328,6 +328,22 @@ def test_list_parent(self): self.assertIsInstance(parent, SContainer) self.assertEqual(parent.name(), "conf") + def test_list_uniques(self): + self.ctx.load_module("yolo-nodetypes") + list1 = next(self.ctx.find_path("/yolo-nodetypes:conf/list1")) + self.assertIsInstance(list1, SList) + uniques = list(list1.uniques()) + self.assertEqual(len(uniques), 1) + elements = [u.name() for u in uniques[0]] + self.assertEqual(len(elements), 2) + self.assertTrue("leaf2" in elements) + self.assertTrue("leaf3" in elements) + + list2 = next(self.ctx.find_path("/yolo-nodetypes:conf/list2")) + self.assertIsInstance(list2, SList) + uniques = list(list2.uniques()) + self.assertEqual(len(uniques), 0) + # ------------------------------------------------------------------------------------- class RpcTest(unittest.TestCase): diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 3dd76832..4ef7d3d6 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -28,5 +28,26 @@ module yolo-nodetypes { default 2.5; default 2.6; } + + list list1 { + key leaf1; + unique "leaf2 leaf3"; + leaf leaf1 { + type string; + } + leaf leaf2 { + type string; + } + leaf leaf3 { + type string; + } + } + + list list2 { + key leaf1; + leaf leaf1 { + type string; + } + } } } From 4b5b425cdef1647a4b6ea6b50765cbccf61c442d Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sat, 11 Nov 2023 23:57:17 +0100 Subject: [PATCH 031/115] schema: add {min,max}_elements for SList and SLeafList This patch add ability to get minumum and maximum number of elements, and add unit tests. The yolo-nodetypes module is used twice and is now loaded in the list test setUp. Fixes: #89 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 16 ++++++++++++++++ tests/test_schema.py | 24 +++++++++++++++++++++++- tests/yang/yolo/yolo-nodetypes.yang | 12 ++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index b1fd416c..c6ffa607 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1325,6 +1325,16 @@ def must_conditions(self) -> Iterator[str]: for must in ly_array_iter(pdata.musts): yield c2str(must.arg.str) + def max_elements(self) -> int: + return ( + self.cdata_leaflist.max + if self.cdata_leaflist.max != (2**32 - 1) + else None + ) + + def min_elements(self) -> int: + return self.cdata_leaflist.min + def __str__(self): return "%s %s" % (self.name(), self.type().name()) @@ -1423,6 +1433,12 @@ def uniques(self) -> Iterator[List[SNode]]: nodes.append(SNode.new(self.context, node)) yield nodes + def max_elements(self) -> int: + return self.cdata_list.max if self.cdata_list.max != (2**32 - 1) else None + + def min_elements(self) -> int: + return self.cdata_list.min + def __str__(self): return "%s [%s]" % (self.name(), ", ".join(k.name() for k in self.keys())) diff --git a/tests/test_schema.py b/tests/test_schema.py index 4524a0a6..ad093c66 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -291,6 +291,7 @@ class ListTest(unittest.TestCase): def setUp(self): self.ctx = Context(YANG_DIR) self.ctx.load_module("yolo-system") + self.ctx.load_module("yolo-nodetypes") self.list = next(self.ctx.find_path(self.PATH["LOG"])) def tearDown(self): @@ -329,7 +330,6 @@ def test_list_parent(self): self.assertEqual(parent.name(), "conf") def test_list_uniques(self): - self.ctx.load_module("yolo-nodetypes") list1 = next(self.ctx.find_path("/yolo-nodetypes:conf/list1")) self.assertIsInstance(list1, SList) uniques = list(list1.uniques()) @@ -344,6 +344,17 @@ def test_list_uniques(self): uniques = list(list2.uniques()) self.assertEqual(len(uniques), 0) + def test_list_min_max(self): + list1 = next(self.ctx.find_path("/yolo-nodetypes:conf/list1")) + self.assertIsInstance(list1, SList) + self.assertEqual(list1.min_elements(), 2) + self.assertEqual(list1.max_elements(), 10) + + list2 = next(self.ctx.find_path("/yolo-nodetypes:conf/list2")) + self.assertIsInstance(list2, SList) + self.assertEqual(list2.min_elements(), 0) + self.assertEqual(list2.max_elements(), None) + # ------------------------------------------------------------------------------------- class RpcTest(unittest.TestCase): @@ -531,3 +542,14 @@ def test_leaflist_defaults(self): leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/ratios")) for d in leaflist.defaults(): self.assertIsInstance(d, float) + + def test_leaf_list_min_max(self): + leaflist1 = next(self.ctx.find_path("/yolo-nodetypes:conf/leaf-list1")) + self.assertIsInstance(leaflist1, SLeafList) + self.assertEqual(leaflist1.min_elements(), 3) + self.assertEqual(leaflist1.max_elements(), 11) + + leaflist2 = next(self.ctx.find_path("/yolo-nodetypes:conf/leaf-list2")) + self.assertIsInstance(leaflist2, SLeafList) + self.assertEqual(leaflist2.min_elements(), 0) + self.assertEqual(leaflist2.max_elements(), None) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 4ef7d3d6..785c4bc7 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -32,6 +32,8 @@ module yolo-nodetypes { list list1 { key leaf1; unique "leaf2 leaf3"; + min-elements 2; + max-elements 10; leaf leaf1 { type string; } @@ -49,5 +51,15 @@ module yolo-nodetypes { type string; } } + + leaf-list leaf-list1 { + type string; + min-elements 3; + max-elements 11; + } + + leaf-list leaf-list2 { + type string; + } } } From 21849df456b587cf556a3f7ec459a014f0f4ff02 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Tue, 28 Nov 2023 15:02:58 +0100 Subject: [PATCH 032/115] data: add insert_sibling function to DNode This patch introduce the insert_sibling function usable for inserting data nodes on the module level. Add a unit test. A presence container is added on the conf container of the yolo-nodetypes module to avoid this error: > libyang.util.LibyangError: failed to parse data tree: Too few "list1" instances. Fixes: #95 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + libyang/data.py | 5 +++++ tests/test_data.py | 17 +++++++++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 5 +++++ 4 files changed, 28 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index ba6bd5d4..7ab29e50 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -971,6 +971,7 @@ LY_ERR lyd_any_value_str(const struct lyd_node *, char **); LY_ERR lyd_merge_tree(struct lyd_node **, const struct lyd_node *, uint16_t); LY_ERR lyd_merge_siblings(struct lyd_node **, const struct lyd_node *, uint16_t); LY_ERR lyd_insert_child(struct lyd_node *, struct lyd_node *); +LY_ERR lyd_insert_sibling(struct lyd_node *, struct lyd_node *, struct lyd_node **); LY_ERR lyd_diff_apply_all(struct lyd_node **, const struct lyd_node *); #define LYD_DUP_NO_META ... diff --git a/libyang/data.py b/libyang/data.py index c2bf95f5..633cf74a 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -322,6 +322,11 @@ def insert_child(self, node): if ret != lib.LY_SUCCESS: raise self.context.error("cannot insert node") + def insert_sibling(self, node): + ret = lib.lyd_insert_sibling(self.cdata, node.cdata, ffi.NULL) + if ret != lib.LY_SUCCESS: + raise self.context.error("cannot insert sibling") + def name(self) -> str: return c2str(self.cdata.schema.name) diff --git a/tests/test_data.py b/tests/test_data.py index 25bec67d..9b774a2e 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -20,6 +20,7 @@ IOType, LibyangError, ) +from libyang.data import dict_to_dnode YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") @@ -32,6 +33,7 @@ def setUp(self): modules = [ self.ctx.load_module("ietf-netconf"), self.ctx.load_module("yolo-system"), + self.ctx.load_module("yolo-nodetypes"), ] for mod in modules: @@ -922,3 +924,18 @@ def test_dnode_unlink(self): self.assertIsNone(child, None) finally: dnode.free() + + def test_dnode_insert_sibling(self): + MAIN = {"yolo-nodetypes:conf": {"percentage": "20.2"}} + SIBLING = {"yolo-nodetypes:test1": 10} + module = self.ctx.get_module("yolo-nodetypes") + dnode1 = dict_to_dnode(MAIN, module, None, validate=False) + dnode2 = dict_to_dnode(SIBLING, module, None, validate=False) + self.assertEqual(len(list(dnode1.siblings(include_self=False))), 0) + self.assertEqual(len(list(dnode2.siblings(include_self=False))), 0) + dnode2.insert_sibling(dnode1) + self.assertEqual(len(list(dnode1.siblings(include_self=False))), 1) + self.assertEqual(len(list(dnode2.siblings(include_self=False))), 1) + sibling = next(dnode1.siblings(include_self=False), None) + self.assertIsInstance(sibling, DLeaf) + self.assertEqual(sibling.cdata, dnode2.cdata) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 785c4bc7..6daab475 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -12,6 +12,7 @@ module yolo-nodetypes { } container conf { + presence "enable conf"; description "Configuration."; leaf percentage { @@ -62,4 +63,8 @@ module yolo-nodetypes { type string; } } + + leaf test1 { + type uint8; + } } From 06d486da18e2a22b9ede0dfb4a28e1bbef6b7a56 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Tue, 28 Nov 2023 15:04:28 +0100 Subject: [PATCH 033/115] data: add only_node option to add_defaults This patch allows user to use only_node option within add_defaults function. This option limits the scope of creating and adding implicit default data nodes to just given tree where DNode is considered as root. Fixes: #96 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + libyang/data.py | 11 ++++++++--- tests/test_data.py | 20 +++++++++++++++----- tests/yang/yolo/yolo-nodetypes.yang | 11 +++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 7ab29e50..8a62118b 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1021,6 +1021,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc #define LYD_IMPLICIT_OUTPUT ... #define LYD_IMPLICIT_NO_DEFAULTS ... +LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **); LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **); LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **); diff --git a/libyang/data.py b/libyang/data.py index 633cf74a..f9d3301d 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -260,6 +260,7 @@ def add_defaults( no_defaults: bool = False, no_state: bool = False, output: bool = False, + only_node: bool = False, ): flags = implicit_flags( no_config=no_config, @@ -267,9 +268,13 @@ def add_defaults( no_state=no_state, output=output, ) - node_p = ffi.new("struct lyd_node **") - node_p[0] = self.cdata - ret = lib.lyd_new_implicit_all(node_p, self.context.cdata, flags, ffi.NULL) + if only_node: + node_p = ffi.cast("struct lyd_node *", self.cdata) + ret = lib.lyd_new_implicit_tree(node_p, flags, ffi.NULL) + else: + node_p = ffi.new("struct lyd_node **") + node_p[0] = self.cdata + ret = lib.lyd_new_implicit_all(node_p, self.context.cdata, flags, ffi.NULL) if ret != lib.LY_SUCCESS: raise self.context.error("cannot get module") diff --git a/tests/test_data.py b/tests/test_data.py index 9b774a2e..1824acba 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -14,6 +14,7 @@ DataType, DContainer, DLeaf, + DList, DNode, DNotif, DRpc, @@ -893,13 +894,22 @@ def test_find_all(self): dnode.free() def test_add_defaults(self): - dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", validate_present=True) - node = dnode.find_path("/yolo-system:conf/speed") + JSON = '{"yolo-nodetypes:records": [{"id": "rec1"}]}' + dnode = self.ctx.parse_data_mem( + JSON, "json", validate_present=True, parse_only=True + ) + self.assertIsInstance(dnode, DList) + node = dnode.find_one("id") self.assertIsInstance(node, DLeaf) - node.free(with_siblings=False) - node = dnode.find_path("/yolo-system:conf/speed") + node = dnode.find_one("name") + self.assertIsNone(node) + dnode.add_defaults(only_node=True) + node = dnode.find_one("name") + self.assertIsInstance(node, DLeaf) + self.assertEqual(node.value(), "ASD") + node = dnode.find_path("/yolo-nodetypes:conf/speed") self.assertIsNone(node) - dnode.add_defaults() + dnode.add_defaults(only_node=False) node = dnode.find_path("/yolo-system:conf/speed") self.assertIsInstance(node, DLeaf) self.assertEqual(node.value(), 4321) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 6daab475..6da6933a 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -11,6 +11,17 @@ module yolo-nodetypes { "Initial version."; } + list records { + key id; + leaf id { + type string; + } + leaf name { + type string; + default "ASD"; + } + } + container conf { presence "enable conf"; description From f26b56a67de204d82c3bc7cf42888d4fc317a88e Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sun, 12 Nov 2023 01:11:23 +0100 Subject: [PATCH 034/115] schema: add require_instance function for leafrefs This patch introduces require_instance function, to allow user to get information whether the leafref requires valid instace prior being instanciated. Fixes: #93 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/schema.py | 6 ++++++ tests/test_schema.py | 7 +++++++ tests/yang/yolo/yolo-system.yang | 1 + 3 files changed, 14 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index c6ffa607..30d4d744 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -681,6 +681,12 @@ def all_patterns(self) -> Iterator[Tuple[str, bool]]: else: yield from self.patterns() + def require_instance(self) -> Optional[bool]: + if self.cdata.basetype != self.LEAFREF: + return None + t = ffi.cast("struct lysc_type_leafref *", self.cdata) + return bool(t.require_instance) + def module(self) -> Module: if not self.cdata_parsed: return None diff --git a/tests/test_schema.py b/tests/test_schema.py index ad093c66..a7a68f54 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -512,6 +512,13 @@ def test_leaf_type_fraction_digits(self): self.assertIsInstance(t, Type) self.assertEqual(next(t.all_fraction_digits(), None), 2) + def test_leaf_type_require_instance(self): + leaf = next(self.ctx.find_path("/yolo-system:conf/hostname-ref")) + self.assertIsInstance(leaf, SLeaf) + t = leaf.type() + self.assertIsInstance(t, Type) + self.assertFalse(t.require_instance()) + # ------------------------------------------------------------------------------------- class LeafTest(unittest.TestCase): diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index a34c421c..78a0e20f 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -43,6 +43,7 @@ module yolo-system { leaf hostname-ref { type leafref { path "../hostname"; + require-instance false; } } From f5b4e7a084a83a44d1996dd152e1a675eeff9cb5 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sun, 12 Nov 2023 00:07:23 +0100 Subject: [PATCH 035/115] schema: add Must class This patches introduces a new Must class that represents a must statement. A new musts method is added to access the must statements of a node. The must_conditions method is updated to use this new class, and factorized in the SNode class. Fixes: #90 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- libyang/__init__.py | 2 ++ libyang/schema.py | 55 ++++++++++++++--------------- tests/test_schema.py | 10 ++++++ tests/yang/yolo/yolo-nodetypes.yang | 3 ++ 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/libyang/__init__.py b/libyang/__init__.py index aa9dcca9..482a0ad0 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -74,6 +74,7 @@ IfNotFeature, IfOrFeatures, Module, + Must, Revision, SContainer, SLeaf, @@ -138,6 +139,7 @@ "MandatoryAdded", "MandatoryRemoved", "Module", + "Must", "MustAdded", "MustRemoved", "NodeTypeAdded", diff --git a/libyang/schema.py b/libyang/schema.py index 30d4d744..b06623c4 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1036,6 +1036,21 @@ def __str__(self): return "(%s OR %s)" % (self.a, self.b) +# ------------------------------------------------------------------------------------- +class Must: + __slots__ = ("context", "cdata") + + def __init__(self, context: "libyang.Context", cdata): + self.context = context + self.cdata = cdata # C type: "struct lysc_must *" + + def condition(self) -> str: + return c2str(lib.lyxp_get_expr(self.cdata.cond)) + + def error_message(self) -> Optional[str]: + return c2str(self.cdata.emsg) if self.cdata.emsg != ffi.NULL else None + + # ------------------------------------------------------------------------------------- class SNode: __slots__ = ("context", "cdata", "cdata_parsed", "__dict__") @@ -1138,7 +1153,17 @@ def extensions(self) -> Iterator[ExtensionCompiled]: yield ExtensionCompiled(self.context, extension) def must_conditions(self) -> Iterator[str]: - return iter(()) + for must in self.musts(): + yield must.condition() + + def musts(self) -> Iterator[Must]: + mc = lib.lysc_node_musts(self.cdata) + if mc == ffi.NULL: + return + for m in ly_array_iter(mc): + if not m: + continue + yield Must(self.context, m) def get_extension( self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None @@ -1269,13 +1294,6 @@ def is_key(self) -> bool: return True return False - def must_conditions(self) -> Iterator[str]: - pdata = self.cdata_leaf_parsed - if pdata.musts == ffi.NULL: - return - for must in ly_array_iter(pdata.musts): - yield c2str(must.arg.str) - def __str__(self): return "%s %s" % (self.name(), self.type().name()) @@ -1324,13 +1342,6 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: else: yield val - def must_conditions(self) -> Iterator[str]: - pdata = self.cdata_leaflist_parsed - if pdata.musts == ffi.NULL: - return - for must in ly_array_iter(pdata.musts): - yield c2str(must.arg.str) - def max_elements(self) -> int: return ( self.cdata_leaflist.max @@ -1363,13 +1374,6 @@ def presence(self) -> Optional[str]: return c2str(self.cdata_container_parsed.presence) - def must_conditions(self) -> Iterator[str]: - pdata = self.cdata_container_parsed - if pdata.musts == ffi.NULL: - return - for must in ly_array_iter(pdata.musts): - yield c2str(must.arg.str) - def __iter__(self) -> Iterator[SNode]: return self.children() @@ -1425,13 +1429,6 @@ def keys(self) -> Iterator[SNode]: yield SLeaf(self.context, node) node = node.next - def must_conditions(self) -> Iterator[str]: - pdata = self.cdata_list_parsed - if pdata.musts == ffi.NULL: - return - for must in ly_array_iter(pdata.musts): - yield c2str(must.arg.str) - def uniques(self) -> Iterator[List[SNode]]: for unique in ly_array_iter(self.cdata_list.uniques): nodes = [] diff --git a/tests/test_schema.py b/tests/test_schema.py index a7a68f54..c83bde7f 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -12,6 +12,7 @@ IOType, LibyangError, Module, + Must, Revision, SContainer, SLeaf, @@ -530,6 +531,15 @@ def tearDown(self): self.ctx.destroy() self.ctx = None + def test_must(self): + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) + self.assertIsInstance(leaf, SLeaf) + must = next(leaf.musts(), None) + self.assertIsInstance(must, Must) + self.assertEqual(must.error_message(), "ERROR1") + must = next(leaf.must_conditions(), None) + self.assertIsInstance(must, str) + def test_leaf_default(self): leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) self.assertIsInstance(leaf.default(), float) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 6da6933a..a456ae1d 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -31,6 +31,9 @@ module yolo-nodetypes { fraction-digits 2; } default 10.2; + must ". = 10.6" { + error-message "ERROR1"; + } } leaf-list ratios { From 0f438382dabd6e1f0850ba5c28cb5f2a3442b80c Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Sun, 12 Nov 2023 00:08:42 +0100 Subject: [PATCH 036/115] schema: add Pattern class This patches introduces a new Pattern class that represents a pattern statement. A new pattern_details method is added to access the pattern statements of a node. Fixes: #92 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 15 +++++++++++++ libyang/__init__.py | 2 ++ libyang/schema.py | 36 ++++++++++++++++++++++++++++++++ tests/test_schema.py | 10 +++++++++ tests/yang/yolo/yolo-system.yang | 4 +++- 5 files changed, 66 insertions(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 8a62118b..53e91695 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -792,6 +792,21 @@ struct lysc_must { struct lysc_ext_instance *exts; }; +struct pcre2_real_code; +typedef struct pcre2_real_code pcre2_code; + +struct lysc_pattern { + const char *expr; + pcre2_code *code; + const char *dsc; + const char *ref; + const char *emsg; + const char *eapptag; + struct lysc_ext_instance *exts; + uint32_t inverted : 1; + uint32_t refcount : 31; +}; + #define LYSP_RESTR_PATTERN_ACK ... #define LYSP_RESTR_PATTERN_NACK ... diff --git a/libyang/__init__.py b/libyang/__init__.py index 482a0ad0..f9dfeccd 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -75,6 +75,7 @@ IfOrFeatures, Module, Must, + Pattern, Revision, SContainer, SLeaf, @@ -146,6 +147,7 @@ "NodeTypeRemoved", "OrderedByUserAdded", "OrderedByUserRemoved", + "Pattern", "PatternAdded", "PatternRemoved", "PresenceAdded", diff --git a/libyang/schema.py b/libyang/schema.py index b06623c4..3ba6d996 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -448,6 +448,24 @@ class Bit(_EnumBit): pass +# ------------------------------------------------------------------------------------- +class Pattern: + __slots__ = ("context", "cdata") + + def __init__(self, context: "libyang.Context", cdata): + self.context = context + self.cdata = cdata # C type: "struct lysc_pattern *" + + def expression(self) -> str: + return c2str(self.cdata.expr) + + def inverted(self) -> bool: + return self.cdata.inverted + + def error_message(self) -> Optional[str]: + return c2str(self.cdata.emsg) if self.cdata.emsg != ffi.NULL else None + + # ------------------------------------------------------------------------------------- class Type: __slots__ = ("context", "cdata", "cdata_parsed", "__dict__") @@ -681,6 +699,24 @@ def all_patterns(self) -> Iterator[Tuple[str, bool]]: else: yield from self.patterns() + def pattern_details(self) -> Iterator[Pattern]: + if self.cdata.basetype != self.STRING: + return + t = ffi.cast("struct lysc_type_str *", self.cdata) + if t.patterns == ffi.NULL: + return + for p in ly_array_iter(t.patterns): + if not p: + continue + yield Pattern(self.context, p) + + def all_pattern_details(self) -> Iterator[Pattern]: + if self.cdata.basetype == lib.LY_TYPE_UNION: + for t in self.union_types(): + yield from t.all_pattern_details() + else: + yield from self.pattern_details() + def require_instance(self) -> Optional[bool]: if self.cdata.basetype != self.LEAFREF: return None diff --git a/tests/test_schema.py b/tests/test_schema.py index c83bde7f..78b1229b 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -13,6 +13,7 @@ LibyangError, Module, Must, + Pattern, Revision, SContainer, SLeaf, @@ -439,6 +440,15 @@ def test_leaf_type_pattern(self): t = leaf.type() self.assertIsInstance(t, Type) self.assertEqual(list(t.patterns()), [("[a-z.]+", False), ("1", True)]) + patterns = list(t.all_pattern_details()) + self.assertEqual(len(patterns), 2) + self.assertIsInstance(patterns[0], Pattern) + self.assertEqual(patterns[0].expression(), "[a-z.]+") + self.assertFalse(patterns[0].inverted()) + self.assertEqual(patterns[0].error_message(), "ERROR1") + self.assertEqual(patterns[1].expression(), "1") + self.assertTrue(patterns[1].inverted()) + self.assertIsNone(patterns[1].error_message()) def test_leaf_type_union(self): leaf = next(self.ctx.find_path("/yolo-system:conf/yolo-system:number")) diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index 78a0e20f..5aa633ad 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -85,7 +85,9 @@ module yolo-system { } leaf host { type string { - pattern "[a-z.]+"; + pattern "[a-z.]+" { + error-message "ERROR1"; + } pattern "1" { modifier "invert-match"; } From a6c7164b743d83041cb0db5363b0863f9a311be1 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Mon, 29 Jan 2024 13:28:21 +0100 Subject: [PATCH 037/115] schema: use Pattern class for patterns method Now that we have the Pattern class, use it for the patterns() method. Fixes: #92 Signed-off-by: Samuel Gauthier --- libyang/schema.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 3ba6d996..c3f0c931 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -677,20 +677,8 @@ def all_lengths(self) -> Iterator[str]: yield length def patterns(self) -> Iterator[Tuple[str, bool]]: - if not self.cdata_parsed or self.cdata.basetype != self.STRING: - return - if self.cdata_parsed.patterns == ffi.NULL: - return - for p in ly_array_iter(self.cdata_parsed.patterns): - if not p: - continue - # in case of pattern restriction, the first byte has a special meaning: - # 0x06 (ACK) for regular match and 0x15 (NACK) for invert-match - invert_match = p.arg.str[0] == b"\x15" - # yield tuples like: - # ('[a-zA-Z_][a-zA-Z0-9\-_.]*', False) - # ('[xX][mM][lL].*', True) - yield c2str(p.arg.str + 1), invert_match + for pattern in self.pattern_details(): + yield pattern.expression(), pattern.inverted() def all_patterns(self) -> Iterator[Tuple[str, bool]]: if self.cdata.basetype == lib.LY_TYPE_UNION: From 5681ef0f50ee14ceaa811c6b36c30f84de0d0f1c Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Fri, 26 Jan 2024 11:13:49 +0100 Subject: [PATCH 038/115] schema: support LYS_GETNEXT_WITH{CASE,CHOICE} Add support for LYS_GETNEXT_WITH{CASE,CHOICE} for Module, Container, Choice and Case. To do so, introduce a new iter_children_options function that translates booleans to lys_getnext options. The Rpc children function is updated to use the iter_children_options mecanism. The lib.LYS_CHOICE and lib.LYS_CASE are added by to the default types in iter_children, as it does not change the behavior. Fixes: #91 Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 1 + libyang/schema.py | 89 +++++++++++++++++++++++++++++++++++++------- tests/test_schema.py | 4 ++ 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 53e91695..78e500a4 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -691,6 +691,7 @@ struct lysc_ext { #define LYS_GETNEXT_WITHCASE ... #define LYS_GETNEXT_INTONPCONT ... #define LYS_GETNEXT_OUTPUT ... +#define LYS_GETNEXT_WITHSCHEMAMOUNT ... const struct lysc_node* lys_find_child(const struct lysc_node *, const struct lys_module *, const char *, size_t, uint16_t, uint32_t); const struct lysc_node* lysc_node_child(const struct lysc_node *); diff --git a/libyang/schema.py b/libyang/schema.py index c3f0c931..b257c8f6 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -137,8 +137,12 @@ def get_module_from_prefix(self, prefix: str) -> Optional["Module"]: def __iter__(self) -> Iterator["SNode"]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator["SNode"]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator["SNode"]: + return iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) def __str__(self) -> str: return self.name() @@ -1401,8 +1405,12 @@ def presence(self) -> Optional[str]: def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator[SNode]: + return iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) # ------------------------------------------------------------------------------------- @@ -1411,8 +1419,10 @@ class SChoice(SNode): def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_case: bool = False + ) -> Iterator[SNode]: + return iter_children(self.context, self.cdata, types=types, with_case=with_case) # ------------------------------------------------------------------------------------- @@ -1421,8 +1431,12 @@ class SCase(SNode): def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator[SNode]: + return iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) # ------------------------------------------------------------------------------------- @@ -1442,9 +1456,18 @@ def __iter__(self) -> Iterator[SNode]: return self.children() def children( - self, skip_keys: bool = False, types: Optional[Tuple[int, ...]] = None + self, + skip_keys: bool = False, + types: Optional[Tuple[int, ...]] = None, + with_choice: bool = False, ) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, skip_keys=skip_keys, types=types) + return iter_children( + self.context, + self.cdata, + skip_keys=skip_keys, + types=types, + with_choice=with_choice, + ) def keys(self) -> Iterator[SNode]: node = lib.lysc_node_child(self.cdata) @@ -1512,9 +1535,7 @@ def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: yield from iter_children(self.context, self.cdata, types=types) # With libyang2, you can get only input or output # To keep behavior, we iter 2 times witt output options - yield from iter_children( - self.context, self.cdata, types=types, options=lib.LYS_GETNEXT_OUTPUT - ) + yield from iter_children(self.context, self.cdata, types=types, output=True) # ------------------------------------------------------------------------------------- @@ -1539,13 +1560,43 @@ class SAnydata(SNode): pass +# ------------------------------------------------------------------------------------- +def iter_children_options( + with_choice: bool = False, + no_choice: bool = False, + with_case: bool = False, + into_non_presence_container: bool = False, + output: bool = False, + with_schema_mount: bool = False, +) -> int: + options = 0 + if with_choice: + options |= lib.LYS_GETNEXT_WITHCHOICE + if no_choice: + options |= lib.LYS_GETNEXT_NOCHOICE + if with_case: + options |= lib.LYS_GETNEXT_WITHCASE + if into_non_presence_container: + options |= lib.LYS_GETNEXT_INTONPCONT + if output: + options |= lib.LYS_GETNEXT_OUTPUT + if with_schema_mount: + options |= lib.LYS_GETNEXT_WITHSCHEMAMOUNT + return options + + # ------------------------------------------------------------------------------------- def iter_children( context: "libyang.Context", parent, # C type: Union["struct lys_module *", "struct lys_node *"] skip_keys: bool = False, types: Optional[Tuple[int, ...]] = None, - options: int = 0, + with_choice: bool = False, + no_choice: bool = False, + with_case: bool = False, + into_non_presence_container: bool = False, + output: bool = False, + with_schema_mount: bool = False, ) -> Iterator[SNode]: if types is None: types = ( @@ -1556,6 +1607,8 @@ def iter_children( lib.LYS_LEAF, lib.LYS_LEAFLIST, lib.LYS_NOTIF, + lib.LYS_CHOICE, + lib.LYS_CASE, ) def _skip(node) -> bool: @@ -1576,6 +1629,14 @@ def _skip(node) -> bool: else: module = ffi.NULL + options = iter_children_options( + with_choice=with_choice, + no_choice=no_choice, + with_case=with_case, + into_non_presence_container=into_non_presence_container, + output=output, + with_schema_mount=with_schema_mount, + ) child = lib.lys_getnext(ffi.NULL, parent, module, options) while child: if not _skip(child): diff --git a/tests/test_schema.py b/tests/test_schema.py index 78b1229b..5b947cd8 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -271,6 +271,10 @@ def test_cont_iter(self): def test_cont_children_leafs(self): leafs = list(self.container.children(types=(SNode.LEAF,))) self.assertEqual(len(leafs), 9) + without_choice = [c.name() for c in self.container.children(with_choice=False)] + with_choice = [c.name() for c in self.container.children(with_choice=True)] + self.assertTrue("pill" not in without_choice) + self.assertTrue("pill" in with_choice) def test_cont_parent(self): self.assertIsNone(self.container.parent()) From ae3b32023ba5f9a5fbf52299785528e49ab8251a Mon Sep 17 00:00:00 2001 From: nvxf <68589039+nvxf@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:23:47 +0100 Subject: [PATCH 039/115] data: fix DNode.new schema handling In case of a call to DNode.new with cdata containing an opaque node the new method tries to access cdata.schema.nodetype, which results in a NULL pointer dereference. To get the schema for an opaque node retrieve the schema from the context using the path of the node. Fixes: #73 Signed-off-by: nvxf <68589039+nvxf@users.noreply.github.com> Acked-by: Samuel Gauthier --- libyang/data.py | 22 ++++++++++++++++------ tests/test_data.py | 11 +++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/libyang/data.py b/libyang/data.py index f9d3301d..abc66cd6 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -417,11 +417,7 @@ def eval_xpath(self, xpath: str): return False def path(self) -> str: - path = lib.lyd_path(self.cdata, lib.LYD_PATH_STD, ffi.NULL, 0) - try: - return c2str(path) - finally: - lib.free(path) + return self._get_path(self.cdata) def validate( self, @@ -923,11 +919,25 @@ def _decorator(nodeclass): @classmethod def new(cls, context: "libyang.Context", cdata) -> "DNode": cdata = ffi.cast("struct lyd_node *", cdata) - nodecls = cls.NODETYPE_CLASS.get(cdata.schema.nodetype, None) + if not cdata.schema: + schemas = list(context.find_path(cls._get_path(cdata))) + if len(schemas) != 1: + raise LibyangError("Unable to determine schema") + nodecls = cls.NODETYPE_CLASS.get(schemas[0].nodetype(), None) + else: + nodecls = cls.NODETYPE_CLASS.get(cdata.schema.nodetype, None) if nodecls is None: raise TypeError("node type %s not implemented" % cdata.schema.nodetype) return nodecls(context, cdata) + @staticmethod + def _get_path(cdata) -> str: + path = lib.lyd_path(cdata, lib.LYD_PATH_STD, ffi.NULL, 0) + try: + return c2str(path) + finally: + lib.free(path) + # ------------------------------------------------------------------------------------- @DNode.register(SNode.CONTAINER) diff --git a/tests/test_data.py b/tests/test_data.py index 1824acba..6a160359 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -949,3 +949,14 @@ def test_dnode_insert_sibling(self): sibling = next(dnode1.siblings(include_self=False), None) self.assertIsInstance(sibling, DLeaf) self.assertEqual(sibling.cdata, dnode2.cdata) + + def test_dnode_new_opaq_find_one(self): + root = self.ctx.create_data_path(path="/yolo-system:conf") + root.new_path( + "hostname", + None, + opt_opaq=True, + ) + dnode = root.find_one("/yolo-system:conf/hostname") + + self.assertIsInstance(dnode, DLeaf) From 1a069b9c8188f329dcc05371d9288b1ad318d936 Mon Sep 17 00:00:00 2001 From: nvxf <68589039+nvxf@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:25:32 +0100 Subject: [PATCH 040/115] data: add support for lyd_attr to DNode Add new DNodeAttrs class mapped to access the lyd_attr structure in a DNode, and unit tests for it. Fixes: #77 Signed-off-by: nvxf <68589039+nvxf@users.noreply.github.com> Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 51 +++++++++++++++++++++++++++++ libyang/__init__.py | 1 + libyang/data.py | 66 ++++++++++++++++++++++++++++++++++++-- tests/test_data.py | 78 +++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 192 insertions(+), 4 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 78e500a4..b9d3b773 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1042,5 +1042,56 @@ LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **); +struct ly_opaq_name { + const char *name; + const char *prefix; + + union { + const char *module_ns; + const char *module_name; + }; +}; + +struct lyd_node_opaq { + union { + struct lyd_node node; + + struct { + uint32_t hash; + uint32_t flags; + const struct lysc_node *schema; + struct lyd_node_inner *parent; + struct lyd_node *next; + struct lyd_node *prev; + struct lyd_meta *meta; + void *priv; + }; + }; + + struct lyd_node *child; + + struct ly_opaq_name name; + const char *value; + uint32_t hints; + LY_VALUE_FORMAT format; + void *val_prefix_data; + + struct lyd_attr *attr; + const struct ly_ctx *ctx; +}; + +struct lyd_attr { + struct lyd_node_opaq *parent; + struct lyd_attr *next; + struct ly_opaq_name name; + const char *value; + uint32_t hints; + LY_VALUE_FORMAT format; + void *val_prefix_data; +}; + +LY_ERR lyd_new_attr(struct lyd_node *, const char *, const char *, const char *, struct lyd_attr **); +void lyd_free_attr_single(const struct ly_ctx *ctx, struct lyd_attr *attr); + /* from libc, needed to free allocated strings */ void free(void *); diff --git a/libyang/__init__.py b/libyang/__init__.py index f9dfeccd..5e3854ad 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -13,6 +13,7 @@ DLeafList, DList, DNode, + DNodeAttrs, DNotif, DRpc, ) diff --git a/libyang/data.py b/libyang/data.py index abc66cd6..f0caf240 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT import logging -from typing import IO, Any, Dict, Iterator, Optional, Union +from typing import IO, Any, Dict, Iterator, Optional, Tuple, Union from _libyang import ffi, lib from .keyed_list import KeyedList @@ -190,13 +190,69 @@ def diff_flags(with_defaults: bool = False) -> int: return flags +# ------------------------------------------------------------------------------------- +class DNodeAttrs: + __slots__ = ("context", "parent", "cdata", "__dict__") + + def __init__(self, context: "libyang.Context", parent: "libyang.DNode"): + self.context = context + self.parent = parent + self.cdata = [] # C type: "struct lyd_attr *" + + def get(self, name: str) -> Optional[str]: + for attr_name, attr_value in self: + if attr_name == name: + return attr_value + return None + + def set(self, name: str, value: str): + attrs = ffi.new("struct lyd_attr **") + ret = lib.lyd_new_attr( + self.parent.cdata, + ffi.NULL, + str2c(name), + str2c(value), + attrs, + ) + if ret != lib.LY_SUCCESS: + raise self.context.error("cannot create attr") + self.cdata.append(attrs[0]) + + def remove(self, name: str): + for attr in self.cdata: + if self._get_attr_name(attr) == name: + lib.lyd_free_attr_single(self.context.cdata, attr) + self.cdata.remove(attr) + break + + def __contains__(self, name: str) -> bool: + for attr_name, _ in self: + if attr_name == name: + return True + return False + + def __iter__(self) -> Iterator[Tuple[str, str]]: + for attr in self.cdata: + name = self._get_attr_name(attr) + yield (name, c2str(attr.value)) + + def __len__(self) -> int: + return len(self.cdata) + + @staticmethod + def _get_attr_name(cdata) -> str: + if cdata.name.prefix != ffi.NULL: + return f"{c2str(cdata.name.prefix)}:{c2str(cdata.name.name)}" + return c2str(cdata.name.name) + + # ------------------------------------------------------------------------------------- class DNode: """ Data tree node. """ - __slots__ = ("context", "cdata", "free_func", "__dict__") + __slots__ = ("context", "cdata", "attributes", "free_func", "__dict__") def __init__(self, context: "libyang.Context", cdata): """ @@ -207,6 +263,7 @@ def __init__(self, context: "libyang.Context", cdata): """ self.context = context self.cdata = cdata # C type: "struct lyd_node *" + self.attributes = None self.free_func = None # type: Callable[DNode] def meta(self): @@ -254,6 +311,11 @@ def new_meta(self, name: str, value: str, clear_dflt: bool = False): if ret != lib.LY_SUCCESS: raise self.context.error("cannot create meta") + def attrs(self) -> DNodeAttrs: + if not self.attributes: + self.attributes = DNodeAttrs(self.context, self) + return self.attributes + def add_defaults( self, no_config: bool = False, diff --git a/tests/test_data.py b/tests/test_data.py index 6a160359..10d9045f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -16,6 +16,7 @@ DLeaf, DList, DNode, + DNodeAttrs, DNotif, DRpc, IOType, @@ -950,13 +951,86 @@ def test_dnode_insert_sibling(self): self.assertIsInstance(sibling, DLeaf) self.assertEqual(sibling.cdata, dnode2.cdata) - def test_dnode_new_opaq_find_one(self): + def _create_opaq_hostname(self): root = self.ctx.create_data_path(path="/yolo-system:conf") root.new_path( "hostname", None, opt_opaq=True, ) - dnode = root.find_one("/yolo-system:conf/hostname") + return root.find_one("/yolo-system:conf/hostname") + + def test_dnode_new_opaq_find_one(self): + dnode = self._create_opaq_hostname() self.assertIsInstance(dnode, DLeaf) + + def test_dnode_attrs(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + self.assertIsInstance(attrs, DNodeAttrs) + + def test_dnode_attrs_set(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + self.assertEqual(len(attrs.cdata), 0) + attrs.set("ietf-netconf:operation", "remove") + + self.assertEqual(len(attrs.cdata), 1) + + def test_dnode_attrs_get(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + attrs.set("ietf-netconf:operation", "remove") + + value = attrs.get("ietf-netconf:operation") + self.assertEqual(value, "remove") + + def test_dnode_attrs__len(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + self.assertEqual(len(attrs), 0) + attrs.set("ietf-netconf:operation", "remove") + + self.assertEqual(len(attrs), 1) + + def test_dnode_attrs__contains(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + attrs.set("ietf-netconf:operation", "remove") + + self.assertTrue("ietf-netconf:operation" in attrs) + + def test_dnode_attrs_remove(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + attrs.set("ietf-netconf:operation", "remove") + attrs.remove("ietf-netconf:operation") + + self.assertEqual(len(attrs), 0) + + def test_dnode_attrs_set_and_remove_multiple(self): + dnode = self._create_opaq_hostname() + attrs = dnode.attrs() + + attrs.set("ietf-netconf:operation", "remove") + attrs.set("something:else", "test") + attrs.set("no_prefix", "test") + self.assertEqual(len(attrs), 3) + + attrs.remove("something:else") + self.assertEqual(len(attrs), 2) + self.assertIn("no_prefix", attrs) + self.assertIn("ietf-netconf:operation", attrs) + + attrs.remove("no_prefix") + self.assertEqual(len(attrs), 1) + + attrs.remove("ietf-netconf:operation") + self.assertEqual(len(attrs), 0) From 6e94f1bd744af16bfc4dd12218e82f98cd356f08 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Fri, 26 Jan 2024 11:13:49 +0100 Subject: [PATCH 041/115] schema: add default function to SChoice class This patches introduces a default function to SChoice class, including unit test. Fixes: #97 Signed-off-by: Stefan Gula Acked-by: Samuel Gauthier --- cffi/cdefs.h | 13 +++++++++++++ libyang/__init__.py | 4 ++++ libyang/schema.py | 11 +++++++++++ tests/test_schema.py | 19 +++++++++++++++++++ tests/yang/yolo/yolo-system.yang | 1 + 5 files changed, 48 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index b9d3b773..304ae55d 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1023,6 +1023,19 @@ struct lysc_when { struct lysc_when** lysc_node_when(const struct lysc_node *); +struct lysc_node_case { + struct lysc_node *child; + struct lysc_when **when; + ...; +}; + +struct lysc_node_choice { + struct lysc_node_case *cases; + struct lysc_when **when; + struct lysc_node_case *dflt; + ...; +}; + #define LYD_DEFAULT ... #define LYD_WHEN_TRUE ... #define LYD_NEW ... diff --git a/libyang/__init__.py b/libyang/__init__.py index 5e3854ad..bc79e2f0 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -78,6 +78,8 @@ Must, Pattern, Revision, + SCase, + SChoice, SContainer, SLeaf, SLeafList, @@ -156,6 +158,8 @@ "RangeAdded", "RangeRemoved", "Revision", + "SCase", + "SChoice", "SContainer", "SLeaf", "SLeafList", diff --git a/libyang/schema.py b/libyang/schema.py index b257c8f6..d49c39a7 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1416,6 +1416,12 @@ def children( # ------------------------------------------------------------------------------------- @SNode.register(SNode.CHOICE) class SChoice(SNode): + __slots__ = ("cdata_choice",) + + def __init__(self, context: "libyang.Context", cdata): + super().__init__(context, cdata) + self.cdata_choice = ffi.cast("struct lysc_node_choice *", cdata) + def __iter__(self) -> Iterator[SNode]: return self.children() @@ -1424,6 +1430,11 @@ def children( ) -> Iterator[SNode]: return iter_children(self.context, self.cdata, types=types, with_case=with_case) + def default(self) -> Optional[SNode]: + if self.cdata_choice.dflt == ffi.NULL: + return None + return SNode.new(self.context, self.cdata_choice.dflt) + # ------------------------------------------------------------------------------------- @SNode.register(SNode.CASE) diff --git a/tests/test_schema.py b/tests/test_schema.py index 5b947cd8..64a8e30e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -15,6 +15,8 @@ Must, Pattern, Revision, + SCase, + SChoice, SContainer, SLeaf, SLeafList, @@ -584,3 +586,20 @@ def test_leaf_list_min_max(self): self.assertIsInstance(leaflist2, SLeafList) self.assertEqual(leaflist2.min_elements(), 0) self.assertEqual(leaflist2.max_elements(), None) + + +# ------------------------------------------------------------------------------------- +class ChoiceTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-system") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_choice_default(self): + conf = next(self.ctx.find_path("/yolo-system:conf")) + choice = next(conf.children((SNode.CHOICE,), with_choice=True)) + self.assertIsInstance(choice, SChoice) + self.assertIsInstance(choice.default(), SCase) diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index 5aa633ad..ef612546 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -72,6 +72,7 @@ module yolo-system { type boolean; } } + default red; } list url { From 52da1c4ee4e0bb6c6e9865466f40effff1e61deb Mon Sep 17 00:00:00 2001 From: Samuel Gauthier Date: Wed, 7 Feb 2024 13:59:19 +0100 Subject: [PATCH 042/115] Revert "schema: use Pattern class for patterns method" This reverts commit a6c7164b743d83041cb0db5363b0863f9a311be1. It actually breaks compatibility, remove this. Signed-off-by: Samuel Gauthier --- libyang/schema.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index d49c39a7..deec466d 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -681,8 +681,20 @@ def all_lengths(self) -> Iterator[str]: yield length def patterns(self) -> Iterator[Tuple[str, bool]]: - for pattern in self.pattern_details(): - yield pattern.expression(), pattern.inverted() + if not self.cdata_parsed or self.cdata.basetype != self.STRING: + return + if self.cdata_parsed.patterns == ffi.NULL: + return + for p in ly_array_iter(self.cdata_parsed.patterns): + if not p: + continue + # in case of pattern restriction, the first byte has a special meaning: + # 0x06 (ACK) for regular match and 0x15 (NACK) for invert-match + invert_match = p.arg.str[0] == b"\x15" + # yield tuples like: + # ('[a-zA-Z_][a-zA-Z0-9\-_.]*', False) + # ('[xX][mM][lL].*', True) + yield c2str(p.arg.str + 1), invert_match def all_patterns(self) -> Iterator[Tuple[str, bool]]: if self.cdata.basetype == lib.LY_TYPE_UNION: From f09ed1164d860fd9f04c008e42bbac13ccdb2ff2 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Thu, 1 Feb 2024 16:15:30 +0800 Subject: [PATCH 043/115] Fix typos, Optionnal -> Optional Found via `codespell -H` --- libyang/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/diff.py b/libyang/diff.py index b2a15118..37441f14 100644 --- a/libyang/diff.py +++ b/libyang/diff.py @@ -23,7 +23,7 @@ def schema_diff( :arg ctx_new: The second context. :arg exclude_node_cb: - Optionnal user callback that will be called with each node that is found in each + Optional user callback that will be called with each node that is found in each context. If the callback returns a "trueish" value, the node will be excluded from the diff (as well as all its children). :arg use_data_path: From 83e76f93bfac3b8132f54cc40ab5ab49443b43e3 Mon Sep 17 00:00:00 2001 From: Robin Jarry Date: Wed, 27 Mar 2024 23:41:32 +0100 Subject: [PATCH 044/115] ci: check commit messages in pull requests Unfortunately Github does not allow commenting on commit messages directly. At least perform basic checks to enforce our rules: * titles should be less than 72 characters * titles should start with a short lower case prefix to mention the "topic" of the commit. * no capitalization nor punctuation in the commit title * all commits should have English prose describing the changes. * all commits should be signed-off-by their author (git commit -s). * the list of sanctioned commit trailers is enforced. * referencing github issues/pull_requests must be done via full urls. * referenced commit ids must be valid. Signed-off-by: Robin Jarry --- .github/workflows/ci.yml | 13 ++++ Makefile | 8 ++- README.rst | 71 ++++++++++++++++----- check-commits.sh | 130 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 17 deletions(-) create mode 100755 check-commits.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c308738..fd6ad584 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,19 @@ jobs: - run: python -m pip install --upgrade tox - run: python -m tox -e lint + check-commits: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + env: + LYPY_START_COMMIT: "${{ github.event.pull_request.base.sha }}" + LYPY_END_COMMIT: "${{ github.event.pull_request.head.sha }}" + steps: + - run: sudo apt-get install git make + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: make check-commits + test: runs-on: ubuntu-20.04 strategy: diff --git a/Makefile b/Makefile index 90147aaa..6e9c436b 100644 --- a/Makefile +++ b/Makefile @@ -12,4 +12,10 @@ tests: format: tox -e format -.PHONY: lint tests format +LYPY_START_COMMIT ?= origin/master +LYPY_END_COMMIT ?= HEAD + +check-commits: + ./check-commits.sh $(LYPY_START_COMMIT)..$(LYPY_END_COMMIT) + +.PHONY: lint tests format check-commits diff --git a/README.rst b/README.rst index 0387cf57..4ea8977f 100644 --- a/README.rst +++ b/README.rst @@ -232,7 +232,7 @@ Here are the steps for submitting a change in the code base: #. Create a new branch named after what your are working on:: - git checkout -b my-topic + git checkout -b my-topic -t origin/master #. Edit the code and call ``make format`` to ensure your modifications comply with the `coding style`__. @@ -251,21 +251,60 @@ Here are the steps for submitting a change in the code base: your changes do not break anything. You can also run ``make`` which will run both. -#. Create commits by following these simple guidelines: - - - Solve only one problem per commit. - - Use a short (less than 72 characters) title on the first line followed by - an blank line and a more thorough description body. - - Wrap the body of the commit message should be wrapped at 72 characters too - unless it breaks long URLs or code examples. - - If the commit fixes a Github issue, include the following line:: - - Fixes: #NNNN - - Inspirations: - - https://chris.beams.io/posts/git-commit/ - https://wiki.openstack.org/wiki/GitCommitMessages +#. Once you are happy with your work, you can create a commit (or several + commits). Follow these general rules: + + - Address only one issue/topic per commit. + - Describe your changes in imperative mood, e.g. *"make xyzzy do frotz"* + instead of *"[This patch] makes xyzzy do frotz"* or *"[I] changed xyzzy to + do frotz"*, as if you are giving orders to the codebase to change its + behaviour. + - Limit the first line (title) of the commit message to 60 characters. + - Use a short prefix for the commit title for readability with ``git log + --oneline``. Do not use the `fix:` nor `feature:` prefixes. See recent + commits for inspiration. + - Only use lower case letters for the commit title except when quoting + symbols or known acronyms. + - Use the body of the commit message to actually explain what your patch + does and why it is useful. Even if your patch is a one line fix, the + description is not limited in length and may span over multiple + paragraphs. Use proper English syntax, grammar and punctuation. + - If you are fixing an issue, use appropriate ``Closes: `` or + ``Fixes: `` trailers. + - If you are fixing a regression introduced by another commit, add a + ``Fixes: ("")`` trailer. + - When in doubt, follow the format and layout of the recent existing + commits. + - The following trailers are accepted in commits. If you are using multiple + trailers in a commit, it's preferred to also order them according to this + list. + + * ``Closes: <URL>``: close the referenced issue or pull request. + * ``Fixes: <SHA> ("<TITLE>")``: reference the commit that introduced + a regression. + * ``Link: <URL>``: any useful link to provide context for your commit. + * ``Suggested-by`` + * ``Requested-by`` + * ``Reported-by`` + * ``Co-authored-by`` + * ``Tested-by`` + * ``Reviewed-by`` + * ``Acked-by`` + * ``Signed-off-by``: Compulsory! + + There is a great reference for commit messages in the `Linux kernel + documentation`__. + + __ https://www.kernel.org/doc/html/latest/process/submitting-patches.html#describe-your-changes + + IMPORTANT: you must sign-off your work using ``git commit --signoff``. Follow + the `Linux kernel developer's certificate of origin`__ for more details. All + contributions are made under the MIT license. If you do not want to disclose + your real name, you may sign-off using a pseudonym. Here is an example:: + + Signed-off-by: Robin Jarry <robin@jarry.cc> + + __ https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin #. Push your topic branch in your forked repository:: diff --git a/check-commits.sh b/check-commits.sh new file mode 100755 index 00000000..acf322d9 --- /dev/null +++ b/check-commits.sh @@ -0,0 +1,130 @@ +#!/bin/sh + +set -e + +revision_range="${1?revision range}" + +valid=0 +revisions=$(git rev-list --reverse "$revision_range") +total=$(echo $revisions | wc -w) +if [ "$total" -eq 0 ]; then + exit 0 +fi +tmp=$(mktemp) +trap "rm -f $tmp" EXIT + +allowed_trailers=" +Closes +Fixes +Link +Suggested-by +Requested-by +Reported-by +Co-authored-by +Signed-off-by +Tested-by +Reviewed-by +Acked-by +" + +n=0 +title= +shortrev= +fail=false +repo=CESNET/libyang-python +repo_url=https://github.com/$repo +api_url=https://api.github.com/repos/$repo + +err() { + + echo "error: commit $shortrev (\"$title\") $*" >&2 + fail=true +} + +check_issue() { + json=$(curl -f -X GET -L --no-progress-meter \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "$api_url/issues/${1##*/}") || return 1 + test $(echo "$json" | jq -r .state) = open +} + +for rev in $revisions; do + n=$((n + 1)) + title=$(git log --format='%s' -1 "$rev") + fail=false + shortrev=$(printf '%-12.12s' $rev) + + if [ "$(echo "$title" | wc -m)" -gt 72 ]; then + err "title is longer than 72 characters, please make it shorter" + fi + if ! echo "$title" | grep -qE '^[a-z0-9,{}/_-]+: '; then + err "title lacks a lowercase topic prefix (e.g. 'data: ')" + fi + if echo "$title" | grep -qE '^[a-z0-9,{}/_-]+: [A-Z][a-z]'; then + err "title starts with an capital letter, please use lower case" + fi + if ! echo "$title" | grep -qE '[A-Za-z0-9]$'; then + err "title ends with punctuation, please remove it" + fi + + author=$(git log --format='%an <%ae>' -1 "$rev") + if ! git log --format="%(trailers:key=Signed-off-by,only,valueonly,unfold)" -1 "$rev" | + grep -qFx "$author"; then + err "'Signed-off-by: $author' trailer is missing" + fi + + for trailer in $(git log --format="%(trailers:only,keyonly)" -1 "$rev"); do + if ! echo "$allowed_trailers" | grep -qFx "$trailer"; then + err "trailer '$trailer' is misspelled or not in the sanctioned list" + fi + done + + git log --format="%(trailers:key=Closes,only,valueonly,unfold)" -1 "$rev" > $tmp + while read -r value; do + if [ -z "$value" ]; then + continue + fi + case "$value" in + $repo_url/*/[0-9]*) + if ! check_issue "$value"; then + err "'$value' does not reference a valid open issue" + fi + ;; + \#[0-9]*) + err "please use the full issue URL: 'Closes: $repo_url/issues/$value'" + ;; + *) + err "invalid trailer value '$value'. The 'Closes:' trailer must only be used to reference issue URLs" + ;; + esac + done < "$tmp" + + git log --format="%(trailers:key=Fixes,only,valueonly,unfold)" -1 "$rev" > $tmp + while read -r value; do + if [ -z "$value" ]; then + continue + fi + fixes_rev=$(echo "$value" | sed -En 's/([A-Fa-f0-9]{7,}[[:space:]]\(".*"\))/\1/p') + if ! git cat-file commit "$fixes_rev" >/dev/null; then + err "trailer '$value' does not refer to a known commit" + fi + done < "$tmp" + + body=$(git log --format='%b' -1 "$rev") + body=${body%$(git log --format='%(trailers)' -1 "$rev")} + if [ "$(echo "$body" | wc -w)" -lt 3 ]; then + err "body has less than three words, please describe your changes" + fi + + if [ "$fail" = true ]; then + continue + fi + echo "ok commit $shortrev (\"$title\")" + valid=$((valid + 1)) +done + +echo "$valid/$total valid commit messages" +if [ "$valid" -ne "$total" ]; then + exit 1 +fi From 2e6261aab6c530d50c32b8f6090db25851e48980 Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Thu, 28 Mar 2024 00:30:45 +0100 Subject: [PATCH 045/115] editorconfig: fix bad syntax The proper indent_style value is "tab" not "tabs". Signed-off-by: Robin Jarry <robin@jarry.cc> --- .editorconfig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.editorconfig b/.editorconfig index b5343938..37f96c4f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -21,13 +21,13 @@ indent_style = space indent_size = 3 [Makefile] -indent_style = tabs +indent_style = tab indent_size = tab [*.sh] -indent_style = tabs +indent_style = tab indent_size = tab [*.{h,c}] -indent_style = tabs +indent_style = tab indent_size = tab From 04b1c81c8057120f2075ab82baae2437fa141cac Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Thu, 28 Mar 2024 12:43:56 +0100 Subject: [PATCH 046/115] ci: disable libyang devel check As libyang 3.x is under development, the devel branch is currently not compatible with the master. Let's disable this check for the meantime until we revert back to a more stable state. Signed-off-by: Robin Jarry <robin@jarry.cc> --- .github/workflows/ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd6ad584..144e73b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,22 +70,6 @@ jobs: - run: python -m pip install --upgrade tox - run: python -m tox -e ${{ matrix.toxenv }} - libyang_devel: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: pip - restore-keys: pip - - run: python -m pip install --upgrade pip setuptools wheel - - run: python -m pip install --upgrade tox - - run: python -m tox -e lydevel - coverage: runs-on: ubuntu-latest steps: From 92b525085b9ce3d1e9d57795624a23f8f1c5ce97 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Sat, 6 Apr 2024 10:42:24 +0200 Subject: [PATCH 047/115] data: remove no_parent_ret flag This flag should have been removed when we ported to libyang 2. It can safely be removed. Although it is an API change, it seems acceptable as the flag was doing nothing. Fixes: 806be4c3bc4f ("Port to libyang 2") Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/context.py | 5 +---- libyang/data.py | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index eefc4e05..f598a225 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -234,7 +234,6 @@ def create_data_path( parent: Optional[DNode] = None, value: Any = None, update: bool = True, - no_parent_ret: bool = True, rpc_output: bool = False, force_return_value: bool = True, ) -> Optional[DNode]: @@ -245,9 +244,7 @@ def create_data_path( value = str(value).lower() elif not isinstance(value, str): value = str(value) - flags = path_flags( - update=update, no_parent_ret=no_parent_ret, rpc_output=rpc_output - ) + flags = path_flags(update=update, rpc_output=rpc_output) dnode = ffi.new("struct lyd_node **") ret = lib.lyd_new_path( parent.cdata if parent else ffi.NULL, diff --git a/libyang/data.py b/libyang/data.py index f0caf240..1c1dfe35 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -77,9 +77,7 @@ def data_format(fmt_string: str) -> int: # ------------------------------------------------------------------------------------- -def path_flags( - update: bool = False, rpc_output: bool = False, no_parent_ret: bool = False -) -> int: +def path_flags(update: bool = False, rpc_output: bool = False) -> int: flags = 0 if update: flags |= lib.LYD_NEW_PATH_UPDATE From ae31525dedc750394f2bd8bb70d9218d191f32f0 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 12 Feb 2024 09:31:05 +0100 Subject: [PATCH 048/115] Port to libyang 3 Refactor the code to switch to libyang 3: - the cdefs.h file is updated to match the new definitions - the flags used in lyd_new_* are regrouped in a newvaloptions bitmap - the log callback management is reworked - the system-ordered lists / leaf-lists are now ordered by key (hence the unit test change) Fixes: https://github.com/CESNET/libyang-python/pull/105 Link: https://github.com/CESNET/libyang/blob/master/doc/transition_2_3.dox Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 43 +++++++++++++++++++------------ cffi/source.c | 7 ++--- libyang/context.py | 15 +++++++---- libyang/data.py | 64 +++++++++++++++++++++++++++++++--------------- libyang/log.py | 21 +++++++++------ tests/test_data.py | 16 +++++++----- 6 files changed, 105 insertions(+), 61 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 304ae55d..e5bcc935 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -179,10 +179,10 @@ int ly_log_options(int); LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL); extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *); -void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *), int); -struct ly_err_item *ly_err_first(const struct ly_ctx *); +void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t)); +const struct ly_err_item *ly_err_first(const struct ly_ctx *); +const struct ly_err_item *ly_err_last(const struct ly_ctx *); void ly_err_clean(struct ly_ctx *, struct ly_err_item *); -LY_VECODE ly_vecode(const struct ly_ctx *); #define LYS_UNKNOWN ... #define LYS_CONTAINER ... @@ -238,14 +238,15 @@ struct lysc_node { struct ly_err_item { LY_LOG_LEVEL level; - LY_ERR no; + LY_ERR err; LY_VECODE vecode; char *msg; - char *path; + char *data_path; + char *schema_path; + uint64_t line; char *apptag; struct ly_err_item *next; struct ly_err_item *prev; - ...; }; struct lyd_node { @@ -261,11 +262,12 @@ struct lyd_node { LY_ERR lys_set_implemented(struct lys_module *, const char **); +#define LYD_NEW_VAL_OUTPUT ... +#define LYD_NEW_VAL_BIN ... +#define LYD_NEW_VAL_CANON ... +#define LYD_NEW_META_CLEAR_DFLT ... #define LYD_NEW_PATH_UPDATE ... -#define LYD_NEW_PATH_OUTPUT ... -#define LYD_NEW_PATH_OPAQ ... -#define LYD_NEW_PATH_BIN_VALUE ... -#define LYD_NEW_PATH_CANON_VALUE ... +#define LYD_NEW_PATH_OPAQ ... LY_ERR lyd_new_path(struct lyd_node *, const struct ly_ctx *, const char *, const char *, uint32_t, struct lyd_node **); LY_ERR lyd_find_xpath(const struct lyd_node *, const char *, struct ly_set **); void lyd_unlink_siblings(struct lyd_node *node); @@ -614,6 +616,7 @@ struct lysp_node_list { }; struct lysc_type { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -641,6 +644,7 @@ struct lysp_type { struct lysp_qname { const char *str; const struct lysp_module *mod; + ...; }; struct lysp_node { @@ -682,7 +686,6 @@ struct lysc_ext { struct lysc_ext_instance *exts; struct lyplg_ext *plugin; struct lys_module *module; - uint32_t refcount; uint16_t flags; }; @@ -703,11 +706,10 @@ typedef enum { LYD_PATH_STD_NO_LAST_PRED } LYD_PATH_TYPE; -LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **); +LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_node **); char* lyd_path(const struct lyd_node *, LYD_PATH_TYPE, char *, size_t); LY_ERR lyd_new_inner(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **); -LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **, ...); -LY_ERR lyd_new_list2(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **); +LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, uint32_t, struct lyd_node **node, ...); struct lyd_node_inner { union { @@ -821,6 +823,7 @@ struct lysp_restr { }; struct lysc_type_num { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -829,6 +832,7 @@ struct lysc_type_num { }; struct lysc_type_dec { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -838,6 +842,7 @@ struct lysc_type_dec { }; struct lysc_type_str { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -859,6 +864,7 @@ struct lysc_type_bitenum_item { }; struct lysc_type_enum { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -867,6 +873,7 @@ struct lysc_type_enum { }; struct lysc_type_bits { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -875,18 +882,19 @@ struct lysc_type_bits { }; struct lysc_type_leafref { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; uint32_t refcount; struct lyxp_expr *path; struct lysc_prefix *prefixes; - const struct lys_module *cur_mod; struct lysc_type *realtype; uint8_t require_instance; }; struct lysc_type_identityref { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -895,6 +903,7 @@ struct lysc_type_identityref { }; struct lysc_type_instanceid { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -903,6 +912,7 @@ struct lysc_type_instanceid { }; struct lysc_type_union { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -911,6 +921,7 @@ struct lysc_type_union { }; struct lysc_type_bin { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -1053,7 +1064,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **); LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **); -LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **); +LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_meta **); struct ly_opaq_name { const char *name; diff --git a/cffi/source.c b/cffi/source.c index 2682dd88..b54ba0de 100644 --- a/cffi/source.c +++ b/cffi/source.c @@ -6,9 +6,6 @@ #include <libyang/libyang.h> #include <libyang/version.h> -#if (LY_VERSION_MAJOR != 2) -#error "This version of libyang bindings only works with libyang 2.x" -#endif -#if (LY_VERSION_MINOR < 37) -#error "Need at least libyang 2.37" +#if (LY_VERSION_MAJOR != 3) +#error "This version of libyang bindings only works with libyang 3.x" #endif diff --git a/libyang/context.py b/libyang/context.py index f598a225..fa5eb5bf 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -11,8 +11,8 @@ DNode, data_format, data_type, + newval_flags, parser_flags, - path_flags, validation_flags, ) from .schema import Module, SNode, schema_in_format @@ -117,8 +117,12 @@ def error(self, msg: str, *args) -> LibyangError: while err: if err.msg: msg += ": %s" % c2str(err.msg) - if err.path: - msg += ": %s" % c2str(err.path) + if err.data_path: + msg += ": Data path: %s" % c2str(err.data_path) + if err.schema_path: + msg += ": Schema path: %s" % c2str(err.schema_path) + if err.line != 0: + msg += " (line %u)" % err.line err = err.next lib.ly_err_clean(self.cdata, ffi.NULL) @@ -244,7 +248,7 @@ def create_data_path( value = str(value).lower() elif not isinstance(value, str): value = str(value) - flags = path_flags(update=update, rpc_output=rpc_output) + flags = newval_flags(update=update, rpc_output=rpc_output) dnode = ffi.new("struct lyd_node **") ret = lib.lyd_new_path( parent.cdata if parent else ffi.NULL, @@ -256,7 +260,8 @@ def create_data_path( ) dnode = dnode[0] if ret != lib.LY_SUCCESS: - if lib.ly_vecode(self.cdata) != lib.LYVE_SUCCESS: + err = lib.ly_err_last(self.cdata) + if err != ffi.NULL and err.vecode != lib.LYVE_SUCCESS: raise self.error("cannot create data path: %s", path) lib.ly_err_clean(self.cdata, ffi.NULL) if not dnode and not force_return_value: diff --git a/libyang/data.py b/libyang/data.py index 1c1dfe35..2895daaf 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -77,12 +77,30 @@ def data_format(fmt_string: str) -> int: # ------------------------------------------------------------------------------------- -def path_flags(update: bool = False, rpc_output: bool = False) -> int: +def newval_flags( + rpc_output: bool = False, + bin_value: bool = False, + canon_value: bool = False, + meta_clear_default: bool = False, + update: bool = False, + opaq: bool = False, +) -> int: + """ + Translate from booleans to newvaloptions flags. + """ flags = 0 + if rpc_output: + flags |= lib.LYD_NEW_VAL_OUTPUT + if bin_value: + flags |= lib.LYD_NEW_VAL_BIN + if canon_value: + flags |= lib.LYD_NEW_VAL_CANON + if meta_clear_default: + flags |= lib.LYD_NEW_META_CLEAR_DFLT if update: flags |= lib.LYD_NEW_PATH_UPDATE - if rpc_output: - flags |= lib.LYD_NEW_PATH_OUTPUT + if opaq: + flags |= lib.LYD_NEW_PATH_OPAQ return flags @@ -297,13 +315,14 @@ def meta_free(self, name): item = item.next def new_meta(self, name: str, value: str, clear_dflt: bool = False): + flags = newval_flags(meta_clear_default=clear_dflt) ret = lib.lyd_new_meta( ffi.NULL, self.cdata, ffi.NULL, str2c(name), str2c(value), - clear_dflt, + flags, ffi.NULL, ) if ret != lib.LY_SUCCESS: @@ -364,20 +383,15 @@ def new_path( opt_bin_value: bool = False, opt_canon_value: bool = False, ): - opt = 0 - if opt_update: - opt |= lib.LYD_NEW_PATH_UPDATE - if opt_output: - opt |= lib.LYD_NEW_PATH_OUTPUT - if opt_opaq: - opt |= lib.LYD_NEW_PATH_OPAQ - if opt_bin_value: - opt |= lib.LYD_NEW_PATH_BIN_VALUE - if opt_canon_value: - opt |= lib.LYD_NEW_PATH_CANON_VALUE - + flags = newval_flags( + update=opt_update, + rpc_output=opt_output, + opaq=opt_opaq, + bin_value=opt_bin_value, + canon_value=opt_canon_value, + ) ret = lib.lyd_new_path( - self.cdata, ffi.NULL, str2c(path), str2c(value), opt, ffi.NULL + self.cdata, ffi.NULL, str2c(path), str2c(value), flags, ffi.NULL ) if ret != lib.LY_SUCCESS: raise self.context.error("cannot get module") @@ -1003,7 +1017,10 @@ def _get_path(cdata) -> str: @DNode.register(SNode.CONTAINER) class DContainer(DNode): def create_path( - self, path: str, value: Any = None, rpc_output: bool = False + self, + path: str, + value: Any = None, + rpc_output: bool = False, ) -> Optional[DNode]: return self.context.create_data_path( path, parent=self, value=value, rpc_output=rpc_output @@ -1177,8 +1194,14 @@ def _create_leaf(_parent, module, name, value, in_rpc_output=False): value = str(value) n = ffi.new("struct lyd_node **") + flags = newval_flags(rpc_output=in_rpc_output) ret = lib.lyd_new_term( - _parent, module.cdata, str2c(name), str2c(value), in_rpc_output, n + _parent, + module.cdata, + str2c(name), + str2c(value), + flags, + n, ) if ret != lib.LY_SUCCESS: @@ -1209,11 +1232,12 @@ def _create_container(_parent, module, name, in_rpc_output=False): def _create_list(_parent, module, name, key_values, in_rpc_output=False): n = ffi.new("struct lyd_node **") + flags = newval_flags(rpc_output=in_rpc_output) ret = lib.lyd_new_list( _parent, module.cdata, str2c(name), - in_rpc_output, + flags, n, *[str2c(str(i)) for i in key_values], ) diff --git a/libyang/log.py b/libyang/log.py index 2b241157..b033ccaa 100644 --- a/libyang/log.py +++ b/libyang/log.py @@ -20,13 +20,18 @@ @ffi.def_extern(name="lypy_log_cb") -def libyang_c_logging_callback(level, msg, path): +def libyang_c_logging_callback(level, msg, data_path, schema_path, line): args = [c2str(msg)] - if path: - fmt = "%s: %s" - args.append(c2str(path)) - else: - fmt = "%s" + fmt = "%s" + if data_path: + fmt += ": %s" + args.append(c2str(data_path)) + if schema_path: + fmt += ": %s" + args.append(c2str(schema_path)) + if line != 0: + fmt += " line %u" + args.append(str(line)) LOG.log(LOG_LEVELS.get(level, logging.NOTSET), fmt, *args) @@ -51,10 +56,10 @@ def configure_logging(enable_py_logger: bool, level: int = logging.ERROR) -> Non break if enable_py_logger: lib.ly_log_options(lib.LY_LOLOG | lib.LY_LOSTORE) - lib.ly_set_log_clb(lib.lypy_log_cb, True) + lib.ly_set_log_clb(lib.lypy_log_cb) else: lib.ly_log_options(lib.LY_LOSTORE) - lib.ly_set_log_clb(ffi.NULL, False) + lib.ly_set_log_clb(ffi.NULL) configure_logging(False, logging.ERROR) diff --git a/tests/test_data.py b/tests/test_data.py index 10d9045f..a40056c0 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -132,17 +132,17 @@ def test_data_parse_config_json_without_yang_lib(self): "path": "/CESNET/libyang-python", "enabled": false }, + { + "proto": "http", + "host": "barfoo.com", + "path": "/barfoo/index.html" + }, { "proto": "http", "host": "foobar.com", "port": 8080, "path": "/index.html", "enabled": true - }, - { - "proto": "http", - "host": "barfoo.com", - "path": "/barfoo/index.html" } ], "number": [ @@ -282,7 +282,9 @@ def test_data_parse_config_xml_multi_error(self): self.assertEqual( str(cm.exception), 'failed to parse data tree: Invalid boolean value "abcd".: ' - 'List instance is missing its key "host".', + "Data path: /yolo-system:conf/url[proto='https']/enabled (line 6): " + 'List instance is missing its key "host".: ' + "Data path: /yolo-system:conf/url[proto='https'] (line 7)", ) XML_STATE = """<state xmlns="urn:yang:yolo:system"> @@ -808,7 +810,7 @@ def test_data_to_dict_keyless_list(self): <host>foobar.com</host> <enabled yang:operation="replace" yang:orig-default="false" yang:orig-value="true">false</enabled> </url> - <url yang:operation="create"> + <url yang:operation="create" yang:key="[proto='http'][host='foobar.com']"> <proto>ftp</proto> <host>github.com</host> <path>/CESNET/libyang-python</path> From e602016200ff3db1c8cde4886dfc1fa8a828ecf8 Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Tue, 16 Apr 2024 13:49:17 +0200 Subject: [PATCH 049/115] cdefs: fix lypy_log_cb prototype The following error is raised when logging: > File "/usr/lib/python3/dist-packages/libyang/log.py", line 59, in configure_logging > TypeError: initializer for ctype 'void(*)(LY_LOG_LEVEL, char *, char *, char *, > uint64_t)' must be a pointer to same type, not cdata 'void(*)(LY_LOG_LEVEL, char *, char > *)' Fix lypy_log_cb's prototype. Fixes: 3849dd523d30 ("Port to libyang3") Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index e5bcc935..921dd8f6 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -178,7 +178,7 @@ enum ly_stmt { int ly_log_options(int); LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL); -extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *); +extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t); void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t)); const struct ly_err_item *ly_err_first(const struct ly_ctx *); const struct ly_err_item *ly_err_last(const struct ly_ctx *); From c3305ab946c951aaa01b59244e9a69e3894a9e9e Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Mon, 10 Jun 2024 12:24:20 +0200 Subject: [PATCH 050/115] schema: fix empty compiled shema in iter_children A module schema can have no compiled schema. In this case, lys_getnext returns an error (module and parent being NULL), leading to this log "Invalid argument parent || module || ext (lys_getnext_()).". Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index deec466d..fb0787ff 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1647,6 +1647,8 @@ def _skip(node) -> bool: return False if ffi.typeof(parent) == ffi.typeof("struct lys_module *"): + if parent.compiled == ffi.NULL: + return module = parent.compiled parent = ffi.NULL else: From 4a9b85e18ef759469179d573248e3767d998554c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20Bevilacqua?= <jean-sebastien.bevilacqua@6wind.com> Date: Tue, 19 Mar 2024 10:00:28 +0100 Subject: [PATCH 051/115] xpath: add support for indexed xpath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds support for indexed xpath to xpath_set. In the case of a leaf-list, it is now possible to apply xpath_set in this way: xpath_set(d, "/lstnum[5]", 33, after="4") This will replace the 5th element of the leaf-list with the new value 33. This feature is important because libyang can return this type of xpath when a callback is called with sr_module_change_subscribe from sysrepo. The tests are updated accordingly. Signed-off-by: Jean-Sébastien Bevilacqua <jean-sebastien.bevilacqua@6wind.com> --- libyang/xpath.py | 105 +++++++++++++++++++++++++++++--------------- tests/test_xpath.py | 7 ++- 2 files changed, 75 insertions(+), 37 deletions(-) diff --git a/libyang/xpath.py b/libyang/xpath.py index facaa8b7..ad5f2a58 100644 --- a/libyang/xpath.py +++ b/libyang/xpath.py @@ -1,6 +1,7 @@ # Copyright (c) 2020 6WIND S.A. # SPDX-License-Identifier: MIT +import contextlib import fnmatch import re from typing import Any, Dict, Iterator, List, Optional, Tuple, Union @@ -56,17 +57,28 @@ def xpath_split(xpath: str) -> Iterator[Tuple[str, str, List[Tuple[str, str]]]]: while i < len(xpath) and xpath[i] == "[": i += 1 # skip opening '[' j = xpath.find("=", i) # find key name end - key_name = xpath[i:j] - quote = xpath[j + 1] # record opening quote character - j = i = j + 2 # skip '=' and opening quote - while True: - if xpath[j] == quote and xpath[j - 1] != "\\": - break - j += 1 - # replace escaped chars by their non-escape version - key_value = xpath[i:j].replace(f"\\{quote}", f"{quote}") - keys.append((key_name, key_value)) - i = j + 2 # skip closing quote and ']' + + if j != -1: # keyed specifier + key_name = xpath[i:j] + quote = xpath[j + 1] # record opening quote character + j = i = j + 2 # skip '=' and opening quote + while True: + if xpath[j] == quote and xpath[j - 1] != "\\": + break + j += 1 + # replace escaped chars by their non-escape version + key_value = xpath[i:j].replace(f"\\{quote}", f"{quote}") + keys.append((key_name, key_value)) + i = j + 2 # skip closing quote and ']' + else: # index specifier + j = i + while True: + if xpath[j] == "]": + break + j += 1 + key_value = xpath[i:j] + keys.append(("", key_value)) + i = j + 2 yield prefix, name, keys @@ -134,6 +146,12 @@ def _list_find_key_index(keys: List[Tuple[str, str]], lst: List) -> int: if py_to_yang(elem) == keys[0][1]: return i + elif keys[0][0] == "": + # keys[0][1] is directly the index + index = int(keys[0][1]) - 1 + if len(lst) > index: + return index + else: for i, elem in enumerate(lst): if not isinstance(elem, dict): @@ -410,32 +428,47 @@ def xpath_set( lst.append(value) return lst[key_val] - if isinstance(lst, list): - # regular python list, need to iterate over it - try: - i = _list_find_key_index(keys, lst) - # found - if force: - lst[i] = value - return lst[i] - except ValueError: - # not found - if after is None: - lst.append(value) - elif after == "": - lst.insert(0, value) - else: - if after[0] != "[": - after = "[.=%r]" % str(after) - _, _, after_keys = next(xpath_split("/*" + after)) - insert_index = _list_find_key_index(after_keys, lst) + 1 - if insert_index == len(lst): - lst.append(value) - else: - lst.insert(insert_index, value) - return value + # regular python list from now + if not isinstance(lst, list): + raise TypeError("expected a list") + + with contextlib.suppress(ValueError): + i = _list_find_key_index(keys, lst) + # found + if force: + lst[i] = value + return lst[i] + + # value not found; handle insertion based on 'after' + if after is None: + lst.append(value) + return value + + if after == "": + lst.insert(0, value) + return value + + # first try to find the value in the leaf list + try: + _, _, after_keys = next( + xpath_split(f"/*{after}" if after[0] == "[" else f"/*[.={after!r}]") + ) + insert_index = _list_find_key_index(after_keys, lst) + 1 + except ValueError: + # handle 'after' as numeric index + if not after.isnumeric(): + raise + + insert_index = int(after) + if insert_index > len(lst): + raise + + if insert_index == len(lst): + lst.append(value) + else: + lst.insert(insert_index, value) - raise TypeError("expected a list") + return value # ------------------------------------------------------------------------------------- diff --git a/tests/test_xpath.py b/tests/test_xpath.py index bf7c7bc7..0901a7ec 100644 --- a/tests/test_xpath.py +++ b/tests/test_xpath.py @@ -41,6 +41,11 @@ def test_xpath_set(self): ) ly.xpath_set(d, "/lstnum[.='100']", 100) ly.xpath_set(d, "/lstnum[.='1']", 1, after="") + ly.xpath_set(d, "/lstnum[5]", 33, after="4") + ly.xpath_set(d, "/lstnum[5]", 34, after="4") + ly.xpath_set(d, "/lstnum[5]", 35, after="4") + ly.xpath_set(d, "/lstnum[7]", 101, after="6") + ly.xpath_set(d, "/lstnum[8]", 102, after="7") with self.assertRaises(ValueError): ly.xpath_set(d, "/lstnum[.='1000']", 1000, after="1000000") with self.assertRaises(ValueError): @@ -101,7 +106,7 @@ def test_xpath_set(self): {"name": "eth3", "mtu": 1000}, ], "lst2": ["a", "b", "c"], - "lstnum": [1, 10, 20, 30, 40, 100], + "lstnum": [1, 10, 20, 30, 35, 100, 101, 102], "val": 43, }, ) From 766ef3c24534290207f701ae8e0109657370cfbe Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 13:35:34 +0200 Subject: [PATCH 052/115] libyang: export SAnydata / SAnyxml Those two classes are missing, export them. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libyang/__init__.py b/libyang/__init__.py index bc79e2f0..02bd26af 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -78,6 +78,8 @@ Must, Pattern, Revision, + SAnydata, + SAnyxml, SCase, SChoice, SContainer, @@ -158,6 +160,8 @@ "RangeAdded", "RangeRemoved", "Revision", + "SAnydata", + "SAnyxml", "SCase", "SChoice", "SContainer", From 3fc55b05cec8c32bc658289a0e34859313248f41 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 13:50:57 +0200 Subject: [PATCH 053/115] schema: fix type for max_elements Those functions can return None, fix their expected type. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index fb0787ff..c0985a4b 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1382,7 +1382,7 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: else: yield val - def max_elements(self) -> int: + def max_elements(self) -> Optional[int]: return ( self.cdata_leaflist.max if self.cdata_leaflist.max != (2**32 - 1) @@ -1506,7 +1506,7 @@ def uniques(self) -> Iterator[List[SNode]]: nodes.append(SNode.new(self.context, node)) yield nodes - def max_elements(self) -> int: + def max_elements(self) -> Optional[int]: return self.cdata_list.max if self.cdata_list.max != (2**32 - 1) else None def min_elements(self) -> int: From fe0b3d0528296390b335d9d01b83f49b9a096008 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 13:51:21 +0200 Subject: [PATCH 054/115] schema: fix comment in Import class The cdata type that is actually lysp_import, fix it. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index c0985a4b..75206bd3 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -300,7 +300,7 @@ class Import: def __init__(self, context: "libyang.Context", cdata, module): self.context = context - self.cdata = cdata # C type: "struct lysp_revision *" + self.cdata = cdata # C type: "struct lysp_import *" self.module = module def name(self) -> str: From 5f05203e9c0c005cc4ae243df891ac46509a44b3 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 14:28:33 +0200 Subject: [PATCH 055/115] util: add ly_list_iter Add a new function to go through a libyang list, equivalent to LY_LIST_FOR. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/util.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libyang/util.py b/libyang/util.py index 9554356e..d640a511 100644 --- a/libyang/util.py +++ b/libyang/util.py @@ -59,6 +59,14 @@ def ly_array_iter(cdata): yield cdata[i] +# ------------------------------------------------------------------------------------- +def ly_list_iter(cdata): + item = cdata + while item != ffi.NULL: + yield item + item = item.next + + # ------------------------------------------------------------------------------------- class IOType(enum.Enum): FD = enum.auto() From b7adea400520477f005a5b8c9857b9daf574de4b Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 1 Mar 2024 13:42:03 +0100 Subject: [PATCH 056/115] schema: adds PNode and its exact variants This patch introduces PNode alternative to SNode, which allows to traverse parsed module tree including groupings, uses and other special nodes. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 124 +++++ libyang/__init__.py | 34 ++ libyang/schema.py | 746 +++++++++++++++++++++++++++- tests/test_schema.py | 252 ++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 31 ++ 5 files changed, 1181 insertions(+), 6 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 921dd8f6..cbe92969 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -528,6 +528,25 @@ typedef enum { char* lysc_path(const struct lysc_node *, LYSC_PATH_TYPE, char *, size_t); +struct lysp_when { + const char *cond; + ...; +}; + +struct lysp_refine { + const char *nodeid; + const char *dsc; + const char *ref; + struct lysp_qname *iffeatures; + struct lysp_restr *musts; + const char *presence; + struct lysp_qname *dflts; + uint32_t min; + uint32_t max; + struct lysp_ext_instance *exts; + uint16_t flags; +}; + struct lysp_node_container { struct lysp_restr *musts; struct lysp_when *when; @@ -615,6 +634,101 @@ struct lysp_node_list { ...; }; +struct lysp_node_choice { + struct lysp_node *child; + struct lysp_when *when; + struct lysp_qname dflt; + ...; +}; + +struct lysp_node_case { + struct lysp_node *child; + struct lysp_when *when; + ...; +}; + +struct lysp_node_anydata { + struct lysp_restr *musts; + struct lysp_when *when; + ...; +}; + +struct lysp_node_uses { + struct lysp_refine *refines; + struct lysp_node_augment *augments; + struct lysp_when *when; + ...; +}; + +struct lysp_node_action_inout { + struct lysp_restr *musts; + struct lysp_tpdf *typedefs; + struct lysp_node_grp *groupings; + struct lysp_node *child; + ...; +}; + +struct lysp_node_action { + union { + struct lysp_node node; + struct { + struct lysp_node_action *next; + ...; + }; + }; + struct lysp_tpdf *typedefs; + struct lysp_node_grp *groupings; + struct lysp_node_action_inout input; + struct lysp_node_action_inout output; + ...; +}; + +struct lysp_node_notif { + union { + struct lysp_node node; + struct { + struct lysp_node_notif *next; + ...; + }; + }; + struct lysp_restr *musts; + struct lysp_tpdf *typedefs; + struct lysp_node_grp *groupings; + struct lysp_node *child; + ...; +}; + +struct lysp_node_grp { + union { + struct lysp_node node; + struct { + struct lysp_node_grp *next; + ...; + }; + }; + struct lysp_tpdf *typedefs; + struct lysp_node_grp *groupings; + struct lysp_node *child; + struct lysp_node_action *actions; + struct lysp_node_notif *notifs; + ...; +}; + +struct lysp_node_augment { + union { + struct lysp_node node; + struct { + struct lysp_node_augment *next; + ...; + }; + }; + struct lysp_node *child; + struct lysp_when *when; + struct lysp_node_action *actions; + struct lysp_node_notif *notifs; + ...; +}; + struct lysc_type { const char *name; struct lysc_ext_instance *exts; @@ -623,6 +737,16 @@ struct lysc_type { uint32_t refcount; }; +struct lysp_type_enum { + const char *name; + const char *dsc; + const char *ref; + int64_t value; + struct lysp_qname *iffeatures; + struct lysp_ext_instance *exts; + uint16_t flags; +}; + struct lysp_type { const char *name; struct lysp_restr *range; diff --git a/libyang/__init__.py b/libyang/__init__.py index 02bd26af..5c931de4 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -76,6 +76,23 @@ IfOrFeatures, Module, Must, + PAction, + PActionInOut, + PAnydata, + PAugment, + PCase, + PChoice, + PContainer, + PEnum, + PGrouping, + PLeaf, + PLeafList, + PList, + PNode, + PNotif, + PRefine, + PType, + PUses, Pattern, Revision, SAnydata, @@ -152,6 +169,23 @@ "NodeTypeRemoved", "OrderedByUserAdded", "OrderedByUserRemoved", + "PAction", + "PActionInOut", + "PAnydata", + "PAugment", + "PCase", + "PChoice", + "PContainer", + "PEnum", + "PGrouping", + "PLeaf", + "PLeafList", + "PList", + "PNode", + "PNotif", + "PRefine", + "PType", + "PUses", "Pattern", "PatternAdded", "PatternRemoved", diff --git a/libyang/schema.py b/libyang/schema.py index 75206bd3..7cd2289b 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -6,8 +6,15 @@ from typing import IO, Any, Dict, Iterator, List, Optional, Tuple, Union from _libyang import ffi, lib -from .util import IOType, LibyangError, c2str, init_output, ly_array_iter, str2c - +from .util import ( + IOType, + LibyangError, + c2str, + init_output, + ly_array_iter, + ly_list_iter, + str2c, +) # ------------------------------------------------------------------------------------- def schema_in_format(fmt_string: str) -> int: @@ -144,6 +151,26 @@ def children( self.context, self.cdata, types=types, with_choice=with_choice ) + def parsed_children(self) -> Iterator["PNode"]: + for c in ly_list_iter(self.cdata.parsed.data): + yield PNode.new(self.context, c, self) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata.parsed.groupings): + yield PGrouping(self.context, g, self) + + def augments(self) -> Iterator["PAugment"]: + for a in ly_array_iter(self.cdata.parsed.augments): + yield PAugment(self.context, a, self) + + def actions(self) -> Iterator["PAction"]: + for a in ly_list_iter(self.cdata.parsed.rpcs): + yield PAction(self.context, a, self) + + def notifications(self) -> Iterator["PNotif"]: + for n in ly_list_iter(self.cdata.parsed.notifs): + yield PNotif(self.context, n, self) + def __str__(self) -> str: return self.name() @@ -454,19 +481,26 @@ class Bit(_EnumBit): # ------------------------------------------------------------------------------------- class Pattern: - __slots__ = ("context", "cdata") + __slots__ = ("context", "cdata", "cdata_parsed") - def __init__(self, context: "libyang.Context", cdata): + def __init__(self, context: "libyang.Context", cdata, cdata_parsed=None): self.context = context self.cdata = cdata # C type: "struct lysc_pattern *" + self.cdata_parsed = cdata_parsed # C type: "struct lysp_restr *" def expression(self) -> str: + if self.cdata is None and self.cdata_parsed: + return c2str(self.cdata_parsed.arg.str + 1) return c2str(self.cdata.expr) def inverted(self) -> bool: + if self.cdata is None and self.cdata_parsed: + return self.cdata_parsed.arg.str[0] == b"\x15" return self.cdata.inverted def error_message(self) -> Optional[str]: + if self.cdata is None and self.cdata_parsed: + return c2str(self.cdata_parsed.emsg) return c2str(self.cdata.emsg) if self.cdata.emsg != ffi.NULL else None @@ -756,6 +790,11 @@ def __repr__(self): def __str__(self): return self.name() + def parsed(self) -> Optional["PType"]: + if self.cdata_parsed is None or self.cdata_parsed == ffi.NULL: + return None + return PType(self.context, self.cdata_parsed, self.module()) + # ------------------------------------------------------------------------------------- class Typedef: @@ -1078,16 +1117,21 @@ def __str__(self): # ------------------------------------------------------------------------------------- class Must: - __slots__ = ("context", "cdata") + __slots__ = ("context", "cdata", "cdata_parsed") - def __init__(self, context: "libyang.Context", cdata): + def __init__(self, context: "libyang.Context", cdata, cdata_parsed=None): self.context = context self.cdata = cdata # C type: "struct lysc_must *" + self.cdata_parsed = cdata_parsed # C type: "struct lysp_must *" def condition(self) -> str: + if self.cdata is None and self.cdata_parsed: + return c2str(self.cdata_parsed.arg.str + 1) return c2str(lib.lyxp_get_expr(self.cdata.cond)) def error_message(self) -> Optional[str]: + if self.cdata is None and self.cdata_parsed: + return c2str(self.cdata_parsed.emsg) return c2str(self.cdata.emsg) if self.cdata.emsg != ffi.NULL else None @@ -1240,6 +1284,11 @@ def when_conditions(self): for cond in ly_array_iter(wh): yield c2str(lib.lyxp_get_expr(cond.cond)) + def parsed(self) -> Optional["PNode"]: + if self.cdata_parsed is None or self.cdata_parsed == ffi.NULL: + return None + return PNode.new(self.context, self.cdata_parsed, self.module()) + def iter_tree(self, full: bool = False) -> Iterator["SNode"]: """ Do a DFS walk of the schema node. @@ -1679,3 +1728,688 @@ def _skip(node) -> bool: Rpc = SRpc RpcInOut = SRpcInOut Anyxml = SAnyxml + + +# ------------------------------------------------------------------------------------- +class PEnum: + __slots__ = ("context", "cdata", "module") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = cdata # C type of "struct lysp_type_enum *" + self.module = module + + def name(self) -> str: + return c2str(self.cdata.name) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def value(self) -> int: + return self.cdata.value + + def if_features(self) -> Iterator[IfFeatureExpr]: + for f in ly_array_iter(self.cdata.iffeatures): + yield IfFeatureExpr(self.context, f, list(self.module.features())) + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + +# ------------------------------------------------------------------------------------- +class PType: + __slots__ = ("context", "cdata", "module") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = cdata # C type of "struct lysp_type *" + self.module = module + + def name(self) -> str: + return c2str(self.cdata.name) + + def range(self) -> Optional[str]: + if self.cdata.range == ffi.NULL: + return None + return c2str(self.cdata.range.arg.str) + + def length(self) -> Optional[str]: + if self.cdata.length == ffi.NULL: + return None + return c2str(self.cdata.length.arg.str) + + def patterns(self) -> Iterator[Pattern]: + for p in ly_array_iter(self.cdata.patterns): + yield Pattern(self.context, None, p) + + def enums(self) -> Iterator[PEnum]: + for e in ly_array_iter(self.cdata.enums): + yield PEnum(self.context, e, self.module) + + def bits(self) -> Iterator[PEnum]: + for b in ly_array_iter(self.cdata.bits): + yield PEnum(self.context, b, self.module) + + def path(self) -> Optional[str]: + if self.cdata.path == ffi.NULL: + return None + return c2str(lib.lyxp_get_expr(self.cdata.path)) + + def bases(self) -> Iterator[str]: + for b in ly_array_iter(self.cdata.bases): + yield c2str(b) + + def types(self) -> Iterator["PType"]: + for t in ly_array_iter(self.cdata.types): + yield PType(self.context, t, self.module) + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + def pmod(self) -> Optional[Module]: + if self.cdata.pmod == ffi.NULL: + return None + return Module(self.context, self.cdata.pmod.mod) + + def compiled(self) -> Optional[Type]: + if self.cdata.compiled == ffi.NULL: + return None + return Type(self.context, self.cdata.compiled, self.cdata) + + def fraction_digits(self) -> int: + return self.cdata.fraction_digits + + def require_instance(self) -> bool: + return self.cdata.require_instance + + +# ------------------------------------------------------------------------------------- +class PRefine: + __slots__ = ("context", "cdata", "module") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = cdata # C type of "struct lysp_refine *" + self.module = module + + def nodeid(self) -> str: + return c2str(self.cdata.nodeid) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def if_features(self) -> Iterator[IfFeatureExpr]: + for f in ly_array_iter(self.cdata.iffeatures): + yield IfFeatureExpr(self.context, f, list(self.module.features())) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata.musts): + yield Must(self.context, None, m) + + def presence(self) -> Optional[str]: + return c2str(self.cdata.presence) + + def defaults(self) -> Iterator[str]: + for d in ly_array_iter(self.cdata.dflts): + yield c2str(d.str) + + def min_elements(self) -> int: + return self.cdata.min + + def max_elements(self) -> Optional[int]: + return self.cdata.max if self.cdata.max != 0 else None + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + +# ------------------------------------------------------------------------------------- +class PNode: + CONTAINER = lib.LYS_CONTAINER + CHOICE = lib.LYS_CHOICE + CASE = lib.LYS_CASE + LEAF = lib.LYS_LEAF + LEAFLIST = lib.LYS_LEAFLIST + LIST = lib.LYS_LIST + RPC = lib.LYS_RPC + ACTION = lib.LYS_ACTION + INPUT = lib.LYS_INPUT + OUTPUT = lib.LYS_OUTPUT + NOTIF = lib.LYS_NOTIF + ANYXML = lib.LYS_ANYXML + ANYDATA = lib.LYS_ANYDATA + AUGMENT = lib.LYS_AUGMENT + USES = lib.LYS_USES + GROUPING = lib.LYS_GROUPING + KEYWORDS = { + CONTAINER: "container", + LEAF: "leaf", + LEAFLIST: "leaf-list", + LIST: "list", + RPC: "rpc", + ACTION: "action", + INPUT: "input", + OUTPUT: "output", + NOTIF: "notification", + ANYXML: "anyxml", + ANYDATA: "anydata", + AUGMENT: "augment", + USES: "uses", + GROUPING: "grouping", + } + + __slots__ = ("context", "cdata", "module", "__dict__") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = ffi.cast("struct lysp_node *", cdata) + self.module = module + + def parent(self) -> Optional["PNode"]: + if self.cdata.parent == ffi.NULL: + return None + return PNode.new(self.context, self.cdata.parent, self.module) + + def nodetype(self) -> int: + return self.cdata.nodetype + + def siblings(self) -> Iterator["PNode"]: + for s in ly_list_iter(self.cdata.next): + yield PNode.new(self.context, s, self.module) + + def name(self) -> str: + return c2str(self.cdata.name) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def if_features(self) -> Iterator[IfFeatureExpr]: + for f in ly_array_iter(self.cdata.iffeatures): + yield IfFeatureExpr(self.context, f, list(self.module.features())) + + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + def get_extension( + self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None + ) -> Optional["ExtensionParsed"]: + for ext in self.extensions(): + if ext.name() != name: + continue + if prefix is not None and ext.module().name() != prefix: + continue + if arg_value is not None and ext.argument() != arg_value: + continue + return ext + return None + + def config_set(self) -> bool: + return bool(self.cdata.flags & lib.LYS_SET_CONFIG) + + def config_false(self) -> bool: + return bool(self.cdata.flags & lib.LYS_CONFIG_R) + + def mandatory(self) -> bool: + return bool(self.cdata.flags & lib.LYS_MAND_TRUE) + + def deprecated(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_DEPRC) + + def obsolete(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_OBSLT) + + def status(self) -> str: + if self.cdata.flags & lib.LYS_STATUS_OBSLT: + return "obsolete" + if self.cdata.flags & lib.LYS_STATUS_DEPRC: + return "deprecated" + return "current" + + def __repr__(self): + cls = self.__class__ + return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) + + def __str__(self): + return self.name() + + NODETYPE_CLASS = {} + + @staticmethod + def register(nodetype): + def _decorator(nodeclass): + PNode.NODETYPE_CLASS[nodetype] = nodeclass + return nodeclass + + return _decorator + + @staticmethod + def new(context: "libyang.Context", cdata, module: Module) -> "PNode": + cdata = ffi.cast("struct lysp_node *", cdata) + nodecls = PNode.NODETYPE_CLASS.get(cdata.nodetype, None) + if nodecls is None: + raise TypeError("node type %s not implemented" % cdata.nodetype) + return nodecls(context, cdata, module) + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.CONTAINER) +class PContainer(PNode): + __slots__ = ("cdata_container",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_container = ffi.cast("struct lysp_node_container *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_container.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_container.when == ffi.NULL: + return None + return c2str(self.cdata_container.when.cond) + + def presence(self) -> Optional[str]: + return c2str(self.cdata_container.presence) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_container.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_container.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_container.child): + yield PNode.new(self.context, c, self.module) + + def actions(self) -> Iterator["PAction"]: + for a in ly_list_iter(self.cdata_container.actions): + yield PAction(self.context, a, self.module) + + def notifications(self) -> Iterator["PNotif"]: + for n in ly_list_iter(self.cdata_container.notifs): + yield PNotif(self.context, n, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.LEAF) +class PLeaf(PNode): + __slots__ = ("cdata_leaf",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_leaf = ffi.cast("struct lysp_node_leaf *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_leaf.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_leaf.when == ffi.NULL: + return None + return c2str(self.cdata_leaf.when.cond) + + def type(self) -> PType: + return PType(self.context, self.cdata_leaf.type, self.module) + + def units(self) -> Optional[str]: + return c2str(self.cdata_leaf.units) + + def default(self) -> Optional[str]: + return c2str(self.cdata_leaf.dflt.str) + + def is_key(self) -> bool: + if self.cdata.flags & lib.LYS_KEY: + return True + return False + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.LEAFLIST) +class PLeafList(PNode): + __slots__ = ("cdata_leaflist",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_leaflist = ffi.cast("struct lysp_node_leaflist *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_leaflist.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_leaflist.when == ffi.NULL: + return None + return c2str(self.cdata_leaflist.when.cond) + + def type(self) -> PType: + return PType(self.context, self.cdata_leaflist.type, self.module) + + def units(self) -> Optional[str]: + return c2str(self.cdata_leaflist.units) + + def defaults(self) -> Iterator[str]: + for d in ly_array_iter(self.cdata_leaflist.dflts): + yield c2str(d.str) + + def min_elements(self) -> int: + return self.cdata_leaflist.min + + def max_elements(self) -> Optional[int]: + return self.cdata_leaflist.max if self.cdata_leaflist.max != 0 else None + + def ordered(self) -> bool: + return bool(self.cdata.flags & lib.LYS_ORDBY_USER) + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.LIST) +class PList(PNode): + __slots__ = ("cdata_list",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_list = ffi.cast("struct lysp_node_list *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_list.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_list.when == ffi.NULL: + return None + return c2str(self.cdata_list.when.cond) + + def key(self) -> Optional[str]: + return c2str(self.cdata_list.key) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_list.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_list.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_list.child): + yield PNode.new(self.context, c, self.module) + + def actions(self) -> Iterator["PAction"]: + for a in ly_list_iter(self.cdata_list.actions): + yield PAction(self.context, a, self.module) + + def notifications(self) -> Iterator["PNotif"]: + for n in ly_list_iter(self.cdata_list.notifs): + yield PNotif(self.context, n, self.module) + + def uniques(self) -> Iterator[str]: + for u in ly_array_iter(self.cdata_list.uniques): + yield c2str(u.str) + + def min_elements(self) -> int: + return self.cdata_list.min + + def max_elements(self) -> Optional[int]: + return self.cdata_list.max if self.cdata_list.max != 0 else None + + def ordered(self) -> bool: + return bool(self.cdata.flags & lib.LYS_ORDBY_USER) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.CASE) +class PCase(PNode): + __slots__ = ("cdata_case",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_case = ffi.cast("struct lysp_node_case *", cdata) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_case.child): + yield PNode.new(self.context, c, self.module) + + def when_condition(self) -> Optional[str]: + if self.cdata_case.when == ffi.NULL: + return None + return c2str(self.cdata_case.when.cond) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.CHOICE) +class PChoice(PNode): + __slots__ = ("cdata_choice",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_choice = ffi.cast("struct lysp_node_choice *", cdata) + + def children(self) -> Iterator[PCase]: + for c in ly_list_iter(self.cdata_choice.child): + yield PCase(self.context, c, self.module) + + def when_condition(self) -> Optional[str]: + if self.cdata_choice.when == ffi.NULL: + return None + return c2str(self.cdata_choice.when.cond) + + def default(self) -> Optional[str]: + return c2str(self.cdata_choice.dflt.str) + + def __iter__(self) -> Iterator[PCase]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.ANYXML) +@PNode.register(PNode.ANYDATA) +class PAnydata(PNode): + __slots__ = ("cdata_anydata",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_anydata = ffi.cast("struct lysp_node_anydata *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_anydata.musts): + yield Must(self.context, None, m) + + def when_condition(self) -> Optional[str]: + if self.cdata_anydata.when == ffi.NULL: + return None + return c2str(self.cdata_anydata.when.cond) + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.AUGMENT) +class PAugment(PNode): + __slots__ = ("cdata_augment",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_augment = ffi.cast("struct lysp_node_augment *", cdata) + + def children(self) -> Iterator["PNode"]: + for c in ly_list_iter(self.cdata_augment.child): + yield PNode.new(self.context, c, self.module) + + def when_condition(self) -> Optional[str]: + if self.cdata_augment.when == ffi.NULL: + return None + return c2str(self.cdata_augment.when.cond) + + def actions(self) -> Iterator["PAction"]: + for a in ly_list_iter(self.cdata_augment.actions): + yield PAction(self.context, a, self.module) + + def notifications(self) -> Iterator["PNotif"]: + for n in ly_list_iter(self.cdata_augment.notifs): + yield PNotif(self.context, n, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.USES) +class PUses(PNode): + __slots__ = ("cdata_uses",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_uses = ffi.cast("struct lysp_node_uses *", cdata) + + def refines(self) -> Iterator[PRefine]: + for r in ly_array_iter(self.cdata_uses.refines): + yield PRefine(self.context, r, self.module) + + def augments(self) -> Iterator[PAugment]: + for a in ly_list_iter(self.cdata_uses.augments): + yield PAugment(self.context, a, self.module) + + def when_condition(self) -> Optional[str]: + if self.cdata_uses.when == ffi.NULL: + return None + return c2str(self.cdata_uses.when.cond) + + +# ------------------------------------------------------------------------------------- +class PActionInOut(PNode): + __slots__ = ("cdata_action_inout",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_action_inout = ffi.cast("struct lysp_node_action_inout *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_action_inout.musts): + yield Must(self.context, None, m) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_action_inout.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_action_inout.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_action_inout.child): + yield PNode.new(self.context, c, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.RPC) +@PNode.register(PNode.ACTION) +class PAction(PNode): + __slots__ = ("cdata_action",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_action = ffi.cast("struct lysp_node_action *", cdata) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_action.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_action.groupings): + yield PGrouping(self.context, g, self.module) + + def input(self) -> PActionInOut: + ptr = ffi.addressof(self.cdata_action.input) + return PActionInOut(self.context, ptr, self.module) + + def output(self) -> PActionInOut: + ptr = ffi.addressof(self.cdata_action.output) + return PActionInOut(self.context, ptr, self.module) + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.NOTIF) +class PNotif(PNode): + __slots__ = ("cdata_notif",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_notif = ffi.cast("struct lysp_node_notif *", cdata) + + def musts(self) -> Iterator[Must]: + for m in ly_array_iter(self.cdata_notif.musts): + yield Must(self.context, None, m) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_notif.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_notif.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_notif.child): + yield PNode.new(self.context, c, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() + + +# ------------------------------------------------------------------------------------- +@PNode.register(PNode.GROUPING) +class PGrouping(PNode): + __slots__ = ("cdata_grouping",) + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + super().__init__(context, cdata, module) + self.cdata_grouping = ffi.cast("struct lysp_node_grp *", cdata) + + def typedefs(self) -> Iterator[Typedef]: + for t in ly_array_iter(self.cdata_grouping.typedefs): + yield Typedef(self.context, t) + + def groupings(self) -> Iterator["PGrouping"]: + for g in ly_list_iter(self.cdata_grouping.groupings): + yield PGrouping(self.context, g, self.module) + + def children(self) -> Iterator[PNode]: + for c in ly_list_iter(self.cdata_grouping.child): + yield PNode.new(self.context, c, self.module) + + def actions(self) -> Iterator[PAction]: + for a in ly_list_iter(self.cdata_grouping.actions): + yield PAction(self.context, a, self.module) + + def notifications(self) -> Iterator[PNotif]: + for n in ly_list_iter(self.cdata_grouping.notifs): + yield PNotif(self.context, n, self.module) + + def __iter__(self) -> Iterator[PNode]: + return self.children() diff --git a/tests/test_schema.py b/tests/test_schema.py index 64a8e30e..87409f90 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -13,8 +13,25 @@ LibyangError, Module, Must, + PAction, + PActionInOut, + PAnydata, Pattern, + PAugment, + PCase, + PChoice, + PContainer, + PGrouping, + PLeaf, + PLeafList, + PList, + PNode, + PNotif, + PRefine, + PType, + PUses, Revision, + SAnydata, SCase, SChoice, SContainer, @@ -287,6 +304,79 @@ def test_iter_tree(self): tree = list(self.container.iter_tree(full=True)) self.assertEqual(len(tree), 25) + def test_container_parsed(self): + pnode = self.container.parsed() + self.assertIsInstance(pnode, PContainer) + self.assertIsNone(next(pnode.musts(), None)) + self.assertIsNone(pnode.when_condition()) + self.assertIsNone(pnode.presence()) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + self.assertIsNotNone(next(iter(pnode))) + self.assertIsNone(next(pnode.actions(), None)) + self.assertIsNone(next(pnode.notifications(), None)) + + +# ------------------------------------------------------------------------------------- +class UsesTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + mod = self.ctx.load_module("yolo-nodetypes") + mod.feature_enable_all() + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_uses_parsed(self): + snode = next(self.ctx.find_path("/yolo-nodetypes:cont2")) + self.assertIsInstance(snode, SContainer) + pnode = snode.parsed() + self.assertIsInstance(pnode, PContainer) + pnode = next(iter(pnode)) + self.assertIsInstance(pnode, PUses) + + ref_pnode = next(pnode.refines()) + self.assertIsInstance(ref_pnode, PRefine) + self.assertEqual("cont3/leaf1", ref_pnode.nodeid()) + self.assertIsNone(ref_pnode.description()) + self.assertIsNone(ref_pnode.reference()) + self.assertIsNone(next(ref_pnode.if_features(), None)) + self.assertIsNone(next(ref_pnode.musts(), None)) + self.assertIsNone(ref_pnode.presence()) + self.assertIsNone(next(ref_pnode.defaults(), None)) + self.assertEqual(0, ref_pnode.min_elements()) + self.assertIsNone(ref_pnode.max_elements()) + self.assertIsNone(next(ref_pnode.extensions(), None)) + + aug_pnode = next(pnode.augments()) + self.assertIsInstance(aug_pnode, PAugment) + self.assertIsNotNone(next(iter(aug_pnode))) + self.assertIsNone(aug_pnode.when_condition()) + self.assertIsNone(next(aug_pnode.actions(), None)) + self.assertIsNone(next(aug_pnode.notifications(), None)) + + +# ------------------------------------------------------------------------------------- +class GroupingTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_grouping_parsed(self): + mod = self.ctx.load_module("yolo-nodetypes") + pnode = next(mod.groupings()) + self.assertIsInstance(pnode, PGrouping) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + child = next(iter(pnode)) + self.assertIsNotNone(child) + self.assertIsNone(next(pnode.actions(), None)) + self.assertIsNone(next(pnode.notifications(), None)) + # ------------------------------------------------------------------------------------- class ListTest(unittest.TestCase): @@ -363,6 +453,25 @@ def test_list_min_max(self): self.assertEqual(list2.min_elements(), 0) self.assertEqual(list2.max_elements(), None) + def test_list_parsed(self): + list1 = next(self.ctx.find_path("/yolo-nodetypes:conf/list1")) + self.assertIsInstance(list1, SList) + pnode = list1.parsed() + self.assertIsInstance(pnode, PList) + self.assertIsNone(next(pnode.musts(), None)) + self.assertIsNone(pnode.when_condition()) + self.assertEqual("leaf1", pnode.key()) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + child = next(iter(pnode)) + self.assertIsInstance(child, PLeaf) + self.assertIsNone(next(pnode.actions(), None)) + self.assertIsNone(next(pnode.notifications(), None)) + self.assertEqual("leaf2 leaf3", next(pnode.uniques())) + self.assertEqual(2, pnode.min_elements()) + self.assertEqual(10, pnode.max_elements()) + self.assertFalse(pnode.ordered()) + # ------------------------------------------------------------------------------------- class RpcTest(unittest.TestCase): @@ -398,6 +507,21 @@ def test_rpc_params(self): def test_rpc_no_parent(self): self.assertIsNone(self.rpc.parent()) + def test_rpc_parsed(self): + self.assertIsInstance(self.rpc, SRpc) + pnode = self.rpc.parsed() + self.assertIsInstance(pnode, PAction) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + pnode2 = pnode.input() + self.assertIsInstance(pnode2, PActionInOut) + self.assertIsInstance(pnode.output(), PActionInOut) + self.assertIsNone(next(pnode2.musts(), None)) + self.assertIsNone(next(pnode2.typedefs(), None)) + self.assertIsNone(next(pnode2.groupings(), None)) + pnode3 = next(iter(pnode2)) + self.assertIsInstance(pnode3, PLeaf) + # ------------------------------------------------------------------------------------- class LeafTypeTest(unittest.TestCase): @@ -536,6 +660,28 @@ def test_leaf_type_require_instance(self): self.assertIsInstance(t, Type) self.assertFalse(t.require_instance()) + def test_leaf_type_parsed(self): + leaf = next(self.ctx.find_path("/yolo-system:conf/yolo-system:hostname")) + self.assertIsInstance(leaf, SLeaf) + t = leaf.type() + self.assertIsInstance(t, Type) + pnode = t.parsed() + self.assertIsInstance(pnode, PType) + self.assertEqual("types:host", pnode.name()) + self.assertIsNone(pnode.range()) + self.assertIsNone(pnode.length()) + self.assertIsNone(next(pnode.patterns(), None)) + self.assertIsNone(next(pnode.enums(), None)) + self.assertIsNone(next(pnode.bits(), None)) + self.assertIsNone(pnode.path()) + self.assertIsNone(next(pnode.bases(), None)) + self.assertIsNone(next(pnode.types(), None)) + self.assertIsNone(next(pnode.extensions(), None)) + self.assertIsNotNone(pnode.pmod()) + self.assertIsNone(pnode.compiled()) + self.assertEqual(0, pnode.fraction_digits()) + self.assertFalse(pnode.require_instance()) + # ------------------------------------------------------------------------------------- class LeafTest(unittest.TestCase): @@ -560,6 +706,41 @@ def test_leaf_default(self): leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) self.assertIsInstance(leaf.default(), float) + def test_leaf_parsed(self): + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) + self.assertIsInstance(leaf, SLeaf) + pnode = leaf.parsed() + self.assertIsInstance(pnode, PLeaf) + must = next(pnode.musts()) + self.assertIsInstance(must, Must) + self.assertEqual(must.error_message(), "ERROR1") + must = next(leaf.must_conditions()) + self.assertIsInstance(must, str) + self.assertIsNone(pnode.when_condition()) + self.assertIsInstance(pnode.type(), PType) + self.assertIsNone(pnode.units()) + self.assertEqual("10.2", pnode.default()) + self.assertFalse(pnode.is_key()) + + # test basic PNode settings + self.assertIsNotNone(pnode.parent()) + self.assertEqual(PNode.LEAF, pnode.nodetype()) + self.assertIsNotNone(next(pnode.siblings())) + self.assertEqual("<libyang.schema.PLeaf: percentage>", repr(pnode)) + self.assertIsNone(pnode.description()) + self.assertIsNone(pnode.reference()) + self.assertIsNone(next(pnode.if_features(), None)) + self.assertIsNone(next(pnode.extensions(), None)) + self.assertIsNone(pnode.get_extension("test", prefix="test")) + self.assertFalse(pnode.config_set()) + self.assertFalse(pnode.config_false()) + self.assertFalse(pnode.mandatory()) + self.assertFalse(pnode.deprecated()) + self.assertFalse(pnode.obsolete()) + self.assertEqual("current", pnode.status()) + + NODETYPE_CLASS = {} + # ------------------------------------------------------------------------------------- class LeafListTest(unittest.TestCase): @@ -587,6 +768,20 @@ def test_leaf_list_min_max(self): self.assertEqual(leaflist2.min_elements(), 0) self.assertEqual(leaflist2.max_elements(), None) + def test_leaf_list_parsed(self): + leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/ratios")) + self.assertIsInstance(leaflist, SLeafList) + pnode = leaflist.parsed() + self.assertIsInstance(pnode, PLeafList) + self.assertIsNone(next(pnode.musts(), None)) + self.assertIsNone(pnode.when_condition()) + self.assertIsInstance(pnode.type(), PType) + self.assertIsNone(pnode.units()) + self.assertEqual("2.5", next(pnode.defaults())) + self.assertEqual(0, pnode.min_elements()) + self.assertIsNone(pnode.max_elements()) + self.assertFalse(pnode.ordered()) + # ------------------------------------------------------------------------------------- class ChoiceTest(unittest.TestCase): @@ -603,3 +798,60 @@ def test_choice_default(self): choice = next(conf.children((SNode.CHOICE,), with_choice=True)) self.assertIsInstance(choice, SChoice) self.assertIsInstance(choice.default(), SCase) + + def test_choice_parsed(self): + conf = next(self.ctx.find_path("/yolo-system:conf")) + choice = next(conf.children((SNode.CHOICE,), with_choice=True)) + self.assertIsInstance(choice, SChoice) + pnode = choice.parsed() + self.assertIsInstance(pnode, PChoice) + + case_pnode = next(iter(pnode)) + self.assertIsInstance(case_pnode, PCase) + self.assertIsNotNone(next(iter(case_pnode))) + self.assertIsNone(case_pnode.when_condition()) + + self.assertIsNone(pnode.when_condition()) + self.assertEqual("red", pnode.default()) + + +# ------------------------------------------------------------------------------------- +class AnydataTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_anydata_parsed(self): + snode = next(self.ctx.find_path("/yolo-nodetypes:any1")) + self.assertIsInstance(snode, SAnydata) + pnode = snode.parsed() + self.assertIsInstance(pnode, PAnydata) + self.assertIsNone(next(pnode.musts(), None)) + self.assertEqual("../cont2", pnode.when_condition()) + + +# ------------------------------------------------------------------------------------- +class NotificationTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_notification_parsed(self): + snode = next(self.ctx.find_path("/yolo-nodetypes:cont2")) + self.assertIsInstance(snode, SContainer) + pnode = snode.parsed() + self.assertIsInstance(pnode, PContainer) + pnode = next(pnode.notifications()) + self.assertIsInstance(pnode, PNotif) + self.assertIsNone(next(pnode.musts(), None)) + self.assertIsNone(next(pnode.typedefs(), None)) + self.assertIsNone(next(pnode.groupings(), None)) + self.assertIsNotNone(next(iter(pnode))) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index a456ae1d..c2690dc9 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -81,4 +81,35 @@ module yolo-nodetypes { leaf test1 { type uint8; } + + grouping grp1 { + container cont3 { + leaf leaf1 { + type string; + } + } + } + + container cont2 { + presence "special container enabled"; + uses grp1 { + refine cont3/leaf1 { + mandatory true; + } + augment cont3 { + leaf leaf2 { + type int8; + } + } + } + notification interface-enabled { + leaf by-user { + type string; + } + } + } + + anydata any1 { + when "../cont2"; + } } From 41a4f74737dba100bbfe5e42b069d58d91a70f8c Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 1 Mar 2024 14:37:04 +0100 Subject: [PATCH 057/115] context: add add_to_dict/remove_from_dict APIs This patch adds add_to_dict and remove_from_dict API, which allows to create internal string reference within the context. Closes: https://github.com/CESNET/libyang-python/pull/102 Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 3 +++ libyang/context.py | 10 ++++++++++ tests/test_context.py | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index cbe92969..8eaaa0a9 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1071,6 +1071,9 @@ LY_ERR lys_parse(struct ly_ctx *, struct ly_in *, LYS_INFORMAT, const char **, s LY_ERR ly_ctx_new_ylpath(const char *, const char *, LYD_FORMAT, int, struct ly_ctx **); LY_ERR ly_ctx_get_yanglib_data(const struct ly_ctx *, struct lyd_node **, const char *, ...); +LY_ERR lydict_insert(const struct ly_ctx *, const char *, size_t, const char **); +LY_ERR lydict_remove(const struct ly_ctx *, const char *); + struct lyd_meta { struct lyd_node *parent; struct lyd_meta *next; diff --git a/libyang/context.py b/libyang/context.py index fa5eb5bf..9e3f89be 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -465,3 +465,13 @@ def __iter__(self) -> Iterator[Module]: while mod: yield Module(self, mod) mod = lib.ly_ctx_get_module_iter(self.cdata, idx) + + def add_to_dict(self, orig_str: str) -> Any: + cstr = ffi.new("char **") + ret = lib.lydict_insert(self.cdata, str2c(orig_str), 0, cstr) + if ret != lib.LY_SUCCESS: + raise LibyangError("Unable to insert string into context dictionary") + return cstr[0] + + def remove_from_dict(self, orig_str: str) -> None: + lib.lydict_remove(self.cdata, str2c(orig_str)) diff --git a/tests/test_context.py b/tests/test_context.py index 59839284..f43bf1cf 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -5,6 +5,7 @@ import unittest from libyang import Context, LibyangError, Module, SLeaf, SLeafList +from libyang.util import c2str YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") @@ -111,3 +112,10 @@ def test_ctx_leafref_extended(self): with Context(YANG_DIR, leafref_extended=True) as ctx: mod = ctx.load_module("yolo-leafref-extended") self.assertIsInstance(mod, Module) + + def test_context_dict(self): + with Context(YANG_DIR) as ctx: + orig_str = "teststring" + handle = ctx.add_to_dict(orig_str) + self.assertEqual(orig_str, c2str(handle)) + ctx.remove_from_dict(orig_str) From 487cec571b3732d87ee68eab7c19227c004b674f Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 14:49:08 +0200 Subject: [PATCH 058/115] schema: fix missing line For some reason, my tox env did not detect it. Fix it. Fixes: b7adea400520 ("schema: adds PNode and its exact variants") Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libyang/schema.py b/libyang/schema.py index 7cd2289b..7c79b866 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -16,6 +16,7 @@ str2c, ) + # ------------------------------------------------------------------------------------- def schema_in_format(fmt_string: str) -> int: if fmt_string == "yang": From dc5a72666f63bd939db151d927c7e0673f15721c Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 14:56:06 +0200 Subject: [PATCH 059/115] test_schema: fix DATA_PATTERN in list test Fix the key order. I guess it was changed on libyang side, it does not seem to have an impact. Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- tests/test_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 87409f90..6b171af2 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -383,7 +383,7 @@ class ListTest(unittest.TestCase): PATH = { "LOG": "/yolo-system:conf/url", "DATA": "/yolo-system:conf/url", - "DATA_PATTERN": "/yolo-system:conf/url[host='%s'][proto='%s']", + "DATA_PATTERN": "/yolo-system:conf/url[proto='%s'][host='%s']", } def setUp(self): From a3132f808bb0e78bc5255dde8acf68c2c549316d Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 15:00:08 +0200 Subject: [PATCH 060/115] libyang: fix import order I prefer as it was, but isort doesn't, let's make him happy. Fixes: b7adea400520 ("schema: adds PNode and its exact variants") Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/__init__.py b/libyang/__init__.py index 5c931de4..26235de1 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -79,6 +79,7 @@ PAction, PActionInOut, PAnydata, + Pattern, PAugment, PCase, PChoice, @@ -93,7 +94,6 @@ PRefine, PType, PUses, - Pattern, Revision, SAnydata, SAnyxml, From 25d7de885d7f82030204cb5b73dbcf37c382e833 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 8 Mar 2024 09:52:42 +0100 Subject: [PATCH 061/115] data: add add_defaults module option This patch adds ability for user to restrict adding off implicit default values based on specified module. Closes: https://github.com/CESNET/libyang-python/pull/110 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 1 + libyang/data.py | 11 ++++++++++- tests/test_data.py | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 8eaaa0a9..7cd37574 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1189,6 +1189,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc #define LYD_IMPLICIT_NO_DEFAULTS ... LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **); +LY_ERR lyd_new_implicit_module(struct lyd_node **, const struct lys_module *, uint32_t, struct lyd_node **); LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **); LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_meta **); diff --git a/libyang/data.py b/libyang/data.py index 2895daaf..d36ffe1b 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -340,6 +340,7 @@ def add_defaults( no_state: bool = False, output: bool = False, only_node: bool = False, + only_module: Optional[Module] = None, ): flags = implicit_flags( no_config=no_config, @@ -353,7 +354,15 @@ def add_defaults( else: node_p = ffi.new("struct lyd_node **") node_p[0] = self.cdata - ret = lib.lyd_new_implicit_all(node_p, self.context.cdata, flags, ffi.NULL) + if only_module is not None: + ret = lib.lyd_new_implicit_module( + node_p, only_module.cdata, flags, ffi.NULL + ) + else: + ret = lib.lyd_new_implicit_all( + node_p, self.context.cdata, flags, ffi.NULL + ) + if ret != lib.LY_SUCCESS: raise self.context.error("cannot get module") diff --git a/tests/test_data.py b/tests/test_data.py index a40056c0..a612646f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -897,7 +897,7 @@ def test_find_all(self): dnode.free() def test_add_defaults(self): - JSON = '{"yolo-nodetypes:records": [{"id": "rec1"}]}' + JSON = '{"yolo-nodetypes:records": [{"id": "rec1"}], "yolo-nodetypes:conf": {}}' dnode = self.ctx.parse_data_mem( JSON, "json", validate_present=True, parse_only=True ) @@ -906,12 +906,25 @@ def test_add_defaults(self): self.assertIsInstance(node, DLeaf) node = dnode.find_one("name") self.assertIsNone(node) + node = dnode.find_one("/yolo-system:conf/speed") + self.assertIsNone(node) + dnode.add_defaults(only_node=True) node = dnode.find_one("name") self.assertIsInstance(node, DLeaf) self.assertEqual(node.value(), "ASD") - node = dnode.find_path("/yolo-nodetypes:conf/speed") + node = dnode.find_one("/yolo-nodetypes:conf/percentage") self.assertIsNone(node) + node = dnode.find_one("/yolo-system:conf/speed") + self.assertIsNone(node) + + dnode.add_defaults(only_module=dnode.module()) + node = dnode.find_one("/yolo-nodetypes:conf/percentage") + self.assertIsInstance(node, DLeaf) + self.assertEqual(node.value(), 10.2) + node = dnode.find_one("/yolo-system:conf/speed") + self.assertIsNone(node) + dnode.add_defaults(only_node=False) node = dnode.find_path("/yolo-system:conf/speed") self.assertIsInstance(node, DLeaf) From 2fc312b93e3db09d15b78c8cec175f10f421bdcf Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 9 May 2024 12:38:07 +0200 Subject: [PATCH 062/115] data: add insert_[before,after] API to lists This patch adds insert_[before,after] functions, which allows user to specify where to insert the sibling in case of ordered (leaf-)lists. Closes: https://github.com/CESNET/libyang-python/pull/120 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 2 ++ libyang/data.py | 10 ++++++++++ tests/test_data.py | 17 +++++++++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 1 + 4 files changed, 30 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 7cd37574..39e44357 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1126,6 +1126,8 @@ LY_ERR lyd_merge_tree(struct lyd_node **, const struct lyd_node *, uint16_t); LY_ERR lyd_merge_siblings(struct lyd_node **, const struct lyd_node *, uint16_t); LY_ERR lyd_insert_child(struct lyd_node *, struct lyd_node *); LY_ERR lyd_insert_sibling(struct lyd_node *, struct lyd_node *, struct lyd_node **); +LY_ERR lyd_insert_after(struct lyd_node *, struct lyd_node *); +LY_ERR lyd_insert_before(struct lyd_node *, struct lyd_node *); LY_ERR lyd_diff_apply_all(struct lyd_node **, const struct lyd_node *); #define LYD_DUP_NO_META ... diff --git a/libyang/data.py b/libyang/data.py index d36ffe1b..96b792ba 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -415,6 +415,16 @@ def insert_sibling(self, node): if ret != lib.LY_SUCCESS: raise self.context.error("cannot insert sibling") + def insert_after(self, node): + ret = lib.lyd_insert_after(self.cdata, node.cdata) + if ret != lib.LY_SUCCESS: + raise self.context.error("cannot insert sibling after") + + def insert_before(self, node): + ret = lib.lyd_insert_before(self.cdata, node.cdata) + if ret != lib.LY_SUCCESS: + raise self.context.error("cannot insert sibling before") + def name(self) -> str: return c2str(self.cdata.schema.name) diff --git a/tests/test_data.py b/tests/test_data.py index a612646f..d43f212b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -966,6 +966,23 @@ def test_dnode_insert_sibling(self): self.assertIsInstance(sibling, DLeaf) self.assertEqual(sibling.cdata, dnode2.cdata) + def test_dnode_insert_sibling_before_after(self): + R1 = {"yolo-nodetypes:records": [{"id": "id1", "name": "name1"}]} + R2 = {"yolo-nodetypes:records": [{"id": "id2", "name": "name2"}]} + R3 = {"yolo-nodetypes:records": [{"id": "id3", "name": "name3"}]} + module = self.ctx.get_module("yolo-nodetypes") + dnode1 = dict_to_dnode(R1, module, None, validate=False) + dnode2 = dict_to_dnode(R2, module, None, validate=False) + dnode3 = dict_to_dnode(R3, module, None, validate=False) + self.assertEqual(dnode1.first_sibling().cdata, dnode1.cdata) + dnode1.insert_before(dnode2) + dnode1.insert_after(dnode3) + self.assertEqual( + [dnode2.cdata, dnode1.cdata, dnode3.cdata], + [s.cdata for s in dnode1.first_sibling().siblings()], + ) + self.assertEqual(dnode1.first_sibling().cdata, dnode2.cdata) + def _create_opaq_hostname(self): root = self.ctx.create_data_path(path="/yolo-system:conf") root.new_path( diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index c2690dc9..d9ef77de 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -20,6 +20,7 @@ module yolo-nodetypes { type string; default "ASD"; } + ordered-by user; } container conf { From e1cefcf64d5374f4c0bcc769e46bfe8758b317bb Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Fri, 2 Aug 2024 15:42:13 +0200 Subject: [PATCH 063/115] schema: use ly_array_iter for SLeafList defaults API It is way clearer this way. Closes: https://github.com/CESNET/libyang-python/pull/119 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 7c79b866..539af886 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1414,15 +1414,12 @@ def type(self) -> Type: def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: if self.cdata_leaflist.dflts == ffi.NULL: return - arr_length = ffi.cast("uint64_t *", self.cdata_leaflist.dflts)[-1] - for i in range(arr_length): - val = lib.lyd_value_get_canonical( - self.context.cdata, self.cdata_leaflist.dflts[i] - ) + for dflt in ly_array_iter(self.cdata_leaflist.dflts): + val = lib.lyd_value_get_canonical(self.context.cdata, dflt) if not val: yield None val = c2str(val) - val_type = Type(self.context, self.cdata_leaflist.dflts[i].realtype, None) + val_type = Type(self.context, dflt.realtype, None) if val_type == Type.BOOL: yield val == "true" elif val_type in Type.NUM_TYPES: From 32165ef8989afb1bb2384ab87f49973127fe5d86 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 18 Apr 2024 09:32:25 +0200 Subject: [PATCH 064/115] schema: fix of SLeafList defaults API This patch fixes defaults API of SLeafList. The returned values are now correctly converted to python types. Closes: https://github.com/CESNET/libyang-python/pull/119 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 4 ++-- tests/test_schema.py | 6 ++++++ tests/yang/yolo/yolo-nodetypes.yang | 11 +++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 539af886..f35a5e9a 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1420,9 +1420,9 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: yield None val = c2str(val) val_type = Type(self.context, dflt.realtype, None) - if val_type == Type.BOOL: + if val_type.base() == Type.BOOL: yield val == "true" - elif val_type in Type.NUM_TYPES: + elif val_type.base() in Type.NUM_TYPES: yield int(val) elif val_type.base() == Type.DEC64: yield float(val) diff --git a/tests/test_schema.py b/tests/test_schema.py index 6b171af2..e2a833ce 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -756,6 +756,12 @@ def test_leaflist_defaults(self): leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/ratios")) for d in leaflist.defaults(): self.assertIsInstance(d, float) + leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/bools")) + for d in leaflist.defaults(): + self.assertIsInstance(d, bool) + leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/integers")) + for d in leaflist.defaults(): + self.assertIsInstance(d, int) def test_leaf_list_min_max(self): leaflist1 = next(self.ctx.find_path("/yolo-nodetypes:conf/leaf-list1")) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index d9ef77de..0732ee72 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -45,6 +45,17 @@ module yolo-nodetypes { default 2.6; } + leaf-list bools { + type boolean; + default true; + } + + leaf-list integers { + type uint32; + default 10; + default 20; + } + list list1 { key leaf1; unique "leaf2 leaf3"; From f437bfb8a3ca4a705c76d8aabb70e815de354960 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Wed, 28 Feb 2024 13:09:20 +0100 Subject: [PATCH 065/115] context: add disable_searchdirs options This patch adds support to disable searching of local directories during loading of module. Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/context.py | 3 +++ tests/test_context.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/libyang/context.py b/libyang/context.py index 9e3f89be..f0471cb6 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -26,6 +26,7 @@ class Context: def __init__( self, search_path: Optional[str] = None, + disable_searchdirs: bool = False, disable_searchdir_cwd: bool = True, explicit_compile: Optional[bool] = False, leafref_extended: bool = False, @@ -38,6 +39,8 @@ def __init__( return # already initialized options = 0 + if disable_searchdirs: + options |= lib.LY_CTX_DISABLE_SEARCHDIRS if disable_searchdir_cwd: options |= lib.LY_CTX_DISABLE_SEARCHDIR_CWD if explicit_compile: diff --git a/tests/test_context.py b/tests/test_context.py index f43bf1cf..02d20a45 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -119,3 +119,8 @@ def test_context_dict(self): handle = ctx.add_to_dict(orig_str) self.assertEqual(orig_str, c2str(handle)) ctx.remove_from_dict(orig_str) + + def test_ctx_disable_searchdirs(self): + with Context(YANG_DIR, disable_searchdirs=True) as ctx: + with self.assertRaises(LibyangError): + ctx.load_module("yolo-nodetypes") From 47b0f0953031aedbd08ba3960c23f97dfecec4f1 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Sun, 3 Mar 2024 11:39:40 +0100 Subject: [PATCH 066/115] context: add leafref_linking options This patch adds leafref_linking context option, which allows the usage of lyd_leafref_get_links and lyd_leafref_link_node_tree functions, whose support will be added in the next commit. Closes: https://github.com/CESNET/libyang-python/pull/108 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 1 + libyang/context.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 39e44357..24688fca 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -15,6 +15,7 @@ struct ly_ctx; #define LY_CTX_REF_IMPLEMENTED ... #define LY_CTX_SET_PRIV_PARSED ... #define LY_CTX_LEAFREF_EXTENDED ... +#define LY_CTX_LEAFREF_LINKING ... typedef enum { diff --git a/libyang/context.py b/libyang/context.py index f0471cb6..1300a09c 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -30,6 +30,7 @@ def __init__( disable_searchdir_cwd: bool = True, explicit_compile: Optional[bool] = False, leafref_extended: bool = False, + leafref_linking: bool = False, yanglib_path: Optional[str] = None, yanglib_fmt: str = "json", cdata=None, # C type: "struct ly_ctx *" @@ -47,6 +48,8 @@ def __init__( options |= lib.LY_CTX_EXPLICIT_COMPILE if leafref_extended: options |= lib.LY_CTX_LEAFREF_EXTENDED + if leafref_linking: + options |= lib.LY_CTX_LEAFREF_LINKING # force priv parsed options |= lib.LY_CTX_SET_PRIV_PARSED From f14116c9216a448cdc2877fcd1e934bd9af1fbd2 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Sun, 3 Mar 2024 11:40:00 +0100 Subject: [PATCH 067/115] data: add leafref_nodes and leafref_link_node_tree This patch adds API, which allows user to: - determine if DNode is being referenced by some other leafref DNodes. - trigger process for creating cross-references between leafref DNodes and target DNodes. Closes: https://github.com/CESNET/libyang-python/pull/108 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 9 +++++++++ libyang/data.py | 24 +++++++++++++++++++++++- tests/test_data.py | 26 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 24688fca..6079ecff 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1248,5 +1248,14 @@ struct lyd_attr { LY_ERR lyd_new_attr(struct lyd_node *, const char *, const char *, const char *, struct lyd_attr **); void lyd_free_attr_single(const struct ly_ctx *ctx, struct lyd_attr *attr); +struct lyd_leafref_links_rec { + const struct lyd_node_term *node; + const struct lyd_node_term **leafref_nodes; + const struct lyd_node_term **target_nodes; +}; + +LY_ERR lyd_leafref_get_links(const struct lyd_node_term *e, const struct lyd_leafref_links_rec **); +LY_ERR lyd_leafref_link_node_tree(struct lyd_node *); + /* from libc, needed to free allocated strings */ void free(void *); diff --git a/libyang/data.py b/libyang/data.py index 96b792ba..00490db6 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -18,7 +18,7 @@ SRpc, Type, ) -from .util import DataType, IOType, LibyangError, c2str, str2c +from .util import DataType, IOType, LibyangError, c2str, ly_array_iter, str2c LOG = logging.getLogger(__name__) @@ -991,6 +991,28 @@ def free(self, with_siblings: bool = True) -> None: finally: self.cdata = ffi.NULL + def leafref_link_node_tree(self) -> None: + """ + Traverse through data tree including root node siblings and adds + leafrefs links to the given nodes. + + Requires leafref_linking to be set on the libyang context. + """ + lib.lyd_leafref_link_node_tree(self.cdata) + + def leafref_nodes(self) -> Iterator["DNode"]: + """ + Gets the leafref links record for given node. + + Requires leafref_linking to be set on the libyang context. + """ + term_node = ffi.cast("struct lyd_node_term *", self.cdata) + out = ffi.new("const struct lyd_leafref_links_rec **") + if lib.lyd_leafref_get_links(term_node, out) != lib.LY_SUCCESS: + return + for n in ly_array_iter(out[0].leafref_nodes): + yield DNode.new(self.context, n) + def __repr__(self): cls = self.__class__ return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) diff --git a/tests/test_data.py b/tests/test_data.py index d43f212b..9ed35dc3 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -21,6 +21,7 @@ DRpc, IOType, LibyangError, + Module, ) from libyang.data import dict_to_dnode @@ -1066,3 +1067,28 @@ def test_dnode_attrs_set_and_remove_multiple(self): attrs.remove("ietf-netconf:operation") self.assertEqual(len(attrs), 0) + + def test_dnode_leafref_linking(self): + MAIN = """{ + "yolo-leafref-extended:list1": [{ + "leaf1": "val1", + "leaflist2": ["val2", "val3"] + }], + "yolo-leafref-extended:ref1": "val1" + }""" + self.ctx.destroy() + self.ctx = Context(YANG_DIR, leafref_extended=True, leafref_linking=True) + mod = self.ctx.load_module("yolo-leafref-extended") + self.assertIsInstance(mod, Module) + dnode1 = self.ctx.parse_data_mem(MAIN, "json", parse_only=True) + self.assertIsInstance(dnode1, DList) + dnode2 = next(dnode1.siblings(include_self=False)) + self.assertIsInstance(dnode2, DLeaf) + dnode3 = next(dnode1.children()) + self.assertIsInstance(dnode3, DLeaf) + self.assertIsNone(next(dnode3.leafref_nodes(), None)) + dnode2.leafref_link_node_tree() + dnode4 = next(dnode3.leafref_nodes()) + self.assertIsInstance(dnode4, DLeaf) + self.assertEqual(dnode4.cdata, dnode2.cdata) + dnode1.free() From 4831b70cadaa7238284f00b123cfe0cd9330c913 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Wed, 28 Feb 2024 13:09:20 +0100 Subject: [PATCH 068/115] context: adding ContextExternalModuleLoader class This patch adds class ContextExternalModuleLoader, which adds ability to add custom module load callback, which allows user to load modules from remote source etc. Closes: https://github.com/CESNET/libyang-python/pull/103 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 5 ++ libyang/context.py | 170 +++++++++++++++++++++++++++++++++++++++++- tests/test_context.py | 23 ++++++ 3 files changed, 196 insertions(+), 2 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 6079ecff..474b4c6a 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1071,6 +1071,11 @@ typedef enum { LY_ERR lys_parse(struct ly_ctx *, struct ly_in *, LYS_INFORMAT, const char **, struct lys_module **); LY_ERR ly_ctx_new_ylpath(const char *, const char *, LYD_FORMAT, int, struct ly_ctx **); LY_ERR ly_ctx_get_yanglib_data(const struct ly_ctx *, struct lyd_node **, const char *, ...); +typedef void (*ly_module_imp_data_free_clb)(void *, void *); +typedef LY_ERR (*ly_module_imp_clb)(const char *, const char *, const char *, const char *, void *, LYS_INFORMAT *, const char **, ly_module_imp_data_free_clb *); +void ly_ctx_set_module_imp_clb(struct ly_ctx *, ly_module_imp_clb, void *); +extern "Python" void lypy_module_imp_data_free_clb(void *, void *); +extern "Python" LY_ERR lypy_module_imp_clb(const char *, const char *, const char *, const char *, void *, LYS_INFORMAT *, const char **, ly_module_imp_data_free_clb *); LY_ERR lydict_insert(const struct ly_ctx *, const char *, size_t, const char **); LY_ERR lydict_remove(const struct ly_ctx *, const char *); diff --git a/libyang/context.py b/libyang/context.py index 1300a09c..94bba597 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: MIT import os -from typing import IO, Any, Iterator, Optional, Union +from typing import IO, Any, Callable, Iterator, Optional, Tuple, Union from _libyang import ffi, lib from .data import ( @@ -19,9 +19,173 @@ from .util import DataType, IOType, LibyangError, c2str, data_load, str2c +# ------------------------------------------------------------------------------------- +@ffi.def_extern(name="lypy_module_imp_data_free_clb") +def libyang_c_module_imp_data_free_clb(cdata, user_data): + instance = ffi.from_handle(user_data) + instance.free_module_data(cdata) + + +# ------------------------------------------------------------------------------------- +@ffi.def_extern(name="lypy_module_imp_clb") +def libyang_c_module_imp_clb( + mod_name, + mod_rev, + submod_name, + submod_rev, + user_data, + fmt, + module_data, + free_module_data, +): + """ + Implement the C callback function for loading modules from any location. + + :arg c_str mod_name: + The YANG module name + :arg c_str mod_rev: + The YANG module revision + :arg c_str submod_name: + The YANG submodule name + :arg c_str submod_rev: + The YANG submodule revision + :arg user_data: + The user data provided by user during registration. In this implementation + it is always considered to be handle of Python object + :arg fmt: + The output pointer where to set the format of schema + :arg module_data: + The output pointer where to set the schema data itself + :arg free_module_data: + The output pointer of callback function which will be called when the schema + data are no longer needed + + :returns: + The LY_SUCCESS in case the needed YANG (sub)module schema was found + The LY_ENOT in case the needed YANG (sub)module schema was not found + """ + fmt[0] = lib.LYS_IN_UNKNOWN + module_data[0] = ffi.NULL + free_module_data[0] = lib.lypy_module_imp_data_free_clb + instance = ffi.from_handle(user_data) + ret = instance.get_module_data( + c2str(mod_name), c2str(mod_rev), c2str(submod_name), c2str(submod_rev) + ) + if ret is None: + return lib.LY_ENOT + in_fmt, content = ret + fmt[0] = schema_in_format(in_fmt) + module_data[0] = content + return lib.LY_SUCCESS + + +# ------------------------------------------------------------------------------------- +class ContextExternalModuleLoader: + __slots__ = ( + "_cdata", + "_module_data_clb", + "_cffi_handle", + "_cdata_modules", + ) + + def __init__(self, cdata) -> None: + self._cdata = cdata # C type: "struct ly_ctx *" + self._module_data_clb = None + self._cffi_handle = ffi.new_handle(self) + self._cdata_modules = [] + + def free_module_data(self, cdata) -> None: + """ + Free previously stored data, obtained after a get_module_data. + + :arg cdata: + The pointer to YANG modelu schema (c_str), which shall be released from memory + """ + self._cdata_modules.remove(cdata) + + def get_module_data( + self, + mod_name: Optional[str], + mod_rev: Optional[str], + submod_name: Optional[str], + submod_rev: Optional[str], + ) -> Optional[Tuple[str, str]]: + """ + Get the YANG module schema data based requirements from libyang_c_module_imp_clb + function and forward that request to user Python based callback function. + + The returned data from callback function are stored within the context to make sure + of no memory access issues. These data a stored until the free_module_data function + is called directly by libyang. + + :arg self + This instance on context + :arg mod_name: + The optional YANG module name + :arg mod_rev: + The optional YANG module revision + :arg submod_name: + The optional YANG submodule name + :arg submod_rev: + The optional YANG submodule revision + + :returns: + Tuple of format string and YANG (sub)module schema + """ + if self._module_data_clb is None: + return "", None + fmt_str, module_data = self._module_data_clb( + mod_name, mod_rev, submod_name, submod_rev + ) + if module_data is None: + return fmt_str, None + module_data_c = str2c(module_data) + self._cdata_modules.append(module_data_c) + return fmt_str, module_data_c + + def set_module_data_clb( + self, + clb: Optional[ + Callable[ + [Optional[str], Optional[str], Optional[str], Optional[str]], + Optional[Tuple[str, str]], + ] + ] = None, + ) -> None: + """ + Set the callback function, which will be called if libyang context would like to + load module or submodule, which is not locally available in context path(s). + + :arg self + This instance on context + :arg clb: + The callback function. The expected arguments are: + mod_name: Module name + mod_rev: Module revision + submod_name: Submodule name + submod_rev: Submodule revision + The expeted return value is either: + tuple of: + format: The string format of the loaded data + data: The YANG (sub)module data as string + or None in case of error + """ + self._module_data_clb = clb + if clb is None: + lib.ly_ctx_set_module_imp_clb(self._cdata, ffi.NULL, ffi.NULL) + else: + lib.ly_ctx_set_module_imp_clb( + self._cdata, lib.lypy_module_imp_clb, self._cffi_handle + ) + + # ------------------------------------------------------------------------------------- class Context: - __slots__ = ("cdata", "__dict__") + __slots__ = ( + "cdata", + "external_module_loader", + "__dict__", + ) def __init__( self, @@ -37,6 +201,7 @@ def __init__( ): if cdata is not None: self.cdata = ffi.cast("struct ly_ctx *", cdata) + self.external_module_loader = ContextExternalModuleLoader(self.cdata) return # already initialized options = 0 @@ -90,6 +255,7 @@ def __init__( ) if not self.cdata: raise self.error("cannot create context") + self.external_module_loader = ContextExternalModuleLoader(self.cdata) def compile_schema(self): ret = lib.ly_ctx_compile(self.cdata) diff --git a/tests/test_context.py b/tests/test_context.py index 02d20a45..8a0412e2 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -124,3 +124,26 @@ def test_ctx_disable_searchdirs(self): with Context(YANG_DIR, disable_searchdirs=True) as ctx: with self.assertRaises(LibyangError): ctx.load_module("yolo-nodetypes") + + def test_ctx_using_clb(self): + def get_module_valid_clb(mod_name, *_): + YOLO_NODETYPES_MOD_PATH = os.path.join(YANG_DIR, "yolo/yolo-nodetypes.yang") + self.assertEqual(mod_name, "yolo-nodetypes") + with open(YOLO_NODETYPES_MOD_PATH, encoding="utf-8") as f: + mod_str = f.read() + return "yang", mod_str + + def get_module_invalid_clb(mod_name, *_): + return None + + with Context(YANG_DIR, disable_searchdirs=True) as ctx: + with self.assertRaises(LibyangError): + ctx.load_module("yolo-nodetypes") + + ctx.external_module_loader.set_module_data_clb(get_module_invalid_clb) + with self.assertRaises(LibyangError): + mod = ctx.load_module("yolo-nodetypes") + + ctx.external_module_loader.set_module_data_clb(get_module_valid_clb) + mod = ctx.load_module("yolo-nodetypes") + self.assertIsInstance(mod, Module) From 012d1448a1da862f5259d8f50d75b53a3ba84640 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Wed, 28 Feb 2024 13:09:20 +0100 Subject: [PATCH 069/115] context: fix of get_module_data This patch fixes broken test related to get_module_data function. Closes: https://github.com/CESNET/libyang-python/pull/103 Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/context.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index 94bba597..b457e7b8 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -133,12 +133,11 @@ def get_module_data( Tuple of format string and YANG (sub)module schema """ if self._module_data_clb is None: - return "", None - fmt_str, module_data = self._module_data_clb( - mod_name, mod_rev, submod_name, submod_rev - ) - if module_data is None: - return fmt_str, None + return None + ret = self._module_data_clb(mod_name, mod_rev, submod_name, submod_rev) + if ret is None: + return None + fmt_str, module_data = ret module_data_c = str2c(module_data) self._cdata_modules.append(module_data_c) return fmt_str, module_data_c From 0c43437c3ead3615a0b3f3a5c23b2bc5603b7f27 Mon Sep 17 00:00:00 2001 From: Samuel Gauthier <samuel.gauthier@6wind.com> Date: Wed, 25 Sep 2024 15:18:09 +0200 Subject: [PATCH 070/115] cdefs: update with libyang master The libyang master branch has changed the lysp_ext_instance structure, update it. Link: https://github.com/CESNET/libyang/commit/35274cfbd Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 1 + 1 file changed, 1 insertion(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 474b4c6a..499a1b32 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -437,6 +437,7 @@ struct lysp_ext_instance { struct lysp_ext_substmt *substmts; void *parsed; struct lysp_stmt *child; + struct lysp_ext_instance *exts; }; struct lysp_import { From 5330a3ff0a01ed4118982f9bbd0c80b62bf07a99 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 12 Feb 2024 09:31:05 +0100 Subject: [PATCH 071/115] data: introduce store_only flags This patch introduces store_only flag, which allows user to only store value without performing any sort of additional type checks as length, range, pattern, etc. Closes: https://github.com/CESNET/libyang-python/pull/107 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 2 ++ libyang/context.py | 11 ++++++++++- libyang/data.py | 24 +++++++++++++++++++----- tests/test_data.py | 8 ++++++++ tests/yang/yolo/yolo-nodetypes.yang | 4 +++- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 499a1b32..6f201878 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -264,6 +264,7 @@ struct lyd_node { LY_ERR lys_set_implemented(struct lys_module *, const char **); #define LYD_NEW_VAL_OUTPUT ... +#define LYD_NEW_VAL_STORE_ONLY ... #define LYD_NEW_VAL_BIN ... #define LYD_NEW_VAL_CANON ... #define LYD_NEW_META_CLEAR_DFLT ... @@ -308,6 +309,7 @@ LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint3 #define LYD_PARSE_LYB_MOD_UPDATE ... #define LYD_PARSE_NO_STATE ... +#define LYD_PARSE_STORE_ONLY ... #define LYD_PARSE_ONLY ... #define LYD_PARSE_OPAQ ... #define LYD_PARSE_OPTS_MASK ... diff --git a/libyang/context.py b/libyang/context.py index b457e7b8..b08df0ca 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -409,6 +409,7 @@ def create_data_path( parent: Optional[DNode] = None, value: Any = None, update: bool = True, + store_only: bool = False, rpc_output: bool = False, force_return_value: bool = True, ) -> Optional[DNode]: @@ -419,7 +420,9 @@ def create_data_path( value = str(value).lower() elif not isinstance(value, str): value = str(value) - flags = newval_flags(update=update, rpc_output=rpc_output) + flags = newval_flags( + update=update, store_only=store_only, rpc_output=rpc_output + ) dnode = ffi.new("struct lyd_node **") ret = lib.lyd_new_path( parent.cdata if parent else ffi.NULL, @@ -513,6 +516,7 @@ def parse_data( strict: bool = False, validate_present: bool = False, validate_multi_error: bool = False, + store_only: bool = False, ) -> Optional[DNode]: if self.cdata is None: raise RuntimeError("context already destroyed") @@ -523,6 +527,7 @@ def parse_data( opaq=opaq, ordered=ordered, strict=strict, + store_only=store_only, ) validation_flgs = validation_flags( no_state=no_state, @@ -580,6 +585,7 @@ def parse_data_mem( strict: bool = False, validate_present: bool = False, validate_multi_error: bool = False, + store_only: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -594,6 +600,7 @@ def parse_data_mem( strict=strict, validate_present=validate_present, validate_multi_error=validate_multi_error, + store_only=store_only, ) def parse_data_file( @@ -609,6 +616,7 @@ def parse_data_file( strict: bool = False, validate_present: bool = False, validate_multi_error: bool = False, + store_only: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -623,6 +631,7 @@ def parse_data_file( strict=strict, validate_present=validate_present, validate_multi_error=validate_multi_error, + store_only=store_only, ) def __iter__(self) -> Iterator[Module]: diff --git a/libyang/data.py b/libyang/data.py index 00490db6..288303d6 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -79,6 +79,7 @@ def data_format(fmt_string: str) -> int: # ------------------------------------------------------------------------------------- def newval_flags( rpc_output: bool = False, + store_only: bool = False, bin_value: bool = False, canon_value: bool = False, meta_clear_default: bool = False, @@ -91,6 +92,8 @@ def newval_flags( flags = 0 if rpc_output: flags |= lib.LYD_NEW_VAL_OUTPUT + if store_only: + flags |= lib.LYD_NEW_VAL_STORE_ONLY if bin_value: flags |= lib.LYD_NEW_VAL_BIN if canon_value: @@ -112,6 +115,7 @@ def parser_flags( opaq: bool = False, ordered: bool = False, strict: bool = False, + store_only: bool = False, ) -> int: flags = 0 if lyb_mod_update: @@ -126,6 +130,8 @@ def parser_flags( flags |= lib.LYD_PARSE_ORDERED if strict: flags |= lib.LYD_PARSE_STRICT + if store_only: + flags |= lib.LYD_PARSE_STORE_ONLY return flags @@ -314,8 +320,10 @@ def meta_free(self, name): break item = item.next - def new_meta(self, name: str, value: str, clear_dflt: bool = False): - flags = newval_flags(meta_clear_default=clear_dflt) + def new_meta( + self, name: str, value: str, clear_dflt: bool = False, store_only: bool = False + ): + flags = newval_flags(meta_clear_default=clear_dflt, store_only=store_only) ret = lib.lyd_new_meta( ffi.NULL, self.cdata, @@ -391,6 +399,7 @@ def new_path( opt_opaq: bool = False, opt_bin_value: bool = False, opt_canon_value: bool = False, + opt_store_only: bool = False, ): flags = newval_flags( update=opt_update, @@ -398,6 +407,7 @@ def new_path( opaq=opt_opaq, bin_value=opt_bin_value, canon_value=opt_canon_value, + store_only=opt_store_only, ) ret = lib.lyd_new_path( self.cdata, ffi.NULL, str2c(path), str2c(value), flags, ffi.NULL @@ -1062,9 +1072,10 @@ def create_path( path: str, value: Any = None, rpc_output: bool = False, + store_only: bool = False, ) -> Optional[DNode]: return self.context.create_data_path( - path, parent=self, value=value, rpc_output=rpc_output + path, parent=self, value=value, rpc_output=rpc_output, store_only=store_only ) def children(self, no_keys=False) -> Iterator[DNode]: @@ -1188,6 +1199,7 @@ def dict_to_dnode( rpc: bool = False, rpcreply: bool = False, notification: bool = False, + store_only: bool = False, ) -> Optional[DNode]: """ Convert a python dictionary to a DNode object given a YANG module object. The return @@ -1214,6 +1226,8 @@ def dict_to_dnode( Data represents RPC or action output parameters. :arg notification: Data represents notification parameters. + :arg store_only: + Data are being stored regardless of type validation (length, range, pattern, etc.) """ if not dic: return None @@ -1235,7 +1249,7 @@ def _create_leaf(_parent, module, name, value, in_rpc_output=False): value = str(value) n = ffi.new("struct lyd_node **") - flags = newval_flags(rpc_output=in_rpc_output) + flags = newval_flags(rpc_output=in_rpc_output, store_only=store_only) ret = lib.lyd_new_term( _parent, module.cdata, @@ -1273,7 +1287,7 @@ def _create_container(_parent, module, name, in_rpc_output=False): def _create_list(_parent, module, name, key_values, in_rpc_output=False): n = ffi.new("struct lyd_node **") - flags = newval_flags(rpc_output=in_rpc_output) + flags = newval_flags(rpc_output=in_rpc_output, store_only=store_only) ret = lib.lyd_new_list( _parent, module.cdata, diff --git a/tests/test_data.py b/tests/test_data.py index 9ed35dc3..9515a58f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1092,3 +1092,11 @@ def test_dnode_leafref_linking(self): self.assertIsInstance(dnode4, DLeaf) self.assertEqual(dnode4.cdata, dnode2.cdata) dnode1.free() + + def test_dnode_store_only(self): + MAIN = {"yolo-nodetypes:test1": 50} + module = self.ctx.load_module("yolo-nodetypes") + dnode = dict_to_dnode(MAIN, module, None, validate=False, store_only=True) + self.assertIsInstance(dnode, DLeaf) + self.assertEqual(dnode.value(), 50) + dnode.free() diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 0732ee72..de4656b1 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -91,7 +91,9 @@ module yolo-nodetypes { } leaf test1 { - type uint8; + type uint8 { + range "2..20"; + } } grouping grp1 { From 551f44b5db419cfb02a1e34acb0ecc35d85615a0 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Sun, 3 Mar 2024 15:56:15 +0100 Subject: [PATCH 072/115] context: add builtin_plugins_only options This patch adds builtin_plugins_only option, which allows user to use only basic YANG type plugins, instead of using advanced plugins as ipv4-address etc. Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 1 + libyang/context.py | 3 +++ tests/test_data.py | 11 +++++++++++ tests/yang/yolo/yolo-nodetypes.yang | 9 +++++++++ 4 files changed, 24 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 6f201878..93f68c79 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -16,6 +16,7 @@ struct ly_ctx; #define LY_CTX_SET_PRIV_PARSED ... #define LY_CTX_LEAFREF_EXTENDED ... #define LY_CTX_LEAFREF_LINKING ... +#define LY_CTX_BUILTIN_PLUGINS_ONLY ... typedef enum { diff --git a/libyang/context.py b/libyang/context.py index b08df0ca..1b1d4cda 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -194,6 +194,7 @@ def __init__( explicit_compile: Optional[bool] = False, leafref_extended: bool = False, leafref_linking: bool = False, + builtin_plugins_only: bool = False, yanglib_path: Optional[str] = None, yanglib_fmt: str = "json", cdata=None, # C type: "struct ly_ctx *" @@ -214,6 +215,8 @@ def __init__( options |= lib.LY_CTX_LEAFREF_EXTENDED if leafref_linking: options |= lib.LY_CTX_LEAFREF_LINKING + if builtin_plugins_only: + options |= lib.LY_CTX_BUILTIN_PLUGINS_ONLY # force priv parsed options |= lib.LY_CTX_SET_PRIV_PARSED diff --git a/tests/test_data.py b/tests/test_data.py index 9515a58f..790b5cd7 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1,6 +1,7 @@ # Copyright (c) 2020 6WIND S.A. # SPDX-License-Identifier: MIT +import gc import json import os import unittest @@ -1100,3 +1101,13 @@ def test_dnode_store_only(self): self.assertIsInstance(dnode, DLeaf) self.assertEqual(dnode.value(), 50) dnode.free() + + def test_dnode_builtin_plugins_only(self): + MAIN = {"yolo-nodetypes:ip-address": "test"} + self.tearDown() + gc.collect() + self.ctx = Context(YANG_DIR, builtin_plugins_only=True) + module = self.ctx.load_module("yolo-nodetypes") + dnode = dict_to_dnode(MAIN, module, None, validate=False, store_only=True) + self.assertIsInstance(dnode, DLeaf) + dnode.free() diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index de4656b1..0ec00a6f 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -3,6 +3,11 @@ module yolo-nodetypes { namespace "urn:yang:yolo:nodetypes"; prefix sys; + import ietf-inet-types { + prefix inet; + revision-date 2013-07-15; + } + description "YOLO Nodetypes."; @@ -126,4 +131,8 @@ module yolo-nodetypes { anydata any1 { when "../cont2"; } + + leaf ip-address { + type inet:ipv4-address; + } } From 8d64c70a21bdb7972c39d645cbb2c1717a0c0f45 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 4 Mar 2024 15:16:44 +0100 Subject: [PATCH 073/115] data: fixing broken DLeaf value and print_dict API This patch fixes broken APIs, which were using no longer valid structs and performing validation step instead of just checking the realtype of data. Closes: https://github.com/CESNET/libyang-python/pull/107 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 6 +++++- libyang/data.py | 52 ++++++++++++++++++++-------------------------- tests/test_data.py | 1 + 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 93f68c79..40d36692 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -902,9 +902,13 @@ struct lyd_value { ...; }; +struct lyd_value_union { + struct lyd_value value; + ...; +}; + const char * lyd_get_value(const struct lyd_node *); struct lyd_node* lyd_child(const struct lyd_node *); -LY_ERR lyd_value_validate(const struct ly_ctx *, const struct lysc_node *, const char *, size_t, const struct lyd_node *, const struct lysc_type **, const char **); LY_ERR lyd_find_path(const struct lyd_node *, const char *, ly_bool, struct lyd_node **); void lyd_free_siblings(struct lyd_node *); struct lyd_node* lyd_first_sibling(const struct lyd_node *); diff --git a/libyang/data.py b/libyang/data.py index 288303d6..19ef0ca7 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -1119,38 +1119,30 @@ def cdata_leaf_value(cdata, context: "libyang.Context" = None) -> Any: return None val = c2str(val) - term_node = ffi.cast("struct lyd_node_term *", cdata) - val_type = ffi.new("const struct lysc_type **", ffi.NULL) - - # get real value type - ctx = context.cdata if context else ffi.NULL - ret = lib.lyd_value_validate( - ctx, - term_node.schema, - str2c(val), - len(val), - ffi.NULL, - val_type, - ffi.NULL, - ) - - if ret in (lib.LY_SUCCESS, lib.LY_EINCOMPLETE): - val_type = val_type[0].basetype - if val_type in Type.STR_TYPES: - return val - if val_type in Type.NUM_TYPES: - return int(val) - if val_type == Type.BOOL: - return val == "true" - if val_type == Type.DEC64: - return float(val) - if val_type == Type.LEAFREF: - return DLeaf.cdata_leaf_value(cdata.value.leafref, context) - if val_type == Type.EMPTY: - return None + if cdata.schema == ffi.NULL: + # opaq node return val - raise TypeError("value type validation error") + node_term = ffi.cast("struct lyd_node_term *", cdata) + + # inspired from libyang lyd_value_validate + val_type = Type(context, node_term.value.realtype, None).base() + if val_type == Type.UNION: + val_type = Type( + context, node_term.value.subvalue.value.realtype, None + ).base() + + if val_type in Type.STR_TYPES: + return val + if val_type in Type.NUM_TYPES: + return int(val) + if val_type == Type.BOOL: + return val == "true" + if val_type == Type.DEC64: + return float(val) + if val_type == Type.EMPTY: + return None + return val # ------------------------------------------------------------------------------------- diff --git a/tests/test_data.py b/tests/test_data.py index 790b5cd7..1728042b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1110,4 +1110,5 @@ def test_dnode_builtin_plugins_only(self): module = self.ctx.load_module("yolo-nodetypes") dnode = dict_to_dnode(MAIN, module, None, validate=False, store_only=True) self.assertIsInstance(dnode, DLeaf) + self.assertEqual(dnode.value(), "test") dnode.free() From adab39c83947c9c08d3a39d212f687f002b8c13e Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 5 Aug 2024 12:25:42 +0200 Subject: [PATCH 074/115] schema: adding parent_node API to extensions This patch adds parent_node API, which allows caller to get parent PNode from PExtension or SNode from Extension instance. Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 2 ++ libyang/__init__.py | 1 + libyang/schema.py | 16 ++++++++++++++++ tests/test_schema.py | 7 +++++++ 4 files changed, 26 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 40d36692..1c1d8f3b 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -174,6 +174,8 @@ enum ly_stmt { LY_STMT_ARG_VALUE }; +#define LY_STMT_NODE_MASK ... + #define LY_LOLOG ... #define LY_LOSTORE ... #define LY_LOSTORE_LAST ... diff --git a/libyang/__init__.py b/libyang/__init__.py index 26235de1..cab99712 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -67,6 +67,7 @@ from .log import configure_logging from .schema import ( Extension, + ExtensionParsed, Feature, IfAndFeatures, IfFeature, diff --git a/libyang/schema.py b/libyang/schema.py index f35a5e9a..a47974ee 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -411,6 +411,14 @@ def name(self) -> str: def module(self) -> Module: return self._module_from_parsed() + def parent_node(self) -> Optional["PNode"]: + if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): + return None + try: + return PNode.new(self.context, self.cdata.parent, self.module()) + except LibyangError: + return None + # ------------------------------------------------------------------------------------- class ExtensionCompiled(Extension): @@ -428,6 +436,14 @@ def module(self) -> Module: raise self.context.error("cannot get module") return Module(self.context, self.cdata_def.module) + def parent_node(self) -> Optional["SNode"]: + if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): + return None + try: + return SNode.new(self.context, self.cdata.parent) + except LibyangError: + return None + # ------------------------------------------------------------------------------------- class _EnumBit: diff --git a/tests/test_schema.py b/tests/test_schema.py index e2a833ce..1ae9fdfc 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -7,6 +7,7 @@ from libyang import ( Context, Extension, + ExtensionParsed, IfFeature, IfOrFeatures, IOType, @@ -496,6 +497,11 @@ def test_rpc_extensions(self): self.assertEqual(len(ext), 1) ext = self.rpc.get_extension("require-admin", prefix="omg-extensions") self.assertIsInstance(ext, Extension) + self.assertIsInstance(ext.parent_node(), SRpc) + parsed = self.rpc.parsed() + ext = parsed.get_extension("require-admin", prefix="omg-extensions") + self.assertIsInstance(ext, ExtensionParsed) + self.assertIsInstance(ext.parent_node(), PAction) def test_rpc_params(self): leaf = next(self.rpc.children()) @@ -609,6 +615,7 @@ def test_leaf_type_extensions(self): "type-desc", prefix="omg-extensions", arg_value="<protocol>" ) self.assertIsInstance(ext, Extension) + self.assertIsNone(ext.parent_node()) def test_leaf_type_enum(self): leaf = next( From 74e08c41e9b6f1f66a3874f1f1d462beda1b440d Mon Sep 17 00:00:00 2001 From: Ariel Otilibili <otilibil@eurecom.fr> Date: Fri, 25 Oct 2024 13:03:25 +0200 Subject: [PATCH 075/115] tox: bumped Python versions * `envlist` looks for Python 3.9 to 3.13; these are the ones being supported [1] * dropped 3.6 to 3.8 from CI/CD tests * removed `py3`: it is never used, since first found wins [2]. ``` $ tox -l format lint py39 py310 py311 py312 py313 pypy3 lydevel coverage ``` [1] https://devguide.python.org/versions/ [2] https://tox.wiki/en/stable/config.html#python-options Signed-off-by: Ariel Otilibili <otilibil@eurecom.fr> --- .github/workflows/ci.yml | 8 ++------ tox.ini | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 144e73b9..48be6a11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,12 +40,6 @@ jobs: strategy: matrix: include: - - python: 3.6 - toxenv: py36 - - python: 3.7 - toxenv: py37 - - python: 3.8 - toxenv: py38 - python: 3.9 toxenv: py39 - python: "3.10" @@ -54,6 +48,8 @@ jobs: toxenv: py311 - python: "3.12" toxenv: py312 + - python: "3.13" + toxenv: py313 - python: pypy3.9 toxenv: pypy3 steps: diff --git a/tox.ini b/tox.ini index 9fd15d15..b6b12c4c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = format,lint,py{36,37,38,39,310,311,312,py3,3},lydevel,coverage +envlist = format,lint,py{39,310,311,312,313,py3},lydevel,coverage skip_missing_interpreters = true isolated_build = true distdir = {toxinidir}/dist From 1afc2a63108e15431b4903d4f7c909bb8c6c1884 Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Wed, 6 Nov 2024 17:29:08 +0100 Subject: [PATCH 076/115] schema: add store_only parameter to parse_data_dict Sometimes it is necessary to store only the data. Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> Acked-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- libyang/schema.py | 2 ++ tests/test_data.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index a47974ee..db0514d8 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -241,6 +241,7 @@ def parse_data_dict( rpc: bool = False, rpcreply: bool = False, notification: bool = False, + store_only: bool = False, ) -> "libyang.data.DNode": """ Convert a python dictionary to a DNode object following the schema of this @@ -276,6 +277,7 @@ def parse_data_dict( rpc=rpc, rpcreply=rpcreply, notification=notification, + store_only=store_only, ) diff --git a/tests/test_data.py b/tests/test_data.py index 1728042b..4b7914e9 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1112,3 +1112,22 @@ def test_dnode_builtin_plugins_only(self): self.assertIsInstance(dnode, DLeaf) self.assertEqual(dnode.value(), "test") dnode.free() + + def test_merge_store_only(self): + MAIN = {"yolo-nodetypes:test1": 50} + module = self.ctx.load_module("yolo-nodetypes") + dnode = module.parse_data_dict(MAIN, validate=False, store_only=True) + self.assertIsInstance(dnode, DLeaf) + self.assertEqual(dnode.value(), 50) + dnode.free() + + def test_merge_builtin_plugins_only(self): + MAIN = {"yolo-nodetypes:ip-address": "test"} + self.tearDown() + gc.collect() + self.ctx = Context(YANG_DIR, builtin_plugins_only=True) + module = self.ctx.load_module("yolo-nodetypes") + dnode = module.parse_data_dict(MAIN, validate=False, store_only=True) + self.assertIsInstance(dnode, DLeaf) + self.assertEqual(dnode.value(), "test") + dnode.free() From a19cab64fa2ea2143d85c3feb903fedbafe16160 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 5 Aug 2024 12:29:34 +0200 Subject: [PATCH 077/115] extension: add ExtensionPlugin class This patch introduces new class ExtensionPlugin, which is wrapper around libyang extension plugin, which allows user to define custom action for parsing, compiling, and freeing parsed or compiled extensions. Custom actions can also raise a new type of exception LibyangExtensionError, which allows proper translation of exception to libyang error codes and logging of error message Closes: https://github.com/CESNET/libyang-python/pull/116 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 55 ++++++++ libyang/__init__.py | 5 + libyang/extension.py | 216 +++++++++++++++++++++++++++++ libyang/log.py | 14 +- libyang/schema.py | 6 +- tests/test_diff.py | 3 + tests/test_extension.py | 193 ++++++++++++++++++++++++++ tests/yang/omg/omg-extensions.yang | 10 ++ tests/yang/yolo/yolo-system.yang | 2 + 9 files changed, 498 insertions(+), 6 deletions(-) create mode 100644 libyang/extension.py create mode 100644 tests/test_extension.py diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 1c1d8f3b..7210adbd 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -174,6 +174,8 @@ enum ly_stmt { LY_STMT_ARG_VALUE }; +#define LY_STMT_OP_MASK ... +#define LY_STMT_DATA_NODE_MASK ... #define LY_STMT_NODE_MASK ... #define LY_LOLOG ... @@ -359,6 +361,7 @@ LY_ERR lys_print_module(struct ly_out *, const struct lys_module *, LYS_OUTFORMA #define LYS_PRINT_SHRINK ... struct lys_module { + struct ly_ctx *ctx; const char *name; const char *revision; const char *ns; @@ -428,6 +431,22 @@ struct lysc_node_container { struct lysc_node_notif *notifs; }; +struct lysp_stmt { + const char *stmt; + const char *arg; + LY_VALUE_FORMAT format; + void *prefix_data; + struct lysp_stmt *next; + struct lysp_stmt *child; + uint16_t flags; + enum ly_stmt kw; +}; + +struct lysp_ext_substmt { + enum ly_stmt stmt; + ...; +}; + struct lysp_ext_instance { const char *name; const char *argument; @@ -1271,6 +1290,42 @@ struct lyd_leafref_links_rec { LY_ERR lyd_leafref_get_links(const struct lyd_node_term *e, const struct lyd_leafref_links_rec **); LY_ERR lyd_leafref_link_node_tree(struct lyd_node *); +const char *lyplg_ext_stmt2str(enum ly_stmt stmt); +const struct lysp_module *lyplg_ext_parse_get_cur_pmod(const struct lysp_ctx *); +struct ly_ctx *lyplg_ext_compile_get_ctx(const struct lysc_ctx *); +void lyplg_ext_parse_log(const struct lysp_ctx *, const struct lysp_ext_instance *, LY_LOG_LEVEL, LY_ERR, const char *, ...); +void lyplg_ext_compile_log(const struct lysc_ctx *, const struct lysc_ext_instance *, LY_LOG_LEVEL, LY_ERR, const char *, ...); +LY_ERR lyplg_ext_parse_extension_instance(struct lysp_ctx *, struct lysp_ext_instance *); +LY_ERR lyplg_ext_compile_extension_instance(struct lysc_ctx *, const struct lysp_ext_instance *, struct lysc_ext_instance *); +void lyplg_ext_pfree_instance_substatements(const struct ly_ctx *ctx, struct lysp_ext_substmt *substmts); +void lyplg_ext_cfree_instance_substatements(const struct ly_ctx *ctx, struct lysc_ext_substmt *substmts); +typedef LY_ERR (*lyplg_ext_parse_clb)(struct lysp_ctx *, struct lysp_ext_instance *); +typedef LY_ERR (*lyplg_ext_compile_clb)(struct lysc_ctx *, const struct lysp_ext_instance *, struct lysc_ext_instance *); +typedef void (*lyplg_ext_parse_free_clb)(const struct ly_ctx *, struct lysp_ext_instance *); +typedef void (*lyplg_ext_compile_free_clb)(const struct ly_ctx *, struct lysc_ext_instance *); +struct lyplg_ext { + const char *id; + lyplg_ext_parse_clb parse; + lyplg_ext_compile_clb compile; + lyplg_ext_parse_free_clb pfree; + lyplg_ext_compile_free_clb cfree; + ...; +}; + +struct lyplg_ext_record { + const char *module; + const char *revision; + const char *name; + struct lyplg_ext plugin; + ...; +}; + +#define LYPLG_EXT_API_VERSION ... +LY_ERR lyplg_add_extension_plugin(struct ly_ctx *, uint32_t, const struct lyplg_ext_record *); +extern "Python" LY_ERR lypy_lyplg_ext_parse_clb(struct lysp_ctx *, struct lysp_ext_instance *); +extern "Python" LY_ERR lypy_lyplg_ext_compile_clb(struct lysc_ctx *, const struct lysp_ext_instance *, struct lysc_ext_instance *); +extern "Python" void lypy_lyplg_ext_parse_free_clb(const struct ly_ctx *, struct lysp_ext_instance *); +extern "Python" void lypy_lyplg_ext_compile_free_clb(const struct ly_ctx *, struct lysc_ext_instance *); /* from libc, needed to free allocated strings */ void free(void *); diff --git a/libyang/__init__.py b/libyang/__init__.py index cab99712..7af2794c 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -63,10 +63,12 @@ UnitsRemoved, schema_diff, ) +from .extension import ExtensionPlugin, LibyangExtensionError from .keyed_list import KeyedList from .log import configure_logging from .schema import ( Extension, + ExtensionCompiled, ExtensionParsed, Feature, IfAndFeatures, @@ -144,6 +146,9 @@ "EnumRemoved", "Extension", "ExtensionAdded", + "ExtensionCompiled", + "ExtensionParsed", + "ExtensionPlugin", "ExtensionRemoved", "Feature", "IfAndFeatures", diff --git a/libyang/extension.py b/libyang/extension.py new file mode 100644 index 00000000..57f7cb2d --- /dev/null +++ b/libyang/extension.py @@ -0,0 +1,216 @@ +# Copyright (c) 2018-2019 Robin Jarry +# Copyright (c) 2020 6WIND S.A. +# Copyright (c) 2021 RACOM s.r.o. +# SPDX-License-Identifier: MIT + +from typing import Callable, Optional + +from _libyang import ffi, lib +from .context import Context +from .log import get_libyang_level +from .schema import ExtensionCompiled, ExtensionParsed, Module +from .util import LibyangError, c2str, str2c + + +# ------------------------------------------------------------------------------------- +extensions_plugins = {} + + +class LibyangExtensionError(LibyangError): + def __init__(self, message: str, ret: int, log_level: int) -> None: + super().__init__(message) + self.ret = ret + self.log_level = log_level + + +@ffi.def_extern(name="lypy_lyplg_ext_parse_clb") +def libyang_c_lyplg_ext_parse_clb(pctx, pext): + plugin = extensions_plugins[pext.record.plugin] + module_cdata = lib.lyplg_ext_parse_get_cur_pmod(pctx).mod + context = Context(cdata=module_cdata.ctx) + module = Module(context, module_cdata) + parsed_ext = ExtensionParsed(context, pext, module) + plugin.set_parse_ctx(pctx) + try: + plugin.parse_clb(module, parsed_ext) + return lib.LY_SUCCESS + except LibyangExtensionError as e: + ly_level = get_libyang_level(e.log_level) + if ly_level is None: + ly_level = lib.LY_EPLUGIN + e_str = str(e) + plugin.add_error_message(e_str) + lib.lyplg_ext_parse_log(pctx, pext, ly_level, e.ret, str2c(e_str)) + return e.ret + + +@ffi.def_extern(name="lypy_lyplg_ext_compile_clb") +def libyang_c_lyplg_ext_compile_clb(cctx, pext, cext): + plugin = extensions_plugins[pext.record.plugin] + context = Context(cdata=lib.lyplg_ext_compile_get_ctx(cctx)) + module = Module(context, cext.module) + parsed_ext = ExtensionParsed(context, pext, module) + compiled_ext = ExtensionCompiled(context, cext) + plugin.set_compile_ctx(cctx) + try: + plugin.compile_clb(parsed_ext, compiled_ext) + return lib.LY_SUCCESS + except LibyangExtensionError as e: + ly_level = get_libyang_level(e.log_level) + if ly_level is None: + ly_level = lib.LY_EPLUGIN + e_str = str(e) + plugin.add_error_message(e_str) + lib.lyplg_ext_compile_log(cctx, cext, ly_level, e.ret, str2c(e_str)) + return e.ret + + +@ffi.def_extern(name="lypy_lyplg_ext_parse_free_clb") +def libyang_c_lyplg_ext_parse_free_clb(ctx, pext): + plugin = extensions_plugins[pext.record.plugin] + context = Context(cdata=ctx) + parsed_ext = ExtensionParsed(context, pext, None) + plugin.parse_free_clb(parsed_ext) + + +@ffi.def_extern(name="lypy_lyplg_ext_compile_free_clb") +def libyang_c_lyplg_ext_compile_free_clb(ctx, cext): + plugin = extensions_plugins[getattr(cext, "def").plugin] + context = Context(cdata=ctx) + compiled_ext = ExtensionCompiled(context, cext) + plugin.compile_free_clb(compiled_ext) + + +class ExtensionPlugin: + ERROR_SUCCESS = lib.LY_SUCCESS + ERROR_MEM = lib.LY_EMEM + ERROR_INVALID_INPUT = lib.LY_EINVAL + ERROR_NOT_VALID = lib.LY_EVALID + ERROR_DENIED = lib.LY_EDENIED + ERROR_NOT = lib.LY_ENOT + + def __init__( + self, + module_name: str, + name: str, + id_str: str, + context: Optional[Context] = None, + parse_clb: Optional[Callable[[Module, ExtensionParsed], None]] = None, + compile_clb: Optional[ + Callable[[ExtensionParsed, ExtensionCompiled], None] + ] = None, + parse_free_clb: Optional[Callable[[ExtensionParsed], None]] = None, + compile_free_clb: Optional[Callable[[ExtensionCompiled], None]] = None, + ) -> None: + """ + Set the callback functions, which will be called if libyang will be processing + given extension defined by name from module defined by module_name. + + :arg self: + This instance of extension plugin + :arg module_name: + The name of module in which the extension is defined + :arg name: + The name of extension itself + :arg id_str: + The unique ID of extension plugin within the libyang context + :arg context: + The context in which the extension plugin will be used. If set to None, + the plugin will be used for all existing and even future contexts + :arg parse_clb: + The optional callback function of which will be called during extension parsing + Expected arguments are: + module: The module which is being parsed + extension: The exact extension instance + Expected raises: + LibyangExtensionError in case of processing error + :arg compile_clb: + The optional callback function of which will be called during extension compiling + Expected arguments are: + extension_parsed: The parsed extension instance + extension_compiled: The compiled extension instance + Expected raises: + LibyangExtensionError in case of processing error + :arg parse_free_clb + The optional callback function of which will be called during freeing of parsed extension + Expected arguments are: + extension: The parsed extension instance to be freed + :arg compile_free_clb + The optional callback function of which will be called during freeing of compiled extension + Expected arguments are: + extension: The compiled extension instance to be freed + """ + self.context = context + self.module_name = module_name + self.module_name_cstr = str2c(self.module_name) + self.name = name + self.name_cstr = str2c(self.name) + self.id_str = id_str + self.id_cstr = str2c(self.id_str) + self.parse_clb = parse_clb + self.compile_clb = compile_clb + self.parse_free_clb = parse_free_clb + self.compile_free_clb = compile_free_clb + self._error_messages = [] + self._pctx = ffi.NULL + self._cctx = ffi.NULL + + self.cdata = ffi.new("struct lyplg_ext_record[2]") + self.cdata[0].module = self.module_name_cstr + self.cdata[0].name = self.name_cstr + self.cdata[0].plugin.id = self.id_cstr + if self.parse_clb is not None: + self.cdata[0].plugin.parse = lib.lypy_lyplg_ext_parse_clb + if self.compile_clb is not None: + self.cdata[0].plugin.compile = lib.lypy_lyplg_ext_compile_clb + if self.parse_free_clb is not None: + self.cdata[0].plugin.pfree = lib.lypy_lyplg_ext_parse_free_clb + if self.compile_free_clb is not None: + self.cdata[0].plugin.cfree = lib.lypy_lyplg_ext_compile_free_clb + ret = lib.lyplg_add_extension_plugin( + context.cdata if context is not None else ffi.NULL, + lib.LYPLG_EXT_API_VERSION, + ffi.cast("const void *", self.cdata), + ) + if ret != lib.LY_SUCCESS: + raise LibyangError("Unable to add extension plugin") + if self.cdata[0].plugin not in extensions_plugins: + extensions_plugins[self.cdata[0].plugin] = self + + def __del__(self) -> None: + if self.cdata[0].plugin in extensions_plugins: + del extensions_plugins[self.cdata[0].plugin] + + @staticmethod + def stmt2str(stmt: int) -> str: + return c2str(lib.lyplg_ext_stmt2str(stmt)) + + def add_error_message(self, err_msg: str) -> None: + self._error_messages.append(err_msg) + + def clear_error_messages(self) -> None: + self._error_messages.clear() + + def set_parse_ctx(self, pctx) -> None: + self._pctx = pctx + + def set_compile_ctx(self, cctx) -> None: + self._cctx = cctx + + def parse_substmts(self, ext: ExtensionParsed) -> int: + return lib.lyplg_ext_parse_extension_instance(self._pctx, ext.cdata) + + def compile_substmts(self, pext: ExtensionParsed, cext: ExtensionCompiled) -> int: + return lib.lyplg_ext_compile_extension_instance( + self._cctx, pext.cdata, cext.cdata + ) + + def free_parse_substmts(self, ext: ExtensionParsed) -> None: + lib.lyplg_ext_pfree_instance_substatements( + self.context.cdata, ext.cdata.substmts + ) + + def free_compile_substmts(self, ext: ExtensionCompiled) -> None: + lib.lyplg_ext_cfree_instance_substatements( + self.context.cdata, ext.cdata.substmts + ) diff --git a/libyang/log.py b/libyang/log.py index b033ccaa..f92c70fd 100644 --- a/libyang/log.py +++ b/libyang/log.py @@ -19,6 +19,13 @@ } +def get_libyang_level(py_level): + for ly_lvl, py_lvl in LOG_LEVELS.items(): + if py_lvl == py_level: + return ly_lvl + return None + + @ffi.def_extern(name="lypy_log_cb") def libyang_c_logging_callback(level, msg, data_path, schema_path, line): args = [c2str(msg)] @@ -50,10 +57,9 @@ def configure_logging(enable_py_logger: bool, level: int = logging.ERROR) -> Non :arg level: Python logging level. By default only ERROR messages are stored/logged. """ - for ly_lvl, py_lvl in LOG_LEVELS.items(): - if py_lvl == level: - lib.ly_log_level(ly_lvl) - break + ly_level = get_libyang_level(level) + if ly_level is not None: + lib.ly_log_level(ly_level) if enable_py_logger: lib.ly_log_options(lib.LY_LOLOG | lib.LY_LOSTORE) lib.ly_set_log_clb(lib.lypy_log_cb) diff --git a/libyang/schema.py b/libyang/schema.py index db0514d8..eb076e23 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -374,7 +374,7 @@ def __str__(self): class Extension: __slots__ = ("context", "cdata", "__dict__") - def __init__(self, context: "libyang.Context", cdata, module_parent: Module = None): + def __init__(self, context: "libyang.Context", cdata): self.context = context self.cdata = cdata @@ -402,6 +402,8 @@ def __init__(self, context: "libyang.Context", cdata, module_parent: Module = No def _module_from_parsed(self) -> Module: prefix = c2str(self.cdata.name).split(":")[0] + if self.module_parent is None: + raise self.context.error("cannot get module") for cdata_imp_mod in ly_array_iter(self.module_parent.cdata.parsed.imports): if ffi.string(cdata_imp_mod.prefix).decode() == prefix: return Module(self.context, cdata_imp_mod.module) @@ -417,7 +419,7 @@ def parent_node(self) -> Optional["PNode"]: if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): return None try: - return PNode.new(self.context, self.cdata.parent, self.module()) + return PNode.new(self.context, self.cdata.parent, self.module_parent) except LibyangError: return None diff --git a/tests/test_diff.py b/tests/test_diff.py index 49bf77a2..d4b7e87e 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -12,6 +12,7 @@ EnumRemoved, EnumStatusAdded, EnumStatusRemoved, + ExtensionAdded, NodeTypeAdded, NodeTypeRemoved, SNodeAdded, @@ -82,6 +83,8 @@ class DiffTest(unittest.TestCase): (EnumRemoved, "/yolo-system:state/url/proto"), (EnumStatusAdded, "/yolo-system:conf/url/proto"), (EnumStatusAdded, "/yolo-system:state/url/proto"), + (ExtensionAdded, "/yolo-system:conf/url/proto"), + (ExtensionAdded, "/yolo-system:state/url/proto"), (EnumStatusRemoved, "/yolo-system:conf/url/proto"), (EnumStatusRemoved, "/yolo-system:state/url/proto"), (SNodeAdded, "/yolo-system:conf/pill/red/out"), diff --git a/tests/test_extension.py b/tests/test_extension.py new file mode 100644 index 00000000..b932788c --- /dev/null +++ b/tests/test_extension.py @@ -0,0 +1,193 @@ +# Copyright (c) 2018-2019 Robin Jarry +# SPDX-License-Identifier: MIT + +import logging +import os +from typing import Any, Optional +import unittest + +from libyang import ( + Context, + ExtensionCompiled, + ExtensionParsed, + ExtensionPlugin, + LibyangError, + LibyangExtensionError, + Module, + PLeaf, + SLeaf, +) + + +YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") + + +# ------------------------------------------------------------------------------------- +class TestExtensionPlugin(ExtensionPlugin): + def __init__(self, context: Context) -> None: + super().__init__( + "omg-extensions", + "type-desc", + "omg-extensions-type-desc-plugin-v1", + context, + parse_clb=self._parse_clb, + compile_clb=self._compile_clb, + parse_free_clb=self._parse_free_clb, + compile_free_clb=self._compile_free_clb, + ) + self.parse_clb_called = 0 + self.compile_clb_called = 0 + self.parse_free_clb_called = 0 + self.compile_free_clb_called = 0 + self.parse_clb_exception: Optional[LibyangExtensionError] = None + self.compile_clb_exception: Optional[LibyangExtensionError] = None + self.parse_parent_stmt = None + + def reset(self) -> None: + self.parse_clb_called = 0 + self.compile_clb_called = 0 + self.parse_free_clb_called = 0 + self.compile_free_clb_called = 0 + self.parse_clb_exception = None + self.compile_clb_exception = None + + def _parse_clb(self, module: Module, ext: ExtensionParsed) -> None: + self.parse_clb_called += 1 + if self.parse_clb_exception is not None: + raise self.parse_clb_exception + self.parse_substmts(ext) + self.parse_parent_stmt = self.stmt2str(ext.cdata.parent_stmt) + + def _compile_clb(self, pext: ExtensionParsed, cext: ExtensionCompiled) -> None: + self.compile_clb_called += 1 + if self.compile_clb_exception is not None: + raise self.compile_clb_exception + self.compile_substmts(pext, cext) + + def _parse_free_clb(self, ext: ExtensionParsed) -> None: + self.parse_free_clb_called += 1 + self.free_parse_substmts(ext) + + def _compile_free_clb(self, ext: ExtensionCompiled) -> None: + self.compile_free_clb_called += 1 + self.free_compile_substmts(ext) + + +# ------------------------------------------------------------------------------------- +class ExtensionTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.plugin = TestExtensionPlugin(self.ctx) + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_extension_basic(self): + self.ctx.load_module("yolo-system") + self.assertEqual(5, self.plugin.parse_clb_called) + self.assertEqual(6, self.plugin.compile_clb_called) + self.assertEqual(0, self.plugin.parse_free_clb_called) + self.assertEqual(0, self.plugin.compile_free_clb_called) + self.assertEqual("type", self.plugin.parse_parent_stmt) + self.ctx.destroy() + self.assertEqual(5, self.plugin.parse_clb_called) + self.assertEqual(6, self.plugin.compile_clb_called) + self.assertEqual(5, self.plugin.parse_free_clb_called) + self.assertEqual(6, self.plugin.compile_free_clb_called) + + def test_extension_invalid_parse(self): + self.plugin.parse_clb_exception = LibyangExtensionError( + "this extension cannot be parsed", + self.plugin.ERROR_NOT_VALID, + logging.ERROR, + ) + with self.assertRaises(LibyangError): + self.ctx.load_module("yolo-system") + + def test_extension_invalid_compile(self): + self.plugin.compile_clb_exception = LibyangExtensionError( + "this extension cannot be compiled", + self.plugin.ERROR_NOT_VALID, + logging.ERROR, + ) + with self.assertRaises(LibyangError): + self.ctx.load_module("yolo-system") + + +# ------------------------------------------------------------------------------------- +class ExampleParseExtensionPlugin(ExtensionPlugin): + def __init__(self, context: Context) -> None: + super().__init__( + "omg-extensions", + "parse-validation", + "omg-extensions-parse-validation-plugin-v1", + context, + parse_clb=self._parse_clb, + ) + + def _verify_single(self, parent: Any) -> None: + count = 0 + for e in parent.extensions(): + if e.name() == self.name and e.module().name() == self.module_name: + count += 1 + if count > 1: + raise LibyangExtensionError( + f"Extension {self.name} is allowed to be defined just once per given " + "parent node context.", + self.ERROR_NOT_VALID, + logging.ERROR, + ) + + def _parse_clb(self, _, ext: ExtensionParsed) -> None: + parent = ext.parent_node() + if not isinstance(parent, PLeaf): + raise LibyangExtensionError( + f"Extension {ext.name()} is allowed only in leaf nodes", + self.ERROR_NOT_VALID, + logging.ERROR, + ) + self._verify_single(parent) + # here you put code to perform something reasonable actions you need for your extension + + +class ExampleCompileExtensionPlugin(ExtensionPlugin): + def __init__(self, context: Context) -> None: + super().__init__( + "omg-extensions", + "compile-validation", + "omg-extensions-compile-validation-plugin-v1", + context, + compile_clb=self._compile_clb, + ) + + def _compile_clb(self, pext: ExtensionParsed, cext: ExtensionCompiled) -> None: + parent = cext.parent_node() + if not isinstance(parent, SLeaf): + raise LibyangExtensionError( + f"Extension {cext.name()} is allowed only in leaf nodes", + self.ERROR_NOT_VALID, + logging.ERROR, + ) + # here you put code to perform something reasonable actions you need for your extension + + +class ExtensionExampleTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.plugins = [] + + def tearDown(self): + self.plugins.clear() + self.ctx.destroy() + self.ctx = None + + def test_parse_validation_example(self): + self.plugins.append(ExampleParseExtensionPlugin(self.ctx)) + self.ctx.load_module("yolo-system") + + def test_compile_validation_example(self): + self.plugins.append(ExampleParseExtensionPlugin(self.ctx)) + self.plugins.append(ExampleCompileExtensionPlugin(self.ctx)) + with self.assertRaises(LibyangError): + self.ctx.load_module("yolo-system") diff --git a/tests/yang/omg/omg-extensions.yang b/tests/yang/omg/omg-extensions.yang index fe20e7e5..926bf3db 100644 --- a/tests/yang/omg/omg-extensions.yang +++ b/tests/yang/omg/omg-extensions.yang @@ -18,4 +18,14 @@ module omg-extensions { "Extend a type to add a desc."; argument name; } + + extension parse-validation { + description + "Example of parse-validation extension which should be put only under leaf nodes."; + } + + extension compile-validation { + description + "Example of compile-validation extension which should be put only under leaf nodes."; + } } diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index ef612546..36c76416 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -83,6 +83,7 @@ module yolo-system { type types:protocol { ext:type-desc "<protocol>"; } + ext:parse-validation; } leaf host { type string { @@ -114,6 +115,7 @@ module yolo-system { type string; } } + ext:compile-validation; } } From 76504c474a4e48762674245717d60dc24ea2d3d5 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 5 Apr 2024 12:08:31 +0200 Subject: [PATCH 078/115] schema: adds Identity and PIdentity classes This patch introduces Identity and PIdentity classes. It also adds identities() API to get list of identities from Module Closes: https://github.com/CESNET/libyang-python/pull/118 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 20 ++++ libyang/__init__.py | 4 + libyang/schema.py | 162 +++++++++++++++++++++++++--- tests/test_schema.py | 51 +++++++++ tests/yang/yolo/yolo-nodetypes.yang | 33 ++++++ 5 files changed, 256 insertions(+), 14 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 7210adbd..e48a8173 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -475,6 +475,16 @@ struct lysp_ext_instance { char rev[LY_REV_SIZE]; }; +struct lysp_ident { + const char *name; + struct lysp_qname *iffeatures; + const char **bases; + const char *dsc; + const char *ref; + struct lysp_ext_instance *exts; + uint16_t flags; +}; + struct lysp_feature { const char *name; struct lysp_qname *iffeatures; @@ -976,6 +986,16 @@ struct lysp_restr { struct lysp_ext_instance *exts; }; +struct lysc_ident { + const char *name; + const char *dsc; + const char *ref; + struct lys_module *module; + struct lysc_ident **derived; + struct lysc_ext_instance *exts; + uint16_t flags; +}; + struct lysc_type_num { const char *name; struct lysc_ext_instance *exts; diff --git a/libyang/__init__.py b/libyang/__init__.py index 7af2794c..ff15755c 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -71,6 +71,7 @@ ExtensionCompiled, ExtensionParsed, Feature, + Identity, IfAndFeatures, IfFeature, IfFeatureExpr, @@ -89,6 +90,7 @@ PContainer, PEnum, PGrouping, + PIdentity, PLeaf, PLeafList, PList, @@ -151,6 +153,7 @@ "ExtensionPlugin", "ExtensionRemoved", "Feature", + "Identity", "IfAndFeatures", "IfFeature", "IfFeatureExpr", @@ -184,6 +187,7 @@ "PContainer", "PEnum", "PGrouping", + "PIdentity", "PLeaf", "PLeafList", "PList", diff --git a/libyang/schema.py b/libyang/schema.py index eb076e23..992db0eb 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -172,6 +172,14 @@ def notifications(self) -> Iterator["PNotif"]: for n in ly_list_iter(self.cdata.parsed.notifs): yield PNotif(self.context, n, self) + def identities(self) -> Iterator["Identity"]: + for i in ly_array_iter(self.cdata.identities): + yield Identity(self.context, i) + + def parsed_identities(self) -> Iterator["PIdentity"]: + for i in ly_array_iter(self.cdata.parsed.identities): + yield PIdentity(self.context, i, self) + def __str__(self) -> str: return self.name() @@ -415,13 +423,16 @@ def name(self) -> str: def module(self) -> Module: return self._module_from_parsed() - def parent_node(self) -> Optional["PNode"]: - if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): - return None - try: - return PNode.new(self.context, self.cdata.parent, self.module_parent) - except LibyangError: - return None + def parent_node(self) -> Optional[Union["PNode", "PIdentity"]]: + if self.cdata.parent_stmt == lib.LY_STMT_IDENTITY: + cdata = ffi.cast("struct lysp_ident *", self.cdata.parent) + return PIdentity(self.context, cdata, self.module_parent) + if bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): + try: + return PNode.new(self.context, self.cdata.parent, self.module_parent) + except LibyangError: + return None + return None # ------------------------------------------------------------------------------------- @@ -440,13 +451,16 @@ def module(self) -> Module: raise self.context.error("cannot get module") return Module(self.context, self.cdata_def.module) - def parent_node(self) -> Optional["SNode"]: - if not bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): - return None - try: - return SNode.new(self.context, self.cdata.parent) - except LibyangError: - return None + def parent_node(self) -> Optional[Union["SNode", "Identity"]]: + if self.cdata.parent_stmt == lib.LY_STMT_IDENTITY: + cdata = ffi.cast("struct lysc_ident *", self.cdata.parent) + return Identity(self.context, cdata) + if bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): + try: + return SNode.new(self.context, self.cdata.parent) + except LibyangError: + return None + return None # ------------------------------------------------------------------------------------- @@ -624,6 +638,13 @@ def leafref_path(self) -> Optional["str"]: lr = ffi.cast("struct lysc_type_leafref *", self.cdata) return c2str(lib.lyxp_get_expr(lr.path)) + def identity_bases(self) -> Iterator["Identity"]: + if self.cdata.basetype != lib.LY_TYPE_IDENT: + return + ident = ffi.cast("struct lysc_type_identityref *", self.cdata) + for b in ly_array_iter(ident.bases): + yield Identity(self.context, b) + def typedef(self) -> "Typedef": if ":" in self.name(): module_prefix, type_name = self.name().split(":") @@ -870,6 +891,68 @@ def __str__(self): return self.name() +# ------------------------------------------------------------------------------------- +class Identity: + __slots__ = ("context", "cdata") + + def __init__(self, context: "libyang.Context", cdata): + self.context = context + self.cdata = cdata # C type: "struct lysc_ident *" + + def name(self) -> str: + return c2str(self.cdata.name) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def module(self) -> Module: + return Module(self.context, self.cdata.module) + + def derived(self) -> Iterator["Identity"]: + for i in ly_array_iter(self.cdata.derived): + yield Identity(self.context, i) + + def extensions(self) -> Iterator[ExtensionCompiled]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionCompiled(self.context, ext) + + def get_extension( + self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None + ) -> Optional[ExtensionCompiled]: + for ext in self.extensions(): + if ext.name() != name: + continue + if prefix is not None and ext.module().name() != prefix: + continue + if arg_value is not None and ext.argument() != arg_value: + continue + return ext + return None + + def deprecated(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_DEPRC) + + def obsolete(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_OBSLT) + + def status(self) -> str: + if self.cdata.flags & lib.LYS_STATUS_OBSLT: + return "obsolete" + if self.cdata.flags & lib.LYS_STATUS_DEPRC: + return "deprecated" + return "current" + + def __repr__(self): + cls = self.__class__ + return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) + + def __str__(self): + return self.name() + + # ------------------------------------------------------------------------------------- class Feature: __slots__ = ("context", "cdata", "__dict__") @@ -1890,6 +1973,57 @@ def extensions(self) -> Iterator["ExtensionParsed"]: yield ExtensionParsed(self.context, ext, self.module) +# ------------------------------------------------------------------------------------- +class PIdentity: + __slots__ = ("context", "cdata", "module") + + def __init__(self, context: "libyang.Context", cdata, module: Module) -> None: + self.context = context + self.cdata = cdata # C type: "struct lysp_ident *" + self.module = module + + def name(self) -> str: + return c2str(self.cdata.name) + + def if_features(self) -> Iterator[IfFeatureExpr]: + for f in ly_array_iter(self.cdata.iffeatures): + yield IfFeatureExpr(self.context, f, list(self.module.features())) + + def bases(self) -> Iterator[str]: + for b in ly_array_iter(self.cdata.bases): + yield c2str(b) + + def description(self) -> Optional[str]: + return c2str(self.cdata.dsc) + + def reference(self) -> Optional[str]: + return c2str(self.cdata.ref) + + def extensions(self) -> Iterator[ExtensionParsed]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module) + + def deprecated(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_DEPRC) + + def obsolete(self) -> bool: + return bool(self.cdata.flags & lib.LYS_STATUS_OBSLT) + + def status(self) -> str: + if self.cdata.flags & lib.LYS_STATUS_OBSLT: + return "obsolete" + if self.cdata.flags & lib.LYS_STATUS_DEPRC: + return "deprecated" + return "current" + + def __repr__(self): + cls = self.__class__ + return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) + + def __str__(self): + return self.name() + + # ------------------------------------------------------------------------------------- class PNode: CONTAINER = lib.LYS_CONTAINER diff --git a/tests/test_schema.py b/tests/test_schema.py index 1ae9fdfc..923f5f07 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -8,6 +8,7 @@ Context, Extension, ExtensionParsed, + Identity, IfFeature, IfOrFeatures, IOType, @@ -23,6 +24,7 @@ PChoice, PContainer, PGrouping, + PIdentity, PLeaf, PLeafList, PList, @@ -868,3 +870,52 @@ def test_notification_parsed(self): self.assertIsNone(next(pnode.typedefs(), None)) self.assertIsNone(next(pnode.groupings(), None)) self.assertIsNotNone(next(iter(pnode))) + + +# ------------------------------------------------------------------------------------- +class IdentityTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.module = self.ctx.load_module("yolo-nodetypes") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_identity_compiled(self): + sidentity = next(self.module.identities()) + self.assertIsInstance(sidentity, Identity) + self.assertEqual(sidentity.name(), "base1") + self.assertEqual(sidentity.description(), "Base 1.") + self.assertEqual(sidentity.reference(), "Some reference.") + self.assertIsInstance(sidentity.module(), Module) + derived = list(sidentity.derived()) + self.assertEqual(2, len(derived)) + for i in derived: + self.assertIsInstance(i, Identity) + self.assertEqual(derived[0].name(), "derived1") + self.assertEqual(derived[1].name(), "derived2") + self.assertEqual(next(derived[1].extensions()).name(), "identity-name") + self.assertIsNone(next(sidentity.extensions(), None)) + self.assertIsNone(sidentity.get_extension("ext1")) + self.assertFalse(sidentity.deprecated()) + self.assertFalse(sidentity.obsolete()) + self.assertEqual("current", sidentity.status()) + + snode = next(self.ctx.find_path("/yolo-nodetypes:identity_ref")) + identities = list(snode.type().identity_bases()) + self.assertEqual(identities[0].name(), sidentity.name()) + self.assertEqual(identities[1].name(), "base2") + + def test_identity_parsed(self): + pidentity = next(self.module.parsed_identities()) + self.assertIsInstance(pidentity, PIdentity) + self.assertEqual(pidentity.name(), "base1") + self.assertIsNone(next(pidentity.if_features(), None)) + self.assertIsNone(next(pidentity.bases(), None)) + self.assertEqual(pidentity.description(), "Base 1.") + self.assertEqual(pidentity.reference(), "Some reference.") + self.assertIsNone(next(pidentity.extensions(), None)) + self.assertFalse(pidentity.deprecated()) + self.assertFalse(pidentity.obsolete()) + self.assertEqual("current", pidentity.status()) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 0ec00a6f..5b994752 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -132,6 +132,39 @@ module yolo-nodetypes { when "../cont2"; } + extension identity-name { + description + "Extend an identity to provide an alternative name."; + argument name; + } + + identity base1 { + description + "Base 1."; + reference "Some reference."; + } + identity base2; + + identity derived1 { + base base1; + } + + identity derived2 { + base base1; + sys:identity-name "Derived2"; + } + + identity derived3 { + base derived1; + } + + leaf identity_ref { + type identityref { + base base1; + base base2; + } + } + leaf ip-address { type inet:ipv4-address; } From 421d3211040087dc5b881c7a5930e37c18ca2864 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 19 Dec 2024 10:26:56 +0100 Subject: [PATCH 079/115] cdefs: removal of unnecessary variable name defintion This patch aligns the cdefs definition with the rest of the file structure Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index e48a8173..fad8d549 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -1308,7 +1308,7 @@ struct lyd_leafref_links_rec { const struct lyd_node_term **target_nodes; }; -LY_ERR lyd_leafref_get_links(const struct lyd_node_term *e, const struct lyd_leafref_links_rec **); +LY_ERR lyd_leafref_get_links(const struct lyd_node_term *, const struct lyd_leafref_links_rec **); LY_ERR lyd_leafref_link_node_tree(struct lyd_node *); const char *lyplg_ext_stmt2str(enum ly_stmt stmt); const struct lysp_module *lyplg_ext_parse_get_cur_pmod(const struct lysp_ctx *); From 407fab3727084e57105af9152f3122aba4d099e5 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 19 Dec 2024 10:25:23 +0100 Subject: [PATCH 080/115] schema: adds nested extensions access This patch introduces access to nested extensions Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/schema.py | 8 ++++++++ tests/test_schema.py | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index 992db0eb..2e3a1154 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -434,6 +434,10 @@ def parent_node(self) -> Optional[Union["PNode", "PIdentity"]]: return None return None + def extensions(self) -> Iterator["ExtensionParsed"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionParsed(self.context, ext, self.module_parent) + # ------------------------------------------------------------------------------------- class ExtensionCompiled(Extension): @@ -462,6 +466,10 @@ def parent_node(self) -> Optional[Union["SNode", "Identity"]]: return None return None + def extensions(self) -> Iterator["ExtensionCompiled"]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionCompiled(self.context, ext) + # ------------------------------------------------------------------------------------- class _EnumBit: diff --git a/tests/test_schema.py b/tests/test_schema.py index 923f5f07..a310aadc 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -7,6 +7,7 @@ from libyang import ( Context, Extension, + ExtensionCompiled, ExtensionParsed, Identity, IfFeature, @@ -498,12 +499,14 @@ def test_rpc_extensions(self): ext = list(self.rpc.extensions()) self.assertEqual(len(ext), 1) ext = self.rpc.get_extension("require-admin", prefix="omg-extensions") - self.assertIsInstance(ext, Extension) + self.assertIsInstance(ext, ExtensionCompiled) self.assertIsInstance(ext.parent_node(), SRpc) + self.assertIsNone(next(ext.extensions(), None)) parsed = self.rpc.parsed() ext = parsed.get_extension("require-admin", prefix="omg-extensions") self.assertIsInstance(ext, ExtensionParsed) self.assertIsInstance(ext.parent_node(), PAction) + self.assertIsNone(next(ext.extensions(), None)) def test_rpc_params(self): leaf = next(self.rpc.children()) From 727927bb90d117baf0598b67e4670b5f5d937f34 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Fri, 5 Apr 2024 11:36:52 +0200 Subject: [PATCH 081/115] data/context: adds json_null parsing option This patch adds options to support JSON 'null' values Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 1 + libyang/context.py | 6 ++++++ libyang/data.py | 3 +++ tests/test_data.py | 6 ++++++ 4 files changed, 16 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index fad8d549..aa750042 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -315,6 +315,7 @@ LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint3 #define LYD_PARSE_LYB_MOD_UPDATE ... #define LYD_PARSE_NO_STATE ... #define LYD_PARSE_STORE_ONLY ... +#define LYD_PARSE_JSON_NULL ... #define LYD_PARSE_ONLY ... #define LYD_PARSE_OPAQ ... #define LYD_PARSE_OPTS_MASK ... diff --git a/libyang/context.py b/libyang/context.py index 1b1d4cda..ba3332a3 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -520,6 +520,7 @@ def parse_data( validate_present: bool = False, validate_multi_error: bool = False, store_only: bool = False, + json_null: bool = False, ) -> Optional[DNode]: if self.cdata is None: raise RuntimeError("context already destroyed") @@ -531,6 +532,7 @@ def parse_data( ordered=ordered, strict=strict, store_only=store_only, + json_null=json_null, ) validation_flgs = validation_flags( no_state=no_state, @@ -589,6 +591,7 @@ def parse_data_mem( validate_present: bool = False, validate_multi_error: bool = False, store_only: bool = False, + json_null: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -604,6 +607,7 @@ def parse_data_mem( validate_present=validate_present, validate_multi_error=validate_multi_error, store_only=store_only, + json_null=json_null, ) def parse_data_file( @@ -620,6 +624,7 @@ def parse_data_file( validate_present: bool = False, validate_multi_error: bool = False, store_only: bool = False, + json_null: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -635,6 +640,7 @@ def parse_data_file( validate_present=validate_present, validate_multi_error=validate_multi_error, store_only=store_only, + json_null=json_null, ) def __iter__(self) -> Iterator[Module]: diff --git a/libyang/data.py b/libyang/data.py index 19ef0ca7..0d63d3c9 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -116,6 +116,7 @@ def parser_flags( ordered: bool = False, strict: bool = False, store_only: bool = False, + json_null: bool = False, ) -> int: flags = 0 if lyb_mod_update: @@ -132,6 +133,8 @@ def parser_flags( flags |= lib.LYD_PARSE_STRICT if store_only: flags |= lib.LYD_PARSE_STORE_ONLY + if json_null: + flags |= lib.LYD_PARSE_JSON_NULL return flags diff --git a/tests/test_data.py b/tests/test_data.py index 4b7914e9..1479eb9a 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1131,3 +1131,9 @@ def test_merge_builtin_plugins_only(self): self.assertIsInstance(dnode, DLeaf) self.assertEqual(dnode.value(), "test") dnode.free() + + def test_dnode_parse_json_null(self): + JSON = """{"yolo-nodetypes:ip-address": null}""" + dnode = self.ctx.parse_data_mem(JSON, "json", json_null=True) + dnode_names = [d.name() for d in dnode.siblings()] + self.assertFalse("ip-address" in dnode_names) From aea34d1e787163248c7b5de38e22d5e0a77c83f4 Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Thu, 6 Feb 2025 09:20:10 +0100 Subject: [PATCH 082/115] tox: fix lint target on python 3.13 Update all packages used in the lint target to their latest release. Explicitly bump the ubuntu image version in all GitHub action jobs. Fix errors reported by new pylint/black versions. Signed-off-by: Robin Jarry <robin@jarry.cc> --- .github/workflows/ci.yml | 24 +++++++++++++++--------- Makefile | 5 ++--- libyang/schema.py | 4 +--- libyang/util.py | 2 ++ libyang/xpath.py | 2 +- pylintrc | 1 + tox.ini | 18 +++++++++--------- 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48be6a11..a7797c88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,11 +3,17 @@ --- name: CI -on: [push, pull_request] +on: + pull_request: + branches: + - master + push: + branches: + - master jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -23,24 +29,24 @@ jobs: - run: python -m tox -e lint check-commits: - if: github.event_name == 'pull_request' + if: ${{ github.event.pull_request.commits }} runs-on: ubuntu-latest env: - LYPY_START_COMMIT: "${{ github.event.pull_request.base.sha }}" - LYPY_END_COMMIT: "${{ github.event.pull_request.head.sha }}" + LYPY_COMMIT_RANGE: "HEAD~${{ github.event.pull_request.commits }}.." steps: - run: sudo apt-get install git make - uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.ref }} - run: make check-commits test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: matrix: include: - - python: 3.9 + - python: "3.9" toxenv: py39 - python: "3.10" toxenv: py310 @@ -56,7 +62,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python }} + python-version: "${{ matrix.python }}" - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -86,7 +92,7 @@ jobs: deploy: needs: [lint, test] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest + runs-on: ubuntu-24.03 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/Makefile b/Makefile index 6e9c436b..fc9a6ea3 100644 --- a/Makefile +++ b/Makefile @@ -12,10 +12,9 @@ tests: format: tox -e format -LYPY_START_COMMIT ?= origin/master -LYPY_END_COMMIT ?= HEAD +LYPY_COMMIT_RANGE ?= origin/master.. check-commits: - ./check-commits.sh $(LYPY_START_COMMIT)..$(LYPY_END_COMMIT) + ./check-commits.sh $(LYPY_COMMIT_RANGE) .PHONY: lint tests format check-commits diff --git a/libyang/schema.py b/libyang/schema.py index 2e3a1154..3f3bd4d0 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1542,9 +1542,7 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: def max_elements(self) -> Optional[int]: return ( - self.cdata_leaflist.max - if self.cdata_leaflist.max != (2**32 - 1) - else None + self.cdata_leaflist.max if self.cdata_leaflist.max != (2**32 - 1) else None ) def min_elements(self) -> int: diff --git a/libyang/util.py b/libyang/util.py index d640a511..c380ae5e 100644 --- a/libyang/util.py +++ b/libyang/util.py @@ -121,4 +121,6 @@ def data_load(in_type, in_data, data, data_keepalive, encode=True): c_str = str2c(in_data, encode=encode) data_keepalive.append(c_str) ret = lib.ly_in_new_memory(c_str, data) + else: + raise ValueError("invalid input") return ret diff --git a/libyang/xpath.py b/libyang/xpath.py index ad5f2a58..e2a33196 100644 --- a/libyang/xpath.py +++ b/libyang/xpath.py @@ -85,7 +85,7 @@ def xpath_split(xpath: str) -> Iterator[Tuple[str, str, List[Tuple[str, str]]]]: # ------------------------------------------------------------------------------------- def _xpath_keys_to_key_name( - keys: List[Tuple[str, str]] + keys: List[Tuple[str, str]], ) -> Optional[Union[str, Tuple[str, ...]]]: """ Extract key name from parsed xpath keys returned by xpath_split. The return value diff --git a/pylintrc b/pylintrc index 16a9f0ae..acf27338 100644 --- a/pylintrc +++ b/pylintrc @@ -74,6 +74,7 @@ disable= too-many-branches, too-many-lines, too-many-locals, + too-many-positional-arguments, too-many-return-statements, too-many-statements, unused-argument, diff --git a/tox.ini b/tox.ini index b6b12c4c..524ad49c 100644 --- a/tox.ini +++ b/tox.ini @@ -36,8 +36,8 @@ basepython = python3 description = Format python code using isort and black. changedir = . deps = - black~=23.12.1 - isort~=5.13.2 + black~=25.1.0 + isort~=6.0.0 skip_install = true install_command = python3 -m pip install {opts} {packages} allowlist_externals = @@ -52,14 +52,14 @@ basepython = python3 description = Run coding style checks. changedir = . deps = - astroid~=3.0.2 - black~=23.12.1 - flake8~=7.0.0 - isort~=5.13.2 - pycodestyle~=2.11.1 + astroid~=3.3.8 + black~=25.1.0 + flake8~=7.1.1 + isort~=6.0.0 + pycodestyle~=2.12.1 pyflakes~=3.2.0 - pylint~=3.0.3 - setuptools~=69.0.3 + pylint~=3.3.4 + setuptools~=75.8.0 allowlist_externals = /bin/sh /usr/bin/sh From 8534053f8a8a542127e8f74acbcc095591b411e9 Mon Sep 17 00:00:00 2001 From: Brad House <brad@brad-house.com> Date: Wed, 5 Feb 2025 19:38:43 -0500 Subject: [PATCH 083/115] context: correct memory leak of `struct ly_in *` objects `data_load()` returns a reference to `struct ly_in *` which must be free'd by `lys_in_free()`. This is done in one of three locations. This corrects this oversight. This was found during unit tests of SONiC during porting to libyang3 from libyang 1.0.73. Signed-off-by: Brad House <brad@brad-house.com> --- libyang/context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libyang/context.py b/libyang/context.py index ba3332a3..fb4a330d 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -325,7 +325,9 @@ def parse_module( mod = ffi.new("struct lys_module **") fmt = schema_in_format(fmt) - if lib.lys_parse(self.cdata, data[0], fmt, feat, mod) != lib.LY_SUCCESS: + ret = lib.lys_parse(self.cdata, data[0], fmt, feat, mod) + lib.ly_in_free(data[0], 0) + if ret != lib.LY_SUCCESS: raise self.error("failed to parse module") return Module(self, mod[0]) @@ -489,6 +491,7 @@ def parse_op( par[0] = parent.cdata ret = lib.lyd_parse_op(self.cdata, par[0], data[0], fmt, dtype, tree, op) + lib.ly_in_free(data[0], 0) if ret != lib.LY_SUCCESS: raise self.error("failed to parse input data") From bdd4cea20a05aa490cff6728b11fa3ba83242e52 Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Mon, 17 Feb 2025 14:19:41 +0100 Subject: [PATCH 084/115] ci: fix issue status check For some reason, storing the output of curl into a bash variable breaks UTF-8 encoding. Use a shell pipeline to prevent any encoding from happening. Signed-off-by: Robin Jarry <robin@jarry.cc> --- .github/workflows/ci.yml | 4 ++-- check-commits.sh | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7797c88..27efc7dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,11 @@ jobs: check-commits: if: ${{ github.event.pull_request.commits }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: LYPY_COMMIT_RANGE: "HEAD~${{ github.event.pull_request.commits }}.." steps: - - run: sudo apt-get install git make + - run: sudo apt-get install git make jq curl - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/check-commits.sh b/check-commits.sh index acf322d9..936236cf 100755 --- a/check-commits.sh +++ b/check-commits.sh @@ -42,11 +42,10 @@ err() { } check_issue() { - json=$(curl -f -X GET -L --no-progress-meter \ + curl -f -X GET -L --no-progress-meter \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "$api_url/issues/${1##*/}") || return 1 - test $(echo "$json" | jq -r .state) = open + "$api_url/issues/${1##*/}" | jq -r .state | grep -Fx open } for rev in $revisions; do From c5af6814f2388e78f4d7f48a322d73cf0b96ee40 Mon Sep 17 00:00:00 2001 From: Matthias Breuninger <matthias.breuninger@etas.com> Date: Mon, 17 Feb 2025 11:57:15 +0100 Subject: [PATCH 085/115] load_module: add missing parameters for ly_ctx_load_module Add parameters to define Yang model revision and features that shall be enabled. Closes: https://github.com/CESNET/libyang-python/issues/101 Signed-off-by: Matthias Breuninger <matthias.breuninger@etas.com> --- libyang/context.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index fb4a330d..f9bd5a57 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: MIT import os -from typing import IO, Any, Callable, Iterator, Optional, Tuple, Union +from typing import IO, Any, Callable, Iterator, Optional, Sequence, Tuple, Union from _libyang import ffi, lib from .data import ( @@ -340,10 +340,19 @@ def parse_module_file( def parse_module_str(self, s: str, fmt: str = "yang", features=None) -> Module: return self.parse_module(s, IOType.MEMORY, fmt, features) - def load_module(self, name: str) -> Module: + def load_module( + self, + name: str, + revision: Optional[str] = None, + enabled_features: Sequence[str] = (), + ) -> Module: if self.cdata is None: raise RuntimeError("context already destroyed") - mod = lib.ly_ctx_load_module(self.cdata, str2c(name), ffi.NULL, ffi.NULL) + if enabled_features: + features = tuple([str2c(f) for f in enabled_features] + [ffi.NULL]) + else: + features = ffi.NULL + mod = lib.ly_ctx_load_module(self.cdata, str2c(name), str2c(revision), features) if mod == ffi.NULL: raise self.error("cannot load module") From 3f4ef9f2dea81f82bc3771ef04921e5148a1e158 Mon Sep 17 00:00:00 2001 From: Matthias Breuninger <matthias.breuninger@etas.com> Date: Mon, 17 Feb 2025 12:04:55 +0100 Subject: [PATCH 086/115] tests: add test to enable all features Expects that all features are enabled due to the "*" input. Closes: https://github.com/CESNET/libyang-python/issues/101 Signed-off-by: Matthias Breuninger <matthias.breuninger@etas.com> --- tests/test_context.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_context.py b/tests/test_context.py index 8a0412e2..db03c329 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -63,6 +63,13 @@ def test_ctx_load_module(self): mod = ctx.load_module("yolo-system") self.assertIsInstance(mod, Module) + def test_ctx_load_module_with_features(self): + with Context(YANG_DIR) as ctx: + mod = ctx.load_module("yolo-system", None, ["*"]) + self.assertIsInstance(mod, Module) + for f in list(mod.features()): + self.assertTrue(f.state()) + def test_ctx_get_module(self): with Context(YANG_DIR) as ctx: ctx.load_module("yolo-system") From d8acc345aeb8d5b9644f7f2197ba6d8733e86378 Mon Sep 17 00:00:00 2001 From: Robin Jarry <robin@jarry.cc> Date: Fri, 7 Mar 2025 17:38:47 +0100 Subject: [PATCH 087/115] github: fix publish on pypi.org when only pushing a tag Also trigger the CI on tag push. Fix a typo in the distro image name. Fixes: aea34d1e7871 ("tox: fix lint target on python 3.13") Signed-off-by: Robin Jarry <robin@jarry.cc> --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27efc7dd..2177a27b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,8 @@ on: push: branches: - master + tags: + - v* jobs: lint: @@ -92,7 +94,7 @@ jobs: deploy: needs: [lint, test] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-24.03 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 From 3072af036386a6bb0fde285eba751e09a4948261 Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Sun, 29 Jun 2025 08:38:32 -0400 Subject: [PATCH 088/115] data: add type hinting to meta functions of DNode The other functions in DNode are properly typed, but the meta functions were missed. Add the missing typing hints. Signed-off-by: Christian Hopps <chopps@labn.net> --- libyang/data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libyang/data.py b/libyang/data.py index 0d63d3c9..9595ea16 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -291,7 +291,7 @@ def __init__(self, context: "libyang.Context", cdata): self.attributes = None self.free_func = None # type: Callable[DNode] - def meta(self): + def meta(self) -> Dict[str, str]: ret = {} item = self.cdata.meta while item != ffi.NULL: @@ -303,7 +303,7 @@ def meta(self): item = item.next return ret - def get_meta(self, name): + def get_meta(self, name: str) -> Optional[str]: item = self.cdata.meta while item != ffi.NULL: if c2str(item.name) == name: @@ -315,7 +315,7 @@ def get_meta(self, name): item = item.next return None - def meta_free(self, name): + def meta_free(self, name: str): item = self.cdata.meta while item != ffi.NULL: if c2str(item.name) == name: From 18faa1e71e67ab81c65ef7450c6ea68976633368 Mon Sep 17 00:00:00 2001 From: Brad House <brad@brad-house.com> Date: Sun, 16 Feb 2025 10:18:20 -0500 Subject: [PATCH 089/115] data: option to allow json int/bool as strings Depends on https://github.com/CESNET/libyang/pull/2344 Add support for new option to python bindings. Signed-off-by: Brad House <brad@brad-house.com> --- cffi/cdefs.h | 1 + cffi/source.c | 4 ++-- libyang/context.py | 6 ++++++ libyang/data.py | 3 +++ pylintrc | 2 +- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index aa750042..de05f4a1 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -321,6 +321,7 @@ LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint3 #define LYD_PARSE_OPTS_MASK ... #define LYD_PARSE_ORDERED ... #define LYD_PARSE_STRICT ... +#define LYD_PARSE_JSON_STRING_DATATYPES ... #define LYD_VALIDATE_NO_STATE ... #define LYD_VALIDATE_PRESENT ... diff --git a/cffi/source.c b/cffi/source.c index b54ba0de..34e18b0e 100644 --- a/cffi/source.c +++ b/cffi/source.c @@ -6,6 +6,6 @@ #include <libyang/libyang.h> #include <libyang/version.h> -#if (LY_VERSION_MAJOR != 3) -#error "This version of libyang bindings only works with libyang 3.x" +#if LY_VERSION_MAJOR * 10000 + LY_VERSION_MINOR * 100 + LY_VERSION_MICRO < 30801 +#error "This version of libyang bindings only works with libyang soversion 3.8.1+" #endif diff --git a/libyang/context.py b/libyang/context.py index f9bd5a57..b50600de 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -533,6 +533,7 @@ def parse_data( validate_multi_error: bool = False, store_only: bool = False, json_null: bool = False, + json_string_datatypes: bool = False, ) -> Optional[DNode]: if self.cdata is None: raise RuntimeError("context already destroyed") @@ -545,6 +546,7 @@ def parse_data( strict=strict, store_only=store_only, json_null=json_null, + json_string_datatypes=json_string_datatypes, ) validation_flgs = validation_flags( no_state=no_state, @@ -604,6 +606,7 @@ def parse_data_mem( validate_multi_error: bool = False, store_only: bool = False, json_null: bool = False, + json_string_datatypes: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -620,6 +623,7 @@ def parse_data_mem( validate_multi_error=validate_multi_error, store_only=store_only, json_null=json_null, + json_string_datatypes=json_string_datatypes, ) def parse_data_file( @@ -637,6 +641,7 @@ def parse_data_file( validate_multi_error: bool = False, store_only: bool = False, json_null: bool = False, + json_string_datatypes: bool = False, ) -> Optional[DNode]: return self.parse_data( fmt, @@ -653,6 +658,7 @@ def parse_data_file( validate_multi_error=validate_multi_error, store_only=store_only, json_null=json_null, + json_string_datatypes=json_string_datatypes, ) def __iter__(self) -> Iterator[Module]: diff --git a/libyang/data.py b/libyang/data.py index 9595ea16..2fc086f6 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -117,6 +117,7 @@ def parser_flags( strict: bool = False, store_only: bool = False, json_null: bool = False, + json_string_datatypes: bool = False, ) -> int: flags = 0 if lyb_mod_update: @@ -135,6 +136,8 @@ def parser_flags( flags |= lib.LYD_PARSE_STORE_ONLY if json_null: flags |= lib.LYD_PARSE_JSON_NULL + if json_string_datatypes: + flags |= lib.LYD_PARSE_JSON_STRING_DATATYPES return flags diff --git a/pylintrc b/pylintrc index acf27338..cd4638b7 100644 --- a/pylintrc +++ b/pylintrc @@ -494,7 +494,7 @@ valid-metaclass-classmethod-first-arg=mcs [DESIGN] # Maximum number of arguments for function / method. -max-args=15 +max-args=20 # Maximum number of attributes for a class (see R0902). max-attributes=20 From c8313b76f4ec3c8c03e16e66d1f4f229e31553fe Mon Sep 17 00:00:00 2001 From: Brad House <brad@brad-house.com> Date: Sun, 16 Feb 2025 11:04:50 -0500 Subject: [PATCH 090/115] schema/context: restore some backlinks support In libyang v1 the schema nodes had a backlinks member to be able to look up dependents of the node. SONiC depends on this to provide functionality it uses and it needs to be exposed via the python module. In theory, exposing the 'dfs' functions could make this work, but it would likely be cost prohibitive since walking the tree would be expensive to create a python node for evaluation in native python. Instead this PR depends on the this libyang PR: https://github.com/CESNET/libyang/pull/2352 And adds thin wrappers. This implementation provides 2 python functions: * Context.find_backlinks_paths() - This function can take the path of the base node and find all dependents. If no path is specified, then it will return all nodes that contain a leafref reference. * Context.find_leafref_path_target_paths() - This function takes an xpath, then returns all target nodes the xpath may reference. Typically only one will be returned, but multiples may be in the case of a union. A user can build a cache by combining Context.find_backlinks_paths() with no path set and building a reverse table using Context.find_leafref_path_target_paths() Signed-off-by: Brad House <brad@brad-house.com> --- cffi/cdefs.h | 2 + libyang/context.py | 110 +++++++++++++++++- tests/test_schema.py | 60 ++++++++++ .../yang/yolo/yolo-leafref-search-extmod.yang | 39 +++++++ tests/yang/yolo/yolo-leafref-search.yang | 36 ++++++ 5 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 tests/yang/yolo/yolo-leafref-search-extmod.yang create mode 100644 tests/yang/yolo/yolo-leafref-search.yang diff --git a/cffi/cdefs.h b/cffi/cdefs.h index de05f4a1..4ccc30e7 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -862,6 +862,8 @@ const struct lysc_node* lys_find_child(const struct lysc_node *, const struct ly const struct lysc_node* lysc_node_child(const struct lysc_node *); const struct lysc_node_action* lysc_node_actions(const struct lysc_node *); const struct lysc_node_notif* lysc_node_notifs(const struct lysc_node *); +LY_ERR lysc_node_lref_targets(const struct lysc_node *, struct ly_set **); +LY_ERR lysc_node_lref_backlinks(const struct ly_ctx *, const struct lysc_node *, ly_bool, struct ly_set **); typedef enum { LYD_PATH_STD, diff --git a/libyang/context.py b/libyang/context.py index b50600de..d8dac48a 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: MIT import os -from typing import IO, Any, Callable, Iterator, Optional, Sequence, Tuple, Union +from typing import IO, Any, Callable, Iterator, List, Optional, Sequence, Tuple, Union from _libyang import ffi, lib from .data import ( @@ -661,6 +661,114 @@ def parse_data_file( json_string_datatypes=json_string_datatypes, ) + def find_leafref_path_target_paths(self, leafref_path: str) -> List[str]: + """ + Fetch all leafref targets of the specified path + + This is an enhanced version of lysc_node_lref_target() which will return + a set of leafref target paths retrieved from the specified schema path. + While lysc_node_lref_target() will only work on nodetype of LYS_LEAF and + LYS_LEAFLIST this function will also evaluate other datatypes that may + contain leafrefs such as LYS_UNION. This does not, however, search for + children with leafref targets. + + :arg self + This instance on context + :arg leafref_path: + Path to node to search for leafref targets + :returns List of target paths that the leafrefs of the specified node + point to. + """ + if self.cdata is None: + raise RuntimeError("context already destroyed") + if leafref_path is None: + raise RuntimeError("leafref_path must be defined") + + out = [] + + node = lib.lys_find_path(self.cdata, ffi.NULL, str2c(leafref_path), 0) + if node == ffi.NULL: + raise self.error("leafref_path not found") + + node_set = ffi.new("struct ly_set **") + if ( + lib.lysc_node_lref_targets(node, node_set) != lib.LY_SUCCESS + or node_set[0] == ffi.NULL + or node_set[0].count == 0 + ): + raise self.error("leafref_path does not contain any leafref targets") + + node_set = node_set[0] + for i in range(node_set.count): + path = lib.lysc_path(node_set.snodes[i], lib.LYSC_PATH_DATA, ffi.NULL, 0) + out.append(c2str(path)) + lib.free(path) + + lib.ly_set_free(node_set, ffi.NULL) + + return out + + def find_backlinks_paths( + self, match_path: str = None, match_ancestors: bool = False + ) -> List[str]: + """ + Search entire schema for nodes that contain leafrefs and return as a + list of schema node paths. + + Perform a complete scan of the schema tree looking for nodes that + contain leafref entries. When a node contains a leafref entry, and + match_path is specified, determine if reference points to match_path, + if so add the node's path to returned list. If no match_path is + specified, the node containing the leafref is always added to the + returned set. When match_ancestors is true, will evaluate if match_path + is self or an ansestor of self. + + This does not return the leafref targets, but the actual node that + contains a leafref. + + :arg self + This instance on context + :arg match_path: + Target path to use for matching + :arg match_ancestors: + Whether match_path is a base ancestor or an exact node + :returns List of paths. Exception of match_path is not found or if no + backlinks are found. + """ + if self.cdata is None: + raise RuntimeError("context already destroyed") + out = [] + + match_node = ffi.NULL + if match_path is not None and match_path == "/" or match_path == "": + match_path = None + + if match_path: + match_node = lib.lys_find_path(self.cdata, ffi.NULL, str2c(match_path), 0) + if match_node == ffi.NULL: + raise self.error("match_path not found") + + node_set = ffi.new("struct ly_set **") + if ( + lib.lysc_node_lref_backlinks( + self.cdata, match_node, match_ancestors, node_set + ) + != lib.LY_SUCCESS + or node_set[0] == ffi.NULL + or node_set[0].count == 0 + ): + raise self.error("backlinks not found") + + node_set = node_set[0] + for i in range(node_set.count): + path = lib.lysc_path(node_set.snodes[i], lib.LYSC_PATH_DATA, ffi.NULL, 0) + out.append(c2str(path)) + lib.free(path) + + lib.ly_set_free(node_set, ffi.NULL) + + return out + def __iter__(self) -> Iterator[Module]: """ Return an iterator that yields all implemented modules from the context diff --git a/tests/test_schema.py b/tests/test_schema.py index a310aadc..b3feba3a 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -801,6 +801,66 @@ def test_leaf_list_parsed(self): self.assertFalse(pnode.ordered()) +# ------------------------------------------------------------------------------------- +class BacklinksTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + self.ctx.load_module("yolo-leafref-search") + self.ctx.load_module("yolo-leafref-search-extmod") + + def tearDown(self): + self.ctx.destroy() + self.ctx = None + + def test_backlinks_all_nodes(self): + expected = [ + "/yolo-leafref-search-extmod:my_extref_list/my_extref", + "/yolo-leafref-search:refstr", + "/yolo-leafref-search:refnum", + "/yolo-leafref-search-extmod:my_extref_list/my_extref_union", + ] + refs = self.ctx.find_backlinks_paths() + expected.sort() + refs.sort() + self.assertEqual(expected, refs) + + def test_backlinks_one(self): + expected = [ + "/yolo-leafref-search-extmod:my_extref_list/my_extref", + "/yolo-leafref-search:refstr", + "/yolo-leafref-search-extmod:my_extref_list/my_extref_union", + ] + refs = self.ctx.find_backlinks_paths( + match_path="/yolo-leafref-search:my_list/my_leaf_string" + ) + expected.sort() + refs.sort() + self.assertEqual(expected, refs) + + def test_backlinks_children(self): + expected = [ + "/yolo-leafref-search-extmod:my_extref_list/my_extref", + "/yolo-leafref-search:refstr", + "/yolo-leafref-search:refnum", + "/yolo-leafref-search-extmod:my_extref_list/my_extref_union", + ] + refs = self.ctx.find_backlinks_paths( + match_path="/yolo-leafref-search:my_list", match_ancestors=True + ) + expected.sort() + refs.sort() + self.assertEqual(expected, refs) + + def test_backlinks_leafref_target_paths(self): + expected = ["/yolo-leafref-search:my_list/my_leaf_string"] + refs = self.ctx.find_leafref_path_target_paths( + "/yolo-leafref-search-extmod:my_extref_list/my_extref" + ) + expected.sort() + refs.sort() + self.assertEqual(expected, refs) + + # ------------------------------------------------------------------------------------- class ChoiceTest(unittest.TestCase): def setUp(self): diff --git a/tests/yang/yolo/yolo-leafref-search-extmod.yang b/tests/yang/yolo/yolo-leafref-search-extmod.yang new file mode 100644 index 00000000..046ceec5 --- /dev/null +++ b/tests/yang/yolo/yolo-leafref-search-extmod.yang @@ -0,0 +1,39 @@ +module yolo-leafref-search-extmod { + yang-version 1.1; + namespace "urn:yang:yolo:leafref-search-extmod"; + prefix leafref-search-extmod; + + import wtf-types { prefix types; } + + import yolo-leafref-search { + prefix leafref-search; + } + + revision 2025-02-11 { + description + "Initial version."; + } + + list my_extref_list { + key my_leaf_string; + leaf my_leaf_string { + type string; + } + leaf my_extref { + type leafref { + path "/leafref-search:my_list/leafref-search:my_leaf_string"; + } + } + leaf my_extref_union { + type union { + type leafref { + path "/leafref-search:my_list/leafref-search:my_leaf_string"; + } + type leafref { + path "/leafref-search:my_list/leafref-search:my_leaf_number"; + } + type types:number; + } + } + } +} diff --git a/tests/yang/yolo/yolo-leafref-search.yang b/tests/yang/yolo/yolo-leafref-search.yang new file mode 100644 index 00000000..5f4af488 --- /dev/null +++ b/tests/yang/yolo/yolo-leafref-search.yang @@ -0,0 +1,36 @@ +module yolo-leafref-search { + yang-version 1.1; + namespace "urn:yang:yolo:leafref-search"; + prefix leafref-search; + + import wtf-types { prefix types; } + + revision 2025-02-11 { + description + "Initial version."; + } + + list my_list { + key my_leaf_string; + leaf my_leaf_string { + type string; + } + leaf my_leaf_number { + description + "A number."; + type types:number; + } + } + + leaf refstr { + type leafref { + path "../my_list/my_leaf_string"; + } + } + + leaf refnum { + type leafref { + path "../my_list/my_leaf_number"; + } + } +} From 91067ebd3d39bd628db7fdd6229e97f5318c5f4d Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Sat, 5 Jul 2025 06:15:52 -0400 Subject: [PATCH 091/115] context: use normal type hint rather than forward reference SNode is imported and available so there's no need to use the "libyang.SNode" forward reference form. (based)pyright complains about this as it considers these forward references as `Unknown` -- perhaps it should be smarter, but that's a different issue. Signed-off-by: Christian Hopps <chopps@labn.net> --- libyang/context.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index d8dac48a..33580d61 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -371,7 +371,7 @@ def find_path( self, path: str, output: bool = False, - root_node: Optional["libyang.SNode"] = None, + root_node: Optional[SNode] = None, ) -> Iterator[SNode]: if self.cdata is None: raise RuntimeError("context already destroyed") @@ -404,9 +404,9 @@ def find_path( def find_jsonpath( self, path: str, - root_node: Optional["libyang.SNode"] = None, + root_node: Optional[SNode] = None, output: bool = False, - ) -> Optional["libyang.SNode"]: + ) -> Optional[SNode]: if root_node is not None: ctx_node = root_node.cdata else: From 1714370d03440641539787c44921bb90b9edbcd4 Mon Sep 17 00:00:00 2001 From: Brad House <brad@brad-house.com> Date: Sun, 6 Jul 2025 16:42:00 -0400 Subject: [PATCH 092/115] tests: test_iffeature_state crash fix As observed on Debian Trixie RC2, test_iffeature_state crashes due to an internal pointer being invalidated. This invalidation appears to be due to a call to lys_set_implemented() possibly causing a full recompile of the ctx. This simply reorders the caching of the path pointers so they are not invalidated when used. Signed-off-by: Brad House <brad@brad-house.com> --- tests/test_schema.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index b3feba3a..bf4c1737 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -197,18 +197,8 @@ def feature_disable_only(feature): continue self.mod.feature_enable(f.name()) - leaf_simple = next(self.ctx.find_path("/yolo-system:conf/yolo-system:speed")) - - self.mod.feature_disable_all() - leaf_not = next(self.ctx.find_path("/yolo-system:conf/yolo-system:offline")) - self.mod.feature_enable_all() - - leaf_and = next(self.ctx.find_path("/yolo-system:conf/yolo-system:full")) - leaf_or = next( - self.ctx.find_path("/yolo-system:conf/yolo-system:isolation-level") - ) - # if-feature is just a feature + leaf_simple = next(self.ctx.find_path("/yolo-system:conf/yolo-system:speed")) tree = next(leaf_simple.if_features()).tree() self.mod.feature_enable_all() self.assertEqual(tree.state(), True) @@ -216,6 +206,8 @@ def feature_disable_only(feature): self.assertEqual(tree.state(), False) # if-feature is "NOT networking" + self.mod.feature_disable_all() + leaf_not = next(self.ctx.find_path("/yolo-system:conf/yolo-system:offline")) tree = next(leaf_not.if_features()).tree() self.mod.feature_enable_all() self.assertEqual(tree.state(), False) @@ -223,6 +215,8 @@ def feature_disable_only(feature): self.assertEqual(tree.state(), True) # if-feature is "turbo-boost AND networking" + self.mod.feature_enable_all() + leaf_and = next(self.ctx.find_path("/yolo-system:conf/yolo-system:full")) tree = next(leaf_and.if_features()).tree() self.mod.feature_enable_all() self.assertEqual(tree.state(), True) @@ -234,6 +228,9 @@ def feature_disable_only(feature): self.assertEqual(tree.state(), False) # if-feature is "turbo-boost OR networking" + leaf_or = next( + self.ctx.find_path("/yolo-system:conf/yolo-system:isolation-level") + ) tree = next(leaf_or.if_features()).tree() self.mod.feature_enable_all() self.assertEqual(tree.state(), True) From aa3f506cf9119e5448a257bbd62fe679c5896246 Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Sun, 31 Aug 2025 01:45:46 +0000 Subject: [PATCH 093/115] data: add missing DNode.is_default API Add access to libyang lyd_is_default() API. Signed-off-by: Christian Hopps <chopps@labn.net> --- cffi/cdefs.h | 1 + libyang/data.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 4ccc30e7..f714e3ba 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -944,6 +944,7 @@ struct lyd_value_union { const char * lyd_get_value(const struct lyd_node *); struct lyd_node* lyd_child(const struct lyd_node *); +ly_bool lyd_is_default(const struct lyd_node *); LY_ERR lyd_find_path(const struct lyd_node *, const char *, ly_bool, struct lyd_node **); void lyd_free_siblings(struct lyd_node *); struct lyd_node* lyd_first_sibling(const struct lyd_node *); diff --git a/libyang/data.py b/libyang/data.py index 2fc086f6..facf932e 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -390,6 +390,9 @@ def flags(self): ret["new"] = True return ret + def is_default(self) -> bool: + return lib.lyd_is_default(self.cdata) + def set_when(self, value: bool): if value: self.cdata.flags |= lib.LYD_WHEN_TRUE From e977f94a319c65a92011cff2dc39903ae4c848dc Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Sun, 31 Aug 2025 01:56:47 +0000 Subject: [PATCH 094/115] data: add missing DataType's Simplify the code to using python enum functionality. Signed-off-by: Christian Hopps <chopps@labn.net> --- cffi/cdefs.h | 5 ++++- libyang/data.py | 20 ++++---------------- libyang/util.py | 17 ++++++++++------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index f714e3ba..06f938b3 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -296,7 +296,10 @@ enum lyd_type { LYD_TYPE_REPLY_YANG, LYD_TYPE_RPC_NETCONF, LYD_TYPE_NOTIF_NETCONF, - LYD_TYPE_REPLY_NETCONF + LYD_TYPE_REPLY_NETCONF, + LYD_TYPE_RPC_RESTCONF, + LYD_TYPE_NOTIF_RESTCONF, + LYD_TYPE_REPLY_RESTCONF }; #define LYD_PRINT_KEEPEMPTYCONT ... diff --git a/libyang/data.py b/libyang/data.py index facf932e..540ce39c 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -177,22 +177,10 @@ def dup_flags( # ------------------------------------------------------------------------------------- -def data_type(dtype): - if dtype == DataType.DATA_YANG: - return lib.LYD_TYPE_DATA_YANG - if dtype == DataType.RPC_YANG: - return lib.LYD_TYPE_RPC_YANG - if dtype == DataType.NOTIF_YANG: - return lib.LYD_TYPE_NOTIF_YANG - if dtype == DataType.REPLY_YANG: - return lib.LYD_TYPE_REPLY_YANG - if dtype == DataType.RPC_NETCONF: - return lib.LYD_TYPE_RPC_NETCONF - if dtype == DataType.NOTIF_NETCONF: - return lib.LYD_TYPE_NOTIF_NETCONF - if dtype == DataType.REPLY_NETCONF: - return lib.LYD_TYPE_REPLY_NETCONF - raise ValueError("Unknown data type") +def data_type(dtype: DataType) -> int: + if not isinstance(dtype, DataType): + dtype = DataType(dtype) + return dtype.value # ------------------------------------------------------------------------------------- diff --git a/libyang/util.py b/libyang/util.py index c380ae5e..3014cc85 100644 --- a/libyang/util.py +++ b/libyang/util.py @@ -77,13 +77,16 @@ class IOType(enum.Enum): # ------------------------------------------------------------------------------------- class DataType(enum.Enum): - DATA_YANG = enum.auto() - RPC_YANG = enum.auto() - NOTIF_YANG = enum.auto() - REPLY_YANG = enum.auto() - RPC_NETCONF = enum.auto() - NOTIF_NETCONF = enum.auto() - REPLY_NETCONF = enum.auto() + DATA_YANG = lib.LYD_TYPE_DATA_YANG + RPC_YANG = lib.LYD_TYPE_RPC_YANG + NOTIF_YANG = lib.LYD_TYPE_NOTIF_YANG + REPLY_YANG = lib.LYD_TYPE_REPLY_YANG + RPC_NETCONF = lib.LYD_TYPE_RPC_NETCONF + NOTIF_NETCONF = lib.LYD_TYPE_NOTIF_NETCONF + REPLY_NETCONF = lib.LYD_TYPE_REPLY_NETCONF + RPC_RESTCONF = lib.LYD_TYPE_RPC_RESTCONF + NOTIF_RESTCONF = lib.LYD_TYPE_NOTIF_RESTCONF + REPLY_RESTCONF = lib.LYD_TYPE_REPLY_RESTCONF # ------------------------------------------------------------------------------------- From 2740fa2ac3272683c362651c55b7f7dd2eae3911 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Tue, 16 Sep 2025 12:49:09 +0200 Subject: [PATCH 095/115] data: adding ability to get target_nodes This patch introduces target_nodes API, which allows user to get all target data nodes that current leafref node is pointing to Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/data.py | 15 ++++++++++++++- tests/test_data.py | 3 +++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/libyang/data.py b/libyang/data.py index 540ce39c..ceb1fa7b 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -1009,7 +1009,7 @@ def leafref_link_node_tree(self) -> None: def leafref_nodes(self) -> Iterator["DNode"]: """ - Gets the leafref links record for given node. + Gets the nodes that are referring to this node. Requires leafref_linking to be set on the libyang context. """ @@ -1020,6 +1020,19 @@ def leafref_nodes(self) -> Iterator["DNode"]: for n in ly_array_iter(out[0].leafref_nodes): yield DNode.new(self.context, n) + def target_nodes(self) -> Iterator["DNode"]: + """ + Gets the target nodes that are referred by this node. + + Requires leafref_linking to be set on the libyang context. + """ + term_node = ffi.cast("struct lyd_node_term *", self.cdata) + out = ffi.new("const struct lyd_leafref_links_rec **") + if lib.lyd_leafref_get_links(term_node, out) != lib.LY_SUCCESS: + return + for n in ly_array_iter(out[0].target_nodes): + yield DNode.new(self.context, n) + def __repr__(self): cls = self.__class__ return "<%s.%s: %s>" % (cls.__module__, cls.__name__, str(self)) diff --git a/tests/test_data.py b/tests/test_data.py index 1479eb9a..820db0ca 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1092,6 +1092,9 @@ def test_dnode_leafref_linking(self): dnode4 = next(dnode3.leafref_nodes()) self.assertIsInstance(dnode4, DLeaf) self.assertEqual(dnode4.cdata, dnode2.cdata) + dnode5 = next(dnode4.target_nodes()) + self.assertIsInstance(dnode5, DLeaf) + self.assertEqual(dnode5.cdata, dnode3.cdata) dnode1.free() def test_dnode_store_only(self): From 39afe5e72f6a876df046da3a318eaa4ecb263449 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Tue, 16 Sep 2025 12:44:15 +0200 Subject: [PATCH 096/115] data: adding dep_tree option for RPC validation This patch introduces dep_tree option to provide config data to which the RPC can referenced to during validation Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/data.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/libyang/data.py b/libyang/data.py index ceb1fa7b..7a5887b0 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -526,6 +526,7 @@ def validate( rpc: bool = False, rpcreply: bool = False, notification: bool = False, + dep_tree: Optional["DNode"] = None, ) -> None: dtype = None if rpc: @@ -538,7 +539,7 @@ def validate( if dtype is None: self.validate_all(no_state, validate_present) else: - self.validate_op(dtype) + self.validate_op(dtype, dep_tree) def validate_all( self, @@ -560,11 +561,15 @@ def validate_all( def validate_op( self, dtype: DataType, + dep_tree: Optional["DNode"] = None, ) -> None: dtype = data_type(dtype) - node_p = ffi.new("struct lyd_node **") - node_p[0] = self.cdata - ret = lib.lyd_validate_op(node_p[0], ffi.NULL, dtype, ffi.NULL) + ret = lib.lyd_validate_op( + self.cdata, + ffi.NULL if dep_tree is None else dep_tree.cdata, + dtype, + ffi.NULL, + ) if ret != lib.LY_SUCCESS: raise self.context.error("validation failed") From 41470168b69b3852525b2b213ec0a4f206e9246b Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Tue, 16 Sep 2025 12:23:41 +0200 Subject: [PATCH 097/115] schema: adds ability to use with_choice option in children calls This patch adds ability to use with_choice within RPC and Notification Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/schema.py | 28 +++++++++++++++++++++------- tests/test_schema.py | 2 ++ tests/yang/yolo/yolo-system.yang | 9 ++++++++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index 3f3bd4d0..76fc9c8f 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1679,8 +1679,12 @@ class SRpcInOut(SNode): def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator[SNode]: + return iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) # ------------------------------------------------------------------------------------- @@ -1710,11 +1714,17 @@ def output(self) -> Optional[SRpcInOut]: def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - yield from iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator[SNode]: + yield from iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) # With libyang2, you can get only input or output # To keep behavior, we iter 2 times witt output options - yield from iter_children(self.context, self.cdata, types=types, output=True) + yield from iter_children( + self.context, self.cdata, types=types, output=True, with_choice=with_choice + ) # ------------------------------------------------------------------------------------- @@ -1723,8 +1733,12 @@ class SNotif(SNode): def __iter__(self) -> Iterator[SNode]: return self.children() - def children(self, types: Optional[Tuple[int, ...]] = None) -> Iterator[SNode]: - return iter_children(self.context, self.cdata, types=types) + def children( + self, types: Optional[Tuple[int, ...]] = None, with_choice: bool = False + ) -> Iterator[SNode]: + return iter_children( + self.context, self.cdata, types=types, with_choice=with_choice + ) # ------------------------------------------------------------------------------------- diff --git a/tests/test_schema.py b/tests/test_schema.py index bf4c1737..d9824e58 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -491,6 +491,8 @@ def test_rpc_attrs(self): self.assertEqual(self.rpc.nodetype(), SNode.RPC) self.assertEqual(self.rpc.keyword(), "rpc") self.assertEqual(self.rpc.schema_path(), "/yolo-system:format-disk") + choice = next(self.rpc.input().children((SNode.CHOICE,), with_choice=True)) + self.assertIsInstance(choice, SChoice) def test_rpc_extensions(self): ext = list(self.rpc.extensions()) diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index 36c76416..b48dcbe8 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -191,7 +191,14 @@ module yolo-system { ext:human-name "Disk"; type types:str; } - anyxml html-info; + choice xml-or-json { + case xml { + anyxml html-info; + } + case json { + anydata json-info; + } + } } output { leaf duration { From d3dd5f017f6f1fc4ee818a55ec77fe055795c991 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Mon, 22 Sep 2025 00:52:49 +0200 Subject: [PATCH 098/115] schema: adds ability to get various outputs as extension parent_node This patch adds ability to get also PRefine, PType and PEnum as a valid output using extension parent_node function Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/schema.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index 76fc9c8f..9b507ab6 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -423,10 +423,21 @@ def name(self) -> str: def module(self) -> Module: return self._module_from_parsed() - def parent_node(self) -> Optional[Union["PNode", "PIdentity"]]: + def parent_node( + self, + ) -> Optional[Union["PNode", "PIdentity", "PRefine", "PType", "PEnum"]]: if self.cdata.parent_stmt == lib.LY_STMT_IDENTITY: cdata = ffi.cast("struct lysp_ident *", self.cdata.parent) return PIdentity(self.context, cdata, self.module_parent) + if self.cdata.parent_stmt == lib.LY_STMT_REFINE: + cdata = ffi.cast("struct lysp_refine *", self.cdata.parent) + return PRefine(self.context, cdata, self.module_parent) + if self.cdata.parent_stmt == lib.LY_STMT_TYPE: + cdata = ffi.cast("struct lysp_type *", self.cdata.parent) + return PType(self.context, cdata, self.module_parent) + if self.cdata.parent_stmt == lib.LY_STMT_ENUM: + cdata = ffi.cast("struct lysp_type_enum *", self.cdata.parent) + return PEnum(self.context, cdata, self.module_parent) if bool(self.cdata.parent_stmt & lib.LY_STMT_NODE_MASK): try: return PNode.new(self.context, self.cdata.parent, self.module_parent) From 90dad07385ea75c1464414357d0926f083fed0ce Mon Sep 17 00:00:00 2001 From: Pepa Hajek <hajekpepa@gmail.com> Date: Mon, 22 Sep 2025 16:29:39 +0200 Subject: [PATCH 099/115] data: add structured error in exception Refactor LibyangError exception handling with structured details Signed-off-by: Pepa Hajek <hajekpepa@gmail.com> --- libyang/context.py | 42 +++++++++++++++++++++++++++++++----------- libyang/util.py | 22 ++++++++++++++++++++-- tests/test_data.py | 12 ++++++++++++ 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index 33580d61..e79f2e2c 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -16,7 +16,15 @@ validation_flags, ) from .schema import Module, SNode, schema_in_format -from .util import DataType, IOType, LibyangError, c2str, data_load, str2c +from .util import ( + DataType, + IOType, + LibyangError, + LibyangErrorItem, + c2str, + data_load, + str2c, +) # ------------------------------------------------------------------------------------- @@ -284,23 +292,35 @@ def __exit__(self, *args, **kwargs): self.destroy() def error(self, msg: str, *args) -> LibyangError: - msg %= args + if args: + msg = msg % args + + parts = [msg] + errors = [] if self.cdata: err = lib.ly_err_first(self.cdata) while err: - if err.msg: - msg += ": %s" % c2str(err.msg) - if err.data_path: - msg += ": Data path: %s" % c2str(err.data_path) - if err.schema_path: - msg += ": Schema path: %s" % c2str(err.schema_path) - if err.line != 0: - msg += " (line %u)" % err.line + m = c2str(err.msg) if err.msg else None + dp = c2str(err.data_path) if err.data_path else None + sp = c2str(err.schema_path) if err.schema_path else None + ln = int(err.line) if err.line else None + parts.extend( + tmpl.format(val) + for val, tmpl in [ + (m, ": {}"), + (dp, ": Data path: {}"), + (sp, ": Schema path: {}"), + (ln, " (line {})"), + ] + if val is not None + ) + errors.append(LibyangErrorItem(m, dp, sp, ln)) err = err.next lib.ly_err_clean(self.cdata, ffi.NULL) - return LibyangError(msg) + msg = "".join(parts) + return LibyangError(msg, errors=errors) def parse_module( self, diff --git a/libyang/util.py b/libyang/util.py index 3014cc85..7b272692 100644 --- a/libyang/util.py +++ b/libyang/util.py @@ -2,16 +2,34 @@ # Copyright (c) 2021 RACOM s.r.o. # SPDX-License-Identifier: MIT +from dataclasses import dataclass import enum -from typing import Optional +from typing import Iterable, Optional import warnings from _libyang import ffi, lib +# ------------------------------------------------------------------------------------- +@dataclass(frozen=True) +class LibyangErrorItem: + msg: Optional[str] + data_path: Optional[str] + schema_path: Optional[str] + line: Optional[int] + + # ------------------------------------------------------------------------------------- class LibyangError(Exception): - pass + def __init__( + self, message: str, *args, errors: Optional[Iterable[LibyangErrorItem]] = None + ): + super().__init__(message, *args) + self.message = message + self.errors = tuple(errors or ()) + + def __str__(self): + return self.message # ------------------------------------------------------------------------------------- diff --git a/tests/test_data.py b/tests/test_data.py index 820db0ca..430873a7 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -289,6 +289,18 @@ def test_data_parse_config_xml_multi_error(self): "Data path: /yolo-system:conf/url[proto='https'] (line 7)", ) + first = cm.exception.errors[0] + self.assertEqual(first.msg, 'Invalid boolean value "abcd".') + self.assertEqual( + first.data_path, "/yolo-system:conf/url[proto='https']/enabled" + ) + self.assertEqual(first.line, 6) + + second = cm.exception.errors[1] + self.assertEqual(second.msg, 'List instance is missing its key "host".') + self.assertEqual(second.data_path, "/yolo-system:conf/url[proto='https']") + self.assertEqual(second.line, 7) + XML_STATE = """<state xmlns="urn:yang:yolo:system"> <hostname>foo</hostname> <url> From b78cdce6c320b6f83a99b015d57e5e233bf5e30f Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Sun, 12 Oct 2025 02:06:22 +0000 Subject: [PATCH 100/115] schema: add missing extensions() for access to module level Currently only have access to the extensions used under schema nodes, need access to extensions used at the module level. Add a test case for the functionality as well. Signed-off-by: Christian Hopps <chopps@labn.net> --- cffi/cdefs.h | 8 ++++++++ libyang/schema.py | 23 +++++++++++++++++++++++ tests/test_schema.py | 13 +++++++++++++ tests/yang/yolo/yolo-system.yang | 4 ++++ 4 files changed, 48 insertions(+) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 06f938b3..671d8cb8 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -365,6 +365,14 @@ LY_ERR lys_print_module(struct ly_out *, const struct lys_module *, LYS_OUTFORMA #define LYS_PRINT_NO_SUBSTMT ... #define LYS_PRINT_SHRINK ... +struct lysc_module { + struct lys_module *mod; + struct lysc_node *data; + struct lysc_node_action *rpcs; + struct lysc_node_notif *notifs; + struct lysc_ext_instance *exts; +}; + struct lys_module { struct ly_ctx *ctx; const char *name; diff --git a/libyang/schema.py b/libyang/schema.py index 9b507ab6..82a1fee1 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -180,6 +180,29 @@ def parsed_identities(self) -> Iterator["PIdentity"]: for i in ly_array_iter(self.cdata.parsed.identities): yield PIdentity(self.context, i, self) + def extensions(self) -> Iterator["ExtensionCompiled"]: + compiled = ffi.cast("struct lysc_module *", self.cdata.compiled) + if compiled == ffi.NULL: + return + exts = ffi.cast("struct lysc_ext_instance *", self.cdata.compiled.exts) + if exts == ffi.NULL: + return + for extension in ly_array_iter(exts): + yield ExtensionCompiled(self.context, extension) + + def get_extension( + self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None + ) -> Optional["ExtensionCompiled"]: + for ext in self.extensions(): + if ext.name() != name: + continue + if prefix is not None and ext.module().name() != prefix: + continue + if arg_value is not None and ext.argument() != arg_value: + continue + return ext + return None + def __str__(self) -> str: return self.name() diff --git a/tests/test_schema.py b/tests/test_schema.py index d9824e58..0a5347ff 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -129,6 +129,19 @@ def test_mod_revisions(self): self.assertEqual(revisions[0].date(), "1999-04-01") self.assertEqual(revisions[1].date(), "1990-04-01") + def test_mod_extensions(self): + assert self.module is not None # pyright doesn't understand assertIsNotNone() + exts = list(self.module.extensions()) + self.assertEqual(len(exts), 1) + ext = self.module.get_extension("compile-validation", prefix="omg-extensions") + self.assertEqual(ext.argument(), "module-level") + sub_exts = list(ext.extensions()) + self.assertEqual(len(sub_exts), 1) + ext = sub_exts[0] + self.assertEqual(ext.name(), "compile-validation") + self.assertEqual(ext.module().name(), "omg-extensions") + self.assertEqual(ext.argument(), "module-sub-level") + # ------------------------------------------------------------------------------------- class RevisionTest(unittest.TestCase): diff --git a/tests/yang/yolo/yolo-system.yang b/tests/yang/yolo/yolo-system.yang index b48dcbe8..a02f310c 100644 --- a/tests/yang/yolo/yolo-system.yang +++ b/tests/yang/yolo/yolo-system.yang @@ -155,6 +155,10 @@ module yolo-system { } } + ext:compile-validation "module-level" { + ext:compile-validation "module-sub-level"; + } + container conf { description "Configuration."; From 2ff17b4767c9d1b8ec7a4663e047cb797e939f82 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 16 Oct 2025 04:11:23 +0200 Subject: [PATCH 101/115] schema: adds extensions and get_extension for Enum class This patch allows to get extensions on Enum type Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/schema.py | 17 +++++++++++++++++ tests/test_schema.py | 3 +++ 2 files changed, 20 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index 82a1fee1..c9f2a5ef 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -525,6 +525,23 @@ def name(self) -> str: def description(self) -> str: return c2str(self.cdata.dsc) + def extensions(self) -> Iterator[ExtensionCompiled]: + for ext in ly_array_iter(self.cdata.exts): + yield ExtensionCompiled(self.context, ext) + + def get_extension( + self, name: str, prefix: Optional[str] = None, arg_value: Optional[str] = None + ) -> Optional[ExtensionCompiled]: + for ext in self.extensions(): + if ext.name() != name: + continue + if prefix is not None and ext.module().name() != prefix: + continue + if arg_value is not None and ext.argument() != arg_value: + continue + return ext + return None + def deprecated(self) -> bool: return bool(self.cdata.flags & lib.LYS_STATUS_DEPRC) diff --git a/tests/test_schema.py b/tests/test_schema.py index 0a5347ff..e27e0010 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -645,6 +645,9 @@ def test_leaf_type_enum(self): self.assertEqual(t.base(), Type.ENUM) enums = [e.name() for e in t.enums()] self.assertEqual(enums, ["http", "https", "ftp", "sftp"]) + enum = next(t.enums()) + self.assertIsNone(next(enum.extensions(), None)) + self.assertIsNone(enum.get_extension("test", prefix="test")) def test_leaf_type_bits(self): leaf = next(self.ctx.find_path("/yolo-system:chmod/yolo-system:perms")) From b35336ba4950a36a0dd9851b61bae6d8d2962bd6 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 16 Oct 2025 04:04:31 +0200 Subject: [PATCH 102/115] context: adds find_xpath_atoms function This patch is adding find_xpath_atoms on context level to allow users to get all nodes needed for xpath evaluation Signed-off-by: Stefan Gula <steweg@gmail.com> --- cffi/cdefs.h | 1 + libyang/context.py | 32 ++++++++++++++++++++++++++++++++ tests/test_context.py | 18 +++++++++++++++++- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 671d8cb8..52f00b24 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -215,6 +215,7 @@ struct lys_module* ly_ctx_get_module_latest(const struct ly_ctx *, const char *) LY_ERR ly_ctx_compile(struct ly_ctx *); LY_ERR lys_find_xpath(const struct ly_ctx *, const struct lysc_node *, const char *, uint32_t, struct ly_set **); +LY_ERR lys_find_xpath_atoms(const struct ly_ctx *, const struct lysc_node *, const char *, uint32_t, struct ly_set **); void ly_set_free(struct ly_set *, void(*)(void *obj)); struct ly_set { diff --git a/libyang/context.py b/libyang/context.py index e79f2e2c..05ad6ec4 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -421,6 +421,38 @@ def find_path( finally: lib.ly_set_free(node_set, ffi.NULL) + def find_xpath_atoms( + self, + path: str, + output: bool = False, + root_node: Optional["libyang.SNode"] = None, + ) -> Iterator[SNode]: + if self.cdata is None: + raise RuntimeError("context already destroyed") + + if root_node is not None: + ctx_node = root_node.cdata + else: + ctx_node = ffi.NULL + + flags = lib.LYS_FIND_XP_OUTPUT if output else 0 + + node_set = ffi.new("struct ly_set **") + if ( + lib.lys_find_xpath_atoms(self.cdata, ctx_node, str2c(path), flags, node_set) + != lib.LY_SUCCESS + ): + raise self.error("cannot find path") + + node_set = node_set[0] + if node_set.count == 0: + raise self.error("cannot find path") + try: + for i in range(node_set.count): + yield SNode.new(self, node_set.snodes[i]) + finally: + lib.ly_set_free(node_set, ffi.NULL) + def find_jsonpath( self, path: str, diff --git a/tests/test_context.py b/tests/test_context.py index db03c329..61c0ae8c 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -4,7 +4,7 @@ import os import unittest -from libyang import Context, LibyangError, Module, SLeaf, SLeafList +from libyang import Context, LibyangError, Module, SContainer, SLeaf, SLeafList from libyang.util import c2str @@ -95,6 +95,22 @@ def test_ctx_find_path(self): node2 = next(ctx.find_path("../number", root_node=node)) self.assertIsInstance(node2, SLeafList) + def test_ctx_find_xpath_atoms(self): + with Context(YANG_DIR) as ctx: + ctx.load_module("yolo-system") + node_iter = ctx.find_xpath_atoms("/yolo-system:conf/offline") + node = next(node_iter) + self.assertIsInstance(node, SContainer) + node = next(node_iter) + self.assertIsInstance(node, SLeaf) + node_iter = ctx.find_xpath_atoms("../number", root_node=node) + node = next(node_iter) + self.assertIsInstance(node, SLeaf) + node = next(node_iter) + self.assertIsInstance(node, SContainer) + node = next(node_iter) + self.assertIsInstance(node, SLeafList) + def test_ctx_iter_modules(self): with Context(YANG_DIR) as ctx: ctx.load_module("yolo-system") From 74b7d1da4fc82bb575c06055b8189dd5410fec48 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Tue, 16 Sep 2025 12:41:01 +0200 Subject: [PATCH 103/115] schema: adds ability to get when context nodes This patch adds ability to get context schema node from which when contition is evaluated. It also fixes memory leak of original when_conditions Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/schema.py | 12 +++++++++++- tests/test_schema.py | 8 ++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/libyang/schema.py b/libyang/schema.py index c9f2a5ef..23c1b1bf 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1440,13 +1440,23 @@ def parent(self) -> Optional["SNode"]: return None def when_conditions(self): - wh = ffi.new("struct lysc_when **") wh = lib.lysc_node_when(self.cdata) if wh == ffi.NULL: return for cond in ly_array_iter(wh): yield c2str(lib.lyxp_get_expr(cond.cond)) + def when_conditions_nodes(self) -> Iterator[Optional["SNode"]]: + wh = lib.lysc_node_when(self.cdata) + if wh == ffi.NULL: + return + for cond in ly_array_iter(wh): + yield ( + None + if cond.context == ffi.NULL + else SNode.new(self.context, cond.context) + ) + def parsed(self) -> Optional["PNode"]: if self.cdata_parsed is None or self.cdata_parsed == ffi.NULL: return None diff --git a/tests/test_schema.py b/tests/test_schema.py index e27e0010..b1862ab8 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -918,6 +918,14 @@ def tearDown(self): self.ctx.destroy() self.ctx = None + def test_anydata(self): + snode = next(self.ctx.find_path("/yolo-nodetypes:any1")) + self.assertIsInstance(snode, SAnydata) + assert next(snode.when_conditions()) is not None + snode2 = next(snode.when_conditions_nodes()) + assert isinstance(snode2, SAnydata) + assert snode2.cdata == snode.cdata + def test_anydata_parsed(self): snode = next(self.ctx.find_path("/yolo-nodetypes:any1")) self.assertIsInstance(snode, SAnydata) From 2cae616d04c79fa3475e7ce63de450bba402e3ae Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Thu, 16 Oct 2025 03:52:33 +0200 Subject: [PATCH 104/115] schema: adds libyang.Enum to package This patch adds missing Enum class to __init__.py to allow imports by library users Signed-off-by: Stefan Gula <steweg@gmail.com> --- libyang/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libyang/__init__.py b/libyang/__init__.py index ff15755c..3d7be2f7 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -67,6 +67,7 @@ from .keyed_list import KeyedList from .log import configure_logging from .schema import ( + Enum, Extension, ExtensionCompiled, ExtensionParsed, @@ -144,6 +145,7 @@ "DefaultRemoved", "DescriptionAdded", "DescriptionRemoved", + "Enum", "EnumAdded", "EnumRemoved", "Extension", From ddb7e997f98cff7d527f561ad96abbbd7db67879 Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Fri, 7 Nov 2025 01:03:17 +0000 Subject: [PATCH 105/115] context: fix libyang abort/crash on parse_data validation error This crash (abort) is hit when the parsed data does not validate: python3: ..validation.c:1998: lyd_validate: Assertion `tree && ctx' failed. Aborted The `tree` mentioned in the assert is from last value ffi.NULL that was passed in in the `parent is not None` case. Instead just pass a pointer for a return value regardless of the parent value. Signed-off-by: Christian Hopps <chopps@labn.net> --- libyang/context.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/libyang/context.py b/libyang/context.py index 05ad6ec4..adab171e 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -615,24 +615,10 @@ def parse_data( if ret != lib.LY_SUCCESS: raise self.error("failed to read input data") - if parent is not None: - ret = lib.lyd_parse_data( - self.cdata, - parent.cdata, - data[0], - fmt, - parser_flgs, - validation_flgs, - ffi.NULL, - ) - lib.ly_in_free(data[0], 0) - if ret != lib.LY_SUCCESS: - raise self.error("failed to parse data tree") - return None - + parent_cdata = parent.cdata if parent is not None else ffi.NULL dnode = ffi.new("struct lyd_node **") ret = lib.lyd_parse_data( - self.cdata, ffi.NULL, data[0], fmt, parser_flgs, validation_flgs, dnode + self.cdata, parent_cdata, data[0], fmt, parser_flgs, validation_flgs, dnode ) lib.ly_in_free(data[0], 0) if ret != lib.LY_SUCCESS: From 4cf965cadaa44cb07a6818fe70d3808548e9b30e Mon Sep 17 00:00:00 2001 From: Esben Laursen <hyber@hyber.dk> Date: Fri, 17 Oct 2025 16:21:03 +0200 Subject: [PATCH 106/115] context: add option all_implemented and enable_imp_features includes the default context options LY_CTX_ALL_IMPLEMENTED and LY_CTX_ENABLE_IMP_FEATURES into the ctor of Context class. Signed-off-by: Esben Laursen <hyber@hyber.dk> --- libyang/context.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libyang/context.py b/libyang/context.py index adab171e..4489680c 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -203,6 +203,8 @@ def __init__( leafref_extended: bool = False, leafref_linking: bool = False, builtin_plugins_only: bool = False, + all_implemented: bool = False, + enable_imp_features: bool = False, yanglib_path: Optional[str] = None, yanglib_fmt: str = "json", cdata=None, # C type: "struct ly_ctx *" @@ -225,6 +227,10 @@ def __init__( options |= lib.LY_CTX_LEAFREF_LINKING if builtin_plugins_only: options |= lib.LY_CTX_BUILTIN_PLUGINS_ONLY + if all_implemented: + options |= lib.LY_CTX_ALL_IMPLEMENTED + if enable_imp_features: + options |= lib.LY_CTX_ENABLE_IMP_FEATURES # force priv parsed options |= lib.LY_CTX_SET_PRIV_PARSED From cec98f85d532833fbe7843ca8147240cc346328f Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Sat, 5 Jul 2025 15:25:20 -0400 Subject: [PATCH 107/115] log: fix bug with logging The arg is added as `str` but the fmt requires `int`, causes exception when turning on python logging -- fix. Signed-off-by: Christian Hopps <chopps@labn.net> --- libyang/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libyang/log.py b/libyang/log.py index f92c70fd..22fc52f6 100644 --- a/libyang/log.py +++ b/libyang/log.py @@ -38,7 +38,7 @@ def libyang_c_logging_callback(level, msg, data_path, schema_path, line): args.append(c2str(schema_path)) if line != 0: fmt += " line %u" - args.append(str(line)) + args.append(line) LOG.log(LOG_LEVELS.get(level, logging.NOTSET), fmt, *args) From ad059d6aa5c4feeaf92339e4c23687ee97bbe095 Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Sat, 5 Jul 2025 15:26:21 -0400 Subject: [PATCH 108/115] log: expose ly_temp_log_options with a context manager When one wishes to disable logging temporarily (e.g., when calling API functions when an error result is expected and OK), libyang provides ly_temp_log_options(). We need access to this in python, so export access to the function and add a python context (i.e., "with") manager API for using it idiomatically as well. ex usage: ``` with temp_log_options(0): ly_unwanted_logging_call(); ``` Signed-off-by: Christian Hopps <chopps@labn.net> --- cffi/cdefs.h | 1 + libyang/__init__.py | 2 +- libyang/log.py | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 52f00b24..8b6d4268 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -182,6 +182,7 @@ enum ly_stmt { #define LY_LOSTORE ... #define LY_LOSTORE_LAST ... int ly_log_options(int); +uint32_t *ly_temp_log_options(uint32_t *); LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL); extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t); diff --git a/libyang/__init__.py b/libyang/__init__.py index 3d7be2f7..d8f95556 100644 --- a/libyang/__init__.py +++ b/libyang/__init__.py @@ -65,7 +65,7 @@ ) from .extension import ExtensionPlugin, LibyangExtensionError from .keyed_list import KeyedList -from .log import configure_logging +from .log import configure_logging, temp_log_options from .schema import ( Enum, Extension, diff --git a/libyang/log.py b/libyang/log.py index 22fc52f6..564cf604 100644 --- a/libyang/log.py +++ b/libyang/log.py @@ -2,6 +2,7 @@ # Copyright (c) 2020 6WIND S.A. # SPDX-License-Identifier: MIT +from contextlib import contextmanager import logging from _libyang import ffi, lib @@ -26,6 +27,15 @@ def get_libyang_level(py_level): return None +@contextmanager +def temp_log_options(opt: int = 0): + opts = ffi.new("uint32_t *", opt) + + lib.ly_temp_log_options(opts) + yield + lib.ly_temp_log_options(ffi.NULL) + + @ffi.def_extern(name="lypy_log_cb") def libyang_c_logging_callback(level, msg, data_path, schema_path, line): args = [c2str(msg)] From 1deecd8ec5422c1c88f39a57c83a09defefb5e0c Mon Sep 17 00:00:00 2001 From: Christian Hopps <chopps@labn.net> Date: Mon, 7 Jul 2025 01:52:03 -0400 Subject: [PATCH 109/115] tests: add a logging test for new temp_log_options API Add a new log unit test. Add a test for the newly exposed temp_log_options() function as well as the already implemented configure_logging() function. Signed-off-by: Christian Hopps <chopps@labn.net> --- tests/test_log.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/test_log.py diff --git a/tests/test_log.py b/tests/test_log.py new file mode 100644 index 00000000..2834414a --- /dev/null +++ b/tests/test_log.py @@ -0,0 +1,55 @@ +# Copyright (c) 2025, LabN Consulting, L.L.C. +# SPDX-License-Identifier: MIT + +import logging +import os +import sys +import unittest + +from libyang import Context, LibyangError, configure_logging, temp_log_options + + +YANG_DIR = os.path.join(os.path.dirname(__file__), "yang") + + +class LogTest(unittest.TestCase): + def setUp(self): + self.ctx = Context(YANG_DIR) + configure_logging(False, logging.INFO) + + def tearDown(self): + if self.ctx is not None: + self.ctx.destroy() + self.ctx = None + + def _cause_log(self): + try: + assert self.ctx is not None + _ = self.ctx.parse_data_mem("bad", fmt="xml") + except LibyangError: + pass + + @unittest.skipIf(sys.version_info < (3, 10), "Test requires Python 3.10+") + def test_configure_logging(self): + """Test configure_logging API.""" + with self.assertNoLogs("libyang", level="ERROR"): + self._cause_log() + + configure_logging(True, logging.INFO) + with self.assertLogs("libyang", level="ERROR"): + self._cause_log() + + @unittest.skipIf(sys.version_info < (3, 10), "Test requires Python 3.10+") + def test_with_temp_log(self): + """Test configure_logging API.""" + configure_logging(True, logging.INFO) + + with self.assertLogs("libyang", level="ERROR"): + self._cause_log() + + with self.assertNoLogs("libyang", level="ERROR"): + with temp_log_options(0): + self._cause_log() + + with self.assertLogs("libyang", level="ERROR"): + self._cause_log() From 0a4b09f461a36f740305f1b5b742de4ff1f88231 Mon Sep 17 00:00:00 2001 From: Stefan Gula <steweg@gmail.com> Date: Sun, 2 Nov 2025 21:26:55 +0100 Subject: [PATCH 110/115] cffi: allows to usage of libyang v4 This patch adjust cffi definitions and also all associated functions based on libyang v4. Closes: https://github.com/CESNET/libyang-python/pull/160 Signed-off-by: Stefan Gula <steweg@gmail.com> Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com> --- cffi/cdefs.h | 61 +++++++++++++++++--------------- cffi/source.c | 4 +-- libyang/context.py | 25 +++++++++----- libyang/data.py | 7 ++-- libyang/extension.py | 24 +++++++++---- libyang/schema.py | 66 ++++++++++++++++++++++++++++------- tests/test_context.py | 6 +++- tests/test_data.py | 67 ++++++++++++++++++++---------------- tests/test_diff.py | 4 ++- tests/test_schema.py | 4 +-- tests/yang/yang-library.json | 15 ++++++++ 11 files changed, 187 insertions(+), 96 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 8b6d4268..82ec9748 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -17,6 +17,7 @@ struct ly_ctx; #define LY_CTX_LEAFREF_EXTENDED ... #define LY_CTX_LEAFREF_LINKING ... #define LY_CTX_BUILTIN_PLUGINS_ONLY ... +#define LY_CTX_COMPILE_OBSOLETE ... typedef enum { @@ -304,20 +305,19 @@ enum lyd_type { LYD_TYPE_REPLY_RESTCONF }; -#define LYD_PRINT_KEEPEMPTYCONT ... #define LYD_PRINT_SHRINK ... +#define LYD_PRINT_EMPTY_CONT ... #define LYD_PRINT_WD_ALL ... #define LYD_PRINT_WD_ALL_TAG ... #define LYD_PRINT_WD_EXPLICIT ... #define LYD_PRINT_WD_IMPL_TAG ... #define LYD_PRINT_WD_MASK ... -#define LYD_PRINT_WITHSIBLINGS ... +#define LYD_PRINT_SIBLINGS ... #define LYD_PRINT_WD_TRIM ... LY_ERR lyd_print_mem(char **, const struct lyd_node *, LYD_FORMAT, uint32_t); LY_ERR lyd_print_tree(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint32_t); LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint32_t); -#define LYD_PARSE_LYB_MOD_UPDATE ... #define LYD_PARSE_NO_STATE ... #define LYD_PARSE_STORE_ONLY ... #define LYD_PARSE_JSON_NULL ... @@ -351,7 +351,7 @@ LY_ERR ly_out_new_file(FILE *, struct ly_out **); LY_ERR ly_out_new_fd(int, struct ly_out **); LY_ERR lyd_parse_data(const struct ly_ctx *, struct lyd_node *, struct ly_in *, LYD_FORMAT, uint32_t, uint32_t, struct lyd_node **); -LY_ERR lyd_parse_op(const struct ly_ctx *, struct lyd_node *, struct ly_in *, LYD_FORMAT, enum lyd_type, struct lyd_node **, struct lyd_node **); +LY_ERR lyd_parse_op(const struct ly_ctx *, struct lyd_node *, struct ly_in *, LYD_FORMAT, enum lyd_type, uint32_t, struct lyd_node **, struct lyd_node **); typedef enum { LYS_OUT_UNKNOWN, @@ -369,6 +369,7 @@ LY_ERR lys_print_module(struct ly_out *, const struct lys_module *, LYS_OUTFORMA struct lysc_module { struct lys_module *mod; + const char **features; struct lysc_node *data; struct lysc_node_action *rpcs; struct lysc_node_notif *notifs; @@ -388,13 +389,15 @@ struct lys_module { const char *ref; struct lysp_module *parsed; struct lysc_module *compiled; + struct lysc_ext *extensions; struct lysc_ident *identities; + struct lysc_submodule *submodules; struct lys_module **augmented_by; struct lys_module **deviated_by; ly_bool implemented; ly_bool to_compile; - uint8_t latest_revision; - ...; + uint8_t version : 2; + uint8_t latest_revision : 4; }; struct lysp_module { @@ -467,12 +470,11 @@ struct lysp_ext_instance { const char *argument; LY_VALUE_FORMAT format; void *prefix_data; - struct lysp_ext *def; + uintptr_t plugin_ref; void *parent; enum ly_stmt parent_stmt; uint64_t parent_stmt_index; uint16_t flags; - const struct lyplg_ext_record *record; struct lysp_ext_substmt *substmts; void *parsed; struct lysp_stmt *child; @@ -610,6 +612,11 @@ struct lysp_node_container { ...; }; +struct lysc_value { + const char *str; + struct lysc_prefix *prefixes; +}; + struct lysc_node_leaf { union { struct lysc_node node; @@ -623,7 +630,7 @@ struct lysc_node_leaf { struct lysc_when **when; struct lysc_type *type; const char *units; - struct lyd_value *dflt; + struct lysc_value dflt; ...; }; @@ -653,7 +660,7 @@ struct lysc_node_leaflist { struct lysc_when **when; struct lysc_type *type; const char *units; - struct lyd_value **dflts; + struct lysc_value *dflts; uint32_t min; uint32_t max; ...; @@ -783,7 +790,7 @@ struct lysp_node_augment { struct lysc_type { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; }; @@ -859,7 +866,7 @@ struct lysc_ext { const char *name; const char *argname; struct lysc_ext_instance *exts; - struct lyplg_ext *plugin; + uintptr_t plugin_ref; struct lys_module *module; uint16_t flags; }; @@ -982,7 +989,6 @@ typedef struct pcre2_real_code pcre2_code; struct lysc_pattern { const char *expr; - pcre2_code *code; const char *dsc; const char *ref; const char *emsg; @@ -1017,7 +1023,7 @@ struct lysc_ident { struct lysc_type_num { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; struct lysc_range *range; @@ -1026,7 +1032,7 @@ struct lysc_type_num { struct lysc_type_dec { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; uint8_t fraction_digits; @@ -1036,7 +1042,7 @@ struct lysc_type_dec { struct lysc_type_str { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; struct lysc_range *length; @@ -1058,7 +1064,7 @@ struct lysc_type_bitenum_item { struct lysc_type_enum { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; struct lysc_type_bitenum_item *enums; @@ -1067,7 +1073,7 @@ struct lysc_type_enum { struct lysc_type_bits { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; struct lysc_type_bitenum_item *bits; @@ -1076,7 +1082,7 @@ struct lysc_type_bits { struct lysc_type_leafref { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; struct lyxp_expr *path; @@ -1088,7 +1094,7 @@ struct lysc_type_leafref { struct lysc_type_identityref { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; struct lysc_ident **bases; @@ -1097,7 +1103,7 @@ struct lysc_type_identityref { struct lysc_type_instanceid { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; uint8_t require_instance; @@ -1106,7 +1112,7 @@ struct lysc_type_instanceid { struct lysc_type_union { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; struct lysc_type **types; @@ -1115,7 +1121,7 @@ struct lysc_type_union { struct lysc_type_bin { const char *name; struct lysc_ext_instance *exts; - struct lyplg_type *plugin; + uintptr_t plugin_ref; LY_DATA_TYPE basetype; uint32_t refcount; struct lysc_range *length; @@ -1159,8 +1165,7 @@ typedef enum { LYD_ANYDATA_DATATREE, LYD_ANYDATA_STRING, LYD_ANYDATA_XML, - LYD_ANYDATA_JSON, - LYD_ANYDATA_LYB + LYD_ANYDATA_JSON } LYD_ANYDATA_VALUETYPE; union lyd_any_value { @@ -1168,7 +1173,6 @@ union lyd_any_value { const char *str; const char *xml; const char *json; - char *mem; }; struct lyd_node_any { @@ -1320,6 +1324,8 @@ struct lyd_attr { LY_ERR lyd_new_attr(struct lyd_node *, const char *, const char *, const char *, struct lyd_attr **); void lyd_free_attr_single(const struct ly_ctx *ctx, struct lyd_attr *attr); +LY_ERR lyd_value_validate_dflt(const struct lysc_node *, const char *, struct lysc_prefix *, const struct lyd_node *, const struct lysc_type **, const char **); + struct lyd_leafref_links_rec { const struct lyd_node_term *node; const struct lyd_node_term **leafref_nodes; @@ -1328,13 +1334,14 @@ struct lyd_leafref_links_rec { LY_ERR lyd_leafref_get_links(const struct lyd_node_term *, const struct lyd_leafref_links_rec **); LY_ERR lyd_leafref_link_node_tree(struct lyd_node *); +struct lyplg_ext *lysc_get_ext_plugin(uintptr_t); const char *lyplg_ext_stmt2str(enum ly_stmt stmt); const struct lysp_module *lyplg_ext_parse_get_cur_pmod(const struct lysp_ctx *); struct ly_ctx *lyplg_ext_compile_get_ctx(const struct lysc_ctx *); void lyplg_ext_parse_log(const struct lysp_ctx *, const struct lysp_ext_instance *, LY_LOG_LEVEL, LY_ERR, const char *, ...); void lyplg_ext_compile_log(const struct lysc_ctx *, const struct lysc_ext_instance *, LY_LOG_LEVEL, LY_ERR, const char *, ...); LY_ERR lyplg_ext_parse_extension_instance(struct lysp_ctx *, struct lysp_ext_instance *); -LY_ERR lyplg_ext_compile_extension_instance(struct lysc_ctx *, const struct lysp_ext_instance *, struct lysc_ext_instance *); +LY_ERR lyplg_ext_compile_extension_instance(struct lysc_ctx *, const struct lysp_ext_instance *, struct lysc_ext_instance *, struct lysc_node *); void lyplg_ext_pfree_instance_substatements(const struct ly_ctx *ctx, struct lysp_ext_substmt *substmts); void lyplg_ext_cfree_instance_substatements(const struct ly_ctx *ctx, struct lysc_ext_substmt *substmts); typedef LY_ERR (*lyplg_ext_parse_clb)(struct lysp_ctx *, struct lysp_ext_instance *); diff --git a/cffi/source.c b/cffi/source.c index 34e18b0e..3c76e984 100644 --- a/cffi/source.c +++ b/cffi/source.c @@ -6,6 +6,6 @@ #include <libyang/libyang.h> #include <libyang/version.h> -#if LY_VERSION_MAJOR * 10000 + LY_VERSION_MINOR * 100 + LY_VERSION_MICRO < 30801 -#error "This version of libyang bindings only works with libyang soversion 3.8.1+" +#if LY_VERSION_MAJOR * 10000 + LY_VERSION_MINOR * 100 + LY_VERSION_MICRO < 40202 +#error "This version of libyang bindings only works with libyang soversion 4.2.2+" #endif diff --git a/libyang/context.py b/libyang/context.py index 4489680c..21db1628 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -205,6 +205,7 @@ def __init__( builtin_plugins_only: bool = False, all_implemented: bool = False, enable_imp_features: bool = False, + compile_obsolete: bool = False, yanglib_path: Optional[str] = None, yanglib_fmt: str = "json", cdata=None, # C type: "struct ly_ctx *" @@ -231,6 +232,8 @@ def __init__( options |= lib.LY_CTX_ALL_IMPLEMENTED if enable_imp_features: options |= lib.LY_CTX_ENABLE_IMP_FEATURES + if compile_obsolete: + options |= lib.LY_CTX_COMPILE_OBSOLETE # force priv parsed options |= lib.LY_CTX_SET_PRIV_PARSED @@ -259,6 +262,7 @@ def __init__( fmt = lib.LYD_JSON else: fmt = lib.LYD_XML + print("steweg", search_path, yanglib_path, yanglib_fmt, options) ret = lib.ly_ctx_new_ylpath( str2c(search_path), str2c(yanglib_path), fmt, options, ctx ) @@ -542,6 +546,8 @@ def parse_op( in_data: Union[IO, str], dtype: DataType, parent: DNode = None, + opaq: bool = False, + strict: bool = False, ) -> DNode: fmt = data_format(fmt) data = ffi.new("struct ly_in **") @@ -551,13 +557,14 @@ def parse_op( if ret != lib.LY_SUCCESS: raise self.error("failed to read input data") + flags = parser_flags(opaq=opaq, strict=strict) tree = ffi.new("struct lyd_node **", ffi.NULL) op = ffi.new("struct lyd_node **", ffi.NULL) par = ffi.new("struct lyd_node **", ffi.NULL) if parent is not None: par[0] = parent.cdata - ret = lib.lyd_parse_op(self.cdata, par[0], data[0], fmt, dtype, tree, op) + ret = lib.lyd_parse_op(self.cdata, par[0], data[0], fmt, dtype, flags, tree, op) lib.ly_in_free(data[0], 0) if ret != lib.LY_SUCCESS: raise self.error("failed to parse input data") @@ -570,9 +577,17 @@ def parse_op_mem( data: str, dtype: DataType = DataType.DATA_YANG, parent: DNode = None, + opaq: bool = False, + strict: bool = False, ): return self.parse_op( - fmt, in_type=IOType.MEMORY, in_data=data, dtype=dtype, parent=parent + fmt, + in_type=IOType.MEMORY, + in_data=data, + dtype=dtype, + parent=parent, + opaq=opaq, + strict=strict, ) def parse_data( @@ -581,7 +596,6 @@ def parse_data( in_type: IOType, in_data: Union[str, bytes, IO], parent: DNode = None, - lyb_mod_update: bool = False, no_state: bool = False, parse_only: bool = False, opaq: bool = False, @@ -596,7 +610,6 @@ def parse_data( if self.cdata is None: raise RuntimeError("context already destroyed") parser_flgs = parser_flags( - lyb_mod_update=lyb_mod_update, no_state=no_state, parse_only=parse_only, opaq=opaq, @@ -640,7 +653,6 @@ def parse_data_mem( data: Union[str, bytes], fmt: str, parent: DNode = None, - lyb_mod_update: bool = False, no_state: bool = False, parse_only: bool = False, opaq: bool = False, @@ -657,7 +669,6 @@ def parse_data_mem( in_type=IOType.MEMORY, in_data=data, parent=parent, - lyb_mod_update=lyb_mod_update, no_state=no_state, parse_only=parse_only, opaq=opaq, @@ -675,7 +686,6 @@ def parse_data_file( fileobj: IO, fmt: str, parent: DNode = None, - lyb_mod_update: bool = False, no_state: bool = False, parse_only: bool = False, opaq: bool = False, @@ -692,7 +702,6 @@ def parse_data_file( in_type=IOType.FD, in_data=fileobj, parent=parent, - lyb_mod_update=lyb_mod_update, no_state=no_state, parse_only=parse_only, opaq=opaq, diff --git a/libyang/data.py b/libyang/data.py index 7a5887b0..230c51af 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -53,11 +53,11 @@ def printer_flags( ) -> int: flags = 0 if with_siblings: - flags |= lib.LYD_PRINT_WITHSIBLINGS + flags |= lib.LYD_PRINT_SIBLINGS if not pretty: flags |= lib.LYD_PRINT_SHRINK if keep_empty_containers: - flags |= lib.LYD_PRINT_KEEPEMPTYCONT + flags |= lib.LYD_PRINT_EMPTY_CONT if trim_default_values: flags |= lib.LYD_PRINT_WD_TRIM if include_implicit_defaults: @@ -109,7 +109,6 @@ def newval_flags( # ------------------------------------------------------------------------------------- def parser_flags( - lyb_mod_update: bool = False, no_state: bool = False, parse_only: bool = False, opaq: bool = False, @@ -120,8 +119,6 @@ def parser_flags( json_string_datatypes: bool = False, ) -> int: flags = 0 - if lyb_mod_update: - flags |= lib.LYD_PARSE_LYB_MOD_UPDATE if no_state: flags |= lib.LYD_PARSE_NO_STATE if parse_only: diff --git a/libyang/extension.py b/libyang/extension.py index 57f7cb2d..f7e4ba93 100644 --- a/libyang/extension.py +++ b/libyang/extension.py @@ -8,7 +8,7 @@ from _libyang import ffi, lib from .context import Context from .log import get_libyang_level -from .schema import ExtensionCompiled, ExtensionParsed, Module +from .schema import ExtensionCompiled, ExtensionParsed, Module, SNode from .util import LibyangError, c2str, str2c @@ -25,7 +25,7 @@ def __init__(self, message: str, ret: int, log_level: int) -> None: @ffi.def_extern(name="lypy_lyplg_ext_parse_clb") def libyang_c_lyplg_ext_parse_clb(pctx, pext): - plugin = extensions_plugins[pext.record.plugin] + plugin = extensions_plugins[lib.lysc_get_ext_plugin(pext.plugin_ref)] module_cdata = lib.lyplg_ext_parse_get_cur_pmod(pctx).mod context = Context(cdata=module_cdata.ctx) module = Module(context, module_cdata) @@ -46,7 +46,7 @@ def libyang_c_lyplg_ext_parse_clb(pctx, pext): @ffi.def_extern(name="lypy_lyplg_ext_compile_clb") def libyang_c_lyplg_ext_compile_clb(cctx, pext, cext): - plugin = extensions_plugins[pext.record.plugin] + plugin = extensions_plugins[lib.lysc_get_ext_plugin(pext.plugin_ref)] context = Context(cdata=lib.lyplg_ext_compile_get_ctx(cctx)) module = Module(context, cext.module) parsed_ext = ExtensionParsed(context, pext, module) @@ -67,7 +67,7 @@ def libyang_c_lyplg_ext_compile_clb(cctx, pext, cext): @ffi.def_extern(name="lypy_lyplg_ext_parse_free_clb") def libyang_c_lyplg_ext_parse_free_clb(ctx, pext): - plugin = extensions_plugins[pext.record.plugin] + plugin = extensions_plugins[lib.lysc_get_ext_plugin(pext.plugin_ref)] context = Context(cdata=ctx) parsed_ext = ExtensionParsed(context, pext, None) plugin.parse_free_clb(parsed_ext) @@ -75,7 +75,9 @@ def libyang_c_lyplg_ext_parse_free_clb(ctx, pext): @ffi.def_extern(name="lypy_lyplg_ext_compile_free_clb") def libyang_c_lyplg_ext_compile_free_clb(ctx, cext): - plugin = extensions_plugins[getattr(cext, "def").plugin] + plugin = extensions_plugins[ + lib.lysc_get_ext_plugin(getattr(cext, "def").plugin_ref) + ] context = Context(cdata=ctx) compiled_ext = ExtensionCompiled(context, cext) plugin.compile_free_clb(compiled_ext) @@ -200,9 +202,17 @@ def set_compile_ctx(self, cctx) -> None: def parse_substmts(self, ext: ExtensionParsed) -> int: return lib.lyplg_ext_parse_extension_instance(self._pctx, ext.cdata) - def compile_substmts(self, pext: ExtensionParsed, cext: ExtensionCompiled) -> int: + def compile_substmts( + self, + pext: ExtensionParsed, + cext: ExtensionCompiled, + parent: Optional[SNode] = None, + ) -> int: return lib.lyplg_ext_compile_extension_instance( - self._cctx, pext.cdata, cext.cdata + self._cctx, + pext.cdata, + cext.cdata, + ffi.NULL if parent is None else parent.cdata, ) def free_parse_substmts(self, ext: ExtensionParsed) -> None: diff --git a/libyang/schema.py b/libyang/schema.py index 23c1b1bf..f104a874 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1522,21 +1522,35 @@ def new(context: "libyang.Context", cdata) -> "SNode": # ------------------------------------------------------------------------------------- @SNode.register(SNode.LEAF) class SLeaf(SNode): - __slots__ = ("cdata_leaf", "cdata_leaf_parsed") + __slots__ = ("cdata_leaf", "cdata_leaf_parsed", "cdata_default_realtype") def __init__(self, context: "libyang.Context", cdata): super().__init__(context, cdata) self.cdata_leaf = ffi.cast("struct lysc_node_leaf *", cdata) self.cdata_leaf_parsed = ffi.cast("struct lysp_node_leaf *", self.cdata_parsed) + self.cdata_default_realtype = None def default(self) -> Union[None, bool, int, str, float]: - if not self.cdata_leaf.dflt: + if not self.cdata_leaf.dflt.str: return None - val = lib.lyd_value_get_canonical(self.context.cdata, self.cdata_leaf.dflt) - if not val: - return None - val = c2str(val) - val_type = Type(self.context, self.cdata_leaf.dflt.realtype, None) + + if self.cdata_default_realtype is None: + # calculate real type of default value just once + val_type_cdata = ffi.new("struct lysc_type **", ffi.NULL) + ret = lib.lyd_value_validate_dflt( + self.cdata, + self.cdata_leaf.dflt.str, + self.cdata_leaf.dflt.prefixes, + ffi.NULL, + val_type_cdata, + ffi.NULL, + ) + if ret != lib.LY_SUCCESS: + raise self.context.error("Unable to get real type of default value") + self.cdata_default_realtype = Type(self.context, val_type_cdata[0], None) + + val = c2str(self.cdata_leaf.dflt.str) + val_type = self.cdata_default_realtype if val_type.base() == Type.BOOL: return val == "true" if val_type.base() in Type.NUM_TYPES: @@ -1563,7 +1577,7 @@ def __str__(self): # ------------------------------------------------------------------------------------- @SNode.register(SNode.LEAFLIST) class SLeafList(SNode): - __slots__ = ("cdata_leaflist", "cdata_leaflist_parsed") + __slots__ = ("cdata_leaflist", "cdata_leaflist_parsed", "cdata_default_realtypes") def __init__(self, context: "libyang.Context", cdata): super().__init__(context, cdata) @@ -1571,6 +1585,7 @@ def __init__(self, context: "libyang.Context", cdata): self.cdata_leaflist_parsed = ffi.cast( "struct lysp_node_leaflist *", self.cdata_parsed ) + self.cdata_default_realtypes = None def ordered(self) -> bool: return bool(self.cdata_parsed.flags & lib.LYS_ORDBY_USER) @@ -1586,12 +1601,37 @@ def type(self) -> Type: def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: if self.cdata_leaflist.dflts == ffi.NULL: return - for dflt in ly_array_iter(self.cdata_leaflist.dflts): - val = lib.lyd_value_get_canonical(self.context.cdata, dflt) - if not val: + + if self.cdata_default_realtypes is None: + # calculate real types of default values just once + val_type_cdata = ffi.new("struct lysc_type **", ffi.NULL) + self.cdata_default_realtypes = [] + for dflt in ly_array_iter(self.cdata_leaflist.dflts): + if not dflt.str: + self.cdata_default_realtypes.append(None) + continue + val_type_cdata[0] = ffi.NULL + ret = lib.lyd_value_validate_dflt( + self.cdata, + dflt.str, + dflt.prefixes, + ffi.NULL, + val_type_cdata, + ffi.NULL, + ) + if ret != lib.LY_SUCCESS: + raise self.context.error("Unable to get real type of default value") + self.cdata_default_realtypes.append( + Type(self.context, val_type_cdata[0], None) + ) + + for dflt, val_type in zip( + ly_array_iter(self.cdata_leaflist.dflts), self.cdata_default_realtypes + ): + if not dflt.str: yield None - val = c2str(val) - val_type = Type(self.context, dflt.realtype, None) + continue + val = c2str(dflt.str) if val_type.base() == Type.BOOL: yield val == "true" elif val_type.base() in Type.NUM_TYPES: diff --git a/tests/test_context.py b/tests/test_context.py index 61c0ae8c..8ffd0454 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -20,7 +20,11 @@ def test_ctx_no_dir(self): self.assertIsNot(ctx, None) def test_ctx_yanglib(self): - ctx = Context(YANG_DIR, yanglib_path=YANG_DIR + "/yang-library.json") + ctx = Context( + YANG_DIR, + yanglib_path=YANG_DIR + "/yang-library.json", + compile_obsolete=True, + ) ctx.load_module("yolo-system") dnode = ctx.get_yanglib_data() j = dnode.print_mem("json", with_siblings=True) diff --git a/tests/test_data.py b/tests/test_data.py index 430873a7..8f56dc19 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -33,7 +33,7 @@ # ------------------------------------------------------------------------------------- class DataTest(unittest.TestCase): def setUp(self): - self.ctx = Context(YANG_DIR) + self.ctx = Context(YANG_DIR, compile_obsolete=True) modules = [ self.ctx.load_module("ietf-netconf"), self.ctx.load_module("yolo-system"), @@ -51,18 +51,18 @@ def tearDown(self): "yolo-system:conf": { "hostname": "foo", "url": [ - { - "proto": "https", - "host": "github.com", - "path": "/CESNET/libyang-python", - "enabled": false - }, { "proto": "http", "host": "foobar.com", "port": 8080, "path": "/index.html", "enabled": true + }, + { + "proto": "https", + "host": "github.com", + "path": "/CESNET/libyang-python", + "enabled": false } ], "number": [ @@ -76,7 +76,9 @@ def tearDown(self): """ def test_data_parse_config_json(self): - dnode = self.ctx.parse_data_mem(self.JSON_CONFIG, "json", no_state=True) + dnode = self.ctx.parse_data_mem( + self.JSON_CONFIG, "json", no_state=True, ordered=True + ) self.assertIsInstance(dnode, DContainer) try: j = dnode.print_mem("json", with_siblings=True) @@ -91,18 +93,18 @@ def test_data_parse_config_json(self): "yolo-system:conf": { "hostname": "foo", "url": [ - { - "proto": "https", - "host": "github.com", - "path": "/CESNET/libyang-python", - "enabled": false - }, { "proto": "http", "host": "foobar.com", "port": 8080, "path": "/index.html", "enabled": true + }, + { + "proto": "https", + "host": "github.com", + "path": "/CESNET/libyang-python", + "enabled": false } ], "number": [ @@ -128,12 +130,6 @@ def test_data_parse_config_json_without_yang_lib(self): "yolo-system:conf": { "hostname": "foo", "url": [ - { - "proto": "https", - "host": "github.com", - "path": "/CESNET/libyang-python", - "enabled": false - }, { "proto": "http", "host": "barfoo.com", @@ -145,6 +141,12 @@ def test_data_parse_config_json_without_yang_lib(self): "port": 8080, "path": "/index.html", "enabled": true + }, + { + "proto": "https", + "host": "github.com", + "path": "/CESNET/libyang-python", + "enabled": false } ], "number": [ @@ -254,7 +256,9 @@ def test_data_parse_state_json(self): """ def test_data_parse_config_xml(self): - dnode = self.ctx.parse_data_mem(self.XML_CONFIG, "xml", validate_present=True) + dnode = self.ctx.parse_data_mem( + self.XML_CONFIG, "xml", validate_present=True, ordered=True + ) self.assertIsInstance(dnode, DContainer) try: xml = dnode.print_mem("xml", with_siblings=True, trim_default_values=True) @@ -509,6 +513,7 @@ def test_data_from_dict_module_free_func(self): } def test_data_from_dict_module_with_prefix(self): + self.maxDiff = None module = self.ctx.get_module("yolo-system") dnode = module.parse_data_dict( self.DICT_CONFIG_WITH_PREFIX, strict=True, validate_present=True @@ -853,17 +858,17 @@ def test_data_diff(self): TREE = [ "/yolo-system:conf", "/yolo-system:conf/hostname", - "/yolo-system:conf/url[proto='https'][host='github.com']", - "/yolo-system:conf/url[proto='https'][host='github.com']/proto", - "/yolo-system:conf/url[proto='https'][host='github.com']/host", - "/yolo-system:conf/url[proto='https'][host='github.com']/path", - "/yolo-system:conf/url[proto='https'][host='github.com']/enabled", "/yolo-system:conf/url[proto='http'][host='foobar.com']", "/yolo-system:conf/url[proto='http'][host='foobar.com']/proto", "/yolo-system:conf/url[proto='http'][host='foobar.com']/host", "/yolo-system:conf/url[proto='http'][host='foobar.com']/port", "/yolo-system:conf/url[proto='http'][host='foobar.com']/path", "/yolo-system:conf/url[proto='http'][host='foobar.com']/enabled", + "/yolo-system:conf/url[proto='https'][host='github.com']", + "/yolo-system:conf/url[proto='https'][host='github.com']/proto", + "/yolo-system:conf/url[proto='https'][host='github.com']/host", + "/yolo-system:conf/url[proto='https'][host='github.com']/path", + "/yolo-system:conf/url[proto='https'][host='github.com']/enabled", "/yolo-system:conf/number[.='1000']", "/yolo-system:conf/number[.='2000']", "/yolo-system:conf/number[.='3000']", @@ -906,7 +911,7 @@ def test_find_all(self): } ] } - self.assertEqual(urls[0].print_dict(absolute=False), expected_url) + self.assertEqual(urls[1].print_dict(absolute=False), expected_url) finally: dnode.free() @@ -1090,7 +1095,9 @@ def test_dnode_leafref_linking(self): "yolo-leafref-extended:ref1": "val1" }""" self.ctx.destroy() - self.ctx = Context(YANG_DIR, leafref_extended=True, leafref_linking=True) + self.ctx = Context( + YANG_DIR, leafref_extended=True, leafref_linking=True, compile_obsolete=True + ) mod = self.ctx.load_module("yolo-leafref-extended") self.assertIsInstance(mod, Module) dnode1 = self.ctx.parse_data_mem(MAIN, "json", parse_only=True) @@ -1121,7 +1128,7 @@ def test_dnode_builtin_plugins_only(self): MAIN = {"yolo-nodetypes:ip-address": "test"} self.tearDown() gc.collect() - self.ctx = Context(YANG_DIR, builtin_plugins_only=True) + self.ctx = Context(YANG_DIR, builtin_plugins_only=True, compile_obsolete=True) module = self.ctx.load_module("yolo-nodetypes") dnode = dict_to_dnode(MAIN, module, None, validate=False, store_only=True) self.assertIsInstance(dnode, DLeaf) @@ -1140,7 +1147,7 @@ def test_merge_builtin_plugins_only(self): MAIN = {"yolo-nodetypes:ip-address": "test"} self.tearDown() gc.collect() - self.ctx = Context(YANG_DIR, builtin_plugins_only=True) + self.ctx = Context(YANG_DIR, builtin_plugins_only=True, compile_obsolete=True) module = self.ctx.load_module("yolo-nodetypes") dnode = module.parse_data_dict(MAIN, validate=False, store_only=True) self.assertIsInstance(dnode, DLeaf) diff --git a/tests/test_diff.py b/tests/test_diff.py index d4b7e87e..2263e344 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -97,7 +97,9 @@ class DiffTest(unittest.TestCase): ) def test_diff(self): - with Context(OLD_YANG_DIR) as ctx_old, Context(NEW_YANG_DIR) as ctx_new: + with Context(OLD_YANG_DIR, compile_obsolete=True) as ctx_old, Context( + NEW_YANG_DIR, compile_obsolete=True + ) as ctx_new: mod = ctx_old.load_module("yolo-system") mod.feature_enable_all() mod = ctx_new.load_module("yolo-system") diff --git a/tests/test_schema.py b/tests/test_schema.py index b1862ab8..a1ca412d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -272,7 +272,7 @@ def test_iffeature_dump(self): # ------------------------------------------------------------------------------------- class ContainerTest(unittest.TestCase): def setUp(self): - self.ctx = Context(YANG_DIR) + self.ctx = Context(YANG_DIR, compile_obsolete=True) mod = self.ctx.load_module("yolo-system") mod.feature_enable_all() self.container = next(self.ctx.find_path("/yolo-system:conf")) @@ -549,7 +549,7 @@ def test_rpc_parsed(self): # ------------------------------------------------------------------------------------- class LeafTypeTest(unittest.TestCase): def setUp(self): - self.ctx = Context(YANG_DIR) + self.ctx = Context(YANG_DIR, compile_obsolete=True) self.ctx.load_module("yolo-system") def tearDown(self): diff --git a/tests/yang/yang-library.json b/tests/yang/yang-library.json index 3dbf2568..f9664845 100644 --- a/tests/yang/yang-library.json +++ b/tests/yang/yang-library.json @@ -11,6 +11,21 @@ ] }, "ietf-yang-library:yang-library": { + "module-set": [ + { + "name": "complete", + "module": [ + { + "name": "yang", + "revision": "2025-01-29", + "namespace": "urn:ietf:params:xml:ns:yang:1", + "location": [ + "file://@internal" + ] + } + ] + } + ], "content-id": "321566" } } From d3a2f352606ae8835168d6a289cdfbc5fe4da103 Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Tue, 13 Jan 2026 16:40:15 +0100 Subject: [PATCH 111/115] schema: remove error after lyd_value_validate_dflt calls libyang returns the realtype field when the store callback return code is LY_SUCCESS and LY_EINCOMPLETE. Don't raise an error when lyd_value_validate_dflt returns LY_EINCOMPLETE. The function can then use the val_type_cdata field. Add a leafref with a default value to yolo-nodetypes.yang. In this case, lyd_value_validate_dflt returns LY_EINCOMPLETE. In test_schema.py, check that the default function returns a correct str value. Link: https://github.com/CESNET/libyang/blob/master/src/tree_data_common.c#L682 Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> --- libyang/schema.py | 4 ++-- tests/test_schema.py | 6 ++++++ tests/yang/yolo/yolo-nodetypes.yang | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index f104a874..e8bbe704 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1545,7 +1545,7 @@ def default(self) -> Union[None, bool, int, str, float]: val_type_cdata, ffi.NULL, ) - if ret != lib.LY_SUCCESS: + if ret not in (lib.LY_SUCCESS, lib.LY_EINCOMPLETE): raise self.context.error("Unable to get real type of default value") self.cdata_default_realtype = Type(self.context, val_type_cdata[0], None) @@ -1619,7 +1619,7 @@ def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: val_type_cdata, ffi.NULL, ) - if ret != lib.LY_SUCCESS: + if ret not in (lib.LY_SUCCESS, lib.LY_EINCOMPLETE): raise self.context.error("Unable to get real type of default value") self.cdata_default_realtypes.append( Type(self.context, val_type_cdata[0], None) diff --git a/tests/test_schema.py b/tests/test_schema.py index a1ca412d..52a7af71 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -732,6 +732,9 @@ def test_must(self): def test_leaf_default(self): leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) self.assertIsInstance(leaf.default(), float) + leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/leafref1")) + self.assertIsInstance(leaf.default(), str) + self.assertEqual("ASD", leaf.default()) def test_leaf_parsed(self): leaf = next(self.ctx.find_path("/yolo-nodetypes:conf/percentage")) @@ -789,6 +792,9 @@ def test_leaflist_defaults(self): leaflist = next(self.ctx.find_path("/yolo-nodetypes:conf/integers")) for d in leaflist.defaults(): self.assertIsInstance(d, int) + leaflist3 = next(self.ctx.find_path("/yolo-nodetypes:conf/leaf-list3")) + for d in leaflist3.defaults(): + self.assertIsInstance(d, str) def test_leaf_list_min_max(self): leaflist1 = next(self.ctx.find_path("/yolo-nodetypes:conf/leaf-list1")) diff --git a/tests/yang/yolo/yolo-nodetypes.yang b/tests/yang/yolo/yolo-nodetypes.yang index 5b994752..9926b1aa 100644 --- a/tests/yang/yolo/yolo-nodetypes.yang +++ b/tests/yang/yolo/yolo-nodetypes.yang @@ -93,6 +93,20 @@ module yolo-nodetypes { leaf-list leaf-list2 { type string; } + + leaf leafref1 { + type leafref { + path "/records/name"; + } + default "ASD"; + } + + leaf-list leaf-list3 { + type leafref { + path "/records/name"; + } + default "ASD"; + } } leaf test1 { From e175b5e9d7d7ef480ca2955a48bccbace5c08dfe Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Fri, 6 Feb 2026 12:05:20 +0100 Subject: [PATCH 112/115] check: fix Fixes trailer regex The sed command with the regex fails to keep the sha1 and remove the commit title from the 'Fixes' trailer. Then the git command fails to find the commit. A commit with 'Fixes' can not be validated by the script. Fix the regex to extract the sha1. Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> --- check-commits.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-commits.sh b/check-commits.sh index 936236cf..99ef06cf 100755 --- a/check-commits.sh +++ b/check-commits.sh @@ -104,7 +104,7 @@ for rev in $revisions; do if [ -z "$value" ]; then continue fi - fixes_rev=$(echo "$value" | sed -En 's/([A-Fa-f0-9]{7,}[[:space:]]\(".*"\))/\1/p') + fixes_rev=$(echo "$value" | sed -En 's/([A-Fa-f0-9]{7,})[[:space:]]\(".*"\)/\1/p') if ! git cat-file commit "$fixes_rev" >/dev/null; then err "trailer '$value' does not refer to a known commit" fi From f1898d5a53970d025ccba10e13330991604034c2 Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Thu, 29 Jan 2026 11:06:27 +0100 Subject: [PATCH 113/115] context: remove debug print This log was introduced for debug. Remove it. Fixes: 0a4b09f461a3 ("cffi: allows to usage of libyang v4") Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> --- libyang/context.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libyang/context.py b/libyang/context.py index 21db1628..d0309d70 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -262,7 +262,6 @@ def __init__( fmt = lib.LYD_JSON else: fmt = lib.LYD_XML - print("steweg", search_path, yanglib_path, yanglib_fmt, options) ret = lib.ly_ctx_new_ylpath( str2c(search_path), str2c(yanglib_path), fmt, options, ctx ) From 6c0727fa2c30de573b88c9806bc3a8d69030de04 Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Tue, 3 Feb 2026 15:46:34 +0100 Subject: [PATCH 114/115] schema: get features from compiled module It is not always possible to get the features from the parsed module. It is the case for a module obtained from a sysrepo printed context. Introduce compiled_enabled_features that returns the features from the compiled object. Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> --- libyang/schema.py | 5 +++++ tests/test_schema.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/libyang/schema.py b/libyang/schema.py index e8bbe704..f7fda210 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -110,6 +110,11 @@ def features(self) -> Iterator["Feature"]: for i in features_list: yield Feature(self.context, i) + def compiled_enabled_features(self) -> Iterator[str]: + if self.cdata.compiled: + for f in ly_array_iter(self.cdata.compiled.features): + yield c2str(f) + def get_feature(self, name: str) -> "Feature": for f in self.features(): if f.name() == name: diff --git a/tests/test_schema.py b/tests/test_schema.py index 52a7af71..2c6ae1bc 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -108,6 +108,11 @@ def test_mod_features(self): features = list(self.module.features()) self.assertEqual(len(features), 2) + def test_mod_compiled_enabled_features(self): + self.module.feature_enable("*") + features = list(self.module.compiled_enabled_features()) + self.assertEqual(len(features), 2) + def test_mod_get_feature(self): self.module.feature_enable("turbo-boost") feature = self.module.get_feature("turbo-boost") From a69eb4a6865069de1a8b73d76b2d7427893214cf Mon Sep 17 00:00:00 2001 From: Jeremie Leska <jeremie.leska@6wind.com> Date: Fri, 20 Feb 2026 15:26:30 +0100 Subject: [PATCH 115/115] schema: handle None parsed objects in type creation In libyang4, when dealing with printed contexts, cdata_parsed is not set. Check it before using it to get the node type. Fixes: 0a4b09f461a3 ("cffi: allows to usage of libyang v4") Signed-off-by: Jeremie Leska <jeremie.leska@6wind.com> --- libyang/schema.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libyang/schema.py b/libyang/schema.py index f7fda210..d3f54654 100644 --- a/libyang/schema.py +++ b/libyang/schema.py @@ -1568,7 +1568,11 @@ def units(self) -> Optional[str]: return c2str(self.cdata_leaf.units) def type(self) -> Type: - return Type(self.context, self.cdata_leaf.type, self.cdata_leaf_parsed.type) + return Type( + self.context, + self.cdata_leaf.type, + self.cdata_leaf_parsed.type if self.cdata_leaf_parsed else None, + ) def is_key(self) -> bool: if self.cdata_leaf.flags & lib.LYS_KEY: @@ -1600,7 +1604,9 @@ def units(self) -> Optional[str]: def type(self) -> Type: return Type( - self.context, self.cdata_leaflist.type, self.cdata_leaflist_parsed.type + self.context, + self.cdata_leaflist.type, + self.cdata_leaflist_parsed.type if self.cdata_leaflist_parsed else None, ) def defaults(self) -> Iterator[Union[None, bool, int, str, float]]: