From 479335e2a261b9751660b046eec22186e10942e9 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Thu, 14 May 2020 07:37:37 +0200 Subject: [PATCH 1/5] Add submodel support --- .gitignore | 2 ++ yangson/datamodel.py | 57 +++++++++++++++++++++++++++++++++++++++++-- yangson/exceptions.py | 4 +++ yangson/instance.py | 32 +++++++++++++++++++++--- yangson/schemadata.py | 13 +++++++++- yangson/schemanode.py | 9 ++++--- yangson/xmlparser.py | 12 ++++----- 7 files changed, 113 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 4c568472..5d555ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ TAGS yangson.egg-info .kdev4 *.kdev4 +.pybuild + diff --git a/yangson/datamodel.py b/yangson/datamodel.py index bcf9d49d..424334dc 100644 --- a/yangson/datamodel.py +++ b/yangson/datamodel.py @@ -27,11 +27,11 @@ from typing import Optional, Tuple import xml.etree.ElementTree as ET from .enumerations import ContentType -from .exceptions import BadYangLibraryData +from .exceptions import BadYangLibraryData, BadRootNode from .instance import (InstanceRoute, InstanceIdParser, ResourceIdParser, RootNode) from .schemadata import SchemaData, SchemaContext -from .schemanode import DataNode, SchemaTreeNode, RawObject, SchemaNode +from .schemanode import DataNode, SchemaTreeNode, RawObject, SchemaNode, ContainerNode from .typealiases import DataPath, SchemaPath @@ -88,6 +88,43 @@ def __init__(self, yltxt: str, mod_path: Tuple[str] = (".",), "Data model ID: " + self.yang_library["ietf-yang-library:modules-state"] ["module-set-id"]) + self.subschema = {} + + def add_submodel(self, container: ContainerNode, submodel: "DataModel"): + if container.schema_root() != self.schema: + raise BadRootNode(container.iname()) + + # update yang library + yl_modules = self.yang_library['ietf-yang-library:modules-state']['module'] + existing = list() + for module in yl_modules: + existing.append((module['name'], module['revision'])) + + sm_modules = submodel.yang_library['ietf-yang-library:modules-state']['module'] + for module in sm_modules: + if (module['name'], module['revision']) not in existing: + yl_modules.append(module) + + # update schema data + self.schema_data.add(submodel.schema_data) + + # update container + for subchild in submodel.schema.children: + container.children.append(subchild) + subchild.parent = container + + self.schema.subschema[(subchild.name, subchild.ns)] = subchild.data_path() + if subchild.mandatory: + container._mandatory_children.add(subchild) + + if self.schema.description.startswith('Data model ID: '): + self.schema.description = ( + "Data model ID: " + + self.yang_library["ietf-yang-library:modules-state"] + ["module-set-id"]) + + # rebuild schema patterns + self.schema._make_schema_patterns() def module_set_id(self) -> str: """Compute unique id of YANG modules comprising the data model. @@ -174,9 +211,25 @@ def clear_val_counters(self): self.schema.clear_val_counters() def parse_instance_id(self, text: str) -> InstanceRoute: + split = text.split('/') + ns, sep, name = split[1].partition(':') + + if (name, ns) in self.schema.subschema: + text = self.schema.subschema[(name, ns)] + if len(split) >= 2: + text = text + '/' + '/'.join(split[2:]) + return InstanceIdParser(text).parse() def parse_resource_id(self, text: str) -> InstanceRoute: + split = text.split('/') + ns, sep, name = split[1].partition(':') + + if (name, ns) in self.schema.subschema: + text = self.schema.subschema[(name, ns)] + if len(split) >= 2: + text = text + '/' + '/'.join(split[2:]) + return ResourceIdParser(text, self.schema).parse() def schema_digest(self) -> str: diff --git a/yangson/exceptions.py b/yangson/exceptions.py index 515b11b9..fd4674d4 100644 --- a/yangson/exceptions.py +++ b/yangson/exceptions.py @@ -387,6 +387,10 @@ def __str__(self) -> str: return f"{prefix}{self.name} under {super().__str__()}" +class BadRootNode(SchemaNodeException): + '''Wrong root node of schema node''' + pass + class BadSchemaNodeType(SchemaNodeException): """A schema node is of a wrong type.""" diff --git a/yangson/instance.py b/yangson/instance.py index 360090e4..0b763c90 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -412,9 +412,10 @@ def to_xml(self, filter: OutputFilter = OutputFilter(), elem: ET.Element = None) if elem is None: element = ET.Element(self.schema_node.name) - module = self.schema_data.modules_by_name.get(self.schema_node.ns) + ns = self.schema_node.ns + module = self.schema_data.modules_by_name.get(ns) if not module: - raise MissingModuleNamespace(self.schema_node.ns) + raise MissingModuleNamespace(ns if ns else 'None') element.attrib['xmlns'] = module.xml_namespace else: element = elem @@ -581,6 +582,7 @@ def __init__(self, value: Value, schema_node: "DataNode", schema_data: "SchemaData", timestamp: datetime): super().__init__("/", value, None, schema_node, timestamp) self.schema_data = schema_data + """Dictionary of subschema root qnames to subschema paths""" def up(self) -> None: """Override the superclass method. @@ -899,6 +901,20 @@ def peek_step(self, val: ObjectValue, val: Current value (object). sn: Current schema node. """ + qn = (self.name, self.namespace) + if isinstance(sn, "SchemaTreeNode") and qn in sn.subschema: + path = sn.subschema.get((self.name, self.namespace)) + c_schema = sn + c_value = val + try: + for dp in path.split('/'): + name, sep, ns = dp.partition(':') + c_schema = c_schema.get_data_child(name, ns if ns else None) + c_value = c_value[c_schema.iname()] + return c_value, c_schema + except (IndexError, KeyError, TypeError): + return (None, c_schema) + cn = sn.get_data_child(self.name, self.namespace) try: return (val[cn.iname()], cn) @@ -911,6 +927,15 @@ def goto_step(self, inst: InstanceNode) -> InstanceNode: Args: inst: Current instance. """ + sn = inst.schema_node + qn = (self.name, self.namespace) + + if isinstance(sn, SchemaTreeNode)and qn in sn.subschema: + path = sn.subschema.get((self.name, self.namespace)) + child = inst + for dp in path[1:].split('/'): + child = child[dp] + return child return inst[self.iname()] @@ -1215,4 +1240,5 @@ def _key_predicates(self) -> EntryKeys: from .schemanode import (AnydataNode, CaseNode, ChoiceNode, DataNode, # NOQA InternalNode, LeafNode, LeafListNode, ListNode, - RpcActionNode, SequenceNode, TerminalNode) + RpcActionNode, SchemaTreeNode, SequenceNode, + TerminalNode) diff --git a/yangson/schemadata.py b/yangson/schemadata.py index ba7cf2a1..bef0dc96 100644 --- a/yangson/schemadata.py +++ b/yangson/schemadata.py @@ -102,11 +102,22 @@ def __init__(self, yang_lib: Dict[str, Any], mod_path: List[str]) -> None: """Dictionary of module data.""" self.modules_by_name = {} # type: Dict[str, ModuleData] """Dictionary of module data by module name.""" - self.modules_by_ns = {} + self.modules_by_ns = {} # type: Dict[str, ModuleData] + """Dictionary of module data by xml namespace.""" self._module_sequence = [] # type: List[ModuleId] """List that defines the order of module processing.""" self._from_yang_library(yang_lib) + def add(self, child: "SchemaData"): + """Add the schemadata of a subschema to this one""" + self.identity_adjs = {**self.identity_adjs, **child.identity_adjs} + self.implement = {**self.implement, **child.implement} + self.module_search_path.extend(child.module_search_path) + self.modules = {**self.modules, **child.modules} + self.modules_by_name = {**self.modules_by_name, **child.modules_by_name} + self.modules_by_ns = {**self.modules_by_ns, **child.modules_by_ns} + self._module_sequence.extend(child._module_sequence) + def _from_yang_library(self, yang_lib: Dict[str, Any]) -> None: """Set the schema structures from YANG library data. diff --git a/yangson/schemanode.py b/yangson/schemanode.py index 867636be..83291618 100644 --- a/yangson/schemanode.py +++ b/yangson/schemanode.py @@ -501,8 +501,6 @@ def _process_xmlobj_child( if ch.iname() not in res: res[ch.iname()] = ch.from_xml(rval, npath, fqn) else: - if isinstance(ch, LeafNode): - print('child schema type:', type(ch.type)) res[ch.iname()] = ch.from_xml(xmlchild, npath) def _process_metadata(self, rmo: RawMetadataObject, @@ -780,11 +778,15 @@ def __init__(self, schemadata: "SchemaData" = None): super().__init__() self.annotations: Dict[QualName, Annotation] = {} self.schema_data = schemadata + self.subschema: Dict[QualName, "InstanceRoute"] = {} def data_parent(self) -> InternalNode: """Override the superclass method.""" return self.parent + def add_subschema(self, container: SchemaNode): + self.subschema[container.qual_name] = container.data_path() + def _annotation_stmt(self, stmt: Statement, sctx: SchemaContext) -> None: """Handle annotation statement.""" if not sctx.schema_data.if_features(stmt, sctx.text_mid): @@ -1086,7 +1088,7 @@ def from_xml(self, rval: ET.Element, jptr: JSONPointer = "", else: for xmlchild in rval: self._process_xmlarray_child( - res, xmlchild, tagname, jptr + "/" + str(i)) + res, xmlchild, jptr + "/" + str(i)) i = i + 1 return res @@ -1108,6 +1110,7 @@ def _process_xmlarray_child( res.append(child) else: child = None + return child def entry_from_raw(self, rval: RawEntry, diff --git a/yangson/xmlparser.py b/yangson/xmlparser.py index ddca82b7..a5a4006f 100644 --- a/yangson/xmlparser.py +++ b/yangson/xmlparser.py @@ -36,8 +36,7 @@ def __init__(self, source: str = None): super().__init__(events=['start', 'start-ns', 'end-ns', 'end']) self._root = None - self._nslist = list() - self._namespaces = {} + self._namespaces = list() if source: self.feed(source) @@ -57,15 +56,14 @@ def parse(self): for ev_type, ev_data in super().read_events(): if ev_type == 'start-ns': ns_name, ns_url = ev_data - self._namespaces[ns_name] = ns_url - self._nslist.append(ns_name) + self._namespaces.append((ns_name, ns_url)) elif ev_type == 'end-ns': - ns_name = self._nslist.pop() - del self._namespaces[ns_name] + ns_name = self._namespaces.pop() + elif ev_type == 'start' and self._root is None: self._root = ev_data elif ev_type == 'end': - for ns_name, ns_url in self._namespaces.items(): + for ns_name, ns_url in self._namespaces: attr = 'xmlns' if ns_name == '' else 'xmlns:'+ns_name ev_data.attrib[attr] = ns_url From a25d030611e423f41ac7c3d2e4023cb3ba299619 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Mon, 18 May 2020 09:51:27 +0200 Subject: [PATCH 2/5] Fix instanceof call --- yangson/instance.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yangson/instance.py b/yangson/instance.py index 8d364541..22f544dc 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -234,7 +234,8 @@ def ita(): return ita() if isinstance(self.value, ObjectValue): return iter(self._member_names()) - raise InstanceValueError(self.json_pointer(), + raise InstanceValueError( + self.json_pointer(), "{} is a scalar instance".format(str(type(self.value)))) def is_internal(self) -> bool: @@ -916,7 +917,7 @@ def peek_step(self, val: ObjectValue, sn: Current schema node. """ qn = (self.name, self.namespace) - if isinstance(sn, "SchemaTreeNode") and qn in sn.subschema: + if isinstance(sn, SchemaTreeNode) and qn in sn.subschema: path = sn.subschema.get((self.name, self.namespace)) c_schema = sn c_value = val From d9b56c81045a983f2cd5fbdb9f441927106ec400 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Mon, 25 May 2020 14:38:06 +0200 Subject: [PATCH 3/5] Add full_peek --- yangson/instance.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/yangson/instance.py b/yangson/instance.py index fcfa288e..9d0288f6 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -339,8 +339,8 @@ def goto(self, iroute: "InstanceRoute") -> "InstanceNode": inst = sel.goto_step(inst) return inst - def peek(self, iroute: "InstanceRoute") -> Optional[Value]: - """Return a value within the receiver's subtree. + def full_peek(self, iroute: "InstanceRoute") -> Optional[Tuple]: + """Return a value and schema within the receiver's subtree. Args: iroute: Instance route (relative to the receiver). @@ -348,10 +348,18 @@ def peek(self, iroute: "InstanceRoute") -> Optional[Value]: val = self.value sn = self.schema_node for sel in iroute: - val, sn = sel.peek_step(val, sn) if val is None: - return None - return val + return (None, None) + val, sn = sel.peek_step(val, sn) + return (val, sn) + + def peek(self, iroute: "InstanceRoute") -> Optional[Value]: + """Return a value within the receiver's subtree. + + Args: + iroute: Instance route (relative to the receiver). + """ + return self.full_peek(iroute)[0] def validate(self, scope: ValidationScope = ValidationScope.all, ctype: ContentType = ContentType.config) -> None: From 12853e4521bf09b89bbf3d5842d2859f9a6361b6 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Thu, 4 Jun 2020 12:16:57 +0200 Subject: [PATCH 4/5] Make sure module path is a list --- yang-modules/test/test-data.json | 34 ++++++++++++++++++++++++++++++++ yang-modules/test/test.py | 13 ++++++++++++ yangson/datamodel.py | 2 +- yangson/instance.py | 3 ++- 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 yang-modules/test/test-data.json create mode 100644 yang-modules/test/test.py diff --git a/yang-modules/test/test-data.json b/yang-modules/test/test-data.json new file mode 100644 index 00000000..2d83b8a0 --- /dev/null +++ b/yang-modules/test/test-data.json @@ -0,0 +1,34 @@ +{ + "test:llistB": ["::1", "127.0.0.1"], + "test:leafX": 53531, + "test:contA": { + "leafB": 9, + "listA": [{ + "leafE": "C0FFEE", + "leafF": true, + "contD": { + "leafG": "foo1-bar", + "contE": { + "leafJ": [null], + "leafP": 10 + } + } + }, + { + "leafE": "ABBA", + "leafW": 9, + "leafF": false + }], + "testb:leafS": + "/test:contA/listA[leafE='C0FFEE'][leafF='true']/contD/contE/leafP", + "testb:leafR": "C0FFEE", + "testb:leafT": "test:CC-BY", + "testb:leafV": 99, + "testb:leafN": "hi!" + }, + "test:contT": { + "bits": "dos cuatro", + "decimal64": 4.50, + "enumeration": "Hearts" + } +} diff --git a/yang-modules/test/test.py b/yang-modules/test/test.py new file mode 100644 index 00000000..1b589205 --- /dev/null +++ b/yang-modules/test/test.py @@ -0,0 +1,13 @@ +import json +from yangson import DataModel +from yangson.enumerations import ContentType +import xml.etree.ElementTree as ET + +dm = DataModel.from_file( + 'yang-library.json', ['.', '../ietf']) +with open('test-data.json') as infile: + ri = json.load(infile) +inst = dm.from_raw(ri) +root = inst.to_xml() + +print(ET.tostring(root)) diff --git a/yangson/datamodel.py b/yangson/datamodel.py index 424334dc..739ca13f 100644 --- a/yangson/datamodel.py +++ b/yangson/datamodel.py @@ -80,7 +80,7 @@ def __init__(self, yltxt: str, mod_path: Tuple[str] = (".",), self.yang_library = json.loads(yltxt) except json.JSONDecodeError as e: raise BadYangLibraryData(str(e)) from None - self.schema_data = SchemaData(self.yang_library, mod_path) + self.schema_data = SchemaData(self.yang_library, list(mod_path)) self.schema = SchemaTreeNode(self.schema_data) self.schema._ctype = ContentType.all self._build_schema() diff --git a/yangson/instance.py b/yangson/instance.py index fa00be96..f28704b3 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -48,7 +48,8 @@ __all__ = ["InstanceNode", "RootNode", "ObjectMember", "ArrayEntry", "InstanceIdParser", "ResourceIdParser", "InstanceRoute", - "InstanceException", "InstanceValueError", "NonexistentInstance"] + "InstanceException", "InstanceValueError", "NonexistentInstance", + "OutputFilter"] class OutputFilter: From b784ef1505e3c763fb15d87fdfb0c6d10d227ade Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Tue, 1 Dec 2020 11:13:33 +0100 Subject: [PATCH 5/5] Don't remove namespace for root node in instance --- yangson/instance.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yangson/instance.py b/yangson/instance.py index 20da1d6c..ffd9cc66 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -575,9 +575,10 @@ def _member_names(self) -> List[InstanceName]: return [m for m in self.value if not m.startswith("@")] def _member(self, name: InstanceName) -> "ObjectMember": - pts = name.partition(":") - if pts[1] and pts[0] == self.namespace: - name = pts[2] + if not type(self) is RootNode: + pts = name.partition(":") + if pts[1] and pts[0] == self.namespace: + name = pts[2] sibs = self.value.copy() try: return ObjectMember(