From 797d69e7747df242b3cfb3a8a33ac7d30c1422e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Walstr=C3=B6m?= Date: Thu, 7 Dec 2023 10:45:31 +0100 Subject: [PATCH] Add support for downloading required yang models automatic --- test/docker/Dockerfile | 4 + test/docker/init-venv.sh | 1 + .../ietf-netconf-monitoring@2010-10-04.yang | 558 ++++++++++++++++++ test/env | 24 +- test/infamy/netconf.py | 59 +- 5 files changed, 624 insertions(+), 22 deletions(-) create mode 100644 test/docker/yang/ietf-netconf-monitoring@2010-10-04.yang diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index 691e5eb5..d1993c59 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -31,4 +31,8 @@ ENV PATH="${PATH}:/usr/lib/ruby/gems/3.2.0/bin" # Install all python packages used by the tests COPY init-venv.sh /root COPY pip-requirements.txt /root + +# Add bootstrap YANG models, the rest will be downloaded from the device +ADD yang /root/yang + RUN ~/init-venv.sh ~/pip-requirements.txt diff --git a/test/docker/init-venv.sh b/test/docker/init-venv.sh index 0ee24e97..8dd23a82 100755 --- a/test/docker/init-venv.sh +++ b/test/docker/init-venv.sh @@ -3,6 +3,7 @@ set -e python3 -m venv ~/.infix-test-venv +cp -r ~/yang ~/.infix-test-venv/yangdir . ~/.infix-test-venv/bin/activate python3 -m pip install --upgrade pip diff --git a/test/docker/yang/ietf-netconf-monitoring@2010-10-04.yang b/test/docker/yang/ietf-netconf-monitoring@2010-10-04.yang new file mode 100644 index 00000000..8d2380e7 --- /dev/null +++ b/test/docker/yang/ietf-netconf-monitoring@2010-10-04.yang @@ -0,0 +1,558 @@ +module ietf-netconf-monitoring { + + namespace "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring"; + prefix "ncm"; + + import ietf-yang-types { prefix yang; } + import ietf-inet-types { prefix inet; } + + organization + "IETF NETCONF (Network Configuration) Working Group"; + + contact + "WG Web: + WG List: + + WG Chair: Mehmet Ersue + + + WG Chair: Bert Wijnen + + + Editor: Mark Scott + + + Editor: Martin Bjorklund + "; + + description + "NETCONF Monitoring Module. + All elements in this module are read-only. + + Copyright (c) 2010 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject + to the license terms contained in, the Simplified BSD + License set forth in Section 4.c of the IETF Trust's + Legal Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 6022; see + the RFC itself for full legal notices."; + + revision 2010-10-04 { + description + "Initial revision."; + reference + "RFC 6022: YANG Module for NETCONF Monitoring"; + } + + typedef netconf-datastore-type { + type enumeration { + enum running; + enum candidate; + enum startup; + } + description + "Enumeration of possible NETCONF datastore types."; + reference + "RFC 4741: NETCONF Configuration Protocol"; + } + + identity transport { + description + "Base identity for NETCONF transport types."; + } + + identity netconf-ssh { + base transport; + description + "NETCONF over Secure Shell (SSH)."; + reference + "RFC 4742: Using the NETCONF Configuration Protocol + over Secure SHell (SSH)"; + } + + identity netconf-soap-over-beep { + base transport; + description + "NETCONF over Simple Object Access Protocol (SOAP) over + Blocks Extensible Exchange Protocol (BEEP)."; + reference + "RFC 4743: Using NETCONF over the Simple Object + Access Protocol (SOAP)"; + } + + identity netconf-soap-over-https { + base transport; + description + "NETCONF over Simple Object Access Protocol (SOAP) + over Hypertext Transfer Protocol Secure (HTTPS)."; + reference + "RFC 4743: Using NETCONF over the Simple Object + Access Protocol (SOAP)"; + } + + identity netconf-beep { + base transport; + description + "NETCONF over Blocks Extensible Exchange Protocol (BEEP)."; + reference + "RFC 4744: Using the NETCONF Protocol over the + Blocks Extensible Exchange Protocol (BEEP)"; + } + + identity netconf-tls { + base transport; + description + "NETCONF over Transport Layer Security (TLS)."; + reference + "RFC 5539: NETCONF over Transport Layer Security (TLS)"; + } + + identity schema-format { + description + "Base identity for data model schema languages."; + } + + identity xsd { + base schema-format; + description + "W3C XML Schema Definition."; + reference + "W3C REC REC-xmlschema-1-20041028: + XML Schema Part 1: Structures"; + } + + identity yang { + base schema-format; + description + "The YANG data modeling language for NETCONF."; + reference + "RFC 6020: YANG - A Data Modeling Language for the + Network Configuration Protocol (NETCONF)"; + } + + identity yin { + base schema-format; + description + "The YIN syntax for YANG."; + reference + "RFC 6020: YANG - A Data Modeling Language for the + Network Configuration Protocol (NETCONF)"; + } + + identity rng { + base schema-format; + description + "Regular Language for XML Next Generation (RELAX NG)."; + reference + "ISO/IEC 19757-2:2008: RELAX NG"; + } + + identity rnc { + base schema-format; + description + "Relax NG Compact Syntax"; + reference + "ISO/IEC 19757-2:2008: RELAX NG"; + } + + grouping common-counters { + description + "Counters that exist both per session, and also globally, + accumulated from all sessions."; + + leaf in-rpcs { + type yang:zero-based-counter32; + description + "Number of correct messages received."; + } + leaf in-bad-rpcs { + type yang:zero-based-counter32; + description + "Number of messages received when an message was expected, + that were not correct messages. This includes XML parse + errors and errors on the rpc layer."; + } + leaf out-rpc-errors { + type yang:zero-based-counter32; + description + "Number of messages sent that contained an + element."; + } + leaf out-notifications { + type yang:zero-based-counter32; + description + "Number of messages sent."; + } + } + + container netconf-state { + config false; + description + "The netconf-state container is the root of the monitoring + data model."; + + container capabilities { + description + "Contains the list of NETCONF capabilities supported by the + server."; + + leaf-list capability { + type inet:uri; + description + "List of NETCONF capabilities supported by the server."; + } + } + + container datastores { + description + "Contains the list of NETCONF configuration datastores."; + + list datastore { + key name; + description + "List of NETCONF configuration datastores supported by + the NETCONF server and related information."; + + leaf name { + type netconf-datastore-type; + description + "Name of the datastore associated with this list entry."; + } + container locks { + presence + "This container is present only if the datastore + is locked."; + description + "The NETCONF and operations allow + a client to lock specific resources in a datastore. The + NETCONF server will prevent changes to the locked + resources by all sessions except the one that acquired + the lock(s). + + Monitoring information is provided for each datastore + entry including details such as the session that acquired + the lock, the type of lock (global or partial) and the + list of locked resources. Multiple locks per datastore + are supported."; + + grouping lock-info { + description + "Lock related parameters, common to both global and + partial locks."; + + leaf locked-by-session { + type uint32; + mandatory true; + description + "The session ID of the session that has locked + this resource. Both a global lock and a partial + lock MUST contain the NETCONF session-id. + + If the lock is held by a session that is not managed + by the NETCONF server (e.g., a CLI session), a session + id of 0 (zero) is reported."; + reference + "RFC 4741: NETCONF Configuration Protocol"; + } + leaf locked-time { + type yang:date-and-time; + mandatory true; + description + "The date and time of when the resource was + locked."; + } + } + + choice lock-type { + description + "Indicates if a global lock or a set of partial locks + are set."; + + container global-lock { + description + "Present if the global lock is set."; + uses lock-info; + } + + list partial-lock { + key lock-id; + description + "List of partial locks."; + reference + "RFC 5717: Partial Lock Remote Procedure Call (RPC) for + NETCONF"; + + leaf lock-id { + type uint32; + description + "This is the lock id returned in the + response."; + } + uses lock-info; + leaf-list select { + type yang:xpath1.0; + min-elements 1; + description + "The xpath expression that was used to request + the lock. The select expression indicates the + original intended scope of the lock."; + } + leaf-list locked-node { + type instance-identifier; + description + "The list of instance-identifiers (i.e., the + locked nodes). + + The scope of the partial lock is defined by the list + of locked nodes."; + } + } + } + } + } + } + + container schemas { + description + "Contains the list of data model schemas supported by the + server."; + + list schema { + key "identifier version format"; + + description + "List of data model schemas supported by the server."; + + leaf identifier { + type string; + description + "Identifier to uniquely reference the schema. The + identifier is used in the operation and may + be used for other purposes such as file retrieval. + + For modeling languages that support or require a data + model name (e.g., YANG module name) the identifier MUST + match that name. For YANG data models, the identifier is + the name of the module or submodule. In other cases, an + identifier such as a filename MAY be used instead."; + } + leaf version { + type string; + description + "Version of the schema supported. Multiple versions MAY be + supported simultaneously by a NETCONF server. Each + version MUST be reported individually in the schema list, + i.e., with same identifier, possibly different location, + but different version. + + For YANG data models, version is the value of the most + recent YANG 'revision' statement in the module or + submodule, or the empty string if no 'revision' statement + is present."; + } + leaf format { + type identityref { + base schema-format; + } + description + "The data modeling language the schema is written + in (currently xsd, yang, yin, rng, or rnc). + For YANG data models, 'yang' format MUST be supported and + 'yin' format MAY also be provided."; + } + leaf namespace { + type inet:uri; + mandatory true; + description + "The XML namespace defined by the data model. + + For YANG data models, this is the module's namespace. + If the list entry describes a submodule, this field + contains the namespace of the module to which the + submodule belongs."; + } + leaf-list location { + type union { + type enumeration { + enum "NETCONF"; + } + type inet:uri; + } + description + "One or more locations from which the schema can be + retrieved. This list SHOULD contain at least one + entry per schema. + + A schema entry may be located on a remote file system + (e.g., reference to file system for ftp retrieval) or + retrieved directly from a server supporting the + operation (denoted by the value 'NETCONF')."; + } + } + } + + container sessions { + description + "The sessions container includes session-specific data for + NETCONF management sessions. The session list MUST include + all currently active NETCONF sessions."; + + list session { + key session-id; + description + "All NETCONF sessions managed by the NETCONF server + MUST be reported in this list."; + + leaf session-id { + type uint32 { + range "1..max"; + } + description + "Unique identifier for the session. This value is the + NETCONF session identifier, as defined in RFC 4741."; + reference + "RFC 4741: NETCONF Configuration Protocol"; + } + leaf transport { + type identityref { + base transport; + } + mandatory true; + description + "Identifies the transport for each session, e.g., + 'netconf-ssh', 'netconf-soap', etc."; + } + leaf username { + type string; + mandatory true; + description + "The username is the client identity that was authenticated + by the NETCONF transport protocol. The algorithm used to + derive the username is NETCONF transport protocol specific + and in addition specific to the authentication mechanism + used by the NETCONF transport protocol."; + } + leaf source-host { + type inet:host; + description + "Host identifier of the NETCONF client. The value + returned is implementation specific (e.g., hostname, + IPv4 address, IPv6 address)"; + } + leaf login-time { + type yang:date-and-time; + mandatory true; + description + "Time at the server at which the session was established."; + } + uses common-counters { + description + "Per-session counters. Zero based with following reset + behaviour: + - at start of a session + - when max value is reached"; + } + } + } + + container statistics { + description + "Statistical data pertaining to the NETCONF server."; + + leaf netconf-start-time { + type yang:date-and-time; + description + "Date and time at which the management subsystem was + started."; + } + leaf in-bad-hellos { + type yang:zero-based-counter32; + description + "Number of sessions silently dropped because an + invalid message was received. This includes + messages with a 'session-id' attribute, bad namespace, and + bad capability declarations."; + } + leaf in-sessions { + type yang:zero-based-counter32; + description + "Number of sessions started. This counter is incremented + when a message with a is sent. + + 'in-sessions' - 'in-bad-hellos' = + 'number of correctly started netconf sessions'"; + } + leaf dropped-sessions { + type yang:zero-based-counter32; + description + "Number of sessions that were abnormally terminated, e.g., + due to idle timeout or transport close. This counter is not + incremented when a session is properly closed by a + operation, or killed by a + operation."; + } + uses common-counters { + description + "Global counters, accumulated from all sessions. + Zero based with following reset behaviour: + - re-initialization of NETCONF server + - when max value is reached"; + } + } + } + + rpc get-schema { + description + "This operation is used to retrieve a schema from the + NETCONF server. + + Positive Response: + The NETCONF server returns the requested schema. + + Negative Response: + If requested schema does not exist, the is + 'invalid-value'. + + If more than one schema matches the requested parameters, the + is 'operation-failed', and is + 'data-not-unique'."; + + input { + leaf identifier { + type string; + mandatory true; + description + "Identifier for the schema list entry."; + } + leaf version { + type string; + description + "Version of the schema requested. If this parameter is not + present, and more than one version of the schema exists on + the server, a 'data-not-unique' error is returned, as + described above."; + } + leaf format { + type identityref { + base schema-format; + } + description + "The data modeling language of the schema. If this + parameter is not present, and more than one formats of + the schema exists on the server, a 'data-not-unique' error + is returned, as described above."; + } + } + output { + anyxml data { + description + "Contains the schema content."; + } + } + } +} diff --git a/test/env b/test/env index 46812db9..b7f0f8b3 100755 --- a/test/env +++ b/test/env @@ -2,21 +2,7 @@ set -e -INFIX_TEST=ghcr.io/kernelkit/infix-test:1.1 - -extract_yang() -{ - local imgfile="$1" - - [ "$imgfile" ] || { true && return; } - - unsquashfs -q -n -f \ - -d $envdir/yangdir \ - "$imgfile" \ - /usr/share/yang - - INFAMY_ARGS="$INFAMY_ARGS -y $envdir/yangdir" -} +INFIX_TEST=ghcr.io/kernelkit/infix-test:1.2 start_topology() { @@ -49,7 +35,7 @@ usage() { cat <] -f -q [...] - test/env [] -C -f -t [...] + test/env [] -C -t [...] Run in a pre-packaged container with all the packages required for running the test suite installed. @@ -61,8 +47,7 @@ usage: test/env [] -f -q [...] namespaces -f - Infix image to test. YANG models from this image are extracted - for use by Infamy. When starting a qeneth network, this image is + Infix image to test. When starting a qeneth network, this image is used on all nodes. -h @@ -167,8 +152,7 @@ fi . "$envdir/bin/activate" export PYTHONPATH="$testdir" - -extract_yang "$imgfile" +INFAMY_ARGS="$INFAMY_ARGS -y $envdir/yangdir" start_topology "$qenethdir" "$imgfile" [ "$topology" ] && INFAMY_ARGS="$INFAMY_ARGS $topology" export INFAMY_ARGS diff --git a/test/infamy/netconf.py b/test/infamy/netconf.py index 955b5d6d..2e85120c 100644 --- a/test/infamy/netconf.py +++ b/test/infamy/netconf.py @@ -7,11 +7,13 @@ import socket import sys import time import uuid # For _ncc_get_data() extension +import os import libyang import lxml import netconf_client.connect import netconf_client.ncclient + from netconf_client.error import RpcError modinfo_fields = ("identifier", "version", "format", "namespace") @@ -53,6 +55,12 @@ class NccGetDataReply: self.data_xml = lxml.etree.tostring(self.data_ele) self.raw_reply = raw +class NccGetSchemaReply: + def __init__(self, raw): + self.ele = lxml.etree.fromstring(raw.xml.decode()) + self.ele = self.ele.find("{urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring}data") + self.schema = self.ele.text + @dataclass class Location: host: str @@ -68,10 +76,13 @@ class Device(object): self.mapping = mapping self.ly = libyang.Context(yangdir) - self._ncc_init(location) + self.modules = {} + self._ly_bootstrap(yangdir) + + del self.ly + self.ly = libyang.Context(yangdir) self._ly_init(yangdir) - # self.update_schema() self.ncc.dispatch('') def _ncc_init(self, location): @@ -88,6 +99,19 @@ class Device(object): password=location.password) self.ncc = Manager(session) + def _ly_bootstrap(self,yangdir): + ly = libyang.Context(yangdir) + self.modules["ietf-netconf-monitoring"] = { "name": "ietf-netconf-monitoring" } + + for v in self.modules.values(): + mod = self.ly.load_module(v["name"]) + mod.feature_enable_all() + schemas = self.get_schemas_list() + for schema in schemas: + if(os.path.exists(yangdir + "/" + schema['filename']) == False): + self.get_schema(schema, yangdir) + print(f"Downloading missing model {schema['identifier']}") + def _ly_init(self, yangdir): self.ly = libyang.Context(yangdir) @@ -249,3 +273,34 @@ class Device(object): mod = self.ly.get_module(modname) lyd = mod.parse_data_dict(action, rpc=True) return self.call_action(lyd.print_mem("xml", with_siblings=True, pretty=False)) + + def get_schemas_list(self): + schemas=[] + + data= self.get_dict("/netconf-state") + for d in data["netconf-state"]["schemas"]["schema"]: + schema={} + schema["identifier"] = d['identifier'] + schema["format"] = d["format"] + if d['version']: + schema["version"] = d['version'] + schema["filename"]= f"{d['identifier']}@{d['version']}.yang" + else: + schema["filename"] = f"{d['identifier']}.yang" + schemas.append(schema) + return schemas + + def get_schema(self, schema, outdir): + query = { + "get-schema": { + "identifier": schema['identifier'], + "version": schema['version'], + "format": "yang" + } + } + rpc_reply = self.call_dict("ietf-netconf-monitoring", query) + data = NccGetSchemaReply(rpc_reply) + + with open(outdir+"/"+schema["filename"], "w") as f: + f.write(data.schema) +