From 60676e21be38f7badd8768bf28e77e8878ee8efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Walstr=C3=B6m?= Date: Thu, 11 Jun 2026 13:11:22 +0200 Subject: [PATCH] test: operational_all: Harden to catch more - Test so the value in operational actually is correct - Move test to last in suite to see that there are no stray data in operational (yangerd) --- test/case/all.yaml | 7 +- test/case/misc/operational_all/test.adoc | 23 ++-- test/case/misc/operational_all/test.py | 111 ++++++++++++++++++-- test/case/misc/operational_all/topology.dot | 1 - test/case/misc/operational_all/topology.svg | 33 ------ 5 files changed, 121 insertions(+), 54 deletions(-) delete mode 120000 test/case/misc/operational_all/topology.dot delete mode 100644 test/case/misc/operational_all/topology.svg diff --git a/test/case/all.yaml b/test/case/all.yaml index 1fdbec3d..79bc4cc6 100644 --- a/test/case/all.yaml +++ b/test/case/all.yaml @@ -17,9 +17,6 @@ infamy: specification: False -- name: "Miscellaneous" - suite: misc/all.yaml - - name: "System" suite: system/all.yaml @@ -62,3 +59,7 @@ - name: "Use Case Tests" suite: use_case/all.yaml + +- name: "Miscellaneous" + suite: misc/all.yaml + diff --git a/test/case/misc/operational_all/test.adoc b/test/case/misc/operational_all/test.adoc index a50ce7df..3cc19ae3 100644 --- a/test/case/misc/operational_all/test.adoc +++ b/test/case/misc/operational_all/test.adoc @@ -4,16 +4,27 @@ ifdef::topdoc[:imagesdir: {topdoc}../../test/case/misc/operational_all] ==== Description -Basic test just to get operational from test-config without errors. +Verify each device's operational datastore against its test-config. +The running config is the source of truth, so the checks hold no matter +how interfaces differ between devices: -==== Topology + 1. The operational interfaces are exactly the configured ones: none + missing, and none present that were never configured. + 2. Features the test-config leaves disabled (NTP, containers, routing + protocols) emit no operational data. -image::topology.svg[Get Operational topology, align=center, scaledwidth=75%] +Both checks use specific-path GETs, which behave the same on NETCONF and +RESTCONF. A full-datastore GET is not portable: RESTCONF does not serve +the operational datastore root, and NETCONF's operational provider errors +when asked for an empty subtree. + +The test has no logical topology; it reads state from whatever DUTs the +physical topology provides. ==== Sequence -. Set up topology and attach to target DUT -. Copy test-config to running configuration -. Get all Operational data from 'target', verify there are no errors +. Attach to all DUTs in the topology +. Verify operational interfaces match the test-config +. Verify unconfigured feature subtrees are absent diff --git a/test/case/misc/operational_all/test.py b/test/case/misc/operational_all/test.py index aec0f188..d784a5ac 100755 --- a/test/case/misc/operational_all/test.py +++ b/test/case/misc/operational_all/test.py @@ -1,23 +1,112 @@ #!/usr/bin/env python3 - -# Test that it is possible to get all operational data """ Get operational -Basic test just to get operational from test-config without errors. +Verify each device's operational datastore against its test-config. +The running config is the source of truth, so the checks hold no matter +how interfaces differ between devices: + + 1. The operational interfaces are exactly the configured ones: none + missing, and none present that were never configured. + 2. Features the test-config leaves disabled (NTP, containers, routing + protocols) emit no operational data. + +Both checks use specific-path GETs, which behave the same on NETCONF and +RESTCONF. A full-datastore GET is not portable: RESTCONF does not serve +the operational datastore root, and NETCONF's operational provider errors +when asked for an empty subtree. + +The test has no logical topology; it reads state from whatever DUTs the +physical topology provides. """ import infamy -import infamy.iface as iface +from infamy.util import parallel, until + +# Feature subtrees that must be absent because the test-config does not +# enable them. /ietf-routing:routing itself is always present (its RIB +# reflects the kernel's connected/local routes), so we target the +# config-gated control-plane-protocols child rather than all of routing. +ABSENT = [ + "/ietf-ntp:ntp", + "/infix-containers:containers", + "/ietf-routing:routing/control-plane-protocols", +] + + +def configured_interfaces(dut): + cfg = dut.get_config_dict("/ietf-interfaces:interfaces") + return {i["name"] for i in cfg["interfaces"]["interface"]} + + +def operational_interfaces(dut): + oper = dut.get_data("/ietf-interfaces:interfaces")["interfaces"]["interface"] + return {i["name"] for i in oper} + + +def interfaces_match(name, dut, want): + """True once operational interfaces equal the configured set. + + A preceding test (e.g. a container use_case) may leave veth endpoints + that are still being torn down when this test starts, so operational + momentarily carries interfaces that are not in the configuration. + Rather than asserting on that transient state, poll until the kernel, + and thus operational, has converged on the configured set. + """ + have = operational_interfaces(dut) + if have != want: + print(f"{name}: operational {sorted(have)} != configured {sorted(want)}, waiting...") + return False + return True + + +def absent(dut, xpath): + # RESTCONF returns nothing for an absent subtree; NETCONF's operational + # provider errors when asked for one. Both mean "not present". + try: + return not dut.get_data(xpath) + except Exception: + return True + + +def features_absent(name, dut): + """True once every unconfigured feature subtree is gone. + + Like the interface set, these subtrees can momentarily linger: a + preceding test (containers, NTP, a routing protocol) leaves state that + yangerd only prunes once the underlying daemon/config is torn down. + Poll until operational has converged rather than asserting on the + transient overlap. + """ + pending = [xpath for xpath in ABSENT if not absent(dut, xpath)] + if pending: + print(f"{name}: feature data still present {pending}, waiting...") + return False + return True + with infamy.Test() as test: - with test.step("Set up topology and attach to target DUT"): - env = infamy.Env() - target = env.attach("target", "mgmt") + with test.step("Attach to all DUTs in the topology"): + env = infamy.Env(ltop=False) + infixen = env.ptop.get_infixen() + assert infixen, "no devices found in topology" + duts = dict(zip(infixen, parallel(*(lambda n=name: env.attach(n, "mgmt") + for name in infixen)))) - with test.step("Copy test-config to running configuration"): - pass + with test.step("Verify operational interfaces match the test-config"): + def check(name, dut): + want = configured_interfaces(dut) + until(lambda: interfaces_match(name, dut, want), attempts=60) - with test.step("Get all Operational data from 'target', verify there are no errors"): - target.get_data(parse=False) + parallel(*(lambda n=name, d=dut: check(n, d) for name, dut in duts.items())) + + with test.step("Verify unconfigured feature subtrees are absent"): + def check_absent(name, dut): + # NTP is pruned by a periodic collector with a 60s poll + # interval, so a stale subtree can take a full cycle to clear; + # wait comfortably past one interval rather than racing it. + until(lambda: features_absent(name, dut), attempts=120) + + parallel(*(lambda n=name, d=dut: check_absent(n, d) + for name, dut in duts.items())) test.succeed() diff --git a/test/case/misc/operational_all/topology.dot b/test/case/misc/operational_all/topology.dot deleted file mode 120000 index 02b78869..00000000 --- a/test/case/misc/operational_all/topology.dot +++ /dev/null @@ -1 +0,0 @@ -../../../infamy/topologies/1x1.dot \ No newline at end of file diff --git a/test/case/misc/operational_all/topology.svg b/test/case/misc/operational_all/topology.svg deleted file mode 100644 index 6fc6f47a..00000000 --- a/test/case/misc/operational_all/topology.svg +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - -1x1 - - - -host - -host - -mgmt - - - -target - -mgmt - -target - - - -host:mgmt--target:mgmt - - - -