Merge pull request #1550 from kernelkit/refactor-misc-operational

test/case/all.yaml

@@ -17,9 +17,6 @@
   infamy:
     specification: False
 
-- name: "Miscellaneous"
-  suite: misc/all.yaml
-
 
 - name: "System"
   suite: system/all.yaml
@@ -62,3 +59,7 @@
 
 - name: "Use Case Tests"
   suite: use_case/all.yaml
+
+- name: "Miscellaneous"
+  suite: misc/all.yaml
+

test/case/misc/operational_all/test.adoc

@@ -4,16 +4,27 @@ ifdef::topdoc[:imagesdir: {topdoc}../../test/case/misc/operational_all]
 
 ==== Description
 
-Basic test just to get operational from test-config without errors.
+Verify each device's operational datastore against its test-config.
+The running config is the source of truth, so the checks hold no matter
+how interfaces differ between devices:
 
-==== Topology
+  1. The operational interfaces are exactly the configured ones: none
+     missing, and none present that were never configured.
+  2. Features the test-config leaves disabled (NTP, containers, routing
+     protocols) emit no operational data.
 
-image::topology.svg[Get Operational topology, align=center, scaledwidth=75%]
+Both checks use specific-path GETs, which behave the same on NETCONF and
+RESTCONF.  A full-datastore GET is not portable: RESTCONF does not serve
+the operational datastore root, and NETCONF's operational provider errors
+when asked for an empty subtree.
+
+The test has no logical topology; it reads state from whatever DUTs the
+physical topology provides.
 
 ==== Sequence
 
-. Set up topology and attach to target DUT
-. Copy test-config to running configuration
-. Get all Operational data from 'target', verify there are no errors
+. Attach to all DUTs in the topology
+. Verify operational interfaces match the test-config
+. Verify unconfigured feature subtrees are absent
 
 

test/case/misc/operational_all/test.py

@@ -1,23 +1,112 @@
 #!/usr/bin/env python3
-
-# Test that it is possible to get all operational data
 """
 Get operational
 
-Basic test just to get operational from test-config without errors.
+Verify each device's operational datastore against its test-config.
+The running config is the source of truth, so the checks hold no matter
+how interfaces differ between devices:
+
+  1. The operational interfaces are exactly the configured ones: none
+     missing, and none present that were never configured.
+  2. Features the test-config leaves disabled (NTP, containers, routing
+     protocols) emit no operational data.
+
+Both checks use specific-path GETs, which behave the same on NETCONF and
+RESTCONF.  A full-datastore GET is not portable: RESTCONF does not serve
+the operational datastore root, and NETCONF's operational provider errors
+when asked for an empty subtree.
+
+The test has no logical topology; it reads state from whatever DUTs the
+physical topology provides.
 """
 import infamy
-import infamy.iface as iface
+from infamy.util import parallel, until
+
+# Feature subtrees that must be absent because the test-config does not
+# enable them.  /ietf-routing:routing itself is always present (its RIB
+# reflects the kernel's connected/local routes), so we target the
+# config-gated control-plane-protocols child rather than all of routing.
+ABSENT = [
+    "/ietf-ntp:ntp",
+    "/infix-containers:containers",
+    "/ietf-routing:routing/control-plane-protocols",
+]
+
+
+def configured_interfaces(dut):
+    cfg = dut.get_config_dict("/ietf-interfaces:interfaces")
+    return {i["name"] for i in cfg["interfaces"]["interface"]}
+
+
+def operational_interfaces(dut):
+    oper = dut.get_data("/ietf-interfaces:interfaces")["interfaces"]["interface"]
+    return {i["name"] for i in oper}
+
+
+def interfaces_match(name, dut, want):
+    """True once operational interfaces equal the configured set.
+
+    A preceding test (e.g. a container use_case) may leave veth endpoints
+    that are still being torn down when this test starts, so operational
+    momentarily carries interfaces that are not in the configuration.
+    Rather than asserting on that transient state, poll until the kernel,
+    and thus operational, has converged on the configured set.
+    """
+    have = operational_interfaces(dut)
+    if have != want:
+        print(f"{name}: operational {sorted(have)} != configured {sorted(want)}, waiting...")
+        return False
+    return True
+
+
+def absent(dut, xpath):
+    # RESTCONF returns nothing for an absent subtree; NETCONF's operational
+    # provider errors when asked for one.  Both mean "not present".
+    try:
+        return not dut.get_data(xpath)
+    except Exception:
+        return True
+
+
+def features_absent(name, dut):
+    """True once every unconfigured feature subtree is gone.
+
+    Like the interface set, these subtrees can momentarily linger: a
+    preceding test (containers, NTP, a routing protocol) leaves state that
+    yangerd only prunes once the underlying daemon/config is torn down.
+    Poll until operational has converged rather than asserting on the
+    transient overlap.
+    """
+    pending = [xpath for xpath in ABSENT if not absent(dut, xpath)]
+    if pending:
+        print(f"{name}: feature data still present {pending}, waiting...")
+        return False
+    return True
+
 
 with infamy.Test() as test:
-    with test.step("Set up topology and attach to target DUT"):
-        env = infamy.Env()
-        target = env.attach("target", "mgmt")
+    with test.step("Attach to all DUTs in the topology"):
+        env = infamy.Env(ltop=False)
+        infixen = env.ptop.get_infixen()
+        assert infixen, "no devices found in topology"
+        duts = dict(zip(infixen, parallel(*(lambda n=name: env.attach(n, "mgmt")
+                                            for name in infixen))))
 
-    with test.step("Copy test-config to running configuration"):
-        pass
+    with test.step("Verify operational interfaces match the test-config"):
+        def check(name, dut):
+            want = configured_interfaces(dut)
+            until(lambda: interfaces_match(name, dut, want), attempts=60)
 
-    with test.step("Get all Operational data from 'target', verify there are no errors"):
-        target.get_data(parse=False)
+        parallel(*(lambda n=name, d=dut: check(n, d) for name, dut in duts.items()))
+
+    with test.step("Verify unconfigured feature subtrees are absent"):
+        def check_absent(name, dut):
+            # NTP is pruned by a periodic collector with a 60s poll
+            # interval, so a stale subtree can take a full cycle to clear;
+            # wait comfortably past one interval rather than racing it.
+            until(lambda: features_absent(name, dut), attempts=120)
+
+        parallel(*(lambda n=name, d=dut: check_absent(n, d)
+                   for name, dut in duts.items()))
 
     test.succeed()

test/case/misc/operational_all/topology.dot

@@ -1 +0,0 @@
-../../../infamy/topologies/1x1.dot

test/case/misc/operational_all/topology.svg

@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
- "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-
-<!-- Title: 1x1 Pages: 1 -->
-<svg width="424pt" height="45pt"
- viewBox="0.00 0.00 424.03 45.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 41)">
-<title>1x1</title>
-<polygon fill="white" stroke="transparent" points="-4,4 -4,-41 420.03,-41 420.03,4 -4,4"/>
-<!-- host -->
-<g id="node1" class="node">
-<title>host</title>
-<polygon fill="none" stroke="black" points="0,-0.5 0,-36.5 100,-36.5 100,-0.5 0,-0.5"/>
-<text text-anchor="middle" x="25" y="-14.8" font-family="DejaVu Sans Mono, Book" font-size="14.00">host</text>
-<polyline fill="none" stroke="black" points="50,-0.5 50,-36.5 "/>
-<text text-anchor="middle" x="75" y="-14.8" font-family="DejaVu Sans Mono, Book" font-size="14.00">mgmt</text>
-</g>
-<!-- target -->
-<g id="node2" class="node">
-<title>target</title>
-<polygon fill="none" stroke="black" points="300.03,-0.5 300.03,-36.5 416.03,-36.5 416.03,-0.5 300.03,-0.5"/>
-<text text-anchor="middle" x="325.03" y="-14.8" font-family="DejaVu Sans Mono, Book" font-size="14.00">mgmt</text>
-<polyline fill="none" stroke="black" points="350.03,-0.5 350.03,-36.5 "/>
-<text text-anchor="middle" x="383.03" y="-14.8" font-family="DejaVu Sans Mono, Book" font-size="14.00">target</text>
-</g>
-<!-- host&#45;&#45;target -->
-<g id="edge1" class="edge">
-<title>host:mgmt&#45;&#45;target:mgmt</title>
-<path fill="none" stroke="lightgray" stroke-width="2" d="M100,-18.5C100,-18.5 300.03,-18.5 300.03,-18.5"/>
-</g>
-</g>
-</svg>

test/spec/generate_spec.py

@@ -140,7 +140,9 @@ class TestCase:
         if title is None:
             title = visitor.name
 
-        self.gen_topology()
+        has_topology = os.path.exists(self.topo_dot)
+        if has_topology:
+            self.gen_topology()
 
         with open(self.spec_path, "w", encoding='utf-8') as spec:
             # This is the test name/title  for the test-specification.pdf,
@@ -156,8 +158,9 @@ class TestCase:
             spec.write("==== Description\n\n")
             spec.write(description + "\n\n")
 
-            spec.write("==== Topology\n\n")
-            spec.write(f"image::topology.svg[{title} topology, align=center, scaledwidth=75%]\n\n")
+            if has_topology:
+                spec.write("==== Topology\n\n")
+                spec.write(f"image::topology.svg[{title} topology, align=center, scaledwidth=75%]\n\n")
 
             spec.write("==== Sequence\n\n")
             spec.writelines([f". {step}\n" for step in test_steps])