From 553d659eacdacd0ab75d9513c5a5e774f136edae Mon Sep 17 00:00:00 2001 From: Tobias Waldekranz Date: Mon, 15 Dec 2025 10:09:11 +0000 Subject: [PATCH] test/case/hardware/watchdog: Add test Verify that if the system encounters a hard lockup, i.e., interrupts are no longer being serviced, then the system's watchdog will correctly reboot it. --- test/case/hardware/Readme.adoc | 3 ++ test/case/hardware/all.yaml | 3 ++ test/case/hardware/watchdog/Readme.adoc | 1 + test/case/hardware/watchdog/test.adoc | 28 +++++++++++ test/case/hardware/watchdog/test.py | 63 ++++++++++++++++++++++++ test/case/hardware/watchdog/topology.dot | 23 +++++++++ test/case/hardware/watchdog/topology.svg | 33 +++++++++++++ test/virt/dual/topology.dot.in | 4 +- test/virt/quad/topology.dot.in | 8 +-- 9 files changed, 160 insertions(+), 6 deletions(-) create mode 120000 test/case/hardware/watchdog/Readme.adoc create mode 100644 test/case/hardware/watchdog/test.adoc create mode 100755 test/case/hardware/watchdog/test.py create mode 100644 test/case/hardware/watchdog/topology.dot create mode 100644 test/case/hardware/watchdog/topology.svg diff --git a/test/case/hardware/Readme.adoc b/test/case/hardware/Readme.adoc index 088069b1..a7fb1de5 100644 --- a/test/case/hardware/Readme.adoc +++ b/test/case/hardware/Readme.adoc @@ -5,7 +5,10 @@ Tests verifying hardware monitoring and management: - USB device detection and enumeration - Multiple USB port management and device handling + - Watchdog reset capability include::usb/Readme.adoc[] include::usb_two_ports/Readme.adoc[] + +include::watchdog/Readme.adoc[] diff --git a/test/case/hardware/all.yaml b/test/case/hardware/all.yaml index 22f4891f..1ca32dd3 100644 --- a/test/case/hardware/all.yaml +++ b/test/case/hardware/all.yaml @@ -4,3 +4,6 @@ - name: USB configuration with two USB ports case: usb_two_ports/test.py + +- name: Watchdog reset on system lockup + case: watchdog/test.py diff --git a/test/case/hardware/watchdog/Readme.adoc b/test/case/hardware/watchdog/Readme.adoc new file mode 120000 index 00000000..ae32c841 --- /dev/null +++ b/test/case/hardware/watchdog/Readme.adoc @@ -0,0 +1 @@ +test.adoc \ No newline at end of file diff --git a/test/case/hardware/watchdog/test.adoc b/test/case/hardware/watchdog/test.adoc new file mode 100644 index 00000000..ca904cd8 --- /dev/null +++ b/test/case/hardware/watchdog/test.adoc @@ -0,0 +1,28 @@ +=== Watchdog reset on system lockup + +ifdef::topdoc[:imagesdir: {topdoc}../../test/case/hardware/watchdog] + +==== Description + +Verify that a system's watchdog trips and successfully reboots the +system back to a working state if a lockup occurs. + +This is tested by using the Linux kernel's `test_lockup` module to +inject a hard lockup (i.e., blocking servicing of all interrupts) on +all CPU cores that lasts for twice as long as the watchdog's reported +timeout. + +==== Topology + +image::topology.svg[Watchdog reset on system lockup topology, align=center, scaledwidth=75%] + +==== Sequence + +. Set up topology and attach to target DUT +. Verify the presence of a watchdog device +. Verify the presence of the test_lockup module +. Trigger a hard lockup on all CPU cores +. Wait for the watchdog to trip +. Verify that the system reboots + + diff --git a/test/case/hardware/watchdog/test.py b/test/case/hardware/watchdog/test.py new file mode 100755 index 00000000..b8854536 --- /dev/null +++ b/test/case/hardware/watchdog/test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Watchdog reset on system lockup + +Verify that a system's watchdog trips and successfully reboots the +system back to a working state if a lockup occurs. + +This is tested by using the Linux kernel's `test_lockup` module to +inject a hard lockup (i.e., blocking servicing of all interrupts) on +all CPU cores that lasts for twice as long as the watchdog's reported +timeout. + +""" +import base64 +import infamy +import json +import subprocess +import time + +with infamy.Test() as test: + with test.step("Set up topology and attach to target DUT"): + env = infamy.Env() + target = env.attach("target", "mgmt") + tgtssh = env.attach("target", "mgmt", "ssh") + + with test.step("Verify the presence of a watchdog device"): + wctl = tgtssh.run(["watchdogctl"], stdout=subprocess.PIPE) + conf = json.loads(wctl.stdout) + + dogs = [ dog for dog in conf.get("device", []) if dog.get("name", "") == "/dev/watchdog" ] + if len(dogs) < 1: + test.fail("No watchdog device available") + else: + dog = dogs[0] + + print(f"Found {dog['name']} ({dog['identity']}), timeout:{dog['timeout']}s") + + with test.step("Verify the presence of the test_lockup module"): + if tgtssh.run(["modprobe", "-q", "-n", "test_lockup"]).returncode != 0: + test.fail("test_lockup module is not available") + + with test.step("Trigger a hard lockup on all CPU cores"): + tgtssh.runsh(f""" + lockup() + {{ + # Give the SSH session some time to properly shut down + sleep 3 + + sudo modprobe test_lockup \ + disable_irq=1 \ + all_cpus=1 \ + time_secs={dog['timeout'] * 2} + }} + + lockup /dev/null & + """) + + with test.step("Wait for the watchdog to trip"): + time.sleep(dog["timeout"]) + + with test.step("Verify that the system reboots"): + infamy.util.wait_boot(target, env) + + test.succeed() diff --git a/test/case/hardware/watchdog/topology.dot b/test/case/hardware/watchdog/topology.dot new file mode 100644 index 00000000..bb497075 --- /dev/null +++ b/test/case/hardware/watchdog/topology.dot @@ -0,0 +1,23 @@ +graph "1x1" { + layout="neato"; + overlap="false"; + esep="+80"; + + node [shape=record, fontname="DejaVu Sans Mono, Book"]; + edge [color="cornflowerblue", penwidth="2", fontname="DejaVu Serif, Book"]; + + host [ + label="host | { mgmt }", + pos="0,12!", + requires="controller", + ]; + + target [ + label="{ mgmt } | target", + pos="10,12!", + + requires="infix watchdog", + ]; + + host:mgmt -- target:mgmt [requires="mgmt", color="lightgray"] +} diff --git a/test/case/hardware/watchdog/topology.svg b/test/case/hardware/watchdog/topology.svg new file mode 100644 index 00000000..6fc6f47a --- /dev/null +++ b/test/case/hardware/watchdog/topology.svg @@ -0,0 +1,33 @@ + + + + + + +1x1 + + + +host + +host + +mgmt + + + +target + +mgmt + +target + + + +host:mgmt--target:mgmt + + + + diff --git a/test/virt/dual/topology.dot.in b/test/virt/dual/topology.dot.in index 736f818c..70dbd741 100644 --- a/test/virt/dual/topology.dot.in +++ b/test/virt/dual/topology.dot.in @@ -24,7 +24,7 @@ graph "dual" { dut1 [ label="{ e1 | e2 | e3 } | dut1 | { e4 | e5 | e6 }", pos="10,18!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9001, qn_mem="384M", @@ -33,7 +33,7 @@ graph "dual" { dut2 [ label="{ e1 | e2 | e3 } | dut2 | { e4 | e5 | e6 }", pos="10,12!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9002, qn_mem="384M", diff --git a/test/virt/quad/topology.dot.in b/test/virt/quad/topology.dot.in index 1242b0ea..609cdced 100644 --- a/test/virt/quad/topology.dot.in +++ b/test/virt/quad/topology.dot.in @@ -23,7 +23,7 @@ graph "quad" { dut1 [ label="{ e1 | e2 | e3 | e4 } | dut1 | { e5 | e6 | e7 | e8}", pos="10,30!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9001, qn_mem="384M", @@ -32,7 +32,7 @@ graph "quad" { dut2 [ label="{ e1 | e2 | e3 | e4 } | dut2 | { e5 | e6 | e7 | e8}", pos="0,20!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9002, qn_mem="384M", @@ -41,7 +41,7 @@ graph "quad" { dut3 [ label="{ e1 | e2 | e3 | e4 } | dut3 | { e5 | e6 | e7 | e8}", pos="0,10!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9003, qn_mem="384M", @@ -51,7 +51,7 @@ graph "quad" { dut4 [ label="{ e1 | e2 | e3 | e4 } | dut4 | { e5 | e6 | e7 | e8}", pos="10,0!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9004, qn_mem="384M",