peter/infix
1
0
Fork 0
mirror of https://github.com/kernelkit/infix.git synced 2026-07-22 01:13:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

test/case/hardware/watchdog: Add test

Browse Source 
Verify that if the system encounters a hard lockup, i.e., interrupts
are no longer being serviced, then the system's watchdog will
correctly reboot it.
This commit is contained in:
Tobias Waldekranz
2025-12-16 09:34:00 +01:00
parent 3460169bc5
commit 553d659eac
9 changed files with 160 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/case/hardware/Readme.adoc
+3
View File
@@ -5,7 +5,10 @@ Tests verifying hardware monitoring and management:
- USB device detection and enumeration
- Multiple USB port management and device handling
- Watchdog reset capability
include::usb/Readme.adoc[]
include::usb_two_ports/Readme.adoc[]
include::watchdog/Readme.adoc[]
test/case/hardware/all.yaml
+3
View File
@@ -4,3 +4,6 @@
- name: USB configuration with two USB ports
case: usb_two_ports/test.py
- name: Watchdog reset on system lockup
case: watchdog/test.py
test/case/hardware/watchdog/Readme.adoc
Symlink
+1
View File
@@ -0,0 +1 @@
test.adoc
test/case/hardware/watchdog/test.adoc
+28
View File
@@ -0,0 +1,28 @@
=== Watchdog reset on system lockup
ifdef::topdoc[:imagesdir: {topdoc}../../test/case/hardware/watchdog]
==== Description
Verify that a system's watchdog trips and successfully reboots the
system back to a working state if a lockup occurs.
This is tested by using the Linux kernel's `test_lockup` module to
inject a hard lockup (i.e., blocking servicing of all interrupts) on
all CPU cores that lasts for twice as long as the watchdog's reported
timeout.
==== Topology
image::topology.svg[Watchdog reset on system lockup topology, align=center, scaledwidth=75%]
==== Sequence
. Set up topology and attach to target DUT
. Verify the presence of a watchdog device
. Verify the presence of the test_lockup module
. Trigger a hard lockup on all CPU cores
. Wait for the watchdog to trip
. Verify that the system reboots
test/case/hardware/watchdog/test.py
Executable
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
"""Watchdog reset on system lockup
Verify that a system's watchdog trips and successfully reboots the
system back to a working state if a lockup occurs.
This is tested by using the Linux kernel's `test_lockup` module to
inject a hard lockup (i.e., blocking servicing of all interrupts) on
all CPU cores that lasts for twice as long as the watchdog's reported
timeout.
"""
import base64
import infamy
import json
import subprocess
import time
with infamy.Test() as test:
with test.step("Set up topology and attach to target DUT"):
env = infamy.Env()
target = env.attach("target", "mgmt")
tgtssh = env.attach("target", "mgmt", "ssh")
with test.step("Verify the presence of a watchdog device"):
wctl = tgtssh.run(["watchdogctl"], stdout=subprocess.PIPE)
conf = json.loads(wctl.stdout)
dogs = [ dog for dog in conf.get("device", []) if dog.get("name", "") == "/dev/watchdog" ]
if len(dogs) < 1:
test.fail("No watchdog device available")
else:
dog = dogs[0]
print(f"Found {dog['name']} ({dog['identity']}), timeout:{dog['timeout']}s")
with test.step("Verify the presence of the test_lockup module"):
if tgtssh.run(["modprobe", "-q", "-n", "test_lockup"]).returncode != 0:
test.fail("test_lockup module is not available")
with test.step("Trigger a hard lockup on all CPU cores"):
tgtssh.runsh(f"""
lockup()
{{
# Give the SSH session some time to properly shut down
sleep 3
sudo modprobe test_lockup \
disable_irq=1 \
all_cpus=1 \
time_secs={dog['timeout'] * 2}
}}
lockup </dev/null &>/dev/null &
""")
with test.step("Wait for the watchdog to trip"):
time.sleep(dog["timeout"])
with test.step("Verify that the system reboots"):
infamy.util.wait_boot(target, env)
test.succeed()
test/case/hardware/watchdog/topology.dot
+23
View File
@@ -0,0 +1,23 @@
graph "1x1" {
layout="neato";
overlap="false";
esep="+80";
node [shape=record, fontname="DejaVu Sans Mono, Book"];
edge [color="cornflowerblue", penwidth="2", fontname="DejaVu Serif, Book"];
host [
label="host | { <mgmt> mgmt }",
pos="0,12!",
requires="controller",
];
target [
label="{ <mgmt> mgmt } | target",
pos="10,12!",
requires="infix watchdog",
];
host:mgmt -- target:mgmt [requires="mgmt", color="lightgray"]
}
test/case/hardware/watchdog/topology.svg
+33
View File
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Title: 1x1 Pages: 1 -->
<svg width="424pt" height="45pt"
viewBox="0.00 0.00 424.03 45.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 41)">
<title>1x1</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-41 420.03,-41 420.03,4 -4,4"/>
<!-- host -->
<g id="node1" class="node">
<title>host</title>
<polygon fill="none" stroke="black" points="0,-0.5 0,-36.5 100,-36.5 100,-0.5 0,-0.5"/>
<text text-anchor="middle" x="25" y="-14.8" font-family="DejaVu Sans Mono, Book" font-size="14.00">host</text>
<polyline fill="none" stroke="black" points="50,-0.5 50,-36.5 "/>
<text text-anchor="middle" x="75" y="-14.8" font-family="DejaVu Sans Mono, Book" font-size="14.00">mgmt</text>
</g>
<!-- target -->
<g id="node2" class="node">
<title>target</title>
<polygon fill="none" stroke="black" points="300.03,-0.5 300.03,-36.5 416.03,-36.5 416.03,-0.5 300.03,-0.5"/>
<text text-anchor="middle" x="325.03" y="-14.8" font-family="DejaVu Sans Mono, Book" font-size="14.00">mgmt</text>
<polyline fill="none" stroke="black" points="350.03,-0.5 350.03,-36.5 "/>
<text text-anchor="middle" x="383.03" y="-14.8" font-family="DejaVu Sans Mono, Book" font-size="14.00">target</text>
</g>
<!-- host&#45;&#45;target -->
<g id="edge1" class="edge">
<title>host:mgmt&#45;&#45;target:mgmt</title>
<path fill="none" stroke="lightgray" stroke-width="2" d="M100,-18.5C100,-18.5 300.03,-18.5 300.03,-18.5"/>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

test/virt/dual/topology.dot.in
+2 -2
View File
@@ -24,7 +24,7 @@ graph "dual" {
dut1 [
label="{ <e1> e1 | <e2> e2 | <e3> e3 } | dut1 | { <e4> e4 | <e5> e5 | <e6> e6 }",
pos="10,18!",
provides="infix",
provides="infix watchdog",
expected_boot="primary",
qn_console=9001,
qn_mem="384M",
@@ -33,7 +33,7 @@ graph "dual" {
dut2 [
label="{ <e1> e1 | <e2> e2 | <e3> e3 } | dut2 | { <e4> e4 | <e5> e5 | <e6> e6 }",
pos="10,12!",
provides="infix",
provides="infix watchdog",
expected_boot="primary",
qn_console=9002,
qn_mem="384M",
test/virt/quad/topology.dot.in
+4 -4
View File
@@ -23,7 +23,7 @@ graph "quad" {
dut1 [
label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut1 | { <e5> e5 | <e6> e6 | <e7> e7 | <e8> e8}",
pos="10,30!",
provides="infix",
provides="infix watchdog",
expected_boot="primary",
qn_console=9001,
qn_mem="384M",
@@ -32,7 +32,7 @@ graph "quad" {
dut2 [
label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut2 | { <e5> e5 | <e6> e6 | <e7> e7 | <e8> e8}",
pos="0,20!",
provides="infix",
provides="infix watchdog",
expected_boot="primary",
qn_console=9002,
qn_mem="384M",
@@ -41,7 +41,7 @@ graph "quad" {
dut3 [
label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut3 | { <e5> e5 | <e6> e6 | <e7> e7 | <e8> e8}",
pos="0,10!",
provides="infix",
provides="infix watchdog",
expected_boot="primary",
qn_console=9003,
qn_mem="384M",
@@ -51,7 +51,7 @@ graph "quad" {
dut4 [
label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut4 | { <e5> e5 | <e6> e6 | <e7> e7 | <e8> e8}",
pos="10,0!",
provides="infix",
provides="infix watchdog",
expected_boot="primary",
qn_console=9004,
qn_mem="384M",