mirror of
https://github.com/kernelkit/infix.git
synced 2026-08-05 23:23:02 +02:00
Tobias Waldekranz
553d659eac
Verify that if the system encounters a hard lockup, i.e., interrupts are no longer being serviced, then the system's watchdog will correctly reboot it.
64 lines
1.9 KiB
Python
Executable File
64 lines
1.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Watchdog reset on system lockup
|
|
|
|
Verify that a system's watchdog trips and successfully reboots the
|
|
system back to a working state if a lockup occurs.
|
|
|
|
This is tested by using the Linux kernel's `test_lockup` module to
|
|
inject a hard lockup (i.e., blocking servicing of all interrupts) on
|
|
all CPU cores that lasts for twice as long as the watchdog's reported
|
|
timeout.
|
|
|
|
"""
|
|
import base64
|
|
import infamy
|
|
import json
|
|
import subprocess
|
|
import time
|
|
|
|
with infamy.Test() as test:
|
|
with test.step("Set up topology and attach to target DUT"):
|
|
env = infamy.Env()
|
|
target = env.attach("target", "mgmt")
|
|
tgtssh = env.attach("target", "mgmt", "ssh")
|
|
|
|
with test.step("Verify the presence of a watchdog device"):
|
|
wctl = tgtssh.run(["watchdogctl"], stdout=subprocess.PIPE)
|
|
conf = json.loads(wctl.stdout)
|
|
|
|
dogs = [ dog for dog in conf.get("device", []) if dog.get("name", "") == "/dev/watchdog" ]
|
|
if len(dogs) < 1:
|
|
test.fail("No watchdog device available")
|
|
else:
|
|
dog = dogs[0]
|
|
|
|
print(f"Found {dog['name']} ({dog['identity']}), timeout:{dog['timeout']}s")
|
|
|
|
with test.step("Verify the presence of the test_lockup module"):
|
|
if tgtssh.run(["modprobe", "-q", "-n", "test_lockup"]).returncode != 0:
|
|
test.fail("test_lockup module is not available")
|
|
|
|
with test.step("Trigger a hard lockup on all CPU cores"):
|
|
tgtssh.runsh(f"""
|
|
lockup()
|
|
{{
|
|
# Give the SSH session some time to properly shut down
|
|
sleep 3
|
|
|
|
sudo modprobe test_lockup \
|
|
disable_irq=1 \
|
|
all_cpus=1 \
|
|
time_secs={dog['timeout'] * 2}
|
|
}}
|
|
|
|
lockup </dev/null &>/dev/null &
|
|
""")
|
|
|
|
with test.step("Wait for the watchdog to trip"):
|
|
time.sleep(dog["timeout"])
|
|
|
|
with test.step("Verify that the system reboots"):
|
|
infamy.util.wait_boot(target, env)
|
|
|
|
test.succeed()
|