From 0298dc3c1ab6aed6c1ad4870af404ccd342424c7 Mon Sep 17 00:00:00 2001 From: Tobias Waldekranz Date: Tue, 16 Dec 2025 08:09:14 +0000 Subject: [PATCH 1/5] test: ssh: Wait for target SSH server to launch, by default By default, ensure that the target device accepts TCP connections on the specified port before returning the device object. This aligns the SSH implementation with {NET,REST}CONF. This solves the issue where tests that use SSH commands early in their execution would sometimes fail as the remote server had not started up yet. --- test/infamy/ssh.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/infamy/ssh.py b/test/infamy/ssh.py index 9883f532..2db6bde5 100644 --- a/test/infamy/ssh.py +++ b/test/infamy/ssh.py @@ -1,7 +1,7 @@ import subprocess from dataclasses import dataclass -from . import env +from . import env, netutil, util @dataclass @@ -15,6 +15,9 @@ class Location: import subprocess import os +def ssh_syn(addr, port=22): + return netutil.tcp_port_is_open(addr, port) + def fetch_file(remote_user, remote_address, remote_file, local_file, key_file, check=False, remove=False): """ Fetches a file over SSH using scp and the provided private key. @@ -62,9 +65,12 @@ def fetch_file(remote_user, remote_address, remote_file, local_file, key_file, c print(f"Error removing fetched file {local_file}: {e}") class Device(object): - def __init__(self, name: str, location: Location): + def __init__(self, name: str, location: Location, wait: bool=True): self.name = name self.location = location + if wait: + util.until(lambda: ssh_syn(location.host, location.port)) + def __str__(self): nm = f"{self.name}" if env.ENV.ltop: From 8eb706b564c212e04b911645b85dcfaaa0c607b3 Mon Sep 17 00:00:00 2001 From: Tobias Waldekranz Date: Fri, 12 Dec 2025 12:52:59 +0000 Subject: [PATCH 2/5] aarch64: kernel: Enable SBSA watchdog Usable by all Server Base System Architecture (SBSA) compliant SoCs, e.g., CN9130. --- board/aarch64/linux_defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/board/aarch64/linux_defconfig b/board/aarch64/linux_defconfig index 0076bfa8..cac80479 100644 --- a/board/aarch64/linux_defconfig +++ b/board/aarch64/linux_defconfig @@ -407,6 +407,7 @@ CONFIG_WATCHDOG=y CONFIG_WATCHDOG_SYSFS=y CONFIG_SOFT_WATCHDOG=y CONFIG_GPIO_WATCHDOG=y +CONFIG_ARM_SBSA_WATCHDOG=y CONFIG_ARMADA_37XX_WATCHDOG=y CONFIG_I6300ESB_WDT=y CONFIG_MFD_MAX77620=y From d720323b504429b879a65bb320be9a5c28fa3e81 Mon Sep 17 00:00:00 2001 From: Tobias Waldekranz Date: Fri, 12 Dec 2025 14:43:16 +0000 Subject: [PATCH 3/5] kernel: Enable test_lockup module This will be us the ability to test a hardware watchdog's ability to trigger on different kinds of lockups. --- board/aarch32/linux_defconfig | 1 + board/aarch64/linux_defconfig | 1 + board/riscv64/linux_defconfig | 1 + board/x86_64/linux_defconfig | 1 + 4 files changed, 4 insertions(+) diff --git a/board/aarch32/linux_defconfig b/board/aarch32/linux_defconfig index c87e4eeb..933daea6 100644 --- a/board/aarch32/linux_defconfig +++ b/board/aarch32/linux_defconfig @@ -394,6 +394,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y CONFIG_WQ_WATCHDOG=y CONFIG_WQ_CPU_INTENSIVE_REPORT=y +CONFIG_TEST_LOCKUP=m # CONFIG_SCHED_DEBUG is not set # CONFIG_RCU_TRACE is not set CONFIG_FUNCTION_TRACER=y diff --git a/board/aarch64/linux_defconfig b/board/aarch64/linux_defconfig index cac80479..cd4a84c1 100644 --- a/board/aarch64/linux_defconfig +++ b/board/aarch64/linux_defconfig @@ -565,6 +565,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y CONFIG_WQ_WATCHDOG=y CONFIG_WQ_CPU_INTENSIVE_REPORT=y +CONFIG_TEST_LOCKUP=m # CONFIG_SCHED_DEBUG is not set # CONFIG_RCU_TRACE is not set CONFIG_FUNCTION_TRACER=y diff --git a/board/riscv64/linux_defconfig b/board/riscv64/linux_defconfig index e3337e65..f3815aef 100644 --- a/board/riscv64/linux_defconfig +++ b/board/riscv64/linux_defconfig @@ -481,6 +481,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y CONFIG_WQ_WATCHDOG=y CONFIG_WQ_CPU_INTENSIVE_REPORT=y +CONFIG_TEST_LOCKUP=m # CONFIG_SCHED_DEBUG is not set CONFIG_STACKTRACE=y CONFIG_RCU_CPU_STALL_TIMEOUT=60 diff --git a/board/x86_64/linux_defconfig b/board/x86_64/linux_defconfig index db815eb1..ffd4b045 100644 --- a/board/x86_64/linux_defconfig +++ b/board/x86_64/linux_defconfig @@ -270,5 +270,6 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y CONFIG_WQ_WATCHDOG=y CONFIG_WQ_CPU_INTENSIVE_REPORT=y +CONFIG_TEST_LOCKUP=m CONFIG_FUNCTION_TRACER=y CONFIG_UNWINDER_FRAME_POINTER=y From 3460169bc53c2fca670e69acd2491bd64dba9625 Mon Sep 17 00:00:00 2001 From: Tobias Waldekranz Date: Fri, 12 Dec 2025 14:44:35 +0000 Subject: [PATCH 4/5] aarch64/styx-dcp-sc-28p-a: Fix warm reset on old rev. a boards Rev. A has a FLASH that Linux reconfigures to 4B-addressing, but must be in 3B-addressing when accessed by the BootROM. Since it has no reset input, this would not work on warm resets. Linux has a workaround for this issue for the common case of a controlled reboot, which this change enables on those boards (which are admittedly few and far between, but still useful as testing vehicles). (This won't take effect until Finit is updated to at least 4.15, which switches over from watchdog- to reboot(2)- based reboot, by default) --- board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts b/board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts index 1286e951..a6838d68 100644 --- a/board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts +++ b/board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts @@ -59,3 +59,9 @@ XSWP(a, "e13", 13, &sfp0); }; }; + +&cp0_spi1 { + spi-flash@0 { + broken-flash-reset; + }; +}; From 553d659eacdacd0ab75d9513c5a5e774f136edae Mon Sep 17 00:00:00 2001 From: Tobias Waldekranz Date: Mon, 15 Dec 2025 10:09:11 +0000 Subject: [PATCH 5/5] test/case/hardware/watchdog: Add test Verify that if the system encounters a hard lockup, i.e., interrupts are no longer being serviced, then the system's watchdog will correctly reboot it. --- test/case/hardware/Readme.adoc | 3 ++ test/case/hardware/all.yaml | 3 ++ test/case/hardware/watchdog/Readme.adoc | 1 + test/case/hardware/watchdog/test.adoc | 28 +++++++++++ test/case/hardware/watchdog/test.py | 63 ++++++++++++++++++++++++ test/case/hardware/watchdog/topology.dot | 23 +++++++++ test/case/hardware/watchdog/topology.svg | 33 +++++++++++++ test/virt/dual/topology.dot.in | 4 +- test/virt/quad/topology.dot.in | 8 +-- 9 files changed, 160 insertions(+), 6 deletions(-) create mode 120000 test/case/hardware/watchdog/Readme.adoc create mode 100644 test/case/hardware/watchdog/test.adoc create mode 100755 test/case/hardware/watchdog/test.py create mode 100644 test/case/hardware/watchdog/topology.dot create mode 100644 test/case/hardware/watchdog/topology.svg diff --git a/test/case/hardware/Readme.adoc b/test/case/hardware/Readme.adoc index 088069b1..a7fb1de5 100644 --- a/test/case/hardware/Readme.adoc +++ b/test/case/hardware/Readme.adoc @@ -5,7 +5,10 @@ Tests verifying hardware monitoring and management: - USB device detection and enumeration - Multiple USB port management and device handling + - Watchdog reset capability include::usb/Readme.adoc[] include::usb_two_ports/Readme.adoc[] + +include::watchdog/Readme.adoc[] diff --git a/test/case/hardware/all.yaml b/test/case/hardware/all.yaml index 22f4891f..1ca32dd3 100644 --- a/test/case/hardware/all.yaml +++ b/test/case/hardware/all.yaml @@ -4,3 +4,6 @@ - name: USB configuration with two USB ports case: usb_two_ports/test.py + +- name: Watchdog reset on system lockup + case: watchdog/test.py diff --git a/test/case/hardware/watchdog/Readme.adoc b/test/case/hardware/watchdog/Readme.adoc new file mode 120000 index 00000000..ae32c841 --- /dev/null +++ b/test/case/hardware/watchdog/Readme.adoc @@ -0,0 +1 @@ +test.adoc \ No newline at end of file diff --git a/test/case/hardware/watchdog/test.adoc b/test/case/hardware/watchdog/test.adoc new file mode 100644 index 00000000..ca904cd8 --- /dev/null +++ b/test/case/hardware/watchdog/test.adoc @@ -0,0 +1,28 @@ +=== Watchdog reset on system lockup + +ifdef::topdoc[:imagesdir: {topdoc}../../test/case/hardware/watchdog] + +==== Description + +Verify that a system's watchdog trips and successfully reboots the +system back to a working state if a lockup occurs. + +This is tested by using the Linux kernel's `test_lockup` module to +inject a hard lockup (i.e., blocking servicing of all interrupts) on +all CPU cores that lasts for twice as long as the watchdog's reported +timeout. + +==== Topology + +image::topology.svg[Watchdog reset on system lockup topology, align=center, scaledwidth=75%] + +==== Sequence + +. Set up topology and attach to target DUT +. Verify the presence of a watchdog device +. Verify the presence of the test_lockup module +. Trigger a hard lockup on all CPU cores +. Wait for the watchdog to trip +. Verify that the system reboots + + diff --git a/test/case/hardware/watchdog/test.py b/test/case/hardware/watchdog/test.py new file mode 100755 index 00000000..b8854536 --- /dev/null +++ b/test/case/hardware/watchdog/test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Watchdog reset on system lockup + +Verify that a system's watchdog trips and successfully reboots the +system back to a working state if a lockup occurs. + +This is tested by using the Linux kernel's `test_lockup` module to +inject a hard lockup (i.e., blocking servicing of all interrupts) on +all CPU cores that lasts for twice as long as the watchdog's reported +timeout. + +""" +import base64 +import infamy +import json +import subprocess +import time + +with infamy.Test() as test: + with test.step("Set up topology and attach to target DUT"): + env = infamy.Env() + target = env.attach("target", "mgmt") + tgtssh = env.attach("target", "mgmt", "ssh") + + with test.step("Verify the presence of a watchdog device"): + wctl = tgtssh.run(["watchdogctl"], stdout=subprocess.PIPE) + conf = json.loads(wctl.stdout) + + dogs = [ dog for dog in conf.get("device", []) if dog.get("name", "") == "/dev/watchdog" ] + if len(dogs) < 1: + test.fail("No watchdog device available") + else: + dog = dogs[0] + + print(f"Found {dog['name']} ({dog['identity']}), timeout:{dog['timeout']}s") + + with test.step("Verify the presence of the test_lockup module"): + if tgtssh.run(["modprobe", "-q", "-n", "test_lockup"]).returncode != 0: + test.fail("test_lockup module is not available") + + with test.step("Trigger a hard lockup on all CPU cores"): + tgtssh.runsh(f""" + lockup() + {{ + # Give the SSH session some time to properly shut down + sleep 3 + + sudo modprobe test_lockup \ + disable_irq=1 \ + all_cpus=1 \ + time_secs={dog['timeout'] * 2} + }} + + lockup /dev/null & + """) + + with test.step("Wait for the watchdog to trip"): + time.sleep(dog["timeout"]) + + with test.step("Verify that the system reboots"): + infamy.util.wait_boot(target, env) + + test.succeed() diff --git a/test/case/hardware/watchdog/topology.dot b/test/case/hardware/watchdog/topology.dot new file mode 100644 index 00000000..bb497075 --- /dev/null +++ b/test/case/hardware/watchdog/topology.dot @@ -0,0 +1,23 @@ +graph "1x1" { + layout="neato"; + overlap="false"; + esep="+80"; + + node [shape=record, fontname="DejaVu Sans Mono, Book"]; + edge [color="cornflowerblue", penwidth="2", fontname="DejaVu Serif, Book"]; + + host [ + label="host | { mgmt }", + pos="0,12!", + requires="controller", + ]; + + target [ + label="{ mgmt } | target", + pos="10,12!", + + requires="infix watchdog", + ]; + + host:mgmt -- target:mgmt [requires="mgmt", color="lightgray"] +} diff --git a/test/case/hardware/watchdog/topology.svg b/test/case/hardware/watchdog/topology.svg new file mode 100644 index 00000000..6fc6f47a --- /dev/null +++ b/test/case/hardware/watchdog/topology.svg @@ -0,0 +1,33 @@ + + + + + + +1x1 + + + +host + +host + +mgmt + + + +target + +mgmt + +target + + + +host:mgmt--target:mgmt + + + + diff --git a/test/virt/dual/topology.dot.in b/test/virt/dual/topology.dot.in index 736f818c..70dbd741 100644 --- a/test/virt/dual/topology.dot.in +++ b/test/virt/dual/topology.dot.in @@ -24,7 +24,7 @@ graph "dual" { dut1 [ label="{ e1 | e2 | e3 } | dut1 | { e4 | e5 | e6 }", pos="10,18!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9001, qn_mem="384M", @@ -33,7 +33,7 @@ graph "dual" { dut2 [ label="{ e1 | e2 | e3 } | dut2 | { e4 | e5 | e6 }", pos="10,12!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9002, qn_mem="384M", diff --git a/test/virt/quad/topology.dot.in b/test/virt/quad/topology.dot.in index 1242b0ea..609cdced 100644 --- a/test/virt/quad/topology.dot.in +++ b/test/virt/quad/topology.dot.in @@ -23,7 +23,7 @@ graph "quad" { dut1 [ label="{ e1 | e2 | e3 | e4 } | dut1 | { e5 | e6 | e7 | e8}", pos="10,30!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9001, qn_mem="384M", @@ -32,7 +32,7 @@ graph "quad" { dut2 [ label="{ e1 | e2 | e3 | e4 } | dut2 | { e5 | e6 | e7 | e8}", pos="0,20!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9002, qn_mem="384M", @@ -41,7 +41,7 @@ graph "quad" { dut3 [ label="{ e1 | e2 | e3 | e4 } | dut3 | { e5 | e6 | e7 | e8}", pos="0,10!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9003, qn_mem="384M", @@ -51,7 +51,7 @@ graph "quad" { dut4 [ label="{ e1 | e2 | e3 | e4 } | dut4 | { e5 | e6 | e7 | e8}", pos="10,0!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9004, qn_mem="384M",