diff --git a/board/aarch32/linux_defconfig b/board/aarch32/linux_defconfig index c87e4eeb..933daea6 100644 --- a/board/aarch32/linux_defconfig +++ b/board/aarch32/linux_defconfig @@ -394,6 +394,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y CONFIG_WQ_WATCHDOG=y CONFIG_WQ_CPU_INTENSIVE_REPORT=y +CONFIG_TEST_LOCKUP=m # CONFIG_SCHED_DEBUG is not set # CONFIG_RCU_TRACE is not set CONFIG_FUNCTION_TRACER=y diff --git a/board/aarch64/linux_defconfig b/board/aarch64/linux_defconfig index 0076bfa8..cd4a84c1 100644 --- a/board/aarch64/linux_defconfig +++ b/board/aarch64/linux_defconfig @@ -407,6 +407,7 @@ CONFIG_WATCHDOG=y CONFIG_WATCHDOG_SYSFS=y CONFIG_SOFT_WATCHDOG=y CONFIG_GPIO_WATCHDOG=y +CONFIG_ARM_SBSA_WATCHDOG=y CONFIG_ARMADA_37XX_WATCHDOG=y CONFIG_I6300ESB_WDT=y CONFIG_MFD_MAX77620=y @@ -564,6 +565,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y CONFIG_WQ_WATCHDOG=y CONFIG_WQ_CPU_INTENSIVE_REPORT=y +CONFIG_TEST_LOCKUP=m # CONFIG_SCHED_DEBUG is not set # CONFIG_RCU_TRACE is not set CONFIG_FUNCTION_TRACER=y diff --git a/board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts b/board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts index 1286e951..a6838d68 100644 --- a/board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts +++ b/board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts @@ -59,3 +59,9 @@ XSWP(a, "e13", 13, &sfp0); }; }; + +&cp0_spi1 { + spi-flash@0 { + broken-flash-reset; + }; +}; diff --git a/board/riscv64/linux_defconfig b/board/riscv64/linux_defconfig index e3337e65..f3815aef 100644 --- a/board/riscv64/linux_defconfig +++ b/board/riscv64/linux_defconfig @@ -481,6 +481,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y CONFIG_WQ_WATCHDOG=y CONFIG_WQ_CPU_INTENSIVE_REPORT=y +CONFIG_TEST_LOCKUP=m # CONFIG_SCHED_DEBUG is not set CONFIG_STACKTRACE=y CONFIG_RCU_CPU_STALL_TIMEOUT=60 diff --git a/board/x86_64/linux_defconfig b/board/x86_64/linux_defconfig index db815eb1..ffd4b045 100644 --- a/board/x86_64/linux_defconfig +++ b/board/x86_64/linux_defconfig @@ -270,5 +270,6 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y CONFIG_WQ_WATCHDOG=y CONFIG_WQ_CPU_INTENSIVE_REPORT=y +CONFIG_TEST_LOCKUP=m CONFIG_FUNCTION_TRACER=y CONFIG_UNWINDER_FRAME_POINTER=y diff --git a/test/case/hardware/Readme.adoc b/test/case/hardware/Readme.adoc index 088069b1..a7fb1de5 100644 --- a/test/case/hardware/Readme.adoc +++ b/test/case/hardware/Readme.adoc @@ -5,7 +5,10 @@ Tests verifying hardware monitoring and management: - USB device detection and enumeration - Multiple USB port management and device handling + - Watchdog reset capability include::usb/Readme.adoc[] include::usb_two_ports/Readme.adoc[] + +include::watchdog/Readme.adoc[] diff --git a/test/case/hardware/all.yaml b/test/case/hardware/all.yaml index 22f4891f..1ca32dd3 100644 --- a/test/case/hardware/all.yaml +++ b/test/case/hardware/all.yaml @@ -4,3 +4,6 @@ - name: USB configuration with two USB ports case: usb_two_ports/test.py + +- name: Watchdog reset on system lockup + case: watchdog/test.py diff --git a/test/case/hardware/watchdog/Readme.adoc b/test/case/hardware/watchdog/Readme.adoc new file mode 120000 index 00000000..ae32c841 --- /dev/null +++ b/test/case/hardware/watchdog/Readme.adoc @@ -0,0 +1 @@ +test.adoc \ No newline at end of file diff --git a/test/case/hardware/watchdog/test.adoc b/test/case/hardware/watchdog/test.adoc new file mode 100644 index 00000000..ca904cd8 --- /dev/null +++ b/test/case/hardware/watchdog/test.adoc @@ -0,0 +1,28 @@ +=== Watchdog reset on system lockup + +ifdef::topdoc[:imagesdir: {topdoc}../../test/case/hardware/watchdog] + +==== Description + +Verify that a system's watchdog trips and successfully reboots the +system back to a working state if a lockup occurs. + +This is tested by using the Linux kernel's `test_lockup` module to +inject a hard lockup (i.e., blocking servicing of all interrupts) on +all CPU cores that lasts for twice as long as the watchdog's reported +timeout. + +==== Topology + +image::topology.svg[Watchdog reset on system lockup topology, align=center, scaledwidth=75%] + +==== Sequence + +. Set up topology and attach to target DUT +. Verify the presence of a watchdog device +. Verify the presence of the test_lockup module +. Trigger a hard lockup on all CPU cores +. Wait for the watchdog to trip +. Verify that the system reboots + + diff --git a/test/case/hardware/watchdog/test.py b/test/case/hardware/watchdog/test.py new file mode 100755 index 00000000..b8854536 --- /dev/null +++ b/test/case/hardware/watchdog/test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Watchdog reset on system lockup + +Verify that a system's watchdog trips and successfully reboots the +system back to a working state if a lockup occurs. + +This is tested by using the Linux kernel's `test_lockup` module to +inject a hard lockup (i.e., blocking servicing of all interrupts) on +all CPU cores that lasts for twice as long as the watchdog's reported +timeout. + +""" +import base64 +import infamy +import json +import subprocess +import time + +with infamy.Test() as test: + with test.step("Set up topology and attach to target DUT"): + env = infamy.Env() + target = env.attach("target", "mgmt") + tgtssh = env.attach("target", "mgmt", "ssh") + + with test.step("Verify the presence of a watchdog device"): + wctl = tgtssh.run(["watchdogctl"], stdout=subprocess.PIPE) + conf = json.loads(wctl.stdout) + + dogs = [ dog for dog in conf.get("device", []) if dog.get("name", "") == "/dev/watchdog" ] + if len(dogs) < 1: + test.fail("No watchdog device available") + else: + dog = dogs[0] + + print(f"Found {dog['name']} ({dog['identity']}), timeout:{dog['timeout']}s") + + with test.step("Verify the presence of the test_lockup module"): + if tgtssh.run(["modprobe", "-q", "-n", "test_lockup"]).returncode != 0: + test.fail("test_lockup module is not available") + + with test.step("Trigger a hard lockup on all CPU cores"): + tgtssh.runsh(f""" + lockup() + {{ + # Give the SSH session some time to properly shut down + sleep 3 + + sudo modprobe test_lockup \ + disable_irq=1 \ + all_cpus=1 \ + time_secs={dog['timeout'] * 2} + }} + + lockup /dev/null & + """) + + with test.step("Wait for the watchdog to trip"): + time.sleep(dog["timeout"]) + + with test.step("Verify that the system reboots"): + infamy.util.wait_boot(target, env) + + test.succeed() diff --git a/test/case/hardware/watchdog/topology.dot b/test/case/hardware/watchdog/topology.dot new file mode 100644 index 00000000..bb497075 --- /dev/null +++ b/test/case/hardware/watchdog/topology.dot @@ -0,0 +1,23 @@ +graph "1x1" { + layout="neato"; + overlap="false"; + esep="+80"; + + node [shape=record, fontname="DejaVu Sans Mono, Book"]; + edge [color="cornflowerblue", penwidth="2", fontname="DejaVu Serif, Book"]; + + host [ + label="host | { mgmt }", + pos="0,12!", + requires="controller", + ]; + + target [ + label="{ mgmt } | target", + pos="10,12!", + + requires="infix watchdog", + ]; + + host:mgmt -- target:mgmt [requires="mgmt", color="lightgray"] +} diff --git a/test/case/hardware/watchdog/topology.svg b/test/case/hardware/watchdog/topology.svg new file mode 100644 index 00000000..6fc6f47a --- /dev/null +++ b/test/case/hardware/watchdog/topology.svg @@ -0,0 +1,33 @@ + + + + + + +1x1 + + + +host + +host + +mgmt + + + +target + +mgmt + +target + + + +host:mgmt--target:mgmt + + + + diff --git a/test/infamy/ssh.py b/test/infamy/ssh.py index 9883f532..2db6bde5 100644 --- a/test/infamy/ssh.py +++ b/test/infamy/ssh.py @@ -1,7 +1,7 @@ import subprocess from dataclasses import dataclass -from . import env +from . import env, netutil, util @dataclass @@ -15,6 +15,9 @@ class Location: import subprocess import os +def ssh_syn(addr, port=22): + return netutil.tcp_port_is_open(addr, port) + def fetch_file(remote_user, remote_address, remote_file, local_file, key_file, check=False, remove=False): """ Fetches a file over SSH using scp and the provided private key. @@ -62,9 +65,12 @@ def fetch_file(remote_user, remote_address, remote_file, local_file, key_file, c print(f"Error removing fetched file {local_file}: {e}") class Device(object): - def __init__(self, name: str, location: Location): + def __init__(self, name: str, location: Location, wait: bool=True): self.name = name self.location = location + if wait: + util.until(lambda: ssh_syn(location.host, location.port)) + def __str__(self): nm = f"{self.name}" if env.ENV.ltop: diff --git a/test/virt/dual/topology.dot.in b/test/virt/dual/topology.dot.in index 736f818c..70dbd741 100644 --- a/test/virt/dual/topology.dot.in +++ b/test/virt/dual/topology.dot.in @@ -24,7 +24,7 @@ graph "dual" { dut1 [ label="{ e1 | e2 | e3 } | dut1 | { e4 | e5 | e6 }", pos="10,18!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9001, qn_mem="384M", @@ -33,7 +33,7 @@ graph "dual" { dut2 [ label="{ e1 | e2 | e3 } | dut2 | { e4 | e5 | e6 }", pos="10,12!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9002, qn_mem="384M", diff --git a/test/virt/quad/topology.dot.in b/test/virt/quad/topology.dot.in index 1242b0ea..609cdced 100644 --- a/test/virt/quad/topology.dot.in +++ b/test/virt/quad/topology.dot.in @@ -23,7 +23,7 @@ graph "quad" { dut1 [ label="{ e1 | e2 | e3 | e4 } | dut1 | { e5 | e6 | e7 | e8}", pos="10,30!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9001, qn_mem="384M", @@ -32,7 +32,7 @@ graph "quad" { dut2 [ label="{ e1 | e2 | e3 | e4 } | dut2 | { e5 | e6 | e7 | e8}", pos="0,20!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9002, qn_mem="384M", @@ -41,7 +41,7 @@ graph "quad" { dut3 [ label="{ e1 | e2 | e3 | e4 } | dut3 | { e5 | e6 | e7 | e8}", pos="0,10!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9003, qn_mem="384M", @@ -51,7 +51,7 @@ graph "quad" { dut4 [ label="{ e1 | e2 | e3 | e4 } | dut4 | { e5 | e6 | e7 | e8}", pos="10,0!", - provides="infix", + provides="infix watchdog", expected_boot="primary", qn_console=9004, qn_mem="384M",