diff --git a/test/infamy/container.py b/test/infamy/container.py index ba5b9bf7..21b7de89 100644 --- a/test/infamy/container.py +++ b/test/infamy/container.py @@ -1,4 +1,7 @@ """Manage Infix containers""" +import time +from infamy.util import warn + class Container: """Helper methods""" @@ -32,4 +35,14 @@ class Container: return False def action(self, name, act): - return self.system.call_action(f"/infix-containers:containers/container[name='{name}']/{act}") + """Call container action (context RPC), retry three times.""" + xpath = f"/infix-containers:containers/container[name='{name}']/{act}" + + for attempt in range(3): + try: + return self.system.call_action(xpath) + except Exception as e: + warn(f"failed {act} {name} ({attempt + 1}/3): {e}") + time.sleep(1) + if attempt == 2: + raise diff --git a/test/infamy/util.py b/test/infamy/util.py index 6aeef2c2..739e2aea 100644 --- a/test/infamy/util.py +++ b/test/infamy/util.py @@ -4,6 +4,7 @@ import infamy.neigh import infamy.netconf as netconf import infamy.restconf as restconf + class ParallelFn(threading.Thread): def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, Verbose=None): @@ -26,11 +27,13 @@ class ParallelFn(threading.Thread): return self._return + def parallel(*fns): ths = [ParallelFn(target=fn) for fn in fns] [th.start() for th in ths] return [th.join() for th in ths] + def until(fn, attempts=10, interval=1): for attempt in range(attempts): if fn(): @@ -40,6 +43,7 @@ def until(fn, attempts=10, interval=1): raise Exception("Expected condition did not materialize") + def wait_boot(target): until(lambda: target.reachable() == False, attempts = 100) print("Device is booting..") @@ -54,3 +58,10 @@ def wait_boot(target): until(lambda: restconf.restconf_reachable(neigh, target.location.password) == True, attempts = 300) return True + + +def warn(msg): + """Print a warning message in yellow to stderr.""" + YELLOW = "\033[93m" + RST = "\033[0m" + print(f"{YELLOW}warn - {msg}{RST}")