test: improve robustness of container actions

If a container has not yet stopped/started we may for proto RESTCONF get
"invalid URI" result back for some container actions.  With this change
we allow the action to be retried up to three times before passing on
the error.

In tests on Qemu (x86_64) this happens very rarely and need at most one
retry before succeeding.  Verified by iterating the same basic test over
night (9000+ iterations).

Fixes #558

Signed-off-by: Joachim Wiberg <troglobit@gmail.com>

test/infamy/container.py

@@ -1,4 +1,7 @@
 """Manage Infix containers"""
+import time
+from infamy.util import warn
+
 
 class Container:
     """Helper methods"""
@@ -32,4 +35,14 @@ class Container:
         return False
 
     def action(self, name, act):
-        return self.system.call_action(f"/infix-containers:containers/container[name='{name}']/{act}")
+        """Call container action (context RPC), retry three times."""
+        xpath = f"/infix-containers:containers/container[name='{name}']/{act}"
+
+        for attempt in range(3):
+            try:
+                return self.system.call_action(xpath)
+            except Exception as e:
+                warn(f"failed {act} {name} ({attempt + 1}/3): {e}")
+                time.sleep(1)
+                if attempt == 2:
+                    raise

test/infamy/util.py

@@ -4,6 +4,7 @@ import infamy.neigh
 import infamy.netconf as netconf
 import infamy.restconf as restconf
 
+
 class ParallelFn(threading.Thread):
     def __init__(self, group=None, target=None, name=None,
                  args=(), kwargs={}, Verbose=None):
@@ -26,11 +27,13 @@ class ParallelFn(threading.Thread):
 
         return self._return
 
+
 def parallel(*fns):
     ths = [ParallelFn(target=fn) for fn in fns]
     [th.start() for th in ths]
     return [th.join() for th in ths]
 
+
 def until(fn, attempts=10, interval=1):
     for attempt in range(attempts):
         if fn():
@@ -40,6 +43,7 @@ def until(fn, attempts=10, interval=1):
 
     raise Exception("Expected condition did not materialize")
 
+
 def wait_boot(target):
     until(lambda: target.reachable() == False, attempts = 100)
     print("Device is booting..")
@@ -54,3 +58,10 @@ def wait_boot(target):
     until(lambda: restconf.restconf_reachable(neigh, target.location.password) == True, attempts = 300)
 
     return True
+
+
+def warn(msg):
+    """Print a warning message in yellow to stderr."""
+    YELLOW = "\033[93m"
+    RST = "\033[0m"
+    print(f"{YELLOW}warn - {msg}{RST}")