From dbcc04c3f89cbb105ff6010018726fb185928816 Mon Sep 17 00:00:00 2001
From: Joachim Wiberg <troglobit@gmail.com>
Date: Sat, 17 Aug 2024 11:16:02 +0200
Subject: [PATCH] test: improve robustness of container actions

If a container has not yet stopped/started we may for proto RESTCONF get
"invalid URI" result back for some container actions.  With this change
we allow the action to be retried up to three times before passing on
the error.

In tests on Qemu (x86_64) this happens very rarely and need at most one
retry before succeeding.  Verified by iterating the same basic test over
night (9000+ iterations).

Fixes #558

Signed-off-by: Joachim Wiberg <troglobit@gmail.com>
---
 test/infamy/container.py | 15 ++++++++++++++-
 test/infamy/util.py      | 11 +++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/test/infamy/container.py b/test/infamy/container.py
index ba5b9bf7..21b7de89 100644
--- a/test/infamy/container.py
+++ b/test/infamy/container.py
@@ -1,4 +1,7 @@
 """Manage Infix containers"""
+import time
+from infamy.util import warn
+
 
 class Container:
     """Helper methods"""
@@ -32,4 +35,14 @@ class Container:
         return False
 
     def action(self, name, act):
-        return self.system.call_action(f"/infix-containers:containers/container[name='{name}']/{act}")
+        """Call container action (context RPC), retry three times."""
+        xpath = f"/infix-containers:containers/container[name='{name}']/{act}"
+
+        for attempt in range(3):
+            try:
+                return self.system.call_action(xpath)
+            except Exception as e:
+                warn(f"failed {act} {name} ({attempt + 1}/3): {e}")
+                time.sleep(1)
+                if attempt == 2:
+                    raise
diff --git a/test/infamy/util.py b/test/infamy/util.py
index 6aeef2c2..739e2aea 100644
--- a/test/infamy/util.py
+++ b/test/infamy/util.py
@@ -4,6 +4,7 @@ import infamy.neigh
 import infamy.netconf as netconf
 import infamy.restconf as restconf
 
+
 class ParallelFn(threading.Thread):
     def __init__(self, group=None, target=None, name=None,
                  args=(), kwargs={}, Verbose=None):
@@ -26,11 +27,13 @@ class ParallelFn(threading.Thread):
 
         return self._return
 
+
 def parallel(*fns):
     ths = [ParallelFn(target=fn) for fn in fns]
     [th.start() for th in ths]
     return [th.join() for th in ths]
 
+
 def until(fn, attempts=10, interval=1):
     for attempt in range(attempts):
         if fn():
@@ -40,6 +43,7 @@ def until(fn, attempts=10, interval=1):
 
     raise Exception("Expected condition did not materialize")
 
+
 def wait_boot(target):
     until(lambda: target.reachable() == False, attempts = 100)
     print("Device is booting..")
@@ -54,3 +58,10 @@ def wait_boot(target):
     until(lambda: restconf.restconf_reachable(neigh, target.location.password) == True, attempts = 300)
 
     return True
+
+
+def warn(msg):
+    """Print a warning message in yellow to stderr."""
+    YELLOW = "\033[93m"
+    RST = "\033[0m"
+    print(f"{YELLOW}warn - {msg}{RST}")