From 0d98ee571686cbb1508e92b2d6a54101fa62ae67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20Walstr=C3=B6m?= <lazzer@gmail.com>
Date: Tue, 1 Oct 2024 08:52:11 +0200
Subject: [PATCH] test-spec: Fail build on failed test specification build

This can fail on two ways:
* Generate topology files
  Unfortunatly the python-graphwiz has no way of detect faults
  it just show 'Warning' on stderr, catch it and if != empty, exit 1

* Generate pdf
  Here the default error level was FATAL, but for example a file
  that is included does not exist it just show an error on stdout.
  the error level is now INFO, it should always be silent.
---
 test/spec/generate_spec.py | 31 +++++++++++++++++++++++++------
 test/test.mk               |  2 +-
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/test/spec/generate_spec.py b/test/spec/generate_spec.py
index 946d8065..2b0b4d9b 100755
--- a/test/spec/generate_spec.py
+++ b/test/spec/generate_spec.py
@@ -3,6 +3,8 @@ import os
 import ast
 import graphviz
 import argparse
+import io
+import sys
 
 from pathlib import Path
 
@@ -90,17 +92,17 @@ class TestCase:
 def parse_directory_tree(directory):
     directories=[]
     for dirpath, dirnames, filenames in os.walk(directory):
-        testscript=False
-        topology=False
+        testscript = False
+        topology = False
         # Search for directories containing a test.py and a topology
         # and define the directory as a test directory
 
         if filenames:
             for filename in filenames:
                 if filename == "test.py":
-                    testscript=True
+                    testscript = True
                 if filename == "topology.dot":
-                    topology=True
+                    topology = True
             if testscript and topology:
                 directories.append(dirpath)
     return directories
@@ -110,7 +112,24 @@ parser.add_argument("-d", "--directory", required=True, help="The directory to p
 parser.add_argument("-r", "--root-dir", help="Path that all paths should be relative to")
 args=parser.parse_args()
 
-directories=parse_directory_tree(args.directory)
+output_capture = io.StringIO()
+sys.stderr = output_capture
+
+directories = parse_directory_tree(args.directory)
+error_string = ""
 for directory in directories:
-    test_case=TestCase(directory, args.root_dir)
+    output_capture.truncate(0)
+    output_capture.seek(0)
+    test_case = TestCase(directory, args.root_dir)
     test_case.generate_specification()
+    if len(output_capture.getvalue()) > 0:
+        error_string = output_capture.getvalue()
+        break
+
+sys.stdout = sys.__stdout__
+
+if len(error_string) > 0:
+    print(error_string)
+    exit(1)
+
+exit(0)
diff --git a/test/test.mk b/test/test.mk
index 07419f34..9f9b8a40 100644
--- a/test/test.mk
+++ b/test/test.mk
@@ -36,7 +36,7 @@ test-sh:
 test-spec:
 	@sed 's/{REPLACE}/$(subst ",,$(INFIX_NAME))/'  $(spec-dir)/Readme.adoc.in > $(spec-dir)/Readme.adoc
 	@$(spec-dir)/generate_spec.py -d $(test-dir)/case -r $(BR2_EXTERNAL_INFIX_PATH)
-	@asciidoctor-pdf --theme $(spec-dir)/theme.yml -a pdf-fontsdir=$(spec-dir)/fonts -o $(test-specification) $(spec-dir)/Readme.adoc
+	@asciidoctor-pdf --failure-level INFO --theme $(spec-dir)/theme.yml -a pdf-fontsdir=$(spec-dir)/fonts -o $(test-specification) $(spec-dir)/Readme.adoc
 
 # Unit tests run with random (-r) hostname and container name to
 # prevent race conditions when running in CI environments.