From 29031be4e9a0975f917f4459c5b1158a3292be3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Walstr=C3=B6m?= Date: Wed, 8 Jan 2025 14:55:46 +0100 Subject: [PATCH] test-specification: Generate test specification from suite file The suite can also contain meta data for the test specification, One meta data that is implemented is title, which is used when the same test code is used twice with parameters. Also if a test case should not be present in the test specification for some case. You can add specification: False as meta data. --- test/case/all.yaml | 5 +- test/spec/generate_spec.py | 116 ++++++++++++++++++++----------------- test/test.mk | 2 +- 3 files changed, 69 insertions(+), 54 deletions(-) diff --git a/test/case/all.yaml b/test/case/all.yaml index 114d836c..5f254c4c 100644 --- a/test/case/all.yaml +++ b/test/case/all.yaml @@ -1,7 +1,10 @@ --- - case: meta/wait.py + infamy: + specification: False - case: meta/reproducible.py - + infamy: + specification: False - name: Misc tests suite: misc/misc.yaml diff --git a/test/spec/generate_spec.py b/test/spec/generate_spec.py index 10279a05..ec181ebd 100755 --- a/test/spec/generate_spec.py +++ b/test/spec/generate_spec.py @@ -5,6 +5,7 @@ import argparse import io import sys import re +import yaml from pathlib import Path @@ -86,14 +87,6 @@ class TestCase: self.topology_image=f"{directory}/topology" self.test_case=f"{directory}/test.py" self.specification=f"{directory}/Readme.adoc" - with open(self.test_case, 'r') as file: - script_content = file.read() - parsed_script = ast.parse(script_content) - visitor = TestStepVisitor() - visitor.visit(parsed_script) - self.name=visitor.name if visitor.name != "" else "Undefined" - self.description=visitor.description if visitor.description != "" else "Undefined" - self.test_steps=visitor.test_steps def generate_topology(self): """Generate SVG file from the topology.dot file""" @@ -110,72 +103,91 @@ class TestCase: f.write(mod_content) - def generate_specification(self): - """Generate a Readme.adoc for the test case""" + def generate_specification(self, name, case_path, spec_path): + """Generate a ASCIIDOC specification for the test case""" + with open(case_path, 'r') as file: + script_content = file.read() + + parsed_script = ast.parse(script_content) + visitor = TestStepVisitor() + visitor.visit(parsed_script) + description=visitor.description if visitor.description != "" else "Undefined" + test_steps=visitor.test_steps + if name is None: + name = visitor.name self.generate_topology() - self.description = replace_image_tag(self.description, self.test_dir) - with open(self.specification, "w") as spec: - spec.write(f"=== {self.name}\n") + description = replace_image_tag(description, self.test_dir) + with open(spec_path, "w") as spec: + spec.write(f"=== {name}\n") spec.write("==== Description\n") - spec.write(self.description + "\n\n") + spec.write(description + "\n\n") spec.write("==== Topology\n") spec.write("ifdef::topdoc[]\n") - spec.write(f"image::../../{self.test_dir}/topology.svg[{self.name} topology]\n") + spec.write(f"image::../../{self.test_dir}/topology.svg[{name} topology]\n") spec.write("endif::topdoc[]\n") spec.write("ifndef::topdoc[]\n") spec.write("ifdef::testgroup[]\n") - spec.write(f"image::{Path(*self.test_dir.parts[3:])}/topology.svg[{self.name} topology]\n") + spec.write(f"image::{Path(*self.test_dir.parts[3:])}/topology.svg[{name} topology]\n") spec.write("endif::testgroup[]\n") spec.write("ifndef::testgroup[]\n") - spec.write(f"image::topology.svg[{self.name} topology]\n") + spec.write(f"image::topology.svg[{name} topology]\n") spec.write("endif::testgroup[]\n") spec.write("endif::topdoc[]\n") spec.write("==== Test sequence\n") - spec.writelines([f". {step}\n" for step in self.test_steps]) + spec.writelines([f". {step}\n" for step in test_steps]) spec.write("\n\n<<<\n\n") # need empty lines to pagebreak -def parse_directory_tree(directory): - """P - arse a directory for subdirectories with a test.py - and a topology.dot files - """ - directories=[] - for dirpath, dirnames, filenames in os.walk(directory): - testscript = False - topology = False - # Search for directories containing a test.py and a topology - # and define the directory as a test directory +def parse_suite(directory, root, suitefile): + test_spec = None + readme = None + num_tests = 0 + with open(suitefile, "r") as f: + data = yaml.safe_load(f) - if filenames: - for filename in filenames: - if filename == "test.py": - testscript = True - if filename == "topology.dot": - topology = True - if testscript and topology: - directories.append(dirpath) - return directories + for entry in data: + if entry.get("suite"): + path = os.path.dirname(entry["suite"]) + parse_suite(f"{directory}/{path}", root, f"{directory}/{entry['suite']}") + elif entry.get("case"): + if not entry.get("infamy"): + test_title = None # Set from the docstring in the test case + else: + specification = entry["infamy"].get("specification", True); + if not specification: + continue + test_title = entry["infamy"]["title"] + path = os.path.dirname(entry["case"]) + if readme is None: + readme = open(f"{directory}/{path}/Readme.adoc.tmp", "w") + test_case_id = entry["name"] + test_file = f"{directory}/{entry['case']}" + test_spec = f"{directory}/{path}/{test_case_id}.adoc" + test_case = TestCase(f"{directory}/{path}", root) + test_case.generate_specification(test_title, test_file, test_spec) + readme.write(f"include::{path}{test_case_id}.adoc[]\n\n") + if path != "": + readme.close() + readme = None + os.unlink(f"{directory}/{path}/Readme.adoc") + os.symlink(f"{test_case_id}.adoc", f"{directory}/{path}/Readme.adoc") + if not readme is None: + readme.close() + os.rename(f"{directory}/Readme.adoc.tmp", f"{directory}/Readme.adoc") parser = argparse.ArgumentParser(description="Generate a test specification for a subtree.") -parser.add_argument("-d", "--directory", required=True, help="The directory to parse.") -parser.add_argument("-r", "--root-dir", help="Path that all paths should be relative to") +parser.add_argument("-s", "--suite", required=True, help="The suite file to parse") +parser.add_argument("-r", "--root-dir", required=True, help="Path that all paths should be relative to") args=parser.parse_args() +error_string = "" output_capture = io.StringIO() sys.stderr = output_capture -directories = parse_directory_tree(args.directory) -error_string = "" -for directory in directories: - # This is hacky, graphviz output error only to stdout and return successful(always). - # If everything goes well, output shall be empty, fail on any output - output_capture.truncate(0) - output_capture.seek(0) - test_case = TestCase(directory, args.root_dir) - test_case.generate_specification() - if len(output_capture.getvalue()) > 0: - error_string = output_capture.getvalue() - break +# This is hacky, graphviz output error only to stdout and return successful(always). +# If everything goes well, output shall be empty, fail on any output +output_capture.truncate(0) +output_capture.seek(0) +parse_suite(os.path.dirname(args.suite), args.root_dir, args.suite) sys.stdout = sys.__stdout__ diff --git a/test/test.mk b/test/test.mk index 0bd3e326..317cdf2b 100644 --- a/test/test.mk +++ b/test/test.mk @@ -35,7 +35,7 @@ test-sh: test-spec: @esc_infix_name="$(echo $(INFIX_NAME) | sed 's/\//\\\//g')"; \ sed 's/{REPLACE}/$(subst ",,$(esc_infix_name)) $(INFIX_VERSION)/' $(spec-dir)/Readme.adoc.in > $(spec-dir)/Readme.adoc - @$(spec-dir)/generate_spec.py -d $(test-dir)/case -r $(BR2_EXTERNAL_INFIX_PATH) + @$(spec-dir)/generate_spec.py -s $(test-dir)/case/all.yaml -r $(BR2_EXTERNAL_INFIX_PATH) @asciidoctor-pdf --failure-level INFO --theme $(spec-dir)/theme.yml -a pdf-fontsdir=$(spec-dir)/fonts -o $(test-specification) $(spec-dir)/Readme.adoc # Unit tests run with random (-r) hostname and container name to