From dadf0e2d1631e705ae248796bd657c62c4c698ec Mon Sep 17 00:00:00 2001 From: Tobias Waldekranz Date: Thu, 9 Jan 2025 13:09:46 +0100 Subject: [PATCH] yanger: Start the modularization process with ietf-system The yanger source grown significantly, code is disorganized, the namespace is all over the place, etc. Take a stab at remedying the situation by splitting out the individual YANG models in separate Python modules, starting with ietf-system. --- src/statd/python/pyproject.toml | 4 +- src/statd/python/yanger/__init__.py | 4 - src/statd/python/yanger/__main__.py | 3 + src/statd/python/yanger/common.py | 27 ++ src/statd/python/yanger/host.py | 133 +++++++++ src/statd/python/yanger/ietf_system.py | 139 +++++++++ src/statd/python/yanger/yanger | 5 + src/statd/python/yanger/yanger.py | 270 +++++------------- test/case/cli/sysrepo-emulator.sh | 2 +- .../chronyc_-c_sources} | 0 .../fw_printenv_BOOT_ORDER} | 0 .../rauc-installation-status} | 0 ...uc_status_--detailed_--output-format=json} | 0 13 files changed, 383 insertions(+), 204 deletions(-) create mode 100644 src/statd/python/yanger/__main__.py create mode 100644 src/statd/python/yanger/common.py create mode 100644 src/statd/python/yanger/host.py create mode 100644 src/statd/python/yanger/ietf_system.py create mode 100755 src/statd/python/yanger/yanger mode change 100755 => 100644 src/statd/python/yanger/yanger.py rename test/case/cli/system-output/{chronyc-sources.txt => run/chronyc_-c_sources} (100%) rename test/case/cli/system-output/{boot-order.txt => run/fw_printenv_BOOT_ORDER} (100%) rename test/case/cli/system-output/{rauc-installer-status.json => run/rauc-installation-status} (100%) rename test/case/cli/system-output/{rauc-status.json => run/rauc_status_--detailed_--output-format=json} (100%) diff --git a/src/statd/python/pyproject.toml b/src/statd/python/pyproject.toml index d5162e70..8f33838a 100644 --- a/src/statd/python/pyproject.toml +++ b/src/statd/python/pyproject.toml @@ -18,6 +18,6 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -yanger = "yanger:main" +yanger = "yanger.__main__:main" cli-pretty = "cli_pretty:main" -ospf-status = "ospf_status:main" \ No newline at end of file +ospf-status = "ospf_status:main" diff --git a/src/statd/python/yanger/__init__.py b/src/statd/python/yanger/__init__.py index 3ab9057f..e69de29b 100644 --- a/src/statd/python/yanger/__init__.py +++ b/src/statd/python/yanger/__init__.py @@ -1,4 +0,0 @@ -from .yanger import main - -if __name__ == "__main__": - main() diff --git a/src/statd/python/yanger/__main__.py b/src/statd/python/yanger/__main__.py new file mode 100644 index 00000000..bc9fc135 --- /dev/null +++ b/src/statd/python/yanger/__main__.py @@ -0,0 +1,3 @@ +from .yanger import main + +main() diff --git a/src/statd/python/yanger/common.py b/src/statd/python/yanger/common.py new file mode 100644 index 00000000..34f59742 --- /dev/null +++ b/src/statd/python/yanger/common.py @@ -0,0 +1,27 @@ +LOG = None + +def lookup(obj, *keys): + """This function returns a value from a nested json object""" + curr = obj + for key in keys: + if isinstance(curr, dict) and key in curr: + curr = curr[key] + else: + return None + return curr + + +def insert(obj, *path_and_value): + """"This function inserts a value into a nested json object""" + if len(path_and_value) < 2: + raise ValueError("Error: insert() takes at least two args") + + *path, value = path_and_value + + curr = obj + for key in path[:-1]: + if key not in curr or not isinstance(curr[key], dict): + curr[key] = {} + curr = curr[key] + + curr[path[-1]] = value diff --git a/src/statd/python/yanger/host.py b/src/statd/python/yanger/host.py new file mode 100644 index 00000000..040851fc --- /dev/null +++ b/src/statd/python/yanger/host.py @@ -0,0 +1,133 @@ +import abc +import datetime +import json +import os +import subprocess + +from . import common + +HOST = None + +class Host(abc.ABC): + """Host system API""" + + @abc.abstractmethod + def now(self): + """Get the current time as a `datetime`""" + pass + + @abc.abstractmethod + def run(self, cmd, default=None, log=True): + """Get stdout of cmd + + Run cmd, provided as an argv, and return the output it + produces. On error, return the default value if provided, + otherwise raise an exception. + + """ + pass + + def run_multiline(self, cmd, default=None): + """Get lines of stdout of cmd""" + try: + txt = self.run(cmd, log=(default is None)) + return txt.splitlines() + except: + if default is not None: + return default + raise + + def run_json(self, cmd, default=None): + """Get JSON object from stdout of cmd""" + try: + txt = self.run(cmd, log=(default is None)) + return json.loads(txt) + except: + if default is not None: + return default + raise + + @abc.abstractmethod + def read(self, path): + """Get the contents of path + + Returns `None` if the file is not readable for any reason. + + """ + pass + + def read_json(self, path, default=None): + """Get JSON object from path """ + try: + txt = self.read(path) + return json.loads(txt) + except: + if default is not None: + return default + raise + +class Localhost(Host): + def now(self): + return datetime.datetime.now(tz=datetime.timezone.utc) + + def run(self, cmd, default=None, log=True): + try: + result = subprocess.run(cmd, check=True, text=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + return result.stdout + except subprocess.CalledProcessError as err: + if default is not None: + return default + + if log: + common.LOG.error(f"Failed to run {err}") + raise + + def read(self, path): + try: + with open(path, 'r') as f: + data = f.read().strip() + return data + except FileNotFoundError: + # This is considered OK + pass + except IOError: + common.LOG.error(f"Failed to read \"{path}\"") + + return None + +class Testhost(Host): + def __init__(self, basedir): + self.basedir = basedir + + def now(self): + return datetime.datetime(2023, 1, 1, 12, 0, 0, tzinfo=datetime.timezone.utc) + + def run(self, cmd, default=None, log=True): + slug = "_".join(cmd).replace("/", "+").replace(" ", "-") + path = os.path.join(self.basedir, "run", slug) + + try: + with open(path, 'r') as f: + return f.read() + except: + if default is not None: + return default + + if log: + common.LOG.error(f"No recording found for run \"{path}\"") + raise + + def read(self, path): + path = os.path.join(self.basedir, "rootfs", path[1:]) + try: + with open(path, 'r') as f: + return f.read() + except FileNotFoundError: + # This is considered OK + pass + except: + common.LOG.error(f"No recording found for file \"{path}\"") + raise diff --git a/src/statd/python/yanger/ietf_system.py b/src/statd/python/yanger/ietf_system.py new file mode 100644 index 00000000..2f424fc5 --- /dev/null +++ b/src/statd/python/yanger/ietf_system.py @@ -0,0 +1,139 @@ +import subprocess + +from .common import insert +from .host import HOST + +def uboot_get_boot_order(): + data = HOST.run_multiline("fw_printenv BOOT_ORDER".split(), []) + for line in data: + if "BOOT_ORDER" in line: + return line.strip().split("=")[1].split() + + raise Exception + +def grub_get_boot_order(): + data = HOST.run_multiline("grub-editenv /mnt/aux/grub/grubenv list".split(), []) + for line in data: + if "ORDER" in line: + return line.split("=")[1].strip().split() + + raise Exception + +def get_boot_order(): + order = None + try: + order = uboot_get_boot_order() + except: + pass + try: + if order is None: + order = grub_get_boot_order() + except: + pass + + return order + +def add_ntp(out): + data = HOST.run_multiline(["chronyc", "-c", "sources"], []) + source = [] + state_mode_map = { + "^": "server", + "=": "peer", + "#": "local-clock" + } + source_state_map = { + "*": "selected", + "+": "candidate", + "-": "outlier", + "?": "unusable", + "x": "falseticker", + "~": "unstable" + } + for line in data: + src = {} + line = line.split(',') + src["address"] = line[2] + src["mode"] = state_mode_map[line[0]] + src["state"] = source_state_map[line[1]] + src["stratum"] = int(line[3]) + src["poll"] = int(line[4]) + source.append(src) + + insert(out, "infix-system:ntp", "sources", "source", source) + +def add_software_slots(out, data): + slots = [] + for slot in data["slots"]: + for key, value in slot.items(): + new = {} + new["name"] = key + new["bootname"] = slot[key].get("bootname") + new["class"] = slot[key].get("class") + new["state"] = slot[key].get("state") + new["bundle"] = {} + slot_status=value.get("slot_status", {}) + if slot_status.get("bundle", {}).get("compatible"): + new["bundle"]["compatible"] = slot_status.get("bundle", {}).get("compatible") + if slot_status.get("bundle", {}).get("version"): + new["bundle"]["version"] = slot_status.get("bundle", {}).get("version") + if slot_status.get("checksum", {}).get("size"): + new["size"] = str(slot_status.get("checksum", {}).get("size")) + if slot_status.get("checksum", {}).get("sha256"): + new["sha256"] = slot_status.get("checksum", {}).get("sha256") + + new["installed"] = {} + if slot_status.get("installed", {}).get("timestamp"): + new["installed"]["datetime"] = slot_status.get("installed", {}).get("timestamp") + + if slot_status.get("installed", {}).get("count"): + new["installed"]["count"] = slot_status.get("installed", {}).get("count") + + new["activated"] = {} + if slot_status.get("activated", {}).get("timestamp"): + new["activated"]["datetime"] = slot_status.get("activated", {}).get("timestamp") + + if slot_status.get("activated", {}).get("count"): + new["activated"]["count"] = slot_status.get("activated", {}).get("count") + slots.append(new) + out["slot"] = slots + +def add_software(out): + software = {} + try: + data = HOST.run_json(["rauc", "status", "--detailed", "--output-format=json"]) + software["compatible"] = data["compatible"] + software["variant"] = data["variant"] + software["booted"] = data["booted"] + boot_order = get_boot_order() + if not boot_order is None: + software["boot-order"] = boot_order + add_software_slots(software, data) + except subprocess.CalledProcessError: + pass # Maybe an upgrade i progress, then rauc does not respond + + installer_status = HOST.run_json(["rauc-installation-status"]) + installer = {} + if installer_status.get("operation"): + installer["operation"] = installer_status["operation"] + if "progress" in installer_status: + progress = {} + + if installer_status["progress"].get("percentage"): + progress["percentage"] = int(installer_status["progress"]["percentage"]) + if installer_status["progress"].get("message"): + progress["message"] = installer_status["progress"]["message"] + installer["progress"] = progress + software["installer"] = installer + + insert(out, "infix-system:software", software) + +def operational(): + out = { + "ietf-system:system-state": { + } + } + out_state = out["ietf-system:system-state"] + + add_software(out_state) + add_ntp(out_state) + return out diff --git a/src/statd/python/yanger/yanger b/src/statd/python/yanger/yanger new file mode 100755 index 00000000..e4f3da36 --- /dev/null +++ b/src/statd/python/yanger/yanger @@ -0,0 +1,5 @@ +#!/bin/sh + +exec env PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=$(readlink -f $(dirname "$0")/../) \ + python3 -m yanger "$@" + diff --git a/src/statd/python/yanger/yanger.py b/src/statd/python/yanger/yanger.py old mode 100755 new mode 100644 index 62fdc125..9d1aea05 --- a/src/statd/python/yanger/yanger.py +++ b/src/statd/python/yanger/yanger.py @@ -9,6 +9,9 @@ import argparse from re import match from datetime import datetime, timedelta, timezone +from . import common +from . import host + TESTPATH = "" logger = None @@ -17,82 +20,6 @@ def datetime_now(): return datetime(2023, 1, 1, 12, 0, 0, tzinfo=timezone.utc) return datetime.now(timezone.utc) -def uboot_get_boot_order(): - data = run_cmd("fw_printenv BOOT_ORDER".split(), "boot-order.txt") - for line in data: - if "BOOT_ORDER" in line: - return line.strip().split("=")[1].split() - - raise Exception - -def grub_get_boot_order(): - data = run_cmd("grub-editenv /mnt/aux/grub/grubenv list".split(), None) # No need for testfile, will be returned from uboot - - for line in data: - if "ORDER" in line: - return line.split("=")[1].strip().split() - - raise Exception - -def json_get_yang_type(iface_in): - if iface_in['link_type'] == "loopback": - return "infix-if-type:loopback" - - if iface_in['link_type'] in ("gre", "gre6"): - return "infix-if-type:gre" - - if iface_in['link_type'] != "ether": - return "infix-if-type:other" - - if 'parentbus' in iface_in and iface_in['parentbus'] == "virtio": - return "infix-if-type:etherlike" - - if 'linkinfo' not in iface_in: - return "infix-if-type:ethernet" - - if 'info_kind' not in iface_in['linkinfo']: - return "infix-if-type:ethernet" - - if iface_in['linkinfo']['info_kind'] == "veth": - return "infix-if-type:veth" - - if iface_in['linkinfo']['info_kind'] in ("gretap", "ip6gretap"): - return "infix-if-type:gretap" - - if iface_in['linkinfo']['info_kind'] == "vlan": - return "infix-if-type:vlan" - - if iface_in['linkinfo']['info_kind'] == "bridge": - return "infix-if-type:bridge" - - if iface_in['linkinfo']['info_kind'] == "dsa": - return "infix-if-type:ethernet" - - if iface_in['linkinfo']['info_kind'] == "dummy": - return "infix-if-type:dummy" - - # Fallback - return "infix-if-type:ethernet" - - -def json_get_yang_origin(addr): - """Translate kernel IP address origin to YANG""" - xlate = { - "kernel_ll": "link-layer", - "kernel_ra": "link-layer", - "static": "static", - "dhcp": "dhcp", - "random": "random", - } - proto = addr['protocol'] - - if proto in ("kernel_ll", "kernel_ra"): - if "stable-privacy" in addr: - return "random" - - return xlate.get(proto, "other") - - def get_proc_value(procfile): """Return contents of /proc file, or None""" try: @@ -176,6 +103,66 @@ def run_json_cmd(cmd, testfile, default=None, check=True): raise return data +def json_get_yang_type(iface_in): + if iface_in['link_type'] == "loopback": + return "infix-if-type:loopback" + + if iface_in['link_type'] in ("gre", "gre6"): + return "infix-if-type:gre" + + if iface_in['link_type'] != "ether": + return "infix-if-type:other" + + if 'parentbus' in iface_in and iface_in['parentbus'] == "virtio": + return "infix-if-type:etherlike" + + if 'linkinfo' not in iface_in: + return "infix-if-type:ethernet" + + if 'info_kind' not in iface_in['linkinfo']: + return "infix-if-type:ethernet" + + if iface_in['linkinfo']['info_kind'] == "veth": + return "infix-if-type:veth" + + if iface_in['linkinfo']['info_kind'] in ("gretap", "ip6gretap"): + return "infix-if-type:gretap" + + if iface_in['linkinfo']['info_kind'] == "vlan": + return "infix-if-type:vlan" + + if iface_in['linkinfo']['info_kind'] == "bridge": + return "infix-if-type:bridge" + + if iface_in['linkinfo']['info_kind'] == "dsa": + return "infix-if-type:ethernet" + + if iface_in['linkinfo']['info_kind'] == "dummy": + return "infix-if-type:dummy" + + # Fallback + return "infix-if-type:ethernet" + + +def json_get_yang_origin(addr): + """Translate kernel IP address origin to YANG""" + xlate = { + "kernel_ll": "link-layer", + "kernel_ra": "link-layer", + "static": "static", + "dhcp": "dhcp", + "random": "random", + } + proto = addr['protocol'] + + if proto in ("kernel_ll", "kernel_ra"): + if "stable-privacy" in addr: + return "random" + + return xlate.get(proto, "other") + + + def iface_is_dsa(iface_in): """Check if interface is a DSA/intra-switch port""" @@ -1090,118 +1077,6 @@ def add_interface(ifname, yang_ifaces): add_container_ifaces(yang_ifaces) -def add_system_ntp(out): - data = run_cmd(["chronyc", "-c", "sources"], "chronyc-sources.txt", "") - source = [] - state_mode_map = { - "^": "server", - "=": "peer", - "#": "local-clock" - } - source_state_map = { - "*": "selected", - "+": "candidate", - "-": "outlier", - "?": "unusable", - "x": "falseticker", - "~": "unstable" - } - for line in data: - src = {} - line = line.split(',') - src["address"] = line[2] - src["mode"] = state_mode_map[line[0]] - src["state"] = source_state_map[line[1]] - src["stratum"] = int(line[3]) - src["poll"] = int(line[4]) - source.append(src) - - insert(out, "infix-system:ntp", "sources", "source", source) - -def add_system_software_slots(out, data): - slots = [] - for slot in data["slots"]: - for key, value in slot.items(): - new = {} - new["name"] = key - new["bootname"] = slot[key].get("bootname") - new["class"] = slot[key].get("class") - new["state"] = slot[key].get("state") - new["bundle"] = {} - slot_status=value.get("slot_status", {}) - if slot_status.get("bundle", {}).get("compatible"): - new["bundle"]["compatible"] = slot_status.get("bundle", {}).get("compatible") - if slot_status.get("bundle", {}).get("version"): - new["bundle"]["version"] = slot_status.get("bundle", {}).get("version") - if slot_status.get("checksum", {}).get("size"): - new["size"] = str(slot_status.get("checksum", {}).get("size")) - if slot_status.get("checksum", {}).get("sha256"): - new["sha256"] = slot_status.get("checksum", {}).get("sha256") - - new["installed"] = {} - if slot_status.get("installed", {}).get("timestamp"): - new["installed"]["datetime"] = slot_status.get("installed", {}).get("timestamp") - - if slot_status.get("installed", {}).get("count"): - new["installed"]["count"] = slot_status.get("installed", {}).get("count") - - new["activated"] = {} - if slot_status.get("activated", {}).get("timestamp"): - new["activated"]["datetime"] = slot_status.get("activated", {}).get("timestamp") - - if slot_status.get("activated", {}).get("count"): - new["activated"]["count"] = slot_status.get("activated", {}).get("count") - slots.append(new) - out["slot"] = slots - -def get_system_software_boot_order(): - order = None - try: - order = uboot_get_boot_order() - except: - pass - try: - if order is None: - order = grub_get_boot_order() - except: - pass - - return order - -def add_system_software(out): - software = {} - try: - data = run_json_cmd(["rauc", "status", "--detailed", "--output-format=json"], "rauc-status.json") - software["compatible"] = data["compatible"] - software["variant"] = data["variant"] - software["booted"] = data["booted"] - boot_order = get_system_software_boot_order() - if not boot_order is None: - software["boot-order"] = boot_order - add_system_software_slots(software, data) - except subprocess.CalledProcessError: - pass # Maybe an upgrade i progress, then rauc does not respond - - installer_status = run_json_cmd(["rauc-installation-status"], "rauc-installer-status.json") - installer = {} - if installer_status.get("operation"): - installer["operation"] = installer_status["operation"] - if "progress" in installer_status: - progress = {} - - if installer_status["progress"].get("percentage"): - progress["percentage"] = int(installer_status["progress"]["percentage"]) - if installer_status["progress"].get("message"): - progress["message"] = installer_status["progress"]["message"] - installer["progress"] = progress - software["installer"] = installer - - insert(out, "infix-system:software", software) - -def add_system(yang_data): - add_system_software(yang_data) - add_system_ntp(yang_data) - def main(): global TESTPATH global logger @@ -1212,10 +1087,6 @@ def main(): parser.add_argument("-t", "--test", default=None, help="Test data base path") args = parser.parse_args() - if args.test: - TESTPATH = args.test - else: - TESTPATH = "" # Set up syslog output for critical errors to aid debugging logger = logging.getLogger('yanger') @@ -1229,6 +1100,14 @@ def main(): log.setFormatter(fmt) logger.setLevel(logging.INFO) logger.addHandler(log) + common.LOG = logger + + if args.test: + TESTPATH = args.test + host.HOST = host.Testhost(args.test) + else: + TESTPATH = "" + host.HOST = host.Localhost() if args.model == 'ietf-interfaces': yang_data = { @@ -1283,11 +1162,8 @@ def main(): add_container(yang_data['infix-containers:containers']['container']) elif args.model == 'ietf-system': - yang_data = { - "ietf-system:system-state": { - } - } - add_system(yang_data['ietf-system:system-state']) + from . import ietf_system + yang_data = ietf_system.operational() else: logger.warning(f"Unsupported model {args.model}", file=sys.stderr) sys.exit(1) diff --git a/test/case/cli/sysrepo-emulator.sh b/test/case/cli/sysrepo-emulator.sh index 49933c4b..f846301d 100755 --- a/test/case/cli/sysrepo-emulator.sh +++ b/test/case/cli/sysrepo-emulator.sh @@ -3,7 +3,7 @@ SCRIPT_PATH="$(dirname "$(readlink -f "$0")")" ROOT_PATH="$SCRIPT_PATH/../../../" -YANGER_TOOL="$ROOT_PATH/src/statd/python/yanger/yanger.py" +YANGER_TOOL="$ROOT_PATH/src/statd/python/yanger/yanger" INTERFACES_OUTPUT_FILE="$(mktemp)" ROUTES_OUTPUT_FILE="$(mktemp)" diff --git a/test/case/cli/system-output/chronyc-sources.txt b/test/case/cli/system-output/run/chronyc_-c_sources similarity index 100% rename from test/case/cli/system-output/chronyc-sources.txt rename to test/case/cli/system-output/run/chronyc_-c_sources diff --git a/test/case/cli/system-output/boot-order.txt b/test/case/cli/system-output/run/fw_printenv_BOOT_ORDER similarity index 100% rename from test/case/cli/system-output/boot-order.txt rename to test/case/cli/system-output/run/fw_printenv_BOOT_ORDER diff --git a/test/case/cli/system-output/rauc-installer-status.json b/test/case/cli/system-output/run/rauc-installation-status similarity index 100% rename from test/case/cli/system-output/rauc-installer-status.json rename to test/case/cli/system-output/run/rauc-installation-status diff --git a/test/case/cli/system-output/rauc-status.json b/test/case/cli/system-output/run/rauc_status_--detailed_--output-format=json similarity index 100% rename from test/case/cli/system-output/rauc-status.json rename to test/case/cli/system-output/run/rauc_status_--detailed_--output-format=json