yanger: Start the modularization process with ietf-system

The yanger source grown significantly, code is disorganized, the
namespace is all over the place, etc.

Take a stab at remedying the situation by splitting out the individual
YANG models in separate Python modules, starting with ietf-system.
This commit is contained in:
Tobias Waldekranz
2025-01-09 23:59:20 +01:00
parent c4a79766b7
commit dadf0e2d16
13 changed files with 383 additions and 204 deletions
+2 -2
View File
@@ -18,6 +18,6 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry.scripts]
yanger = "yanger:main"
yanger = "yanger.__main__:main"
cli-pretty = "cli_pretty:main"
ospf-status = "ospf_status:main"
ospf-status = "ospf_status:main"
-4
View File
@@ -1,4 +0,0 @@
from .yanger import main
if __name__ == "__main__":
main()
+3
View File
@@ -0,0 +1,3 @@
from .yanger import main
main()
+27
View File
@@ -0,0 +1,27 @@
LOG = None
def lookup(obj, *keys):
"""This function returns a value from a nested json object"""
curr = obj
for key in keys:
if isinstance(curr, dict) and key in curr:
curr = curr[key]
else:
return None
return curr
def insert(obj, *path_and_value):
""""This function inserts a value into a nested json object"""
if len(path_and_value) < 2:
raise ValueError("Error: insert() takes at least two args")
*path, value = path_and_value
curr = obj
for key in path[:-1]:
if key not in curr or not isinstance(curr[key], dict):
curr[key] = {}
curr = curr[key]
curr[path[-1]] = value
+133
View File
@@ -0,0 +1,133 @@
import abc
import datetime
import json
import os
import subprocess
from . import common
HOST = None
class Host(abc.ABC):
"""Host system API"""
@abc.abstractmethod
def now(self):
"""Get the current time as a `datetime`"""
pass
@abc.abstractmethod
def run(self, cmd, default=None, log=True):
"""Get stdout of cmd
Run cmd, provided as an argv, and return the output it
produces. On error, return the default value if provided,
otherwise raise an exception.
"""
pass
def run_multiline(self, cmd, default=None):
"""Get lines of stdout of cmd"""
try:
txt = self.run(cmd, log=(default is None))
return txt.splitlines()
except:
if default is not None:
return default
raise
def run_json(self, cmd, default=None):
"""Get JSON object from stdout of cmd"""
try:
txt = self.run(cmd, log=(default is None))
return json.loads(txt)
except:
if default is not None:
return default
raise
@abc.abstractmethod
def read(self, path):
"""Get the contents of path
Returns `None` if the file is not readable for any reason.
"""
pass
def read_json(self, path, default=None):
"""Get JSON object from path """
try:
txt = self.read(path)
return json.loads(txt)
except:
if default is not None:
return default
raise
class Localhost(Host):
def now(self):
return datetime.datetime.now(tz=datetime.timezone.utc)
def run(self, cmd, default=None, log=True):
try:
result = subprocess.run(cmd, check=True, text=True,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
return result.stdout
except subprocess.CalledProcessError as err:
if default is not None:
return default
if log:
common.LOG.error(f"Failed to run {err}")
raise
def read(self, path):
try:
with open(path, 'r') as f:
data = f.read().strip()
return data
except FileNotFoundError:
# This is considered OK
pass
except IOError:
common.LOG.error(f"Failed to read \"{path}\"")
return None
class Testhost(Host):
def __init__(self, basedir):
self.basedir = basedir
def now(self):
return datetime.datetime(2023, 1, 1, 12, 0, 0, tzinfo=datetime.timezone.utc)
def run(self, cmd, default=None, log=True):
slug = "_".join(cmd).replace("/", "+").replace(" ", "-")
path = os.path.join(self.basedir, "run", slug)
try:
with open(path, 'r') as f:
return f.read()
except:
if default is not None:
return default
if log:
common.LOG.error(f"No recording found for run \"{path}\"")
raise
def read(self, path):
path = os.path.join(self.basedir, "rootfs", path[1:])
try:
with open(path, 'r') as f:
return f.read()
except FileNotFoundError:
# This is considered OK
pass
except:
common.LOG.error(f"No recording found for file \"{path}\"")
raise
+139
View File
@@ -0,0 +1,139 @@
import subprocess
from .common import insert
from .host import HOST
def uboot_get_boot_order():
data = HOST.run_multiline("fw_printenv BOOT_ORDER".split(), [])
for line in data:
if "BOOT_ORDER" in line:
return line.strip().split("=")[1].split()
raise Exception
def grub_get_boot_order():
data = HOST.run_multiline("grub-editenv /mnt/aux/grub/grubenv list".split(), [])
for line in data:
if "ORDER" in line:
return line.split("=")[1].strip().split()
raise Exception
def get_boot_order():
order = None
try:
order = uboot_get_boot_order()
except:
pass
try:
if order is None:
order = grub_get_boot_order()
except:
pass
return order
def add_ntp(out):
data = HOST.run_multiline(["chronyc", "-c", "sources"], [])
source = []
state_mode_map = {
"^": "server",
"=": "peer",
"#": "local-clock"
}
source_state_map = {
"*": "selected",
"+": "candidate",
"-": "outlier",
"?": "unusable",
"x": "falseticker",
"~": "unstable"
}
for line in data:
src = {}
line = line.split(',')
src["address"] = line[2]
src["mode"] = state_mode_map[line[0]]
src["state"] = source_state_map[line[1]]
src["stratum"] = int(line[3])
src["poll"] = int(line[4])
source.append(src)
insert(out, "infix-system:ntp", "sources", "source", source)
def add_software_slots(out, data):
slots = []
for slot in data["slots"]:
for key, value in slot.items():
new = {}
new["name"] = key
new["bootname"] = slot[key].get("bootname")
new["class"] = slot[key].get("class")
new["state"] = slot[key].get("state")
new["bundle"] = {}
slot_status=value.get("slot_status", {})
if slot_status.get("bundle", {}).get("compatible"):
new["bundle"]["compatible"] = slot_status.get("bundle", {}).get("compatible")
if slot_status.get("bundle", {}).get("version"):
new["bundle"]["version"] = slot_status.get("bundle", {}).get("version")
if slot_status.get("checksum", {}).get("size"):
new["size"] = str(slot_status.get("checksum", {}).get("size"))
if slot_status.get("checksum", {}).get("sha256"):
new["sha256"] = slot_status.get("checksum", {}).get("sha256")
new["installed"] = {}
if slot_status.get("installed", {}).get("timestamp"):
new["installed"]["datetime"] = slot_status.get("installed", {}).get("timestamp")
if slot_status.get("installed", {}).get("count"):
new["installed"]["count"] = slot_status.get("installed", {}).get("count")
new["activated"] = {}
if slot_status.get("activated", {}).get("timestamp"):
new["activated"]["datetime"] = slot_status.get("activated", {}).get("timestamp")
if slot_status.get("activated", {}).get("count"):
new["activated"]["count"] = slot_status.get("activated", {}).get("count")
slots.append(new)
out["slot"] = slots
def add_software(out):
software = {}
try:
data = HOST.run_json(["rauc", "status", "--detailed", "--output-format=json"])
software["compatible"] = data["compatible"]
software["variant"] = data["variant"]
software["booted"] = data["booted"]
boot_order = get_boot_order()
if not boot_order is None:
software["boot-order"] = boot_order
add_software_slots(software, data)
except subprocess.CalledProcessError:
pass # Maybe an upgrade i progress, then rauc does not respond
installer_status = HOST.run_json(["rauc-installation-status"])
installer = {}
if installer_status.get("operation"):
installer["operation"] = installer_status["operation"]
if "progress" in installer_status:
progress = {}
if installer_status["progress"].get("percentage"):
progress["percentage"] = int(installer_status["progress"]["percentage"])
if installer_status["progress"].get("message"):
progress["message"] = installer_status["progress"]["message"]
installer["progress"] = progress
software["installer"] = installer
insert(out, "infix-system:software", software)
def operational():
out = {
"ietf-system:system-state": {
}
}
out_state = out["ietf-system:system-state"]
add_software(out_state)
add_ntp(out_state)
return out
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
exec env PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=$(readlink -f $(dirname "$0")/../) \
python3 -m yanger "$@"
+73 -197
View File
@@ -9,6 +9,9 @@ import argparse
from re import match
from datetime import datetime, timedelta, timezone
from . import common
from . import host
TESTPATH = ""
logger = None
@@ -17,82 +20,6 @@ def datetime_now():
return datetime(2023, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
return datetime.now(timezone.utc)
def uboot_get_boot_order():
data = run_cmd("fw_printenv BOOT_ORDER".split(), "boot-order.txt")
for line in data:
if "BOOT_ORDER" in line:
return line.strip().split("=")[1].split()
raise Exception
def grub_get_boot_order():
data = run_cmd("grub-editenv /mnt/aux/grub/grubenv list".split(), None) # No need for testfile, will be returned from uboot
for line in data:
if "ORDER" in line:
return line.split("=")[1].strip().split()
raise Exception
def json_get_yang_type(iface_in):
if iface_in['link_type'] == "loopback":
return "infix-if-type:loopback"
if iface_in['link_type'] in ("gre", "gre6"):
return "infix-if-type:gre"
if iface_in['link_type'] != "ether":
return "infix-if-type:other"
if 'parentbus' in iface_in and iface_in['parentbus'] == "virtio":
return "infix-if-type:etherlike"
if 'linkinfo' not in iface_in:
return "infix-if-type:ethernet"
if 'info_kind' not in iface_in['linkinfo']:
return "infix-if-type:ethernet"
if iface_in['linkinfo']['info_kind'] == "veth":
return "infix-if-type:veth"
if iface_in['linkinfo']['info_kind'] in ("gretap", "ip6gretap"):
return "infix-if-type:gretap"
if iface_in['linkinfo']['info_kind'] == "vlan":
return "infix-if-type:vlan"
if iface_in['linkinfo']['info_kind'] == "bridge":
return "infix-if-type:bridge"
if iface_in['linkinfo']['info_kind'] == "dsa":
return "infix-if-type:ethernet"
if iface_in['linkinfo']['info_kind'] == "dummy":
return "infix-if-type:dummy"
# Fallback
return "infix-if-type:ethernet"
def json_get_yang_origin(addr):
"""Translate kernel IP address origin to YANG"""
xlate = {
"kernel_ll": "link-layer",
"kernel_ra": "link-layer",
"static": "static",
"dhcp": "dhcp",
"random": "random",
}
proto = addr['protocol']
if proto in ("kernel_ll", "kernel_ra"):
if "stable-privacy" in addr:
return "random"
return xlate.get(proto, "other")
def get_proc_value(procfile):
"""Return contents of /proc file, or None"""
try:
@@ -176,6 +103,66 @@ def run_json_cmd(cmd, testfile, default=None, check=True):
raise
return data
def json_get_yang_type(iface_in):
if iface_in['link_type'] == "loopback":
return "infix-if-type:loopback"
if iface_in['link_type'] in ("gre", "gre6"):
return "infix-if-type:gre"
if iface_in['link_type'] != "ether":
return "infix-if-type:other"
if 'parentbus' in iface_in and iface_in['parentbus'] == "virtio":
return "infix-if-type:etherlike"
if 'linkinfo' not in iface_in:
return "infix-if-type:ethernet"
if 'info_kind' not in iface_in['linkinfo']:
return "infix-if-type:ethernet"
if iface_in['linkinfo']['info_kind'] == "veth":
return "infix-if-type:veth"
if iface_in['linkinfo']['info_kind'] in ("gretap", "ip6gretap"):
return "infix-if-type:gretap"
if iface_in['linkinfo']['info_kind'] == "vlan":
return "infix-if-type:vlan"
if iface_in['linkinfo']['info_kind'] == "bridge":
return "infix-if-type:bridge"
if iface_in['linkinfo']['info_kind'] == "dsa":
return "infix-if-type:ethernet"
if iface_in['linkinfo']['info_kind'] == "dummy":
return "infix-if-type:dummy"
# Fallback
return "infix-if-type:ethernet"
def json_get_yang_origin(addr):
"""Translate kernel IP address origin to YANG"""
xlate = {
"kernel_ll": "link-layer",
"kernel_ra": "link-layer",
"static": "static",
"dhcp": "dhcp",
"random": "random",
}
proto = addr['protocol']
if proto in ("kernel_ll", "kernel_ra"):
if "stable-privacy" in addr:
return "random"
return xlate.get(proto, "other")
def iface_is_dsa(iface_in):
"""Check if interface is a DSA/intra-switch port"""
@@ -1090,118 +1077,6 @@ def add_interface(ifname, yang_ifaces):
add_container_ifaces(yang_ifaces)
def add_system_ntp(out):
data = run_cmd(["chronyc", "-c", "sources"], "chronyc-sources.txt", "")
source = []
state_mode_map = {
"^": "server",
"=": "peer",
"#": "local-clock"
}
source_state_map = {
"*": "selected",
"+": "candidate",
"-": "outlier",
"?": "unusable",
"x": "falseticker",
"~": "unstable"
}
for line in data:
src = {}
line = line.split(',')
src["address"] = line[2]
src["mode"] = state_mode_map[line[0]]
src["state"] = source_state_map[line[1]]
src["stratum"] = int(line[3])
src["poll"] = int(line[4])
source.append(src)
insert(out, "infix-system:ntp", "sources", "source", source)
def add_system_software_slots(out, data):
slots = []
for slot in data["slots"]:
for key, value in slot.items():
new = {}
new["name"] = key
new["bootname"] = slot[key].get("bootname")
new["class"] = slot[key].get("class")
new["state"] = slot[key].get("state")
new["bundle"] = {}
slot_status=value.get("slot_status", {})
if slot_status.get("bundle", {}).get("compatible"):
new["bundle"]["compatible"] = slot_status.get("bundle", {}).get("compatible")
if slot_status.get("bundle", {}).get("version"):
new["bundle"]["version"] = slot_status.get("bundle", {}).get("version")
if slot_status.get("checksum", {}).get("size"):
new["size"] = str(slot_status.get("checksum", {}).get("size"))
if slot_status.get("checksum", {}).get("sha256"):
new["sha256"] = slot_status.get("checksum", {}).get("sha256")
new["installed"] = {}
if slot_status.get("installed", {}).get("timestamp"):
new["installed"]["datetime"] = slot_status.get("installed", {}).get("timestamp")
if slot_status.get("installed", {}).get("count"):
new["installed"]["count"] = slot_status.get("installed", {}).get("count")
new["activated"] = {}
if slot_status.get("activated", {}).get("timestamp"):
new["activated"]["datetime"] = slot_status.get("activated", {}).get("timestamp")
if slot_status.get("activated", {}).get("count"):
new["activated"]["count"] = slot_status.get("activated", {}).get("count")
slots.append(new)
out["slot"] = slots
def get_system_software_boot_order():
order = None
try:
order = uboot_get_boot_order()
except:
pass
try:
if order is None:
order = grub_get_boot_order()
except:
pass
return order
def add_system_software(out):
software = {}
try:
data = run_json_cmd(["rauc", "status", "--detailed", "--output-format=json"], "rauc-status.json")
software["compatible"] = data["compatible"]
software["variant"] = data["variant"]
software["booted"] = data["booted"]
boot_order = get_system_software_boot_order()
if not boot_order is None:
software["boot-order"] = boot_order
add_system_software_slots(software, data)
except subprocess.CalledProcessError:
pass # Maybe an upgrade i progress, then rauc does not respond
installer_status = run_json_cmd(["rauc-installation-status"], "rauc-installer-status.json")
installer = {}
if installer_status.get("operation"):
installer["operation"] = installer_status["operation"]
if "progress" in installer_status:
progress = {}
if installer_status["progress"].get("percentage"):
progress["percentage"] = int(installer_status["progress"]["percentage"])
if installer_status["progress"].get("message"):
progress["message"] = installer_status["progress"]["message"]
installer["progress"] = progress
software["installer"] = installer
insert(out, "infix-system:software", software)
def add_system(yang_data):
add_system_software(yang_data)
add_system_ntp(yang_data)
def main():
global TESTPATH
global logger
@@ -1212,10 +1087,6 @@ def main():
parser.add_argument("-t", "--test", default=None, help="Test data base path")
args = parser.parse_args()
if args.test:
TESTPATH = args.test
else:
TESTPATH = ""
# Set up syslog output for critical errors to aid debugging
logger = logging.getLogger('yanger')
@@ -1229,6 +1100,14 @@ def main():
log.setFormatter(fmt)
logger.setLevel(logging.INFO)
logger.addHandler(log)
common.LOG = logger
if args.test:
TESTPATH = args.test
host.HOST = host.Testhost(args.test)
else:
TESTPATH = ""
host.HOST = host.Localhost()
if args.model == 'ietf-interfaces':
yang_data = {
@@ -1283,11 +1162,8 @@ def main():
add_container(yang_data['infix-containers:containers']['container'])
elif args.model == 'ietf-system':
yang_data = {
"ietf-system:system-state": {
}
}
add_system(yang_data['ietf-system:system-state'])
from . import ietf_system
yang_data = ietf_system.operational()
else:
logger.warning(f"Unsupported model {args.model}", file=sys.stderr)
sys.exit(1)