import requests import json import warnings import os import sys import libyang import re import time from requests.auth import HTTPBasicAuth from urllib3.exceptions import InsecureRequestWarning from dataclasses import dataclass from infamy.transport import Transport, infer_put_dict from . import env # We know we have a self-signed certificate, silence warning about it warnings.simplefilter('ignore', InsecureRequestWarning) @dataclass class Location: interface: str host: str username: str password: str port: int = 443 def xpath_to_uri(xpath, extra=None): """Convert xpath to HTTP URI""" pattern = r'\[(.*?)=["\'](.*?)["\']\]' matches = re.findall(pattern, xpath) uri_path = xpath if matches: for key, value in matches: # replace [key=value] with =value uri_path = re.sub(rf'\[{re.escape(key)}=["\']{re.escape(value)}["\']\]', f'={value}', uri_path) # Append extra if provided if extra is not None: uri_path = f"{uri_path}/{extra}" return uri_path # Workaround for bug in requests 2.32.x: https://github.com/psf/requests/issues/6735 def requests_workaround(method, url, json, headers, auth, verify=False, retry=0): # Create a session session = requests.Session() # Prepare the request request = requests.Request(method, url, json=json, headers=headers, auth=auth) prepared_request = session.prepare_request(request) prepared_request.url = prepared_request.url.replace('%25', '%') prepared_request.url = prepared_request.url.replace('%3A', ':') response = session.send(prepared_request, verify=verify) try: # Raise exceptions for HTTP errors response.raise_for_status() except Exception as e: # most likely caused by nginx up, but not yet rousette if e.response.status_code == 502 and retry < 10: retry = retry + 1 print(f"{method} {url}: HTTP error 502, retrying({retry})") time.sleep(1) response = requests_workaround(method, url, json, headers, auth, verify, retry) else: raise e return response def requests_workaround_put(url, json, headers, auth, verify=False): return requests_workaround('PUT', url, json, headers, auth, verify=False) def requests_workaround_delete(url, headers, auth, verify=False): return requests_workaround('DELETE', url, None, headers, auth, verify=False) def requests_workaround_post(url, json, headers, auth, verify=False): return requests_workaround('POST', url, json, headers, auth, verify=False) def requests_workaround_get(url, headers, auth, verify=False): return requests_workaround('GET', url, None, headers, auth, verify=False) def restconf_reachable(neigh, password): try: headers = { 'Content-Type': 'application/yang-data+json', 'Accept': 'application/yang-data+json' } url = f"https://[{neigh}]/restconf/data/ietf-system:system/hostname" auth = HTTPBasicAuth("admin", password) response = requests_workaround_get(url, headers=headers, auth=auth, verify=False) if response.status_code == 200: return True except: return False return False class Device(Transport): def __init__(self, name: str, location: Location, mapping: dict, yangdir: None | str = None): print("Testing using RESTCONF") self.name = name self.location = location self.mapping = mapping self.url_base = f"https://[{location.host}]:{location.port}" self.restconf_url = f"{self.url_base}/restconf" self.yang_url = f"{self.url_base}/yang" self.rpc_url = f"{self.url_base}/restconf/operations" self.headers = { 'Content-Type': 'application/yang-data+json', 'Accept': 'application/yang-data+json' } self.auth = HTTPBasicAuth(location.username, location.password) self.modules = {} self.lyctx = libyang.Context(yangdir) self._ly_bootstrap(yangdir) self._ly_init(yangdir) def __str__(self): nm = f"{self.name}" if env.ENV.ltop: nm += f"({env.ENV.ltop.xlate(self.name)})" return nm + " [RESTCONF]" def get_schemas_list(self): modules = self.get_operational("/ietf-yang-library:modules-state") data = modules.print_dict() return data["modules-state"]["module"] def get_schema(self, name, revision, yangdir): schema_name = f"{name}@{revision}.yang" url = f"{self.yang_url}/{schema_name}" data = self._get_raw(url=url, parse=False) data = data.decode('utf-8') with open(f"{yangdir}/{schema_name}", 'w') as json_file: json_file.write(data) def schema_exist(self, name, revision, yangdir): schema_name = f"{name}@{revision}.yang" schema_path = f"{yangdir}/{schema_name}" return os.path.exists(schema_path) def _ly_bootstrap(self, yangdir): schemas = self.get_schemas_list() for schema in schemas: if not self.schema_exist(schema["name"], schema["revision"], yangdir): self.get_schema(schema["name"], schema["revision"], yangdir) if schema.get("submodule"): for submodule in schema["submodule"]: if not self.schema_exist(submodule["name"], submodule["revision"], yangdir): self.get_schema(submodule["name"], submodule["revision"], yangdir) if not any("submodule" in x and schema["name"] in x["submodule"] for x in schemas): self.modules.update({schema["name"]: schema}) print("YANG models downloaded.") def _ly_init(self, yangdir): for ms in self.modules.values(): if ms["conformance-type"] != "implement": continue mod = self.lyctx.load_module(ms["name"]) # TODO: ms["feature"] contains the list of enabled # features, so ideally we should only enable the supported" # ones. However, features can depend on each other, so the # naïve looping approach doesn't work. mod.feature_enable_all() def _get_raw(self, url, parse=True): """Actually send a GET to RESTCONF server""" response = requests_workaround_get(url, headers=self.headers, auth=self.auth, verify=False) # Raise exceptions for HTTP errors response.raise_for_status() if parse: data = response.json() data = self.lyctx.parse_data_mem(json.dumps(data), "json", parse_only=True) return data return response.content def get_datastore(self, datastore="operational", path="", parse=True): """Get a datastore""" dspath = f"/ds/ietf-datastores:{datastore}" if path is not None: dspath = f"{dspath}{path}" url = f"{self.restconf_url}{dspath}" try: return self._get_raw(url, parse) except requests.exceptions.HTTPError as e: if e.response.status_code == 404: return None else: raise e def get_running(self, path=None): """Wrapper function to get running datastore""" return self.get_datastore("running", path) def get_operational(self, path=None, parse=True): """Wrapper function to get operational datastore""" return self.get_datastore("operational", path, parse) def get_factory(self, path=None): """Wrapper function to get factory defaults""" return self.get_datastore("factory-default", path) def post_datastore(self, datastore, data): """Actually send a POST to RESTCONF server""" url = f"{self.restconf_url}/ds/ietf-datastores:{datastore}" # Directly pass the dictionary without using json.dumps response = requests_workaround_post(url, json=data, headers=self.headers, auth=self.auth, verify=False) # Raise exceptions for HTTP errors response.raise_for_status() def put_datastore(self, datastore, data): """Actually send a PUT to RESTCONF server""" # Directly pass the dictionary without using json.dumps response = requests_workaround_put( f"{self.restconf_url}/ds/ietf-datastores:{datastore}", json=data, headers=self.headers, auth=self.auth, verify=False ) # Raise exceptions for HTTP errors response.raise_for_status() def get_config_dict(self, modname): """Get all configuration for module @modname as dictionary""" ds = self.get_running(modname) ds = json.loads(ds.print_mem("json", with_siblings=True, pretty=False)) model, container = modname.split(":") for k, v in ds.items(): return {container: v} def put_config_dicts(self, models): """PUT full configuration of all models to running-config""" infer_put_dict(self.name, models) running = self.get_running() for model in models.keys(): mod = self.lyctx.get_module(model) lyd = mod.parse_data_dict(models[model], no_state=True, validate=False) running.merge(lyd) cfg = running.print_mem("json", with_siblings=True, pretty=True) # print(f"PUT new running-config: {cfg}") return self.put_datastore("running", json.loads(cfg)) def put_config_dict(self, modname, edit): """Add @edit to running config and put the whole configuration""" # This is hacky, refactor when rousette have PATCH support. running = self.get_running() mod = self.lyctx.get_module(modname) for k, _ in edit.items(): module = modname + ":" + k break # Ugly hack, but this function should be refactored when patch # is available in rousette anyway. rundict = json.loads(running.print_mem("json", with_siblings=True, pretty=False)) if rundict.get(module) is None: rundict[module] = {} running = self.lyctx.parse_data_mem(json.dumps(rundict), "json", parse_only=True) change = mod.parse_data_dict(edit, no_state=True, validate=False) running.merge_module(change) cfg = running.print_mem("json", with_siblings=True, pretty=False) # print(f"PUT new running-config: {cfg}") data = json.loads(cfg) return self.put_datastore("running", data) def call_dict(self, model, call): pass # Need implementation def call_rpc(self, rpc): """Actually send a POST to RESTCONF server""" url = f"{self.rpc_url}/{rpc}" response = requests_workaround_post( url, json=None, headers=self.headers, auth=self.auth, verify=False ) response.raise_for_status() # Raise an exception for HTTP errors def get_dict(self, xpath=None, parse=True): """NETCONF compat function, just wraps get_data""" return self.get_data(xpath, parse) def get_data(self, xpath=None, parse=True): """Get operational data""" uri = xpath_to_uri(xpath) if xpath is not None else None data = self.get_operational(uri, parse) if parse and data: return data.print_dict() return data def copy(self, source, target): factory = self.get_datastore(source) data = factory.print_mem("json", with_siblings=True, pretty=False) self.put_datastore(target, json.loads(data)) def reboot(self): self.call_rpc("ietf-system:system-restart") def call_action(self, xpath): path = xpath_to_uri(xpath) url = f"{self.restconf_url}/data{path}" response = requests_workaround_post( url, json=None, headers=self.headers, auth=self.auth, verify=False ) # Raise exceptions for HTTP errors response.raise_for_status() return response.content def delete_xpath(self, xpath): """Delete XPath from running config""" path = f"/ds/ietf-datastores:running{xpath_to_uri(xpath)}" url = f"{self.restconf_url}{path}" response = requests_workaround_delete(url, headers=self.headers, auth=self.auth, verify=False) # Raise exceptions for HTTP errors response.raise_for_status() return True