confd: add support for generating /cfg/failure-config.cfg

This rather huge change is a refactor of the factory-config generataion to add
support for also generating a failure-config.

The confd bootstrap script has been given an rc file.  This both eases manual
testing, when modifying the script(s), and also makes it easier to override
from a br2-external.  Infix default is router/end-device, but a br2-external
may be a switch firmware and want to default to all ports in a bridge.

The generated failure-config creates a fail-safe "do no harm" config to boot
with in case startup-config for some reason is broken or cannot be applied,
e.g., bug in confd.  Meaning, for both the router and switch use-cases the
device will start up with all interfaces isolated¹, with an IPv6 SLAAC (EUI64)
address per interface.

Services enabled in this fail-safe mode are: LLDP, mDNS/SD, SSH, and NETCONF.
All to facilitate diagnostics, troubleshooting and device recovery.

Other noteworthy changes include:

 - rename factory/failure directories again -> factory.d/failure.d.  Use
   same naming as we do on target for directories holding generated files
 - The bootstrap script no longer regenerates /cfg/factory.d on each boot
 - The bootstrap script copies all static templates to /cfg/factory.d in
   case a newer image changes the contents of them.  For troubleshooting
 - Support for overriding the 20-interfaces.json generation by br2-external
 - Support for additional 30-config.json (ovrride/extend) by br2-external
 - Expand gen-interfaces to support bridge use-case.
_____
¹ For a switch this means "no switchport", i.e., no switching between ports
  otherwise connected to a switchcore (or bridge) in startup-config.

Signed-off-by: Joachim Wiberg <troglobit@gmail.com>

src/confd/Makefile.am

@@ -1,5 +1,6 @@
 SUBDIRS             = src yang share
 dist_doc_DATA       = README.md LICENSE
+dist_sysconf_DATA   = confdrc
 pkglibexec_SCRIPTS  = $(wildcard bin/*)
 DISTCLEANFILES      = *~ *.d
 ACLOCAL_AMFLAGS     = -I m4

src/confd/bin/.confdrc

@@ -0,0 +1,28 @@
+# Defaults for testing bootstrap script                       -*-conf-*-
+TESTING=true
+
+PATH=.:$PATH
+INIT_DATA=/etc/sysrepo/factory-default.json
+SEARCH=/usr/share/yang/modules/confd:/usr/share/yang/modules/libnetconf2:/usr/share/yang/modules/libyang:/usr/share/yang/modules/netopeer2:/usr/share/yang/modules/sysrepo
+
+CFG_PATH_=/tmp/confd
+
+FACTORY_DEFAULTS_D=../share/factory.d
+FAILURE_DEFAULTS_D=../share/failure.d
+
+FACTORY_D=$CFG_PATH_/factory.d
+FAILURE_D=$CFG_PATH_/failure.d
+
+FACTORY_CFG=$CFG_PATH_/factory-config.cfg
+FAILURE_CFG=$CFG_PATH_/failure-config.cfg
+STARTUP_CFG=$CFG_PATH_/startup-config.cfg
+
+# Uncomment this line in to create a bridge (br0) with all (classified
+# 'gruop port') interfaces as bridge ports.  The br0 interface will get
+# an IPv6 EUI64 SLAAC address as well as an IPv4 ZeroConf address, and a
+# DHCPv4 address if the device is attached to a LAN with a DHCP server.
+#GEN_IFACE_OPTS="-b -4 -d"
+
+# Default hostname in Fail Secure mode, plus last three octets in the base
+# MAC address, e.g. "failed-c0-ff-ee".
+FAIL_HOSTNAME="failed"

src/confd/bin/bootstrap

@@ -11,42 +11,87 @@
 # It's all really simple ... this script takes care of 1-4
 #
 
-PATH=/usr/libexec/confd/:$PATH
+# /etc/confdrc controls the behavior or most of the gen-scripts,
-INIT_DATA=/etc/sysrepo/factory-default.json
+# customize in an overlay when using Infix as an br2-external.
-SEARCH=/usr/share/yang/modules/confd:/usr/share/yang/modules/libnetconf2:/usr/share/yang/modules/libyang:/usr/share/yang/modules/netopeer2:/usr/share/yang/modules/sysrepo
+RC=/etc/confdrc
+if [ "$1" = "-f" ] && [ -f "$2" ]; then
+    RC=$2
+fi
 
-[ "$FACTORY_D" ] || FACTORY_D=/usr/share/confd/factory
+if [ ! -f "$RC" ]; then
-[ "$CFG_D" ]     || CFG_D=/cfg
+    logger -sik -p user.error -t bootstrap "Missing rc file $RC" 2>/dev/null \
-[ "$GENCFG_D" ]  || GENCFG_D=/cfg/factory.d
+        || echo "Missing rc file $RC"
+    exit 99
+fi
 
-FACTORY=$CFG_D/factory-config.cfg
+# shellcheck disable=SC1090
-STARTUP=$CFG_D/startup-config.cfg
+. "$RC"
 
-# Generate device's factory-config at first boot
+FACTORY_GEN="$(dirname "$FACTORY_CFG")/$(basename "$FACTORY_CFG" .cfg).gen"
-if [ ! -d "$GENCFG_D" ]; then
+FAILURE_GEN="$(dirname "$FAILURE_CFG")/$(basename "$FAILURE_CFG" .cfg).gen"
-    mkdir -p $GENCFG_D
 
-    gen-hostname   >$GENCFG_D/20-auto-hostname.json
+# Generate device's factory-config at first boot or on override
-    gen-interfaces >$GENCFG_D/20-auto-interfaces.json
+if [ ! -d "$FACTORY_D" ] || [ "$FACTORY_OVERRIDE" = "true" ]; then
+    mkdir -p "$FACTORY_D"
 
-    [ -s $GENCFG_D/20-auto-hostkey.json ] || gen-hostkeys >$GENCFG_D/20-auto-hostkey.json
+    # Save the original templates (from the original image) used to create factory-config
+    for file in $(find "$FACTORY_DEFAULTS_D" -type f); do
+	cp "$file" "$FACTORY_D"
+    done
 
-    # Allow regenerating default factory-config, but keep it read-ony
+    # Create an overlay for /etc/hostname to change the default in an br2-external
-    # to prevent it from being overwritten by mistake by users.
+    gen-hostname                                            >"$FACTORY_D/20-hostname.json"
-    rm -f $CFG_D/auto-factory-config.cfg
+    gen-interfaces "$GEN_IFACE_OPTS"                        >"$FACTORY_D/20-interfaces.json"
+
+    [ -s "$FACTORY_D/20-hostkey.json"   ] || gen-hostkeys   >"$FACTORY_D/20-hostkey.json"
+
+    # Optional commands (from an overlay) to run for br2-externals
+    [ -x "$(command -v gen-ifs-custom)" ] && gen-ifs-custom >"$FACTORY_D/20-interfaces.json"
+    [ -x "$(command -v gen-cfg-custom)" ] && gen-cfg-custom >"$FACTORY_D/30-config.json"
+
+    rm -f "$FACTORY_GEN"
     # shellcheck disable=SC2046
-    jq -s 'reduce .[] as $item ({}; . * $item)' \
+    jq -s 'reduce .[] as $item ({}; . * $item)'	\
-       $(find $FACTORY_D -name '*.json') \
+       $(find "$FACTORY_D" -name '*.json' | sort) >"$FACTORY_GEN"
-       $(find $GENCFG_D  -name '*.json') \
+    chmod 444 "$FACTORY_GEN"
-       >$CFG_D/auto-factory-config.cfg
-    chmod 444 $CFG_D/auto-factory-config.cfg
 
     # TODO: Look for statically defined factory-config, based on the
     #       system's product ID, or just custom site-specific factory.
 
-    # If we haven't found a better one, settle for auto-factory-config as
+    # If we haven't found a more specific, better match, settle for
-    # the system's factory-config.
+    # factory-config.gen as the system's factory-config.
-    [ -h $FACTORY ] || ln -sf auto-factory-config.cfg $FACTORY
+    [ -h "$FACTORY_CFG" ] || ln -sf "$(basename "$FACTORY_GEN")" "$FACTORY_CFG"
+fi
+
+# Generate device's failure-config at first boot
+if [ ! -d "$FAILURE_D" ] || [ "$FAILURE_OVERRIDE" = "true" ]; then
+    mkdir -p "$FAILURE_D"
+
+    # Save the original templates (from the original image) used to create failure-config
+    for file in $(find "$FAILURE_DEFAULTS_D" -type f); do
+	cp "$file" "$FAILURE_D"
+    done
+
+    gen-hostname   "$FAIL_HOSTNAME"                         >"$FAILURE_D/20-hostname.json"
+    gen-interfaces                                          >"$FAILURE_D/20-interfaces.json"
+
+    [ -s "$FAILURE_D/20-hostkey.json"   ] || gen-hostkeys   >"$FAILURE_D/20-hostkey.json"
+
+    # Optional failure/error config to generate (or override) for br2-externals
+    [ -x "$(command -v gen-err-custom)" ] && gen-err-custom >"$FAILURE_D/30-error.json"
+
+    rm -f "$FAILURE_GEN"
+    # shellcheck disable=SC2046
+    jq -s 'reduce .[] as $item ({}; . * $item)'	\
+       $(find "$FAILURE_D" -name '*.json' | sort) >"$FAILURE_GEN"
+    chmod 444 "$FAILURE_GEN"
+
+    [ -h "$FAILURE_CFG" ] || ln -sf "$(basename "$FAILURE_GEN")" "$FAILURE_CFG"
+fi
+
+if [ -n "$TESTING" ]; then
+	echo "Done."
+	exit 0
 fi
 
 # Drop all pre-initialized data from netopeer2 install, then re-create
@@ -54,7 +99,7 @@ fi
 # and then we initialize it all with our factory defaults.
 rm -rf /etc/sysrepo/* /dev/shm/sr_*
 mkdir -p /etc/sysrepo/
-cp "$FACTORY" "$INIT_DATA"
+cp "$FACTORY_CFG" "$INIT_DATA"
 sysrepoctl -s $SEARCH							\
 	   -i ietf-system@2014-08-06.yang      -g wheel -p 0660		\
            	-e authentication					\
@@ -138,8 +183,8 @@ sysrepoctl -c ietf-netconf-acm -g wheel -p 0660
 # On first boot, install factory-config as startup-config.  Due to a
 # limitation in sysrepo we cannot initialize ietf-netconf-acm, so we
 # cheat, see sysrepo#3079
-if [ ! -f "$STARTUP" ]; then
+if [ ! -f "$STARTUP_CFG" ]; then
-    sysrepocfg -f json -X"$STARTUP"
+    sysrepocfg -f json -X"$STARTUP_CFG"
 fi
 
 # Clear running-config so we can load startup in the next step

src/confd/bin/gen-interfaces

@@ -1,7 +1,41 @@
 #!/bin/sh
+# Generates ietf-intefaces + ietf-ip settings for all detected interfaces
+# Interfaces must be of type Ethernet (copper/sfp), not be "internal" DSA
+# ports, and must have a "parentbus", i.e., be physical ports.
+#
+# Usage: gen-interfaces [-4bd]
+#
+#   -b brN  Bridged mode, set all interfaces as ports in brN
+#   -4      Generic option for both modes, enables ZeroConf IPv4
+#   -d      Generic option for both modes, enables DHCPv4 client
+#   -g grp  Filter interfaces using 'ip link show group grp'
+#
+# By default this script generates plain interfaces with IPv6 autoconfig
+# (EUI64) enabled.  Useful both in a plain router and a Fail Secure mode
+# where any switchcore-backed ports should be prevented from switching.
+#
+# The default mode currently does not support the '-4' or '-d' options,
+# for IPv4 ZeroConf or DHCPv4 address, beacause they require source
+# routing, or similar, to work.  IPv6 does not require that and is
+# therefore the recommended access method.  A separate (and optional)
+# 'gen-cfg-custom' script is checked for in the bootstrap script which
+# can be used to enable IPv4 and DHCP on a single service interface if
+# needed.
+#
+# The -b option triggers the bridge mode, creating a 'brN' interface
+# using all interfaces classified in 'group port' by the nameif script.
+# In the bridge mode port interfaces do not have any IP address, the
+# IPv6 autoconfig address is instead set on 'brN'.  If the '-4' option
+# is set, ZeroConfig (169.254.x.y) is anabled on 'brN'.  If the '-d'
+# option is set, a DHCPv4 client is enabled on 'brN'.
 
 set -e
 
+bridge=
+ipv4=false
+ipv6=true
+dhcp=
+
 gen_interface()
 {
     cat <<EOF
@@ -9,13 +43,45 @@ gen_interface()
         "name": "$1",
         "type": "infix-if-type:ethernet",
         "ietf-ip:ipv6": {
-	    "enabled": true
+	    "enabled": $2
+EOF
+    if [ -n "$3" ]; then
+	cat <<EOF
+	},
+	"infix-interfaces:bridge-port": {
+	    "bridge": "$3"
+EOF
+    fi
+    cat <<EOF
 	}
       }
 EOF
 }
 
-phys_ifaces=$(ip -d -j link show | jq -r '
+while [ "$1" != "" ]; do
+    case $1 in
+	-4)
+	    ipv4=true
+	    ;;
+	-b)
+	    bridge="$2"
+	    shift
+	    ipv6=false		# for bridge port interfaces
+	    ;;
+	-d)
+	    dhcp=true
+	    ;;
+	-g)
+	    group="group $2"	# -g port => 'ip link show group port'
+	    shift
+	    ;;
+	*)
+	    break
+    esac
+    shift
+done
+
+phys_ifaces=$(ip -d -j link show $group | jq -r '
 	 .[] |
 	 select(.link_type == "ether") |
 	 select(.group != "internal")  |
@@ -35,9 +101,42 @@ cat <<EOF
         "ietf-ip:ipv6": {
           "address": [{ "ip": "::1", "prefix-length": 128 }]
         }
+EOF
+if [ -n "$bridge" ]; then
+    cat <<EOF
+      },
+      {
+        "name": "$bridge",
+        "type": "infix-if-type:bridge",
+        "ietf-ip:ipv4": {
+	    "infix-ip:autoconf": {
+                "enabled": $ipv4
+            }
+        },
+	"ietf-ip:ipv6": {
+	    "enabled": true
+        }
+EOF
+fi
+
+cat <<EOF
       }
-$(for iface in $phys_ifaces; do gen_interface $iface; done)
+$(for iface in $phys_ifaces; do gen_interface $iface $ipv6 $bridge; done)
     ]
+EOF
+if [ "$dhcp" = "true" ] && [ -n "$bridge" ]; then
+cat <<EOF
+  },
+  "infix-dhcp-client:dhcp-client": {
+    "enabled": true,
+    "client-if": [
+      {
+        "if-name": "$bridge"
+      }
+    ]
+EOF
+fi
+cat <<EOF
   }
 }
 EOF

src/confd/confdrc

@@ -0,0 +1,34 @@
+# Defaults for /usr/libexec/confd/bootstrap                   -*-conf-*-
+# Copy and mofify this file to your br2-external overlay
+
+PATH=/usr/libexec/confd/:$PATH
+INIT_DATA=/etc/sysrepo/factory-default.json
+SEARCH=/usr/share/yang/modules/confd:/usr/share/yang/modules/libnetconf2:/usr/share/yang/modules/libyang:/usr/share/yang/modules/netopeer2:/usr/share/yang/modules/sysrepo
+
+CFG_PATH_=/cfg
+
+FACTORY_DEFAULTS_D=/usr/share/confd/factory.d
+FAILURE_DEFAULTS_D=/usr/share/confd/failure.d
+
+FACTORY_D=$CFG_PATH_/factory.d
+FAILURE_D=$CFG_PATH_/failure.d
+
+FACTORY_CFG=$CFG_PATH_/factory-config.cfg
+FAILURE_CFG=$CFG_PATH_/failure-config.cfg
+STARTUP_CFG=$CFG_PATH_/startup-config.cfg
+
+# Uncomment this line in to create a bridge (br0) with all (classified
+# 'gruop port') interfaces as bridge ports.  The br0 interface will get
+# an IPv6 EUI64 SLAAC address as well as an IPv4 ZeroConf address, and a
+# DHCPv4 address if the device is attached to a LAN with a DHCP server.
+#GEN_IFACE_OPTS="-g port -b br0 -4 -d"
+
+# Default hostname in Fail Secure mode, plus last three octets in the base
+# MAC address, e.g. "failed-c0-ff-ee".
+FAIL_HOSTNAME="failed"
+
+# Uncomment to regenerate factory-config (except hostkeys) on each boot.
+#FACTORY_OVERRIDE=true
+
+# Uncomment to regenerate failure-config (except hostkeys) on each boot.
+#FAILURE_OVERRIDE=true

src/confd/configure.ac

@@ -6,8 +6,8 @@ AM_SILENT_RULES(yes)
 AC_CONFIG_FILES([
 	Makefile
 	share/Makefile
-	share/factory/Makefile
+	share/factory.d/Makefile
-	share/failure/Makefile
+	share/failure.d/Makefile
 	src/Makefile
 	yang/Makefile
 ])

src/confd/share/Makefile.am

@@ -1,2 +1,2 @@
-SUBDIRS             = factory failure
+SUBDIRS             = factory.d failure.d
-pkgdata_DATA        = $(wildcard share/*)
+dist_pkgdata_DATA   = README

src/confd/share/factory.d/Makefile.am

@@ -0,0 +1,3 @@
+factorydir          = $(pkgdatadir)/factory.d
+dist_factory_DATA   = 10-authentication.json 10-nacm.json \
+		      10-netconf-server.json 10-system.json

src/confd/share/factory/Makefile.am

@@ -1,2 +0,0 @@
-factorydir          = $(pkgdatadir)/factory
-factory_DATA        = $(wildcard *.json)

src/confd/share/failure.d/10-nacm.json

@@ -0,0 +1 @@
+../factory.d/10-nacm.json

src/confd/share/failure.d/10-netconf-server.json

@@ -0,0 +1 @@
+../factory.d/10-netconf-server.json

src/confd/share/failure.d/Makefile.am

@@ -0,0 +1,4 @@
+failuredir          = $(pkgdatadir)/failure.d
+dist_failure_DATA   = 10-authentication.json 10-nacm.json \
+		      10-netconf-server.json 10-system.json
+

src/confd/share/failure/10-nacm.json

@@ -1 +0,0 @@
-../factory/10-nacm.json

src/confd/share/failure/10-netconf-server.json

@@ -1 +0,0 @@
-../factory/10-netconf-server.json

src/confd/share/failure/Makefile.am

@@ -1,2 +0,0 @@
-failuredir          = $(pkgdatadir)/failure
-failure_DATA        = $(wildcard *.json)