diff --git a/configs/snippets/mirror.conf b/configs/snippets/mirror.conf new file mode 100644 index 00000000..d359c022 --- /dev/null +++ b/configs/snippets/mirror.conf @@ -0,0 +1,5 @@ +# Prefer internal download mirror over official upstream sites. If +# the mirror is unreachable, e.g., off-site without VPN, Buildroot +# falls back to the upstream URL and then sources.buildroot.net +# Set up a local mirror and add IP to /etc/hosts to override +BR2_PRIMARY_SITE="http://mirror.internal/pub" diff --git a/doc/developers-guide.md b/doc/developers-guide.md index b1cdd2e4..8598bbe5 100644 --- a/doc/developers-guide.md +++ b/doc/developers-guide.md @@ -190,6 +190,9 @@ To apply a single snippet to the current output directory: make apply-ext4 # build an ext4 rootfs (needed for boards # whose bootloader lacks squashfs support, # e.g. Marvell ESPRESSObin) + make apply-mirror # prefer an internal download mirror + # (BR2_PRIMARY_SITE) over upstream sites, + # see utils/mirror-sync.sh for populating it The `apply-*` targets require an existing `.config` (i.e. you must have already run a `make _defconfig`). The snippet is merged using diff --git a/utils/mirror-sync.sh b/utils/mirror-sync.sh new file mode 100755 index 00000000..5c4a4d5b --- /dev/null +++ b/utils/mirror-sync.sh @@ -0,0 +1,86 @@ +#!/bin/sh +# Populate the shared Buildroot download cache (dl/) with the source +# artifacts needed by all defconfigs, then optionally publish it to a +# download mirror using rsync. The dl/ directory layout is exactly what +# BR2_PRIMARY_SITE expects (dl//), the mirror root is +# a plain copy of dl/, see configs/snippets/mirror.conf +# +# Intended to run nightly from cron on the file server, on every branch +# that should remain buildable from the mirror: +# +# 0 3 * * * cd $HOME/src/infix && git pull --ff-only -q && \ +# ./utils/mirror-sync.sh /srv/pub +# + +set -u + +usage() +{ + cat <<-EOF >&2 + Usage: $0 [-o DIR] [DEST] + + Run 'make source' for every defconfig in configs/, + downloading all required source artifacts to the shared dl/ + directory. All tarball hashes are verified by Buildroot. + + If DEST is given, dl/ is then published there with rsync. Git + caches (dl/*/git) are skipped and nothing is ever deleted from + DEST -- old release branches still reference old tarballs. + + Options: + -o DIR Directory for scratch build trees. Speeds up + repeated runs (default: temporary, removed on exit) + EOF + exit "$1" +} + +outdir= +while getopts "ho:" opt; do + case $opt in + h) usage 0;; + o) outdir=$OPTARG;; + *) usage 1;; + esac +done +shift $((OPTIND - 1)) +dest=${1:-} + +cd "$(dirname "$0")/.." || exit 1 + +exec 9> "${TMPDIR:-/tmp}/infix-mirror-sync.lock" +if ! flock -n 9; then + echo "$0: another instance is already running, skipping." >&2 + exit 0 +fi + +if [ -z "$outdir" ]; then + outdir=$(mktemp -d) + trap 'rm -rf "$outdir"' EXIT INT TERM +fi + +rc=0 +for cfg in configs/*_defconfig; do + cfg=$(basename "$cfg") + O="$outdir/${cfg%_defconfig}" + + echo "=== $cfg" + if ! make O="$O" "$cfg" >/dev/null; then + echo "$0: failed configuring $cfg, skipping." >&2 + rc=1 + continue + fi + if ! make O="$O" source; then + echo "$0: failed downloading sources for $cfg." >&2 + rc=1 + fi + + # Drop this defconfig's build tree; the downloads live in the + # shared dl/ dir, and keeping every tree would exhaust the disk. + rm -rf "$O" +done + +if [ -n "$dest" ]; then + rsync -a --exclude='/*/git' --exclude='*.lock' dl/ "$dest" || rc=1 +fi + +exit $rc