Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
10 kB · 314 lines
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315#!/usr/bin/env nix-shell#! nix-shell --pure#! nix-shell -i bash#! nix-shell -p curl cacert#! nix-shell -p git#! nix-shell -p nix#! nix-shell -p jq
set -o pipefail
# How the refresher works:## For a given list of <targets>:# 1. fetch latest successful '.build` job# 2. fetch oldest evaluation that contained that '.build', extract nixpkgs commit# 3. fetch all the `.build` artifacts from '$out/on-server/' directory# 4. calculate hashes and craft the commit message with the details on# how to upload the result to 'tarballs.nixos.org'
scratch_dir=$(mktemp -d)trap 'rm -rf -- "${scratch_dir}"' EXIT
usage() { cat >&2 <<EOFUsage: $0 [ --commit ] --targets=<target>[,<target>,...]
The tool must be ran from the root directory of 'nixpkgs' repository.
Synopsis: 'refresh-tarballs.bash' script fetches latest bootstrapFiles built by hydra, registers them in 'nixpkgs' and provides commands to upload seed files to 'tarballs.nixos.org'.
This is usually done in the following cases:
1. Single target fix: current bootstrap files for a single target are problematic for some reason (target-specific bug). In this case we can refresh just that target as:
\$ $0 --commit --targets=i686-unknown-linux-gnu
2. Routine refresh: all bootstrap files should be refreshed to avoid debugging problems that only occur on very old binaries.
\$ $0 --commit --all-targets
To get help on uploading refreshed binaries to 'tarballs.nixos.org'please have a look at <maintainers/scripts/bootstrap-files/README.md>.EOF exit 1}
# log helpers
die() { echo "ERROR: $*" >&2 exit 1}
info() { echo "INFO: $*" >&2}
[[ ${#@} -eq 0 ]] && usage
# known targets
NATIVE_TARGETS=( aarch64-unknown-linux-gnu aarch64-unknown-linux-musl i686-unknown-linux-gnu x86_64-unknown-linux-gnu x86_64-unknown-linux-musl aarch64-apple-darwin x86_64-apple-darwin)
is_native() { local t target=$1 for t in "${NATIVE_TARGETS[@]}"; do [[ $t == $target ]] && return 0 done return 1}
CROSS_TARGETS=( armv5tel-unknown-linux-gnueabi armv6l-unknown-linux-gnueabihf armv6l-unknown-linux-musleabihf armv7l-unknown-linux-gnueabihf mips64el-unknown-linux-gnuabi64 mips64el-unknown-linux-gnuabin32 mipsel-unknown-linux-gnu powerpc64-unknown-linux-gnuabielfv2 powerpc64le-unknown-linux-gnu riscv64-unknown-linux-gnu s390x-unknown-linux-gnu x86_64-unknown-freebsd loongarch64-unknown-linux-gnu)
is_cross() { local t target=$1 for t in "${CROSS_TARGETS[@]}"; do [[ $t == $target ]] && return 0 done return 1}
nar_sri_get() { local restore_path store_path ((${#@} != 2)) && die "nar_sri_get /path/to/name.nar.xz name" restore_path="${scratch_dir}/$2" xz -d < "$1" | nix-store --restore "${restore_path}" [[ $? -ne 0 ]] && die "Failed to unpack '$1'"
store_path=$(nix-store --add "${restore_path}") [[ $? -ne 0 ]] && die "Failed to add '$restore_path' to store" rm -rf -- "${restore_path}"
nix-hash --to-sri "$(nix-store --query --hash "${store_path}")"}
# collect passed options
targets=()commit=no
for arg in "$@"; do case "$arg" in --all-targets) targets+=( ${CROSS_TARGETS[@]} ${NATIVE_TARGETS[@]} ) ;; --targets=*) # Convert "--targets=a,b,c" to targets=(a b c) bash array. comma_targets=${arg#--targets=} targets+=(${comma_targets//,/ }) ;; --commit) commit=yes ;; *) usage ;; esacdone
for target in "${targets[@]}"; do # Native and cross jobsets differ a bit. We'll have to pick the # one based on target name: if is_native $target; then jobset=nixpkgs/trunk job="stdenvBootstrapTools.${target}.build" elif is_cross $target; then jobset=nixpkgs/cross-trunk job="bootstrapTools.${target}.build" else die "'$target' is not present in either of 'NATIVE_TARGETS' or 'CROSS_TARGETS'. Please add one." fi
# 'nixpkgs' prefix where we will write new tarball hashes case "$target" in *linux*) nixpkgs_prefix="pkgs/stdenv/linux" ;; *darwin*) nixpkgs_prefix="pkgs/stdenv/darwin" ;; *freebsd*) nixpkgs_prefix="pkgs/stdenv/freebsd" ;; *) die "don't know where to put '$target'" ;; esac
# We enforce s3 prefix for all targets here. This slightly differs # from manual uploads targets where names were chosen inconsistently. s3_prefix="stdenv/$target"
# resolve 'latest' build to the build 'id', construct the link. latest_build_uri="https://hydra.nixos.org/job/$jobset/$job/latest" latest_build="$target.latest-build" info "Fetching latest successful build from '${latest_build_uri}'" curl -s -H "Content-Type: application/json" -L "$latest_build_uri" > "$latest_build" [[ $? -ne 0 ]] && die "Failed to fetch latest successful build" latest_build_id=$(jq '.id' < "$latest_build") [[ $? -ne 0 ]] && die "Did not find 'id' in latest build" build_uri="https://hydra.nixos.org/build/${latest_build_id}"
# We pick oldest jobset evaluation and extract the 'nicpkgs' commit. # # We use oldest instead of latest to make the result more stable # across unrelated 'nixpkgs' updates. Ideally two subsequent runs of # this refresher should produce the same output (provided there are # no bootstrapTools updates committed between the two runs). oldest_eval_id=$(jq '.jobsetevals|min' < "$latest_build") [[ $? -ne 0 ]] && die "Did not find 'jobsetevals' in latest build" eval_uri="https://hydra.nixos.org/eval/${oldest_eval_id}" eval_meta="$target.eval-meta" info "Fetching oldest eval details from '${eval_uri}' (can take a minute)" curl -s -H "Content-Type: application/json" -L "${eval_uri}" > "$eval_meta" [[ $? -ne 0 ]] && die "Failed to fetch eval metadata" nixpkgs_revision=$(jq --raw-output ".jobsetevalinputs.nixpkgs.revision" < "$eval_meta") [[ $? -ne 0 ]] && die "Failed to fetch revision"
# Extract the build paths out of the build metadata drvpath=$(jq --raw-output '.drvpath' < "${latest_build}") [[ $? -ne 0 ]] && die "Did not find 'drvpath' in latest build" outpath=$(jq --raw-output '.buildoutputs.out.path' < "${latest_build}") [[ $? -ne 0 ]] && die "Did not find 'buildoutputs' in latest build" build_timestamp=$(jq --raw-output '.timestamp' < "${latest_build}") [[ $? -ne 0 ]] && die "Did not find 'timestamp' in latest build" build_time=$(TZ=UTC LANG=C date --date="@${build_timestamp}" --rfc-email) [[ $? -ne 0 ]] && die "Failed to format timestamp"
info "Fetching bootstrap tools to calculate hashes from '${outpath}'" nix-store --realize "$outpath" [[ $? -ne 0 ]] && die "Failed to fetch '${outpath}' from hydra"
fnames=()
target_file="${nixpkgs_prefix}/bootstrap-files/${target}.nix" info "Writing '${target_file}'" { # header cat <<EOF# Autogenerated by maintainers/scripts/bootstrap-files/refresh-tarballs.bash as:# $ ./refresh-tarballs.bash --targets=${target}## Metadata:# - nixpkgs revision: ${nixpkgs_revision}# - hydra build: ${latest_build_uri}# - resolved hydra build: ${build_uri}# - instantiated derivation: ${drvpath}# - output directory: ${outpath}# - build time: ${build_time}{EOF for p in "${outpath}/on-server"/*; do fname=$(basename "$p") fnames+=("$fname") case "$fname" in bootstrap-tools.tar.xz) attr=bootstrapTools ;; busybox) attr=$fname ;; unpack.nar.xz) attr=unpack ;; *) die "Don't know how to map '$fname' to attribute name. Please update me." esac
executable_arg= executable_nix= if [[ -x "$p" ]]; then executable_arg="--executable" executable_nix="executable = true;" fi unpack_nix= name_nix= if [[ $fname = *.nar.xz ]]; then unpack_nix="unpack = true;" name_nix="name = \"${fname%.nar.xz}\";" sri=$(nar_sri_get "$p" "${fname%.nar.xz}") [[ $? -ne 0 ]] && die "Failed to get hash of '$p'" else sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p") [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'" sri=$(nix-hash --to-sri "sha256:$sha256") [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form" fi
# individual file entries cat <<EOF $attr = import <nix/fetchurl.nix> { url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname"; hash = "${sri}";$( [[ -n ${executable_nix} ]] && printf "\n %s" "${executable_nix}" [[ -n ${name_nix} ]] && printf "\n %s" "${name_nix}" [[ -n ${unpack_nix} ]] && printf "\n %s" "${unpack_nix}" ) };EOF done # footer cat <<EOF}EOF } > "${target_file}"
target_file_commit_msg=${target}.commit_message cat > "$target_file_commit_msg" <<EOF${nixpkgs_prefix}: update ${target} bootstrap-files
sha256sum of files to be uploaded:
$(echo "$ sha256sum ${outpath}/on-server/*"sha256sum ${outpath}/on-server/*)
Suggested commands to upload files to 'tarballs.nixos.org':
$ nix-store --realize ${outpath} $ aws s3 cp --recursive --acl public-read ${outpath}/on-server/ s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision} $ aws s3 cp --recursive s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision} ./ $ sha256sum ${fnames[*]} $ sha256sum ${outpath}/on-server/*EOF
cat "$target_file_commit_msg" if [[ $commit == yes ]]; then git commit "${target_file}" -F "$target_file_commit_msg" else info "DRY RUN: git commit ${target_file} -F $target_file_commit_msg" fi rm -- "$target_file_commit_msg"
# delete temp files rm -- "$latest_build" "$eval_meta"done