Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
15 kB · 507 lines
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508#! /usr/bin/env bash
set -e -o pipefail
url=rev=expHash=hashType=$NIX_HASH_ALGOdeepClone=$NIX_PREFETCH_GIT_DEEP_CLONEleaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GITfetchSubmodules=fetchLFS=builder=fetchTags=branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
# ENV paramsout=${out:-}http_proxy=${http_proxy:-}
# NOTE: use of NIX_GIT_SSL_CAINFO is for backwards compatibility; NIX_SSL_CERT_FILE is preferred# as of PR#303307GIT_SSL_CAINFO=${NIX_GIT_SSL_CAINFO:-$NIX_SSL_CERT_FILE}
# populated by clone_user_rev()fullRev=humanReadableRev=commitDate=commitDateStrict8601=
if test -n "$deepClone"; then deepClone=trueelse deepClone=fi
if test "$leaveDotGit" != 1; then leaveDotGit=else leaveDotGit=truefi
usage(){ echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
Options: --out path Path where the output would be stored. --url url Any url understood by 'git clone'. --rev ref Any sha1 or references (such as refs/heads/master) --hash h Expected hash. --name n Symbolic store path name to use for the result (default: based on URL) --branch-name Branch name to check out into --sparse-checkout Only fetch and checkout part of the repository. --non-cone-mode Use non-cone mode for sparse checkouts. --deepClone Clone the entire repository. --no-deepClone Make a shallow clone of just the required ref. --leave-dotGit Keep the .git directories. --fetch-lfs Fetch git Large File Storage (LFS) files. --fetch-submodules Fetch submodules. --fetch-tags Fetch all tags (useful for git describe). --builder Clone as fetchgit does, but url, rev, and out option are mandatory. --quiet Only print the final json summary." exit 1}
# some git commands print to stdout, which would contaminate our JSON outputclean_git(){ git "$@" >&2}
argi=0argfun=""for arg; do if test -z "$argfun"; then case $arg in --out) argfun=set_out;; --url) argfun=set_url;; --rev) argfun=set_rev;; --hash) argfun=set_hashType;; --name) argfun=set_symbolicName;; --branch-name) argfun=set_branchName;; --deepClone) deepClone=true;; --sparse-checkout) argfun=set_sparseCheckout;; --non-cone-mode) nonConeMode=true;; --quiet) QUIET=true;; --no-deepClone) deepClone=;; --leave-dotGit) leaveDotGit=true;; --fetch-lfs) fetchLFS=true;; --fetch-submodules) fetchSubmodules=true;; --fetch-tags) fetchTags=true;; --builder) builder=true;; -h|--help) usage; exit;; *) : $((++argi)) case $argi in 1) url=$arg;; 2) rev=$arg;; 3) expHash=$arg;; *) exit 1;; esac ;; esac else case $argfun in set_*) var=${argfun#set_} eval "$var=$(printf %q "$arg")" ;; esac argfun="" fidone
if test -z "$url"; then usagefi
init_remote(){ local url=$1 clean_git init --initial-branch=master clean_git remote add origin "$url" if [ -n "$sparseCheckout" ]; then git config remote.origin.partialclonefilter "blob:none" echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone} fi ( [ -n "$http_proxy" ] && clean_git config --global http.proxy "$http_proxy" ) || true local proxy_pairs i read -a proxy_pairs <<< "${FETCHGIT_HTTP_PROXIES:-}" for ((i = 1; i < ${#proxy_pairs[@]}; i += 2)); do clean_git config --global "http.${proxy_pairs[$i - 1]}.proxy" "${proxy_pairs[$i]}" done}
# Return the reference of an hash if it exists on the remote repository.ref_from_hash(){ local hash=$1 git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"}
# Return the hash of a reference if it exists on the remote repository.hash_from_ref(){ local ref=$1 git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"}
# Returns a name based on the url and reference## This function needs to be in sync with nix's fetchgit implementation# of urlToName() to re-use the same nix store paths.url_to_name(){ local url=$1 local ref=$2 local base base=$(basename "$url" .git | cut -d: -f2)
if [[ $ref =~ ^[a-z0-9]+$ ]]; then echo "$base-${ref:0:7}" else echo "$base" fi}
# Fetch and checkout the right sha1checkout_hash(){ local hash="$1" local ref="$2"
if test -z "$hash"; then hash=$(hash_from_ref "$ref") fi
[[ -z "$deepClone" ]] && \ clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \ clean_git fetch -t ${builder:+--progress} origin || return 1
local object_type=$(git cat-file -t "$hash") if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then clean_git checkout -b "$branchName" "$hash" || return 1 elif [[ "$object_type" == "tree" ]]; then clean_git config user.email "nix-prefetch-git@localhost" clean_git config user.name "nix-prefetch-git" local commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash") clean_git checkout -b "$branchName" "$commit_id" || return 1 else echo "Unrecognized git object type: $object_type" return 1 fi}
# Fetch only a branch/tag and checkout it.checkout_ref(){ local hash="$1" local ref="$2"
if [[ -n "$deepClone" ]]; then # The caller explicitly asked for a deep clone. Deep clones # allow "git describe" and similar tools to work. See # https://marc.info/?l=nix-dev&m=139641582514772 # for a discussion. return 1 fi
if test -z "$ref"; then ref=$(ref_from_hash "$hash") fi
if test -n "$ref"; then # --depth option is ignored on http repository. clean_git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1 clean_git checkout -b "$branchName" FETCH_HEAD || return 1 else return 1 fi}
# Update submodulesinit_submodules(){ # shallow with leaveDotGit will change hashes [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \ clean_git submodule update --init --recursive --checkout -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \ clean_git submodule update --init --recursive --checkout -j ${NIX_BUILD_CORES:-1} --progress}
clone(){ local top=$PWD local dir="$1" local url="$2" local hash="$3" local ref="$4"
cd "$dir"
# Initialize the repository. init_remote "$url"
# Download data from the repository. checkout_ref "$hash" "$ref" || checkout_hash "$hash" "$ref" || ( echo 1>&2 "Unable to checkout $hash$ref from $url." exit 1 )
# Fetch all tags if requested if test -n "$fetchTags"; then echo "fetching all tags..." >&2 clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2 fi
# Checkout linked sources. if test -n "$fetchSubmodules"; then init_submodules fi
if [ -z "$builder" ] && [ -f .topdeps ]; then if tg help &>/dev/null; then echo "populating TopGit branches..." tg remote --populate origin else echo "WARNING: would populate TopGit branches but TopGit is not available" >&2 echo "WARNING: install TopGit to fix the problem" >&2 fi fi
cd "$top"}
# Remove all remote branches, remove tags not reachable from HEAD, do a full# repack and then garbage collect unreferenced objects.make_deterministic_repo(){ local repo="$1"
# run in sub-shell to not touch current working directory ( cd "$repo" # Remove files that contain timestamps or otherwise have non-deterministic # properties. if [ -f .git ]; then local dotgit_content=$(<.git) local dotgit_dir="${dotgit_content#gitdir: }" else local dotgit_dir=".git" fi pushd "$dotgit_dir" >/dev/null rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config popd >/dev/null # Remove all remote branches. git branch -r | while read -r branch; do clean_git branch -rD "$branch" done
# Remove tags not reachable from HEAD. If we're exactly on a tag, don't # delete it. maybe_tag=$(git tag --points-at HEAD) git tag --contains HEAD | while read -r tag; do if [ "$tag" != "$maybe_tag" ]; then clean_git tag -d "$tag" fi done
# Do a full repack. Must run single-threaded, or else we lose determinism. clean_git config pack.threads 1 clean_git repack -A -d -f rm -f "$dotgit_dir/config"
# Garbage collect unreferenced objects. # Note: --keep-largest-pack prevents non-deterministic ordering of packs # listed in .git/objects/info/packs by only using a single pack clean_git gc --prune=all --keep-largest-pack )}
clone_user_rev() { local dir="$1" local url="$2" local rev="${3:-HEAD}"
if [ -n "$fetchLFS" ]; then clean_git lfs install fi
# Perform the checkout. case "$rev" in HEAD|refs/*) clone "$dir" "$url" "" "$rev" 1>&2;; *) if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then clone "$dir" "$url" "$rev" "" 1>&2 else # if revision is not hexadecimal it might be a tag clone "$dir" "$url" "" "refs/tags/$rev" 1>&2 fi;; esac
pushd "$dir" >/dev/null fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1) humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --) commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev") commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev") popd >/dev/null
# Allow doing additional processing before .git removal eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK" if test -z "$leaveDotGit"; then echo "removing \`.git'..." >&2 find "$dir" -name .git -print0 | xargs -0 rm -rf else find "$dir" -name .git | while read -r gitdir; do make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")" done fi}
exit_handlers=()
run_exit_handlers() { exit_status=$? for handler in "${exit_handlers[@]}"; do eval "$handler $exit_status" done}
trap run_exit_handlers EXIT
quiet_exit_handler() { exec 2>&3 3>&- if [ $1 -ne 0 ]; then cat "$errfile" >&2 fi rm -f "$errfile"}
quiet_mode() { errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")" exit_handlers+=(quiet_exit_handler) exec 3>&2 2>"$errfile"}
json_escape() { local s="$1" s="${s//\\/\\\\}" # \ s="${s//\"/\\\"}" # " s="${s//^H/\\\b}" # \b (backspace) s="${s//^L/\\\f}" # \f (form feed) s="${s///\\\n}" # \n (newline) s="${s//^M/\\\r}" # \r (carriage return) s="${s// /\\t}" # \t (tab) echo "$s"}
print_results() { hash="$1" if ! test -n "$QUIET"; then echo "" >&2 echo "git revision is $fullRev" >&2 if test -n "$finalPath"; then echo "path is $finalPath" >&2 fi echo "git human-readable version is $humanReadableRev" >&2 echo "Commit date is $commitDate" >&2 if test -n "$hash"; then echo "hash is $hash" >&2 fi fi if test -n "$hash"; then cat <<EOF{ "url": "$(json_escape "$url")", "rev": "$(json_escape "$fullRev")", "date": "$(json_escape "$commitDateStrict8601")", "path": "$(json_escape "$finalPath")", "$(json_escape "$hashType")": "$(json_escape "$hash")", "hash": "$(nix-hash --to-sri --type $hashType $hash)", "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false), "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false), "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false), "fetchTags": $([[ -n "$fetchTags" ]] && echo true || echo false), "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false)}EOF fi}
remove_tmpPath() { rm -rf "$tmpPath"}
remove_tmpHomePath() { chmod -R u+w "$tmpHomePath" rm -rf "$tmpHomePath"}
if test -n "$QUIET"; then quiet_modefi
if test -z "$branchName"; then branchName=fetchgitfi
if [ -v symbolicName ]; then storePathName="$symbolicName"else storePathName="$(url_to_name "$url" "$rev")"fi
tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")"exit_handlers+=(remove_tmpHomePath)ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc"HOME="$tmpHomePath"unset XDG_CONFIG_HOMEexport GIT_CONFIG_NOSYSTEM=1
if test -n "$builder"; then test -n "$out" -a -n "$url" -a -n "$rev" || usage mkdir -p "$out" clone_user_rev "$out" "$url" "$rev"else if test -z "$hashType"; then hashType=sha256 fi
# If the hash was given, a file with that hash may already be in the # store. if test -n "$expHash"; then finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$storePathName") if ! nix-store --check-validity "$finalPath" 2> /dev/null; then finalPath= fi hash=$expHash fi
# If we don't know the hash or a path with that hash doesn't exist, # download the file and add it to the store. if test -z "$finalPath"; then # nix>=2.20 rejects adding symlinked paths to the store, so use realpath # to resolve to a physical path. https://github.com/NixOS/nix/issues/11941 tmpPath="$(realpath "$(mktemp -d --tmpdir git-checkout-tmp-XXXXXXXX)")" exit_handlers+=(remove_tmpPath)
tmpFile="$tmpPath/$storePathName" mkdir -p "$tmpFile"
# Perform the checkout. clone_user_rev "$tmpFile" "$url" "$rev"
# Compute the hash. hash=$(nix-hash --type $hashType --base32 "$tmpFile")
# Add the downloaded file to the Nix store. finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
if test -n "$expHash" -a "$expHash" != "$hash"; then echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2 exit 1 fi fi
print_results "$hash"
if test -n "$PRINT_PATH"; then echo "$finalPath" fifi