Something went wrong. Try again.
lol
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327# Evaluates all the accessible paths in nixpkgs.# *This only builds on Linux* since it requires the Linux sandbox isolation to# be able to write in various places while evaluating inside the sandbox.## This file is used by nixpkgs CI (see .github/workflows/eval.yml) as well as# being used directly as an entry point in Lix's CI (in `flake.nix` in the Lix# repo).## If you know you are doing a breaking API change, please ping the nixpkgs CI# maintainers and the Lix maintainers (`nix eval -f . lib.teams.lix`).{ callPackage, lib, runCommand, writeShellScript, symlinkJoin, busybox, jq, nix,}:
{ # The number of attributes per chunk, see ./README.md for more info. chunkSize ? 5000, # Whether to just evaluate a single chunk for quick testing quickTest ? false, # Don't try to eval packages marked as broken. includeBroken ? false, # Customize the config used to evaluate nixpkgs extraNixpkgsConfig ? { },}:
let nixpkgs = with lib.fileset; toSource { root = ../..; fileset = unions ( map (lib.path.append ../..) [ ".version" "ci/eval/attrpaths.nix" "ci/eval/chunk.nix" "ci/eval/outpaths.nix" "default.nix" "doc" "lib" "maintainers" "modules" "nixos" "pkgs" ] ); };
supportedSystems = builtins.fromJSON ( builtins.readFile ../../pkgs/top-level/release-supported-systems.json );
attrpathsSuperset = { evalSystem, }: runCommand "attrpaths-superset.json" { src = nixpkgs; # Don't depend on -dev outputs to reduce closure size for CI. nativeBuildInputs = map lib.getBin [ busybox nix ]; } '' export NIX_STATE_DIR=$(mktemp -d) mkdir $out export GC_INITIAL_HEAP_SIZE=4g command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \ nix-instantiate --eval --strict --json --show-trace \ "$src/ci/eval/attrpaths.nix" \ -A paths \ -I "$src" \ --argstr extraNixpkgsConfigJson ${lib.escapeShellArg (builtins.toJSON extraNixpkgsConfig)} \ --option restrict-eval true \ --option allow-import-from-derivation false \ --option eval-system "${evalSystem}" > $out/paths.json '';
singleSystem = { # The system to evaluate. # Note that this is intentionally not called `system`, # because `--argstr system` would only be passed to the ci/default.nix file! evalSystem ? builtins.currentSystem, # The path to the `paths.json` file from `attrpathsSuperset` attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json", }: let singleChunk = writeShellScript "single-chunk" '' set -euo pipefail chunkSize=$1 myChunk=$2 system=$3 outputDir=$4
# Default is 5, higher values effectively disable the warning. # This randomly breaks Eval. export GC_LARGE_ALLOC_WARN_INTERVAL=1000
export NIX_SHOW_STATS=1 export NIX_SHOW_STATS_PATH="$outputDir/stats/$myChunk" echo "Chunk $myChunk on $system start" set +e command time -o "$outputDir/timestats/$myChunk" \ -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \ nix-env -f "${nixpkgs}/ci/eval/chunk.nix" \ --eval-system "$system" \ --option restrict-eval true \ --option allow-import-from-derivation false \ --query --available \ --out-path --json \ --meta \ --show-trace \ --arg chunkSize "$chunkSize" \ --arg myChunk "$myChunk" \ --arg attrpathFile "${attrpathFile}" \ --arg systems "[ \"$system\" ]" \ --arg includeBroken ${lib.boolToString includeBroken} \ --argstr extraNixpkgsConfigJson ${lib.escapeShellArg (builtins.toJSON extraNixpkgsConfig)} \ -I ${nixpkgs} \ -I ${attrpathFile} \ > "$outputDir/result/$myChunk" \ 2> "$outputDir/stderr/$myChunk" exitCode=$? set -e cat "$outputDir/stderr/$myChunk" cat "$outputDir/timestats/$myChunk" if (( exitCode != 0 )); then echo "Evaluation failed with exit code $exitCode" # This immediately halts all xargs processes kill $PPID elif [[ -s "$outputDir/stderr/$myChunk" ]]; then echo "Nixpkgs on $system evaluated with warnings, aborting" kill $PPID fi ''; in runCommand "nixpkgs-eval-${evalSystem}" { # Don't depend on -dev outputs to reduce closure size for CI. nativeBuildInputs = map lib.getBin [ busybox jq nix ]; env = { inherit evalSystem chunkSize; }; __structuredAttrs = true; unsafeDiscardReferences.out = true; } '' export NIX_STATE_DIR=$(mktemp -d) nix-store --init
echo "System: $evalSystem" cores=$NIX_BUILD_CORES echo "Cores: $cores" attrCount=$(jq length "${attrpathFile}") echo "Attribute count: $attrCount" echo "Chunk size: $chunkSize" # Same as `attrCount / chunkSize` but rounded up chunkCount=$(( (attrCount - 1) / chunkSize + 1 )) echo "Chunk count: $chunkCount"
mkdir -p $out/${evalSystem}
# Record and print stats on free memory and swap in the background ( while true; do availMemory=$(free -m | grep Mem | awk '{print $7}') freeSwap=$(free -m | grep Swap | awk '{print $4}') echo "Available memory: $(( availMemory )) MiB, free swap: $(( freeSwap )) MiB"
if [[ ! -f "$out/${evalSystem}/min-avail-memory" ]] || (( availMemory < $(<$out/${evalSystem}/min-avail-memory) )); then echo "$availMemory" > $out/${evalSystem}/min-avail-memory fi if [[ ! -f $out/${evalSystem}/min-free-swap ]] || (( freeSwap < $(<$out/${evalSystem}/min-free-swap) )); then echo "$freeSwap" > $out/${evalSystem}/min-free-swap fi sleep 4 done ) &
seq_end=$(( chunkCount - 1 ))
${lib.optionalString quickTest '' seq_end=0 ''}
chunkOutputDir=$(mktemp -d) mkdir "$chunkOutputDir"/{result,stats,timestats,stderr}
seq -w 0 "$seq_end" | command time -f "%e" -o "$out/${evalSystem}/total-time" \ xargs -I{} -P"$cores" \ ${singleChunk} "$chunkSize" {} "$evalSystem" "$chunkOutputDir"
cp -r "$chunkOutputDir"/stats $out/${evalSystem}/stats-by-chunk
if (( chunkSize * chunkCount != attrCount )); then # A final incomplete chunk would mess up the stats, don't include it rm "$chunkOutputDir"/stats/"$seq_end" fi
cat "$chunkOutputDir"/result/* | jq -s 'add | map_values(.outputs)' > $out/${evalSystem}/paths.json cat "$chunkOutputDir"/result/* | jq -s 'add | map_values(.meta)' > $out/${evalSystem}/meta.json '';
diff = callPackage ./diff.nix { };
combine = { diffDir, }: runCommand "combined-eval" { # Don't depend on -dev outputs to reduce closure size for CI. nativeBuildInputs = map lib.getBin [ jq ]; } '' mkdir -p $out
# Combine output paths from all systems cat ${diffDir}/*/diff.json | jq -s ' reduce .[] as $item ({}; { added: (.added + $item.added), changed: (.changed + $item.changed), removed: (.removed + $item.removed), rebuilds: (.rebuilds + $item.rebuilds) }) ' > $out/combined-diff.json
# Combine maintainers from all systems cat ${diffDir}/*/maintainers.json | jq -s ' add | group_by(.package) | map({ key: .[0].package, value: map(.maintainers) | flatten | unique }) | from_entries ' > $out/maintainers.json
mkdir -p $out/before/stats for d in ${diffDir}/before/*; do cp -r "$d"/stats-by-chunk $out/before/stats/$(basename "$d") done
mkdir -p $out/after/stats for d in ${diffDir}/after/*; do cp -r "$d"/stats-by-chunk $out/after/stats/$(basename "$d") done '';
compare = callPackage ./compare { };
baseline = { # Whether to evaluate on a specific set of systems, by default all are evaluated evalSystems ? if quickTest then [ "x86_64-linux" ] else supportedSystems, }: symlinkJoin { name = "nixpkgs-eval-baseline"; paths = map ( evalSystem: singleSystem { inherit evalSystem; } ) evalSystems; };
full = { # Whether to evaluate on a specific set of systems, by default all are evaluated evalSystems ? if quickTest then [ "x86_64-linux" ] else supportedSystems, baseline, # What files have been touched? Defaults to none; use the expression below to calculate it. # ``` # git diff --name-only --merge-base master HEAD \ # | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json # ``` touchedFilesJson ? builtins.toFile "touched-files.json" "[ ]", }: let diffs = symlinkJoin { name = "nixpkgs-eval-diffs"; paths = map ( evalSystem: diff { inherit evalSystem; beforeDir = baseline; afterDir = singleSystem { inherit evalSystem; }; } ) evalSystems; }; comparisonReport = compare { combinedDir = combine { diffDir = diffs; }; inherit touchedFilesJson; }; in comparisonReport;
in{ inherit attrpathsSuperset singleSystem diff combine compare # The above three are used by separate VMs in a GitHub workflow, # while the below are intended for testing on a single local machine baseline full ;}