Something went wrong. Try again.
lol
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480name: Eval
on: workflow_call: inputs: artifact-prefix: required: true type: string mergedSha: required: true type: string headSha: required: false # only required when testVersions is true type: string targetSha: required: true type: string systems: required: true type: string testVersions: required: false default: false type: boolean secrets: # Should only be provided in the merge queue, not in pull requests, # where we're evaluating untrusted code. CACHIX_AUTH_TOKEN_GHA: required: false
permissions: {}
defaults: run: shell: bash
jobs: versions: if: inputs.testVersions runs-on: ubuntu-slim outputs: versions: ${{ steps.versions.outputs.versions }} ciPinBumpCommit: ${{ steps.find-pinned-commit.outputs.ciPinBumpCommit }} ciPinBumpCommitShort: ${{ steps.find-pinned-commit.outputs.ciPinBumpCommitShort }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false path: trusted sparse-checkout: | ci/supportedVersions.nix
- name: Check out the PR at the test merge commit uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: ${{ inputs.mergedSha }} path: untrusted sparse-checkout: | ci/pinned.json
- name: Find commit that touched ci/pinned.json id: find-pinned-commit uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: TARGET_SHA: ${{ inputs.targetSha }} HEAD_SHA: ${{ inputs.headSha }} with: script: | const targetSha = process.env.TARGET_SHA const headSha = process.env.HEAD_SHA
if (!targetSha || !headSha) { core.setFailed('Error: Both targetSha and headSha inputs are required when testVersions is true.') return }
// Compare the two commits to get the list of commits in between const comparison = await github.rest.repos.compareCommitsWithBasehead({ ...context.repo, basehead: `${targetSha}...${headSha}`, })
if(comparison.data.commits.length > 50) { core.setFailed('Error: Too many commits in comparison, cannot reliably find pinned.json change.') return }
const logRateLimit = async (label) => { const { data } = await github.rest.rateLimit.get() const { remaining, limit, used } = data.rate core.info(`[Rate Limit ${label}] ${remaining}/${limit} remaining (${used} used)`) }
await logRateLimit('before commit filtering')
// Filter commits that modified ci/pinned.json const commitsModifyingPinned = ( await Promise.all( comparison.data.commits.map(async (commit) => { const commitDetails = await github.rest.repos.getCommit({ ...context.repo, ref: commit.sha, }) const modifiesPinned = commitDetails.data.files?.some( (file) => file.filename === "ci/pinned.json" ) return modifiesPinned ? commit.sha : null }) ) ).filter((sha) => sha !== null)
await logRateLimit('after commit filtering')
if (commitsModifyingPinned.length === 0) { // This should not happen as testVersions should only be true // when ci/pinned.json was modified in the PR. core.setFailed("Error: ci/pinned.json was not modified in this PR") return } else if (commitsModifyingPinned.length > 1) { core.setFailed([ "Error: Multiple commits touch ci/pinned.json in this PR:", ...commitsModifyingPinned, "Please ensure only a single commit modifies ci/pinned.json for accurate version matrix evaluation." ].join("\n")) return }
const ciPinBumpCommit = commitsModifyingPinned[0] core.setOutput("ciPinBumpCommit", ciPinBumpCommit) core.setOutput("ciPinBumpCommitShort", ciPinBumpCommit.substring(0, 7)) core.info(`Found pinned.json commit: ${ciPinBumpCommit}`)
- name: Install Nix uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5
- name: Load supported versions id: versions run: | echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
eval: runs-on: ubuntu-24.04-arm needs: versions if: ${{ !cancelled() && !failure() }} strategy: fail-fast: false matrix: system: ${{ fromJSON(inputs.systems) }} version: - "" # Default Eval triggering rebuild labels and such. - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates. # Failures for versioned Evals will be collected in a separate job below # to not interrupt main Eval's compare step. continue-on-error: ${{ matrix.version != '' }} name: ${{ matrix.system }}${{ matrix.version && format(' @ {0} ({1})', matrix.version, needs.versions.outputs.ciPinBumpCommitShort) || '' }} timeout-minutes: 20 steps: # This is not supposed to be used and just acts as a fallback. # Without swap, when Eval runs OOM, it will fail badly with a # job that is sometimes not interruptible anymore. # If Eval starts swapping, decrease chunkSize to keep it fast. - name: Enable swap run: | sudo fallocate -l 10G /swap sudo chmod 600 /swap sudo mkswap /swap sudo swapon /swap
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false sparse-checkout: .github/actions - name: Check out the PR at merged and target commits uses: ./.github/actions/checkout with: # For versioned evals, use the target as the untrusted base and apply the pin-bump commit merged-as-untrusted-at: ${{ matrix.version && inputs.targetSha || inputs.mergedSha }} untrusted-pin-bump: ${{ matrix.version && needs.versions.outputs.ciPinBumpCommit }} target-as-trusted-at: ${{ inputs.targetSha }}
- name: Install Nix uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5
- uses: cachix/cachix-action@1eb2ef646ac0255473d23a5907ad7b04ce94065c # v17 continue-on-error: true with: # The nixpkgs-gha cache should not be trusted or used outside of Nixpkgs and its forks' CI. name: ${{ vars.CACHIX_NAME || 'nixpkgs-gha' }} extraPullNames: nixpkgs-gha authToken: ${{ secrets.CACHIX_AUTH_TOKEN_GHA }} pushFilter: '(-source|-single-chunk)$'
- name: Evaluate the ${{ matrix.system }} output paths at the merge commit env: MATRIX_SYSTEM: ${{ matrix.system }} MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }} run: | nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \ --argstr evalSystem "$MATRIX_SYSTEM" \ --arg chunkSize 8000 \ --argstr nixPath "$MATRIX_VERSION" \ --out-link merged # If it uses too much memory, slightly decrease chunkSize. # Note: Keep the same further down in sync!
- name: Evaluate the ${{ matrix.system }} output paths at the target commit env: MATRIX_SYSTEM: ${{ matrix.system }} run: | TARGET_DRV=$(nix-instantiate nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \ --argstr evalSystem "$MATRIX_SYSTEM" \ --arg chunkSize 8000 \ --argstr nixPath "nixVersions.latest")
# Try to fetch this from Cachix a few times, for up to 30 seconds. This avoids running Eval # twice in the Merge Queue, when a later item finishes Eval at the merge commit earlier. for _i in {1..6}; do # Using --max-jobs 0 will cause nix-build to fail if this can't be substituted from cachix. if nix-build "$TARGET_DRV" --max-jobs 0; then break fi sleep 5 done
# Either fetches from Cachix or runs Eval itself. The fallback is required # for pull requests into wip-branches without merge queue. nix-build "$TARGET_DRV" --out-link target
- name: Compare outpaths against the target branch env: MATRIX_SYSTEM: ${{ matrix.system }} run: | nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \ --arg beforeDir ./target \ --arg afterDir ./merged \ --argstr evalSystem "$MATRIX_SYSTEM" \ --out-link diff
- name: Upload outpaths diff and stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ inputs.artifact-prefix }}${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }} path: diff/*
compare: runs-on: ubuntu-24.04-arm needs: [eval] if: ${{ !cancelled() && !failure() }} permissions: pull-requests: write # submitting 'wrong branch' reviews statuses: write # creating 'Eval Summary' commit statuses timeout-minutes: 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false sparse-checkout: .github/actions - name: Check out the PR at the target commit uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }} target-as-trusted-at: ${{ inputs.targetSha }}
- name: Download output paths and eval stats for all systems uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: ${{ inputs.artifact-prefix }}diff-* path: diff merge-multiple: true
- name: Install Nix uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5
- name: Combine all output paths and eval stats run: | nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \ --arg diffDir ./diff \ --out-link combined
- name: Upload the maintainer list uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ inputs.artifact-prefix }}maintainers path: combined/maintainers.json
- name: Compare against the target branch env: TARGET_SHA: ${{ inputs.mergedSha }} run: | git -C nixpkgs/trusted diff --name-only "$TARGET_SHA" \ | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
# Use the target branch to get accurate maintainer info nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \ --arg combinedDir ./combined \ --arg touchedFilesJson ./touched-files.json \ --out-link comparison
cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
- name: Upload the comparison results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ inputs.artifact-prefix }}comparison path: comparison/*
- name: Add eval summary to commit statuses if: ${{ github.event_name == 'pull_request_target' }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { readFile } = require('node:fs/promises') const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8')) const removedByKernel = Object.fromEntries( Object.entries(changed.attrdiffByKernel ?? {}).map(([kernel, diff]) => [ kernel, diff.removed.length, ]), ) const description = 'Package: ' + [ `added ${changed.attrdiff.added.length}`, `removed ${changed.attrdiff.removed.length}`, `changed ${changed.attrdiff.changed.length}` ].join(', ') + ' — Rebuild: ' + [ `linux ${changed.rebuildCountByKernel.linux}`, `darwin ${changed.rebuildCountByKernel.darwin}` ].join(', ') + ( Object.values(removedByKernel).some((count) => count > 0) ? ' — Removed: ' + [ `linux ${removedByKernel.linux ?? 0}`, `darwin ${removedByKernel.darwin ?? 0}` ].join(', ') : '' )
const { serverUrl, repo, runId, payload } = context const target_url = `${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
await github.rest.repos.createCommitStatus({ ...repo, sha: payload.pull_request.head.sha, context: 'Eval Summary', state: 'success', description, target_url }) - name: Request changes if PR is against an inappropriate branch if: ${{ github.event_name == 'pull_request_target' }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | require('./nixpkgs/trusted/ci/github-script/check-target-branch.js')({ github, context, core, dry: context.eventName == 'pull_request', })
# Creates a matrix of Eval performance for various versions and systems. report: runs-on: ubuntu-slim needs: [versions, eval] steps: - name: Download output paths and eval stats for all versions uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: "*-diff-*" path: versions
- name: Add version comparison table to job summary uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: ARTIFACT_PREFIX: ${{ inputs.artifact-prefix }} SYSTEMS: ${{ inputs.systems }} VERSIONS: ${{ needs.versions.outputs.versions }} CI_PIN_BUMP_COMMIT: ${{ needs.versions.outputs.ciPinBumpCommit }} with: script: | const { readFileSync } = require('node:fs') const path = require('node:path')
const prefix = process.env.ARTIFACT_PREFIX const systems = JSON.parse(process.env.SYSTEMS) const versions = JSON.parse(process.env.VERSIONS) const ciPinBumpCommit = process.env.CI_PIN_BUMP_COMMIT
core.summary.addHeading('Lix/Nix version comparison') core.summary.addRaw(`\n*Evaluated at commit: \`${ciPinBumpCommit}\` (commit that modified ci/pinned.json)*\n`, true) core.summary.addTable( [].concat( [ [{ data: 'Version', header: true }].concat( systems.map((system) => ({ data: system, header: true })), ), ], versions.map((version) => [{ data: version }].concat( systems.map((system) => { try { const artifact = path.join('versions', `${prefix}${version}-diff-${system}`) const time = Math.round( parseFloat( readFileSync( path.join(artifact, 'after', system, 'total-time'), 'utf-8', ), ), ) const diff = JSON.parse( readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'), ) const attrs = [] .concat(diff.added, diff.removed, diff.changed, diff.rebuilds) // There are some special attributes, which are ignored for rebuilds. // These only have a single path component, because they lack the `.<system>` suffix. .filter((attr) => attr.split('.').length > 1) if (attrs.length > 0) { core.setFailed( `${version} on ${system} has changed outpaths!\n` + `Note: This indicates that commit ${ciPinBumpCommit} ` + `(which modified ci/pinned.json) also contains other ` + `changes affecting package outputs. ` + `Please ensure ci/pinned.json is updated in a standalone commit.` ) return { data: ':x:' } } return { data: time } } catch { core.warning(`${version} on ${system} did not produce artifact.`) return { data: ':warning:' } } }), ), ), ), ) core.summary.addRaw( '\n*Evaluation time in seconds without downloading dependencies.*', true, ) core.summary.addRaw('\n*:warning: Job did not report a result.*', true) core.summary.addRaw( '\n*:x: Job produced different outpaths than the target branch.*', true, ) core.summary.write()
misc: if: ${{ github.event_name != 'push' }} runs-on: ubuntu-24.04-arm timeout-minutes: 10 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false sparse-checkout: .github/actions - name: Checkout the merge commit uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }}
- name: Install Nix uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5
- name: Ensure flake outputs on all systems still evaluate run: nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1'
- name: Query nixpkgs with aliases enabled to check for basic syntax errors run: | time nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null
- name: Ensure NixOS modules meta is valid run: | time nix-instantiate -I ./nixpkgs/untrusted --strict --eval --json ./nixpkgs/untrusted/nixos --arg configuration '{}' --attr config.meta --option restrict-eval true --option allow-import-from-derivation false