nixpkgs/.github/workflows/eval.yml

name: Eval

on:
  workflow_call:
    inputs:
      artifact-prefix:
        required: true
        type: string
      mergedSha:
        required: true
        type: string
      headSha:
        required: false # only required when testVersions is true
        type: string
      targetSha:
        required: true
        type: string
      baseBranch:
        required: true
        type: string
      systems:
        required: true
        type: string
      testVersions:
        required: false
        default: false
        type: boolean
    secrets:
      # Can be provided in pull requests because the job it is used in does
      # not evaluate untrusted code.
      NIXPKGS_BRANCH_CHECK_APP_PRIVATE_KEY:
        required: false
      # Should only be provided in the merge queue, not in pull requests,
      # where we're evaluating untrusted code.
      CACHIX_AUTH_TOKEN_GHA:
        required: false

permissions: {}

defaults:
  run:
    shell: bash

jobs:
  versions:
    if: inputs.testVersions
    runs-on: ubuntu-slim
    outputs:
      versions: ${{ steps.versions.outputs.versions }}
      ciPinBumpCommit: ${{ steps.find-pinned-commit.outputs.ciPinBumpCommit }}
      ciPinBumpCommitShort: ${{ steps.find-pinned-commit.outputs.ciPinBumpCommitShort }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
          path: trusted
          sparse-checkout: |
            ci/supportedVersions.nix

      - name: Check out the PR at the test merge commit
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
          ref: ${{ inputs.mergedSha }}
          path: untrusted
          sparse-checkout: |
            ci/pinned.json

      - name: Find commit that touched ci/pinned.json
        id: find-pinned-commit
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          TARGET_SHA: ${{ inputs.targetSha }}
          HEAD_SHA: ${{ inputs.headSha }}
        with:
          script: |
            const targetSha = process.env.TARGET_SHA
            const headSha = process.env.HEAD_SHA

            if (!targetSha || !headSha) {
              core.setFailed('Error: Both targetSha and headSha inputs are required when testVersions is true.')
              return
            }

            // Compare the two commits to get the list of commits in between
            const comparison = await github.rest.repos.compareCommitsWithBasehead({
              ...context.repo,
              basehead: `${targetSha}...${headSha}`,
            })

            if(comparison.data.commits.length > 50) {
              core.setFailed('Error: Too many commits in comparison, cannot reliably find pinned.json change.')
              return
            }

            const logRateLimit = async (label) => {
              const { data } = await github.rest.rateLimit.get()
              const { remaining, limit, used } = data.rate
              core.info(`[Rate Limit ${label}] ${remaining}/${limit} remaining (${used} used)`)
            }

            await logRateLimit('before commit filtering')

            // Filter commits that modified ci/pinned.json
            const commitsModifyingPinned = (
              await Promise.all(
                comparison.data.commits.map(async (commit) => {
                  const commitDetails = await github.rest.repos.getCommit({
                    ...context.repo,
                    ref: commit.sha,
                  })
                  const modifiesPinned = commitDetails.data.files?.some(
                    (file) => file.filename === "ci/pinned.json"
                  )
                  return modifiesPinned ? commit.sha : null
                })
              )
            ).filter((sha) => sha !== null)

            await logRateLimit('after commit filtering')

            if (commitsModifyingPinned.length === 0) {
              // This should not happen as testVersions should only be true
              // when ci/pinned.json was modified in the PR.
              core.setFailed("Error: ci/pinned.json was not modified in this PR")
              return
            } else if (commitsModifyingPinned.length > 1) {
              core.setFailed([
                "Error: Multiple commits touch ci/pinned.json in this PR:",
                ...commitsModifyingPinned,
                "Please ensure only a single commit modifies ci/pinned.json for accurate version matrix evaluation."
              ].join("\n"))
              return
            }

            const ciPinBumpCommit = commitsModifyingPinned[0]
            core.setOutput("ciPinBumpCommit", ciPinBumpCommit)
            core.setOutput("ciPinBumpCommitShort", ciPinBumpCommit.substring(0, 7))
            core.info(`Found pinned.json commit: ${ciPinBumpCommit}`)

      - name: Install Nix
        uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6

      - name: Load supported versions
        id: versions
        run: |
          echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"

  eval:
    runs-on: ubuntu-24.04-arm
    needs: versions
    if: ${{ !cancelled() && !failure() }}
    strategy:
      fail-fast: false
      matrix:
        system: ${{ fromJSON(inputs.systems) }}
        version:
          - "" # Default Eval triggering rebuild labels and such.
          - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
    # Failures for versioned Evals will be collected in a separate job below
    # to not interrupt main Eval's compare step.
    continue-on-error: ${{ matrix.version != '' }}
    name: ${{ matrix.system }}${{ matrix.version && format(' @ {0} ({1})', matrix.version, needs.versions.outputs.ciPinBumpCommitShort) || '' }}
    timeout-minutes: 20
    steps:
      # This is not supposed to be used and just acts as a fallback.
      # Without swap, when Eval runs OOM, it will fail badly with a
      # job that is sometimes not interruptible anymore.
      # If Eval starts swapping, decrease chunkSize to keep it fast.
      - name: Enable swap
        run: |
          sudo fallocate -l 10G /swap
          sudo chmod 600 /swap
          sudo mkswap /swap
          sudo swapon /swap

      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
          sparse-checkout: .github/actions
      - name: Check out the PR at merged and target commits
        uses: ./.github/actions/checkout
        with:
          # For versioned evals, use the target as the untrusted base and apply the pin-bump commit
          merged-as-untrusted-at: ${{ matrix.version && inputs.targetSha || inputs.mergedSha }}
          untrusted-pin-bump: ${{ matrix.version && needs.versions.outputs.ciPinBumpCommit }}
          target-as-trusted-at: ${{ inputs.targetSha }}

      - name: Install Nix
        uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6

      - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
        continue-on-error: true
        with:
          # The nixpkgs-gha cache should not be trusted or used outside of Nixpkgs and its forks' CI.
          name: ${{ vars.CACHIX_NAME || 'nixpkgs-gha' }}
          extraPullNames: nixpkgs-gha
          authToken: ${{ secrets.CACHIX_AUTH_TOKEN_GHA }}
          pushFilter: '(-source|-single-chunk)$'

      - name: Evaluate the ${{ matrix.system }} output paths at the merge commit
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
          MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
        run: |
          nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --arg chunkSize 8000 \
            --argstr nixPath "$MATRIX_VERSION" \
            --out-link merged
          # If it uses too much memory, slightly decrease chunkSize.
          # Note: Keep the same further down in sync!

      - name: Evaluate the ${{ matrix.system }} output paths at the target commit
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
        run: |
          TARGET_DRV=$(nix-instantiate nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --arg chunkSize 8000 \
            --argstr nixPath "nixVersions.latest")

          # Try to fetch this from Cachix a few times, for up to 30 seconds. This avoids running Eval
          # twice in the Merge Queue, when a later item finishes Eval at the merge commit earlier.
          for _i in {1..6}; do
            # Using --max-jobs 0 will cause nix-build to fail if this can't be substituted from cachix.
            if nix-build "$TARGET_DRV" --max-jobs 0; then
              break
            fi
            sleep 5
          done

          # Either fetches from Cachix or runs Eval itself. The fallback is required
          # for pull requests into wip-branches without merge queue.
          nix-build "$TARGET_DRV" --out-link target

      - name: Compare outpaths against the target branch
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
        run: |
          nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
            --arg beforeDir ./target \
            --arg afterDir ./merged \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --out-link diff

      - name: Upload outpaths diff and stats
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ inputs.artifact-prefix }}${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
          path: diff/*

  compare:
    runs-on: ubuntu-24.04-arm
    needs: [eval]
    if: ${{ !cancelled() && !failure() }}
    permissions:
      pull-requests: write # submitting 'wrong branch' reviews
      statuses: write # creating 'Eval Summary' commit statuses
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
          sparse-checkout: .github/actions
      - name: Check out the PR at the target commit
        uses: ./.github/actions/checkout
        with:
          merged-as-untrusted-at: ${{ inputs.mergedSha }}
          target-as-trusted-at: ${{ inputs.targetSha }}

      - name: Download output paths and eval stats for all systems
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: ${{ inputs.artifact-prefix }}diff-*
          path: diff
          merge-multiple: true

      - name: Install Nix
        uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6

      - name: Combine all output paths and eval stats
        run: |
          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
            --arg diffDir ./diff \
            --out-link combined

      - name: Upload the maintainer list
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ inputs.artifact-prefix }}maintainers
          path: combined/maintainers.json

      - name: Compare against the target branch
        env:
          TARGET_SHA: ${{ inputs.mergedSha }}
          BASE_BRANCH: ${{ fromJSON(inputs.baseBranch).branch }}
        run: |
          git -C nixpkgs/trusted diff --name-only "$TARGET_SHA" \
            | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json

          # Use the target branch to get accurate maintainer info
          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
            --arg combinedDir ./combined \
            --arg touchedFilesJson ./touched-files.json \
            --argstr baseBranch "$BASE_BRANCH" \
            --out-link comparison

          cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload the comparison results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ inputs.artifact-prefix }}comparison
          path: comparison/*

      - name: Add eval summary to commit statuses
        if: ${{ github.event_name == 'pull_request_target' }}
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const { readFile } = require('node:fs/promises')
            const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
            const removedByKernel = Object.fromEntries(
              Object.entries(changed.attrdiffByKernel ?? {}).map(([kernel, diff]) => [
                kernel,
                diff.removed.length,
              ]),
            )
            const description =
              'Package: ' + [
                `added ${changed.attrdiff.added.length}`,
                `removed ${changed.attrdiff.removed.length}`,
                `changed ${changed.attrdiff.changed.length}`
              ].join(', ') +
              ' — Rebuild: ' + [
                `linux ${changed.rebuildCountByKernel.linux}`,
                `darwin ${changed.rebuildCountByKernel.darwin}`
              ].join(', ') +
              (
                Object.values(removedByKernel).some((count) => count > 0)
                  ? ' — Removed: ' + [
                      `linux ${removedByKernel.linux ?? 0}`,
                      `darwin ${removedByKernel.darwin ?? 0}`
                    ].join(', ')
                  : ''
              )

            const { serverUrl, repo, runId, payload } = context
            const target_url =
              `${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`

            await github.rest.repos.createCommitStatus({
              ...repo,
              sha: payload.pull_request.head.sha,
              context: 'Eval Summary',
              state: 'success',
              description,
              target_url
            })

      - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
        if: github.event_name == 'pull_request_target' && vars.NIXPKGS_BRANCH_CHECK_CLIENT_ID
        id: app-token
        with:
          client-id: ${{ vars.NIXPKGS_BRANCH_CHECK_CLIENT_ID }}
          private-key: ${{ secrets.NIXPKGS_BRANCH_CHECK_APP_PRIVATE_KEY }}
          permission-pull-requests: write

      # It's fine to reuse this app in the 'pull-request-target / prepare' job,
      # because that job has to run before this one.
      - name: Request changes if PR is against an inappropriate branch
        if: ${{ github.event_name == 'pull_request_target' }}
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          github-token: ${{ steps.app-token.outputs.token || github.token }}
          script: |
            require('./nixpkgs/trusted/ci/github-script/check-target-branch.js')({
              github,
              context,
              core,
              dry: context.eventName == 'pull_request',
            })

  # Creates a matrix of Eval performance for various versions and systems.
  report:
    runs-on: ubuntu-slim
    needs: [versions, eval]
    steps:
      - name: Download output paths and eval stats for all versions
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: "*-diff-*"
          path: versions

      - name: Add version comparison table to job summary
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          ARTIFACT_PREFIX: ${{ inputs.artifact-prefix }}
          SYSTEMS: ${{ inputs.systems }}
          VERSIONS: ${{ needs.versions.outputs.versions }}
          CI_PIN_BUMP_COMMIT: ${{ needs.versions.outputs.ciPinBumpCommit }}
        with:
          script: |
            const { readFileSync } = require('node:fs')
            const path = require('node:path')

            const prefix = process.env.ARTIFACT_PREFIX
            const systems = JSON.parse(process.env.SYSTEMS)
            const versions = JSON.parse(process.env.VERSIONS)
            const ciPinBumpCommit = process.env.CI_PIN_BUMP_COMMIT

            core.summary.addHeading('Lix/Nix version comparison')
            core.summary.addRaw(`\n*Evaluated at commit: \`${ciPinBumpCommit}\` (commit that modified ci/pinned.json)*\n`, true)
            core.summary.addTable(
              [].concat(
                [
                  [{ data: 'Version', header: true }].concat(
                    systems.map((system) => ({ data: system, header: true })),
                  ),
                ],
                versions.map((version) =>
                  [{ data: version }].concat(
                    systems.map((system) => {
                      try {
                        const artifact = path.join('versions', `${prefix}${version}-diff-${system}`)
                        const time = Math.round(
                          parseFloat(
                            readFileSync(
                              path.join(artifact, 'after', system, 'total-time'),
                              'utf-8',
                            ),
                          ),
                        )
                        const diff = JSON.parse(
                          readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
                        )
                        const attrs = []
                          .concat(diff.added, diff.removed, diff.changed, diff.rebuilds)
                          // There are some special attributes, which are ignored for rebuilds.
                          // These only have a single path component, because they lack the `.<system>` suffix.
                          .filter((attr) => attr.split('.').length > 1)
                        if (attrs.length > 0) {
                          core.setFailed(
                            `${version} on ${system} has changed outpaths!\n` +
                              `Note: This indicates that commit ${ciPinBumpCommit} ` +
                              `(which modified ci/pinned.json) also contains other ` +
                              `changes affecting package outputs. ` +
                              `Please ensure ci/pinned.json is updated in a standalone commit.`
                          )
                          return { data: ':x:' }
                        }
                        return { data: time }
                      } catch {
                        core.warning(`${version} on ${system} did not produce artifact.`)
                        return { data: ':warning:' }
                      }
                    }),
                  ),
                ),
              ),
            )
            core.summary.addRaw(
              '\n*Evaluation time in seconds without downloading dependencies.*',
              true,
            )
            core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
            core.summary.addRaw(
              '\n*:x: Job produced different outpaths than the target branch.*',
              true,
            )
            core.summary.write()

  misc:
    if: ${{ github.event_name != 'push' }}
    runs-on: ubuntu-24.04-arm
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
          sparse-checkout: .github/actions
      - name: Checkout the merge commit
        uses: ./.github/actions/checkout
        with:
          merged-as-untrusted-at: ${{ inputs.mergedSha }}

      - name: Install Nix
        uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6

      - name: Ensure flake outputs on all systems still evaluate
        run: nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1'

      - name: Query nixpkgs with aliases enabled to check for basic syntax errors
        run: |
          time nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null

      - name: Ensure NixOS modules meta is valid
        run: |
          time nix-instantiate -I ./nixpkgs/untrusted --strict --eval --json ./nixpkgs/untrusted/nixos --arg configuration '{}' --attr config.meta --option restrict-eval true --option allow-import-from-derivation false