From a3922c83c6d6dd8f80c67097bb3deb89e0e03b3b Mon Sep 17 00:00:00 2001
From: usertam <code@usertam.dev>
Date: Sun, 13 Jul 2025 01:44:34 +0800
Subject: [PATCH 1/3] build.yml: make determinism checks actually functional

Previously it will just rebuild symlinks in 4s and isn't actually
rebuilding anything. Give it a new nix store to motivate it.
---
 .github/workflows/build.yml | 62 +++++++++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f4e6947..e2f5504 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1,12 +1,21 @@
 name: "Build tests on platforms"
+
+# Only scheduled builds enforce determinism checks by default.
 on:
   workflow_dispatch:
+    inputs:
+      do-determinism-check:
+        description: 'Enable determinism checks'
+        required: true
+        type: boolean
+        default: false
   push:
     branches:
     - master
   pull_request:
   schedule:
   - cron: '0 0 * * MON'
+
 jobs:
   build-linux:
     name: Build Linux
@@ -28,6 +37,7 @@ jobs:
       with:
         name: usertam-nixfiles
         authToken: ${{secrets.CACHIX_AUTH_TOKEN}}
+        skipAddingSubstituter: ${{ github.event_name == 'schedule' || inputs.do-determinism-check == true }}
     - name: Build ${{matrix.system}}.${{matrix.config}}.toplevel
       run: |
         nix build -L .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.toplevel
@@ -37,25 +47,40 @@ jobs:
         nix build -L .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release
         find -L result -name '*.xz' -exec cp -a {} . \;
         echo "artifact=$(ls *.xz | tee /dev/stderr)" >> "$GITHUB_ENV"
-    - name: Deterministic check on ${{matrix.system}}.${{matrix.config}}.toplevel
-      if: github.event_name == 'schedule'
+    - name: Determinism check on ${{matrix.system}}.${{matrix.config}}.toplevel
+      if: github.event_name == 'schedule' || inputs.do-determinism-check == true
       run: |
-        if ! nix build -L --rebuild --keep-failed .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.toplevel 2>&1 | tee stderr; then
-          echo "::warning::$(grep 'error: ' stderr)"
-          diff -ar $(nix eval --raw){,.check} || true
+        # Move previous build results to new directory.
+        mkdir prev-build && mv result* prev-build
+        # Rebuild everything, enforced by a different store path.
+        nix build -L --store ./nix-store .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.toplevel
+        # Compare the new build results with the previous ones.
+        if ! diff -ar prev-build/result result; then
+          echo "::warning::Build of toplevel may not be deterministic, check toplevel-diffoscope.log for details"
+          nix run nixpkgs#diffoscopeMinimal -- --exclude-directory-metadata recursive prev-build/result result | tee toplevel-diffoscope.log
         fi
-    - name: Deterministic check on ${{matrix.system}}.${{matrix.config}}.release
-      if: github.event_name == 'schedule' && contains(matrix.config, 'generic')
+    - name: Determinism check on ${{matrix.system}}.${{matrix.config}}.release
+      if: (github.event_name == 'schedule' || inputs.do-determinism-check == true) && contains(matrix.config, 'generic')
       run: |
+        # Here checks if the image building process is deterministic.
         if ! nix build -L --rebuild --keep-failed .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release 2>&1 | tee stderr; then
+          echo "::warning::Build of release may not be deterministic, check release-diffoscope.log for details"
           echo "::warning::$(grep 'error: ' stderr)"
-          diff -ar $(nix eval --raw){,.check} || true
+          nix run nixpkgs#diffoscopeMinimal -- --exclude-directory-metadata recursive \
+            $(nix eval --raw .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release){,.check} | tee release-diffoscope.log
         fi
     - uses: actions/upload-artifact@main
       if: env.artifact != ''
       with:
         name: ${{env.artifact}}
         path: ${{env.artifact}}
+    - uses: actions/upload-artifact@main
+      if: github.event_name == 'schedule' || inputs.do-determinism-check == true
+      with:
+        name: ${{matrix.system}}-${{matrix.config}}-diffoscope
+        path: |
+          toplevel-diffoscope.log
+          release-diffoscope.log
 
   build-darwin:
     name: Build Darwin
@@ -71,6 +96,7 @@ jobs:
       with:
         name: usertam-nixfiles
         authToken: ${{secrets.CACHIX_AUTH_TOKEN}}
+        skipAddingSubstituter: ${{ github.event_name == 'schedule' || inputs.do-determinism-check == true }}
     - name: Switch to darwin-runner
       run: |
         nix build .#darwinConfigurations.darwin-runner.config.system.build.toplevel
@@ -82,13 +108,23 @@ jobs:
     - name: Build gale.toplevel
       run: |
         nix build -L .#darwinConfigurations.gale.config.system.build.toplevel
-    - name: Deterministic check on gale.toplevel
-      if: github.event_name == 'schedule'
+    - name: Determinism check on gale.toplevel
+      if: github.event_name == 'schedule' || inputs.do-determinism-check == true
       run: |
-        if ! nix build --rebuild --keep-failed .#darwinConfigurations.gale.config.system.build.toplevel 2>stderr; then
-          echo "::warning::$(grep 'error: ' stderr)"
-          diff -ar $(nix eval --raw){,.check} || true
+        # Move previous build results to new directory.
+        mkdir prev-build && mv result* prev-build
+        # Rebuild everything, enforced by a different store path.
+        nix build -L --store ./nix-store .#darwinConfigurations.gale.config.system.build.toplevel
+        # Compare the new build results with the previous ones.
+        if ! diff -ar prev-build/result result; then
+          echo "::warning::Build of toplevel may not be deterministic, check toplevel-diffoscope.log for details"
+          nix run nixpkgs#diffoscopeMinimal -- --exclude-directory-metadata recursive prev-build/result result | tee toplevel-diffoscope.log
         fi
+    - uses: actions/upload-artifact@main
+      if: github.event_name == 'schedule' || inputs.do-determinism-check == true
+      with:
+        name: darwin-gale-diffoscope
+        path: toplevel-diffoscope.log
 
   build-status:
     name: Build Status

From c623b5173fe820c1097ec74232b7b42f842c6b99 Mon Sep 17 00:00:00 2001
From: usertam <code@usertam.dev>
Date: Sun, 13 Jul 2025 20:52:25 +0800
Subject: [PATCH 2/3] build.yml: new implementation of determinism checks

---
 .github/workflows/build.yml | 137 ++++++++++++++++++++++++------------
 1 file changed, 91 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e2f5504..4df45c5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -5,7 +5,7 @@ on:
   workflow_dispatch:
     inputs:
       do-determinism-check:
-        description: 'Enable determinism checks'
+        description: 'Enable determinism-check builds'
         required: true
         type: boolean
         default: false
@@ -37,7 +37,6 @@ jobs:
       with:
         name: usertam-nixfiles
         authToken: ${{secrets.CACHIX_AUTH_TOKEN}}
-        skipAddingSubstituter: ${{ github.event_name == 'schedule' || inputs.do-determinism-check == true }}
     - name: Build ${{matrix.system}}.${{matrix.config}}.toplevel
       run: |
         nix build -L .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.toplevel
@@ -47,40 +46,104 @@ jobs:
         nix build -L .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release
         find -L result -name '*.xz' -exec cp -a {} . \;
         echo "artifact=$(ls *.xz | tee /dev/stderr)" >> "$GITHUB_ENV"
-    - name: Determinism check on ${{matrix.system}}.${{matrix.config}}.toplevel
-      if: github.event_name == 'schedule' || inputs.do-determinism-check == true
-      run: |
-        # Move previous build results to new directory.
-        mkdir prev-build && mv result* prev-build
-        # Rebuild everything, enforced by a different store path.
-        nix build -L --store ./nix-store .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.toplevel
-        # Compare the new build results with the previous ones.
-        if ! diff -ar prev-build/result result; then
-          echo "::warning::Build of toplevel may not be deterministic, check toplevel-diffoscope.log for details"
-          nix run nixpkgs#diffoscopeMinimal -- --exclude-directory-metadata recursive prev-build/result result | tee toplevel-diffoscope.log
-        fi
-    - name: Determinism check on ${{matrix.system}}.${{matrix.config}}.release
-      if: (github.event_name == 'schedule' || inputs.do-determinism-check == true) && contains(matrix.config, 'generic')
-      run: |
-        # Here checks if the image building process is deterministic.
-        if ! nix build -L --rebuild --keep-failed .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release 2>&1 | tee stderr; then
-          echo "::warning::Build of release may not be deterministic, check release-diffoscope.log for details"
-          echo "::warning::$(grep 'error: ' stderr)"
-          nix run nixpkgs#diffoscopeMinimal -- --exclude-directory-metadata recursive \
-            $(nix eval --raw .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release){,.check} | tee release-diffoscope.log
-        fi
     - uses: actions/upload-artifact@main
       if: env.artifact != ''
       with:
         name: ${{env.artifact}}
         path: ${{env.artifact}}
+
+  build-linux-determinism-check:
+    name: Build Linux with determinism checks
+    if: github.event_name == 'schedule' || inputs.do-determinism-check == true
+    strategy:
+      matrix:
+        config: [generic.installer, generic.docker]
+        system: [x86_64-linux, aarch64-linux]
+        include:
+        - config: tsrvbld
+          system: x86_64-linux
+      fail-fast: false
+    runs-on: ${{matrix.system == 'x86_64-linux' && 'ubuntu-24.04' || 'ubuntu-24.04-arm'}} 
+    steps:
+    - uses: actions/checkout@main
+      with:
+        ref: ${{github.ref}}
+    - uses: DeterminateSystems/nix-installer-action@main
+    - uses: cachix/cachix-action@master
+      with:
+        name: usertam-nixfiles
+        authToken: ${{secrets.CACHIX_AUTH_TOKEN}}
+        skipAddingSubstituter: true
+    - name: Set up local overlays
+      run: |
+        sudo mkdir -p /overlay/{a,b}{/nix,-work,-upper}
+        sudo mount -t overlay overlay \
+          -o lowerdir=/nix \
+          -o upperdir=/overlay/a-upper \
+          -o workdir=/overlay/a-work \
+          /overlay/a/nix
+        sudo mount -t overlay overlay \
+          -o lowerdir=/nix \
+          -o upperdir=/overlay/b-upper \
+          -o workdir=/overlay/b-work \
+          /overlay/b/nix
+    - name: Prefetch prebuilt dependencies
+      run: |
+        nix-store --query --references \
+          $(nix eval --raw .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.toplevel.drvPath) \
+          | xargs nix-store --realise --max-jobs 0 || true
+    - name: Bind mount /nix -> /overlay/a/nix
+      run: |
+        sudo systemctl stop nix-daemon
+        sudo mount --bind /overlay/a/nix /nix
+        sudo systemctl start nix-daemon
+        sudo systemctl status nix-daemon
+    - name: First build ${{matrix.system}}.${{matrix.config}}.toplevel
+      run: |
+        sudo $(which nix) build -L \
+          .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.toplevel
+    - name: First build ${{matrix.system}}.${{matrix.config}}.release
+      if: contains(matrix.config, 'generic')
+      run: |
+        sudo $(which nix) build -L \
+          .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release
+    - name: Bind mount /nix -> /overlay/b/nix
+      run: |
+        sudo systemctl stop nix-daemon
+        sudo umount /nix
+        sudo mount --bind /overlay/b/nix /nix
+        sudo systemctl start nix-daemon
+        sudo systemctl status nix-daemon
+    - name: Second build ${{matrix.system}}.${{matrix.config}}.toplevel
+      run: |
+        sudo $(which nix) build -L \
+          .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.toplevel
+    - name: Second build ${{matrix.system}}.${{matrix.config}}.release
+      if: contains(matrix.config, 'generic')
+      run: |
+        sudo $(which nix) build -L \
+          .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release
+    - name: Remove bind mounts on /nix
+      run: |
+        sudo systemctl stop nix-daemon
+        sudo umount /nix
+        sudo systemctl start nix-daemon
+        sudo systemctl status nix-daemon
+    - name: Run diffoscope on the overlays
+      run: |
+        sudo $(which nix) run nixpkgs#diffoscopeMinimal -- \
+          --markdown diffoscope.md --status-fd 2 \
+          --exclude-directory-metadata recursive /overlay/a-upper/store /overlay/b-upper/store
+        if [ -f diffoscope.md ]; then
+          echo "::warning::Build of toplevel may not be deterministic, check toplevel-diffoscope.log for details"
+        fi
+        echo "diff=$(! test -f diffoscope.md; echo $?)" >> $GITHUB_ENV
     - uses: actions/upload-artifact@main
-      if: github.event_name == 'schedule' || inputs.do-determinism-check == true
+      if: env.diff == '1'
       with:
         name: ${{matrix.system}}-${{matrix.config}}-diffoscope
         path: |
-          toplevel-diffoscope.log
-          release-diffoscope.log
+          diffoscope.md
 
   build-darwin:
     name: Build Darwin
@@ -96,7 +159,6 @@ jobs:
       with:
         name: usertam-nixfiles
         authToken: ${{secrets.CACHIX_AUTH_TOKEN}}
-        skipAddingSubstituter: ${{ github.event_name == 'schedule' || inputs.do-determinism-check == true }}
     - name: Switch to darwin-runner
       run: |
         nix build .#darwinConfigurations.darwin-runner.config.system.build.toplevel
@@ -108,28 +170,11 @@ jobs:
     - name: Build gale.toplevel
       run: |
         nix build -L .#darwinConfigurations.gale.config.system.build.toplevel
-    - name: Determinism check on gale.toplevel
-      if: github.event_name == 'schedule' || inputs.do-determinism-check == true
-      run: |
-        # Move previous build results to new directory.
-        mkdir prev-build && mv result* prev-build
-        # Rebuild everything, enforced by a different store path.
-        nix build -L --store ./nix-store .#darwinConfigurations.gale.config.system.build.toplevel
-        # Compare the new build results with the previous ones.
-        if ! diff -ar prev-build/result result; then
-          echo "::warning::Build of toplevel may not be deterministic, check toplevel-diffoscope.log for details"
-          nix run nixpkgs#diffoscopeMinimal -- --exclude-directory-metadata recursive prev-build/result result | tee toplevel-diffoscope.log
-        fi
-    - uses: actions/upload-artifact@main
-      if: github.event_name == 'schedule' || inputs.do-determinism-check == true
-      with:
-        name: darwin-gale-diffoscope
-        path: toplevel-diffoscope.log
 
   build-status:
     name: Build Status
     runs-on: ubuntu-latest
-    needs: [build-linux, build-darwin]
+    needs: [build-linux, build-darwin, build-linux-determinism-check]
     if: always()
     steps:
     - run: ${{!contains(needs.*.result, 'failure')}}

From 4d59413f88aeb20f2513e6a20e40ca2ed235e4e2 Mon Sep 17 00:00:00 2001
From: usertam <code@usertam.dev>
Date: Mon, 14 Jul 2025 14:44:35 +0800
Subject: [PATCH 3/3] test

---
 .github/workflows/build.yml | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4df45c5..87ec642 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -123,6 +123,9 @@ jobs:
       run: |
         sudo $(which nix) build -L \
           .#packages.${{matrix.system}}.nixosConfigurations.${{matrix.config}}.config.system.build.release
+    - name: Second build some garbage
+      run: |
+        sudo $(which nix) build nixpkgs#hello
     - name: Remove bind mounts on /nix
       run: |
         sudo systemctl stop nix-daemon
@@ -134,12 +137,16 @@ jobs:
         sudo $(which nix) run nixpkgs#diffoscopeMinimal -- \
           --markdown diffoscope.md --status-fd 2 \
           --exclude-directory-metadata recursive /overlay/a-upper/store /overlay/b-upper/store
-        if [ -f diffoscope.md ]; then
-          echo "::warning::Build of toplevel may not be deterministic, check toplevel-diffoscope.log for details"
+    - name: Verdict on diffoscope output
+      run: |
+        diff_exists=$(test -f diffoscope.md && echo true || echo false)
+        echo "diff_exists=$diff_exists" >> $GITHUB_ENV
+        if $diff_exists; then
+          echo "::warning::Builds may not be deterministic, check diffoscope.md for details"
+          false
         fi
-        echo "diff=$(! test -f diffoscope.md; echo $?)" >> $GITHUB_ENV
     - uses: actions/upload-artifact@main
-      if: env.diff == '1'
+      if: always() && env.diff_exists == 'true'
       with:
         name: ${{matrix.system}}-${{matrix.config}}-diffoscope
         path: |