|
1 | | -#!/bin/sh |
2 | | - |
3 | | -usage () { |
4 | | - echo 1>&2 " |
5 | | -usage: |
6 | | - $0 |
7 | | - [--git commit..commit | --git commit] |
8 | | - [--svn rev:rev | --svn rev] |
9 | | - [--path path[:path]*] |
10 | | - [--help] |
11 | | -
|
12 | | -This program is used to investigate how any changes inside your nixpkgs |
13 | | -repository may hurt. With these kind of information you may choose wisely |
14 | | -where you should commit your changes. |
15 | | -
|
16 | | -This program adapts it-self to your versionning system to avoid too much |
17 | | -effort on your Internet bandwidth. If you need to check more than one |
18 | | -commits / revisions, you may use the following commands: |
19 | | -
|
20 | | - --git remotes/trunk..master |
21 | | - --svn 17670:17677 |
22 | | -
|
23 | | - Check the differences between each commit separating the first and the |
24 | | - last commit. |
25 | | -
|
26 | | - --path /etc/nixos/nixpkgs:/tmp/nixpkgs_1:/tmp/nixpkgs_2 |
27 | | -
|
28 | | - Check the differences between multiple directories containing different |
29 | | - versions of nixpkgs. |
30 | | -
|
31 | | -All these options exist with one commit / revision argument. Such options |
32 | | -are used to compare your \$NIXPKGS path with the specified version. |
33 | | -
|
34 | | -If you omit to mention any other commit / revision, then your \$NIXPKGS path |
35 | | -is compared with its last update. This command is useful to test code from |
36 | | -a dirty repository. |
37 | | -
|
38 | | -" |
39 | | - |
40 | | - exit 1; |
41 | | -} |
42 | | - |
43 | | -##################### |
44 | | -# Process Arguments # |
45 | | -##################### |
46 | | - |
47 | | -: ${NIXPKGS=/etc/nixos/nixpkgs/} |
48 | | - |
49 | | -vcs="" |
50 | | -gitCommits="" |
51 | | -svnRevisions="" |
52 | | -pathLocations="" |
53 | | -verbose=false |
54 | | - |
55 | | -argfun="" |
56 | | -for arg; do |
57 | | - if test -z "$argfun"; then |
58 | | - case $arg in |
59 | | - --git) vcs="git"; argfun="set_gitCommits";; |
60 | | - --svn) vcs="svn"; argfun="set_svnRevisions";; |
61 | | - --path) vcs="path"; argfun="set_pathLocations";; |
62 | | - --verbose) verbose=true;; |
63 | | - --help) usage;; |
64 | | - *) usage;; |
65 | | - esac |
66 | | - else |
67 | | - case $argfun in |
68 | | - set_*) |
69 | | - var=$(echo $argfun | sed 's,^set_,,') |
70 | | - eval $var=$arg |
71 | | - ;; |
72 | | - esac |
73 | | - argfun="" |
74 | | - fi |
75 | | -done |
76 | | - |
77 | | -if $verbose; then |
78 | | - set -x |
79 | | -else |
80 | | - set +x |
| 1 | +#!/usr/bin/env bash |
| 2 | +set -e |
| 3 | + |
| 4 | +if [ "$#" != 1 ] && [ "$#" != 2 ]; then |
| 5 | + cat <<-EOF |
| 6 | + Usage: $0 commit-spec [commit-spec] |
| 7 | + You need to be in a git-controlled nixpkgs tree. |
| 8 | + The current state of the tree will be used if the second commit is missing. |
| 9 | + EOF |
| 10 | + exit 1 |
81 | 11 | fi |
82 | 12 |
|
83 | | -############################ |
84 | | -# Find the repository type # |
85 | | -############################ |
| 13 | +# A slightly hacky way to get the config. |
| 14 | +parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \ |
| 15 | + | grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)" |
86 | 16 |
|
87 | | -if test -z "$vcs"; then |
88 | | - if test -x "$NIXPKGS/.git"; then |
89 | | - if git --git-dir="$NIXPKGS/.git" branch > /dev/null 2>&1; then |
90 | | - vcs="git" |
91 | | - gitCommits=$(git --git-dir="$NIXPKGS/.git" log -n 1 --pretty=format:%H 2> /dev/null) |
92 | | - fi |
93 | | - elif test -x "$NIXPKGS/.svn"; then |
94 | | - cd "$NIXPKGS" |
95 | | - if svn info > /dev/null 2>&1; then |
96 | | - vcs="svn"; |
97 | | - svnRevisions=$(svn info | sed -n 's,Revision: ,,p') |
98 | | - fi |
99 | | - cd - |
100 | | - else |
101 | | - usage |
102 | | - fi |
103 | | -fi |
| 17 | +echo "Estimating rebuild amount by counting changed Hydra jobs." |
104 | 18 |
|
105 | | -############################### |
106 | | -# Define a storage directory. # |
107 | | -############################### |
| 19 | +toRemove=() |
108 | 20 |
|
109 | | -pkgListDir="" |
110 | | -exitCode=1 |
111 | | -cleanup(){ |
112 | | - test -e "$pkgListDir" && rm -rf "$pkgListDir" |
113 | | - exit $exitCode; |
| 21 | +cleanup() { |
| 22 | + rm -rf "${toRemove[@]}" |
114 | 23 | } |
115 | | - |
116 | 24 | trap cleanup EXIT SIGINT SIGQUIT ERR |
117 | 25 |
|
118 | | -pkgListDir=$(mktemp --tmpdir -d rebuild-amount-XXXXXXXX) |
119 | | -vcsDir="$pkgListDir/.vcs" |
120 | | - |
121 | | -########################### |
122 | | -# Versionning for Dummies # |
123 | | -########################### |
124 | | - |
125 | | -path_init() { |
126 | | - if test "${pathLocations#*:}" = "$pathLocations"; then |
127 | | - pathLocations="$NIXPKGS:$pathLocations" |
128 | | - fi |
129 | | - pathLocations="${pathLocations}:" |
| 26 | +MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX' |
| 27 | + |
| 28 | +nixexpr() { |
| 29 | + cat <<-EONIX |
| 30 | + let |
| 31 | + lib = import $1/lib; |
| 32 | + hydraJobs = import $1/pkgs/top-level/release.nix |
| 33 | + # Compromise: accuracy vs. resources needed for evaluation. |
| 34 | + { supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; }; |
| 35 | + cfg = (import $1 {}).config.rebuild-amount or {}; |
| 36 | +
|
| 37 | + recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; }; |
| 38 | +
|
| 39 | + # hydraJobs leaves recurseForDerivations as empty attrmaps; |
| 40 | + # that would break nix-env and we also need to recurse everywhere. |
| 41 | + tweak = lib.mapAttrs |
| 42 | + (name: val: |
| 43 | + if name == "recurseForDerivations" then true |
| 44 | + else if lib.isAttrs val && val.type or null != "derivation" |
| 45 | + then recurseIntoAttrs (tweak val) |
| 46 | + else val |
| 47 | + ); |
| 48 | +
|
| 49 | + # Some of these contain explicit references to platform(s) we want to avoid; |
| 50 | + # some even (transitively) depend on ~/.nixpkgs/config.nix (!) |
| 51 | + blacklist = [ |
| 52 | + "tarball" "metrics" "manual" |
| 53 | + "darwin-tested" "unstable" "stdenvBootstrapTools" |
| 54 | + "moduleSystem" "lib-tests" # these just confuse the output |
| 55 | + ]; |
| 56 | + |
| 57 | + in |
| 58 | + tweak (builtins.removeAttrs hydraJobs blacklist) |
| 59 | + EONIX |
130 | 60 | } |
131 | 61 |
|
132 | | -path_getNext() { |
133 | | - pathLoc="${pathLocations%%:*}" |
134 | | - pathLocations="${pathLocations#*:}" |
| 62 | +# Output packages in tree $2 that weren't in $1. |
| 63 | +# Changing the output hash or name is taken as a change. |
| 64 | +# Extra nix-env parameters can be in $3 |
| 65 | +newPkgs() { |
| 66 | + # We use files instead of pipes, as running multiple nix-env processes |
| 67 | + # could eat too much memory for a standard 4GiB machine. |
| 68 | + local -a list |
| 69 | + for i in 1 2; do |
| 70 | + local l="$($MKTEMP)" |
| 71 | + list[$i]="$l" |
| 72 | + toRemove+=("$l") |
| 73 | + |
| 74 | + local expr="$($MKTEMP)" |
| 75 | + toRemove+=("$expr") |
| 76 | + nixexpr "${!i}" > "$expr" |
| 77 | + |
| 78 | + nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \ |
| 79 | + | sort > "${list[$i]}" & |
| 80 | + |
| 81 | + if [ "$parallel" != "true" ]; then |
| 82 | + wait |
| 83 | + fi |
| 84 | + done |
| 85 | + |
| 86 | + wait |
| 87 | + comm -13 "${list[@]}" |
135 | 88 | } |
136 | 89 |
|
137 | | -path_setPath() { |
138 | | - path="$pathLoc" |
139 | | -} |
140 | | - |
141 | | -path_setName() { |
142 | | - name=$(echo "$pathLoc" | tr '/' '_') |
143 | | -} |
144 | | - |
145 | | -################ |
146 | | -# Git Commands # |
147 | | -################ |
148 | | - |
149 | | -git_init() { |
150 | | - git clone "$NIXPKGS/.git" "$vcsDir" > /dev/null 2>&1 |
151 | | - if echo "gitCommits" | grep -c "\.\." > /dev/null 2>&1; then |
152 | | - gitCommits=$(git --git-dir="$vcsDir/.git" log --reverse --pretty=format:%H $gitCommits 2> /dev/null) |
153 | | - else |
154 | | - pathLocations="$vcsDir:$NIXPKGS" |
155 | | - vcs="path" |
156 | | - path_init |
157 | | - fi |
158 | | -} |
159 | | - |
160 | | -git_getNext() { |
161 | | - git --git-dir="$vcsDir/.git" checkout $(echo "$gitCommits" | head -n 1) > /dev/null 2>&1 |
162 | | - gitCommits=$(echo "$gitCommits" | sed '1 d') |
163 | | -} |
164 | | - |
165 | | -git_setPath() { |
166 | | - path="$vcsDir" |
167 | | -} |
168 | | - |
169 | | -git_setName() { |
170 | | - name=$(git --git-dir="$vcsDir/.git" log -n 1 --pretty=format:%H 2> /dev/null) |
171 | | -} |
172 | | - |
173 | | -####################### |
174 | | -# Subversion Commands # |
175 | | -####################### |
176 | | - |
177 | | -svn_init() { |
178 | | - cp -r "$NIXPKGS" "$vcsDir" > /dev/null 2>&1 |
179 | | - if echo "svnRevisions" | grep -c ":" > /dev/null 2>&1; then |
180 | | - svnRevisions=$(seq ${svnRevisions%:*} ${svnRevisions#*:}) |
181 | | - else |
182 | | - pathLocations="$vcsDir:$NIXPKGS" |
183 | | - vcs="path" |
184 | | - path_init |
185 | | - fi |
186 | | -} |
187 | | - |
188 | | -svn_getNext() { |
189 | | - cd "$vcsDir" |
190 | | - svn checkout $(echo "$svnRevisions" | head -n 1) > /dev/null 2>&1 |
191 | | - cd - |
192 | | - svnRevisions=$(echo "$svnRevisions" | sed '1 d') |
193 | | -} |
194 | | - |
195 | | -svn_setPath() { |
196 | | - path="$vcsDir" |
197 | | -} |
198 | | - |
199 | | -svn_setName() { |
200 | | - name=$(svn info 2> /dev/null | sed -n 's,Revision: ,,p') |
201 | | -} |
202 | | - |
203 | | -#################### |
204 | | -# Logical Commands # |
205 | | -#################### |
206 | | - |
207 | | -init () { ${vcs}_init; } |
208 | | -getNext () { ${vcs}_getNext; } |
209 | | -setPath () { ${vcs}_setPath; } |
210 | | -setName () { ${vcs}_setName; } |
211 | | - |
212 | | - |
213 | | -##################### |
214 | | -# Check for Rebuild # |
215 | | -##################### |
216 | | - |
217 | | -# Generate the list of all derivations that could be build from a nixpkgs |
218 | | -# respository. This list of derivation hashes is compared with previous |
219 | | -# lists and a brief summary is produced on the output. |
220 | | - |
221 | | -compareNames () { |
222 | | - nb=$(diff -y --suppress-common-lines --speed-large-files "$pkgListDir/$1.drvs" "$pkgListDir/$2.drvs" 2> /dev/null | wc -l) |
223 | | - echo "$1 -> $2: $nb" |
224 | | -} |
225 | | - |
226 | | -echo "Please wait, this may take some minutes ..." |
227 | | - |
228 | | -init |
229 | | -first="" |
230 | | -oldPrev="" |
231 | | - |
232 | | -prev="" |
233 | | -curr="" |
234 | | - |
235 | | -while true; do |
236 | | - getNext |
237 | | - setPath # set path=... |
238 | | - setName # set name=... |
239 | | - curr="$name" |
240 | | - |
241 | | - test -z "$curr" && break || true |
242 | | - |
243 | | - nix-instantiate "$path" > "$pkgListDir/$curr.drvs" > /dev/null 2>&1 || true |
244 | | - |
245 | | - if test -n "$prev"; then |
246 | | - compareNames "$prev" "$curr" |
247 | | - else |
248 | | - echo "Number of package to rebuild:" |
249 | | - first="$curr" |
250 | | - fi |
251 | | - oldPrev="$prev" |
252 | | - prev="$curr" |
| 90 | +# Prepare nixpkgs trees. |
| 91 | +declare -a tree |
| 92 | +for i in 1 2; do |
| 93 | + if [ -n "${!i}" ]; then # use the given commit |
| 94 | + dir="$($MKTEMP -d)" |
| 95 | + tree[$i]="$dir" |
| 96 | + toRemove+=("$dir") |
| 97 | + |
| 98 | + git clone --shared --no-checkout --quiet . "${tree[$i]}" |
| 99 | + (cd "${tree[$i]}" && git checkout --quiet "${!i}") |
| 100 | + else #use the current tree |
| 101 | + tree[$i]="$(pwd)" |
| 102 | + fi |
253 | 103 | done |
254 | 104 |
|
255 | | -if test "$first" != "$oldPrev"; then |
256 | | - echo "Number of package to rebuild (first -> last):" |
257 | | - compareNames "$first" "$curr" |
258 | | -fi |
| 105 | +newlist="$($MKTEMP)" |
| 106 | +toRemove+=("$newlist") |
| 107 | +# Notes: |
| 108 | +# - the evaluation is done on x86_64-linux, like on Hydra. |
| 109 | +# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove) |
| 110 | +newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist" |
| 111 | + |
| 112 | +# Hacky: keep only the last word of each attribute path and sort. |
| 113 | +sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \ |
| 114 | + | sort | uniq -c |
259 | 115 |
|
260 | | -exitCode=0 |
0 commit comments