commit | author | age
|
8ea4ef
|
1 |
#!/bin/bash |
f6854c
|
2 |
#A basic bash script to automate the building of arch packages |
dc0641
|
3 |
# Usage: main.sh init|check|add|remove|build_all |
8ea4ef
|
4 |
|
e6f0af
|
5 |
source $(dirname "$(realpath $0)")/vars.sh |
8ea4ef
|
6 |
|
54808c
|
7 |
ERRORFILE=$(mktemp) |
C |
8 |
WAITLIST=$(mktemp) |
|
9 |
WAITLIST_LCK=$(mktemp) |
|
10 |
|
8ea4ef
|
11 |
#Helper for finding newest and oldest files |
JG |
12 |
#Sourced from stack overflow |
f6854c
|
13 |
# Usage: newold_matching_file [n/o] [filename] |
8ea4ef
|
14 |
function newold_matching_file |
JG |
15 |
{ |
7c4087
|
16 |
# Use ${1-} instead of $1 in case 'nounset' is set |
JG |
17 |
local -r glob_pattern=${2-} |
8ea4ef
|
18 |
|
7c4087
|
19 |
# To avoid printing garbage if no files match the pattern, set |
JG |
20 |
# 'nullglob' if necessary |
|
21 |
local -i need_to_unset_nullglob=0 |
|
22 |
if [[ ":$BASHOPTS:" != *:nullglob:* ]] ; then |
|
23 |
shopt -s nullglob |
|
24 |
need_to_unset_nullglob=1 |
|
25 |
fi |
8ea4ef
|
26 |
|
7c4087
|
27 |
file= |
JG |
28 |
for f in $glob_pattern ; do |
8ea4ef
|
29 |
if [ $1 == "n" ]; then |
JG |
30 |
[[ -z $f || $f -nt $_file ]] && file=$f |
|
31 |
elif [ $1 == "o" ]; then |
|
32 |
[[ -z $f || $f -ot $_file ]] && file=$f |
|
33 |
fi |
7c4087
|
34 |
done |
8ea4ef
|
35 |
|
7c4087
|
36 |
# To avoid unexpected behaviour elsewhere, unset nullglob if it was |
JG |
37 |
# set by this function |
|
38 |
(( need_to_unset_nullglob )) && shopt -u nullglob |
8ea4ef
|
39 |
|
7c4087
|
40 |
# Use printf instead of echo in case the file name begins with '-' |
JG |
41 |
[[ -n $file ]] && printf '%s\n' "$file" |
8ea4ef
|
42 |
|
7c4087
|
43 |
return 0 |
8ea4ef
|
44 |
} |
JG |
45 |
|
|
46 |
#Build latest version of a package |
9837c8
|
47 |
# Usage: build_pkg [package name] [-f force] |
8ea4ef
|
48 |
function build_pkg { |
JG |
49 |
#check if PKGBUILD has updated, don't rebuild if hasn't changed |
b59712
|
50 |
if [[ -n $(git pull | grep 'Already up to date.') && -z $(grep 'pkgver() {' PKGBUILD) && -z "$2" ]]; then |
8ea4ef
|
51 |
return 2 |
JG |
52 |
fi |
f6854c
|
53 |
|
JG |
54 |
#make and force rebuild if is git package |
9837c8
|
55 |
# Mictosoft fonts have problems with checksums and need a seperate argument |
C |
56 |
if [[ "$1" == "ttf-ms-win10" || |
|
57 |
"$1" == "ttf-office-2007-fonts" || |
|
58 |
"$1" == "ttf-ms-win8" || |
|
59 |
"$1" == "ttf-win7-fonts" ]]; then |
cb04c8
|
60 |
makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) --skipchecksums 2>&1 |
9837c8
|
61 |
else |
e74aea
|
62 |
makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) 2>&1 |
9837c8
|
63 |
fi |
266025
|
64 |
if [[ $? != 0 && $? != 13 ]]; then |
8ea4ef
|
65 |
#Register error |
54808c
|
66 |
echo $1 >> $ERRORFILE |
8ea4ef
|
67 |
return 1 |
JG |
68 |
fi |
|
69 |
|
2bcbc6
|
70 |
#Remove old packages from build directory |
88a91e
|
71 |
source PKGBUILD |
96e462
|
72 |
srcdir="$(pwd)/src" |
37434c
|
73 |
if grep -q 'pkgver() {' PKGBUILD; then |
JG |
74 |
ver=$(pkgver) |
|
75 |
else |
|
76 |
ver=$pkgver |
|
77 |
fi |
266025
|
78 |
find . -mindepth 1 -maxdepth 1 -type f \( -name "*.pkg.tar.*" -o -name "*.src.tar.*" \) -not -name "*$ver-$pkgrel*" -delete |
3d9186
|
79 |
|
2bcbc6
|
80 |
#Get build artifact names from PKGBUILD and build artifacts |
JG |
81 |
#Remove duplicates from the list |
37434c
|
82 |
pkgs=() |
5f18f4
|
83 |
ipkgs=() |
88a91e
|
84 |
for i in ${pkgname[@]}; do |
b2c2b5
|
85 |
#pkgs+=("$i-$pkgver-$pkgrel") |
5f18f4
|
86 |
ipkgs+=($(find . -mindepth 1 -maxdepth 1 -type f \( -name "$i*.pkg.tar.*" -o -name "$i*.src.tar.*" \) -not -name "*.sig" | sed 's/^\.\///')) |
88a91e
|
87 |
done |
5f18f4
|
88 |
while read -r -d '' x; do pkgs+=("$x"); done < <(printf "%s\0" "${ipkgs[@]}" | sort -uz) |
88a91e
|
89 |
|
94852c
|
90 |
# Weird exceptions |
C |
91 |
if [[ "$1" == "zoom" ]]; then |
|
92 |
rm zoom*_orig* |
|
93 |
for i in ${pkgs[@]}; do |
|
94 |
if [ -z "${i##*_orig*}" ]; then |
|
95 |
pkgs=(${pkgs[@]/$i}) |
|
96 |
fi |
|
97 |
done |
|
98 |
fi |
|
99 |
|
e72b38
|
100 |
#Move package to repodir and add to repo db |
d4b7a7
|
101 |
#Dont change the database if rebuilt the same package at same release and version |
5f18f4
|
102 |
flag=0 |
88a91e
|
103 |
for i in ${pkgs[@]}; do |
e4347d
|
104 |
if [[ ! -f $REPODIR/$i ]]; then |
5f18f4
|
105 |
flag=1 |
d4b7a7
|
106 |
fi |
88a91e
|
107 |
done |
5f18f4
|
108 |
if [[ $flag == 1 ]]; then |
C |
109 |
rm -f $REPODIR/*$1*.tar.* |
|
110 |
for i in ${pkgs[@]}; do |
|
111 |
cp $i $REPODIR/ |
|
112 |
[[ "$SIGN" == "Y" ]] && cp $i.sig $REPODIR/ |
|
113 |
done |
3d9186
|
114 |
else |
5f18f4
|
115 |
return; |
9837c8
|
116 |
fi |
C |
117 |
|
e72b38
|
118 |
# Add package to waiting list to be added to repo db |
9837c8
|
119 |
while true; do |
54808c
|
120 |
if [[ $(cat $WAITLIST_LCK) == 1 ]]; then |
e6f0af
|
121 |
sleep 1 |
4ebf3d
|
122 |
else |
54808c
|
123 |
echo 1 > $WAITLIST_LCK |
C |
124 |
echo $1 >> $WAITLIST |
|
125 |
echo 0 > $WAITLIST_LCK |
e6f0af
|
126 |
break |
C |
127 |
fi |
|
128 |
done |
9837c8
|
129 |
while true; do |
e72b38
|
130 |
# Wait until package is at the top of the queue and add to db |
54808c
|
131 |
if [[ "$(head -n1 $WAITLIST)" == "$1" ]]; then |
3d9186
|
132 |
repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] || echo $COMPRESSION && echo zst) ${pkgs[@]} |
9837c8
|
133 |
while true; do |
54808c
|
134 |
if [[ $(cat $WAITLIST_LCK) == 1 ]]; then |
e6f0af
|
135 |
sleep 1 |
4ebf3d
|
136 |
else |
e72b38
|
137 |
# Remove self from top of queue |
5e843f
|
138 |
echo 1 > $WAITLIST_LCK |
54808c
|
139 |
TEMP=$(mktemp) |
C |
140 |
tail -n +2 $WAITLIST > $TEMP |
|
141 |
cp $TEMP $WAITLIST |
|
142 |
rm $TEMP |
|
143 |
unset TEMP |
|
144 |
echo 0 > $WAITLIST_LCK |
e6f0af
|
145 |
break |
C |
146 |
fi |
|
147 |
done |
|
148 |
break |
|
149 |
else |
54808c
|
150 |
if [[ -z "$(grep $1 $WAITLIST)" ]]; then |
b2c2b5
|
151 |
# Not on waitlist for some reason, need to readd |
54808c
|
152 |
if [[ $(cat $WAITLIST_LCK) == 1 ]]; then |
e74aea
|
153 |
sleep 1 |
C |
154 |
else |
54808c
|
155 |
echo 1 > $WAITLIST_LCK |
C |
156 |
echo $1 >> $WAITLIST |
|
157 |
echo 0 > $WAITLIST_LCK |
e74aea
|
158 |
fi |
C |
159 |
fi |
e6f0af
|
160 |
sleep 10 |
4ebf3d
|
161 |
fi |
e6f0af
|
162 |
done |
8ea4ef
|
163 |
|
JG |
164 |
#Remove old versions of packages |
f6854c
|
165 |
#TODO: Want to be able to keep multiple versions of old packages, future work |
JG |
166 |
#Currently old package versions stay in the repodir indefinately |
|
167 |
# while [ $NUM_OLD \< $(find . -name '*.pkg.tar.xz' | wc -l) ] |
|
168 |
# do |
7c4087
|
169 |
# old=$(newold_matching_file o '*.pkg.tar.xz') |
JG |
170 |
# rm $REPODIR/$old $old |
f6854c
|
171 |
# done |
8ea4ef
|
172 |
return 0 |
JG |
173 |
} |
|
174 |
|
|
175 |
#Update packages in BUILDDIR |
f6854c
|
176 |
# Usage: build_all [-f force] |
8ea4ef
|
177 |
function build_all { |
JG |
178 |
#system update |
9837c8
|
179 |
if [[ $UPDATE == "Y" ]]; then |
8ea4ef
|
180 |
sudo pacman -Syu --noconfirm |
JG |
181 |
fi |
ac333f
|
182 |
|
8ea4ef
|
183 |
#update every package currently stored |
c3d80e
|
184 |
for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -type d) |
8ea4ef
|
185 |
do |
JG |
186 |
cd $d |
9837c8
|
187 |
if [[ "$PARALLEL" == "Y" ]]; then |
C |
188 |
build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" || echo "/dev/tty") & |
|
189 |
else |
|
190 |
build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" || echo "/dev/tty") |
|
191 |
fi |
8ea4ef
|
192 |
done |
e6f0af
|
193 |
wait |
8ea4ef
|
194 |
|
JG |
195 |
return 0 |
|
196 |
} |
|
197 |
|
|
198 |
#Add a new package to be built |
7c4087
|
199 |
#Adding build dependencies is |
f6854c
|
200 |
# Usage: add [package name] |
8ea4ef
|
201 |
function add { |
37434c
|
202 |
local i j k |
67ef73
|
203 |
for i in $@; do |
JG |
204 |
cd $BUILDDIR |
ca0d8a
|
205 |
if [[ -z $(git ls-remote https://aur.archlinux.org/$i.git) ]]; then |
C |
206 |
echo "Not a package" |
|
207 |
exit 2 |
|
208 |
fi |
67ef73
|
209 |
git clone https://aur.archlinux.org/$i.git |
JG |
210 |
cd $i |
37434c
|
211 |
unset depends |
JG |
212 |
unset makedepends |
|
213 |
local makedeps |
|
214 |
source PKGBUILD |
ca0d8a
|
215 |
|
3d9186
|
216 |
#Check for all build dependencies |
37434c
|
217 |
for j in ${makedepends[@]}; do |
JG |
218 |
k=$(echo $j | sed 's/[>]=.*//g') |
|
219 |
if ! pacman -Si $k; then |
|
220 |
makedeps+=($k) |
ca0d8a
|
221 |
fi &>/dev/null |
C |
222 |
done |
37434c
|
223 |
for j in ${depends[@]}; do |
JG |
224 |
k=$(echo $j | sed 's/[>]=.*//g') |
|
225 |
if ! pacman -Si $k; then |
|
226 |
makedeps+=($k) |
|
227 |
fi &>/dev/null |
ca0d8a
|
228 |
done |
3d9186
|
229 |
|
JG |
230 |
#Add dependencies and update so overall build can work |
37434c
|
231 |
for j in ${makedeps[@]}; do |
JG |
232 |
add $j |
|
233 |
done |
|
234 |
if [[ -n "${makedeps[@]}" ]]; then |
3b9f12
|
235 |
sudo pacman -Sy |
C |
236 |
fi |
3d9186
|
237 |
|
ca0d8a
|
238 |
#Actually build wanted package |
37434c
|
239 |
cd $BUILDDIR/$i |
9837c8
|
240 |
build_pkg $i -f |
67ef73
|
241 |
done |
8ea4ef
|
242 |
return 0 |
c3d80e
|
243 |
} |
C |
244 |
|
|
245 |
#Remove a package from the build list and repository |
e6670f
|
246 |
#Usage of -a removes all packages moved to official repos |
JG |
247 |
# Usage remove [-a|package name] |
c3d80e
|
248 |
function remove { |
e6670f
|
249 |
if [[ "$1" == "-a" ]]; then |
JG |
250 |
rmlist="" |
|
251 |
rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq core | sort) | tr '\n' ' ')" |
|
252 |
rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq extra | sort) | tr '\n' ' ')" |
|
253 |
rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq community | sort) | tr '\n' ' ')" |
|
254 |
for i in $rmlist; do |
|
255 |
rm -rf $BUILDDIR/$i |
|
256 |
repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] || echo $COMPRESSION && echo zst) $i |
|
257 |
rm -f $REPODIR/*$i* |
|
258 |
done |
|
259 |
else |
|
260 |
for i in $@; do |
|
261 |
rm -rf $BUILDDIR/$i |
|
262 |
repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] || echo $COMPRESSION && echo zst) $i |
|
263 |
rm -f $REPODIR/*$i* |
|
264 |
done |
|
265 |
fi |
8ea4ef
|
266 |
} |
JG |
267 |
|
7aa510
|
268 |
#Check for packages moved to official repos or removed from the AUR |
C |
269 |
function check { |
|
270 |
rmlist="" |
|
271 |
rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq core | sort) | tr '\n' ' ')" |
|
272 |
rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq extra | sort) | tr '\n' ' ')" |
|
273 |
rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq community | sort) | tr '\n' ' ')" |
|
274 |
TMPFILE=$(mktemp) |
|
275 |
for i in $(find $BUILDDIR -mindepth 1 -maxdepth 1 -type d); do |
|
276 |
check_pkg $TMPFILE "$(echo $i | rev | cut -d'/' -f1 | rev)" & |
|
277 |
done |
|
278 |
wait |
|
279 |
echo "Merged into official repos: $rmlist" |
|
280 |
echo "Not in AUR: $(cat $TMPFILE | tr '\n' ' ')" |
|
281 |
rm -f $TMPFILE |
|
282 |
} |
|
283 |
|
3d9186
|
284 |
#Check helper function |
7aa510
|
285 |
function check_pkg { |
C |
286 |
if [[ -z "$(curl -sI "https://aur.archlinux.org/packages/$2" | head -n1 | grep 200)" ]]; then |
|
287 |
echo "$2" >> $1 |
|
288 |
fi |
|
289 |
} |
|
290 |
|
|
291 |
|
8ea4ef
|
292 |
#Check config and create build folders |
f6854c
|
293 |
#Set variables before usage |
JG |
294 |
# Usage: init |
8ea4ef
|
295 |
function init { |
9837c8
|
296 |
if [[ $uid != 1 ]]; then |
4ebf3d
|
297 |
echo "This must be run as root" |
JG |
298 |
fi |
|
299 |
|
8ea4ef
|
300 |
#check for configuration here |
9837c8
|
301 |
[[ -z $REPODIR ]] && echo "Enter REPODIR" && return 1 |
C |
302 |
[[ -z $BUILDDIR ]] && echo "Enter BUILDDIR" && return 2 |
|
303 |
[[ -z $REPONAME ]] && echo "Enter REPONAME" && return 3 |
8ea4ef
|
304 |
|
JG |
305 |
#make build directories |
9837c8
|
306 |
[[ ! -d $REPODIR ]] && mkdir -p $REPODIR |
C |
307 |
[[ ! -d $BUILDDIR ]] && mkdir -p $BUILDDIR |
8ea4ef
|
308 |
|
JG |
309 |
#packages required to build others |
4ebf3d
|
310 |
pacman -S --noconfirm base-devel git |
8ea4ef
|
311 |
|
JG |
312 |
#add repo to pacman.conf so can install own packages |
9837c8
|
313 |
if [[ -z $(grep "$REPONAME" /etc/pacman.conf) ]]; then |
8ea4ef
|
314 |
printf "[$REPONAME]\nSigLevel = Optional TrustAll\nServer = file://$REPODIR\n" >> /etc/pacman.conf |
JG |
315 |
fi |
|
316 |
|
|
317 |
#create GPG key for package signing |
4ebf3d
|
318 |
if [[ "$SIGN" == "Y" && "$KEY" == "" ]]; then |
8ea4ef
|
319 |
( |
JG |
320 |
echo "Key-Type: RSA" |
|
321 |
echo "Key-Length: 2048" |
|
322 |
echo "Subkey-Type: RSA" |
|
323 |
echo "Subkey-Length: 2048" |
|
324 |
echo "Passphrase: \"\"" |
|
325 |
echo "Expire-Date: 0" |
|
326 |
echo "Name-Real: John Doe" |
|
327 |
echo "Name-Comment: Arch buildbot" |
|
328 |
echo "Name-Email: $(whoami)@localhost" |
|
329 |
echo "%commit" |
|
330 |
) | gpg --batch --generate-key |
|
331 |
gpg --export --output $REPONAME.key --armor "John Doe" |
|
332 |
gpg --export-secret-keys --output $REPONAME.secret.key --armor "John Doe" |
|
333 |
echo "Please change the key information in this file" |
|
334 |
fi |
|
335 |
|
|
336 |
return 0 |
|
337 |
} |
|
338 |
|
86d762
|
339 |
function send_email { |
C |
340 |
( |
|
341 |
echo "From: build@localhost" |
|
342 |
echo "To: $EMAIL" |
|
343 |
echo "Subject: Build errors" |
e6f0af
|
344 |
echo "There were build errors for the build of $REPONAME at $(date), please address them soon." |
be70fd
|
345 |
echo "The errors were: $@" |
86d762
|
346 |
) | sendmail -t |
C |
347 |
} |
|
348 |
|
8ea4ef
|
349 |
case $1 in |
JG |
350 |
"init") |
|
351 |
init;; |
|
352 |
"add") |
9837c8
|
353 |
add ${@:2};; |
8ea4ef
|
354 |
"build-all") |
9837c8
|
355 |
build_all $([[ "$2" == "-f" ]] && echo "-f");; |
c3d80e
|
356 |
"remove") |
9837c8
|
357 |
remove ${@:2};; |
7aa510
|
358 |
"check") |
C |
359 |
check;; |
8ea4ef
|
360 |
*) |
dc0641
|
361 |
echo -e "\033[0;31mInvalid usage\033[0m" |
C |
362 |
echo -e "Usage: $0 init|check|add|remove|build-all" |
|
363 |
echo -e "\033[0;32minit\033[0m - initialise repository for use" |
|
364 |
echo -e "\033[0;32mcheck\033[0m - check if packages have been moved into the official repositories or removed from the AUR" |
|
365 |
echo -e "\033[0;32madd package ...\033[0m - add a package to \$BUILDDIR and repository, also used to rebuild failed packages" |
|
366 |
echo -e "\033[0;32mremove -a | package ...\033[0m - remove package from \$BUILDDIR and repository, \"-a\" removes packages added to official repos" |
|
367 |
echo -e "\033[0;32mbuild-all [-f]\033[0m - build all packages in \$BUILDDIR, \"-f\" force builds whole repository" |
8ea4ef
|
368 |
esac |
9837c8
|
369 |
|
C |
370 |
# Error reporting, send email only for build-all as assuming an batch job for that |
7aa510
|
371 |
if [[ $1 == "build-all" || $1 == "add" ]]; then |
54808c
|
372 |
if [[ -n $(cat $ERRORFILE) ]]; then |
C |
373 |
ERRORS=$(cat $ERRORFILE | tr '\n' ' ') |
7aa510
|
374 |
echo "Errors in packages: $ERRORS" |
C |
375 |
if [[ "$EMAIL" != "" && "$1" == "build-all" ]]; then |
|
376 |
send_email $ERRORS |
|
377 |
fi |
|
378 |
else |
|
379 |
echo "All packages built successfully" |
9837c8
|
380 |
fi |
C |
381 |
fi |
54808c
|
382 |
|
C |
383 |
rm $ERRORFILE $WAITLIST $WAITLIST_LCK |