Arch-autobuild-repo.git

Joel Grunbaum

2023-06-17 9c892f59d916f85b8825646d58516ac49904a464

commit \| author \| age
8ea4ef	1	#!/bin/bash
f6854c	2	#A basic bash script to automate the building of arch packages
dc0641	3	# Usage: main.sh init\|check\|add\|remove\|build_all
8ea4ef	4
e6f0af	5	source $(dirname "$(realpath $0)")/vars.sh
8ea4ef	6
54808c	7	ERRORFILE=$(mktemp)
C	8	WAITLIST=$(mktemp)
	9	WAITLIST_LCK=$(mktemp)
	10
8ea4ef	11	#Helper for finding newest and oldest files
JG	12	#Sourced from stack overflow
f6854c	13	# Usage: newold_matching_file [n/o] [filename]
8ea4ef	14	function newold_matching_file
JG	15	{
7c4087	16	# Use ${1-} instead of $1 in case 'nounset' is set
JG	17	local -r glob_pattern=${2-}
8ea4ef	18
7c4087	19	# To avoid printing garbage if no files match the pattern, set
JG	20	# 'nullglob' if necessary
	21	local -i need_to_unset_nullglob=0
	22	if [[ ":$BASHOPTS:" != :nullglob: ]] ; then
	23	shopt -s nullglob
	24	need_to_unset_nullglob=1
	25	fi
8ea4ef	26
7c4087	27	file=
JG	28	for f in $glob_pattern ; do
8ea4ef	29	if [ $1 == "n" ]; then
JG	30	[[ -z $f \|\| $f -nt $_file ]] && file=$f
	31	elif [ $1 == "o" ]; then
	32	[[ -z $f \|\| $f -ot $_file ]] && file=$f
	33	fi
7c4087	34	done
8ea4ef	35
7c4087	36	# To avoid unexpected behaviour elsewhere, unset nullglob if it was
JG	37	# set by this function
	38	(( need_to_unset_nullglob )) && shopt -u nullglob
8ea4ef	39
7c4087	40	# Use printf instead of echo in case the file name begins with '-'
JG	41	[[ -n $file ]] && printf '%s\n' "$file"
8ea4ef	42
7c4087	43	return 0
8ea4ef	44	}
JG	45
	46	#Build latest version of a package
9837c8	47	# Usage: build_pkg [package name] [-f force]
8ea4ef	48	function build_pkg {
b2dc9d	49	#first need to reset repo, revert any changes so update can happen
JG	50	git reset --hard
8ea4ef	51	#check if PKGBUILD has updated, don't rebuild if hasn't changed
b59712	52	if [[ -n $(git pull \| grep 'Already up to date.') && -z $(grep 'pkgver() {' PKGBUILD) && -z "$2" ]]; then
8ea4ef	53	return 2
JG	54	fi
f6854c	55
5f1f4f	56	#Remove old packages from build directory
JG	57	source PKGBUILD
	58	srcdir="$(pwd)/src"
	59	if grep -q 'pkgver() {' PKGBUILD; then
	60	ver=$(pkgver)
	61	else
	62	ver=$pkgver
	63	fi
	64	find . -mindepth 1 -maxdepth 1 -type f $ -name ".pkg.tar." -o -name ".src.tar." $ -not -name "$ver-$pkgrel" -delete
	65
f6854c	66	#make and force rebuild if is git package
9837c8	67	# Mictosoft fonts have problems with checksums and need a seperate argument
C	68	if [[ "$1" == "ttf-ms-win10" \|\|
	69	"$1" == "ttf-office-2007-fonts" \|\|
	70	"$1" == "ttf-ms-win8" \|\|
	71	"$1" == "ttf-win7-fonts" ]]; then
cb04c8	72	makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) --skipchecksums 2>&1
9837c8	73	else
e74aea	74	makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) 2>&1
9837c8	75	fi
266025	76	if [[ $? != 0 && $? != 13 ]]; then
fc805e	77	if [[ -n $(find . -iname "*cmake") && "$2" != "-s" ]]; then
JG	78	find ./src -iname "cmake" -type f -delete
	79	build_pkg "$1" -s
	80	return $?
	81	fi
8ea4ef	82	#Register error
54808c	83	echo $1 >> $ERRORFILE
8ea4ef	84	return 1
JG	85	fi
3d9186	86
2bcbc6	87	#Get build artifact names from PKGBUILD and build artifacts
JG	88	#Remove duplicates from the list
37434c	89	pkgs=()
5f18f4	90	ipkgs=()
88a91e	91	for i in ${pkgname[@]}; do
b2c2b5	92	#pkgs+=("$i-$pkgver-$pkgrel")
5f18f4	93	ipkgs+=($(find . -mindepth 1 -maxdepth 1 -type f $ -name "$i.pkg.tar." -o -name "$i.src.tar." $ -not -name "*.sig" \| sed 's/^\.\///'))
88a91e	94	done
5f18f4	95	while read -r -d '' x; do pkgs+=("$x"); done < <(printf "%s\0" "${ipkgs[@]}" \| sort -uz)
88a91e	96
94852c	97	# Weird exceptions
C	98	if [[ "$1" == "zoom" ]]; then
	99	rm zoom_orig
	100	for i in ${pkgs[@]}; do
	101	if [ -z "${i##_orig}" ]; then
	102	pkgs=(${pkgs[@]/$i})
	103	fi
	104	done
	105	fi
	106
e72b38	107	#Move package to repodir and add to repo db
d4b7a7	108	#Dont change the database if rebuilt the same package at same release and version
5f18f4	109	flag=0
88a91e	110	for i in ${pkgs[@]}; do
e4347d	111	if [[ ! -f $REPODIR/$i ]]; then
5f18f4	112	flag=1
d4b7a7	113	fi
88a91e	114	done
5f18f4	115	if [[ $flag == 1 ]]; then
C	116	rm -f $REPODIR/$1.tar.*
	117	for i in ${pkgs[@]}; do
	118	cp $i $REPODIR/
	119	[[ "$SIGN" == "Y" ]] && cp $i.sig $REPODIR/
	120	done
3d9186	121	else
5f18f4	122	return;
9837c8	123	fi
C	124
e72b38	125	# Add package to waiting list to be added to repo db
9837c8	126	while true; do
54808c	127	if [[ $(cat $WAITLIST_LCK) == 1 ]]; then
e6f0af	128	sleep 1
4ebf3d	129	else
54808c	130	echo 1 > $WAITLIST_LCK
C	131	echo $1 >> $WAITLIST
	132	echo 0 > $WAITLIST_LCK
e6f0af	133	break
C	134	fi
	135	done
9837c8	136	while true; do
e72b38	137	# Wait until package is at the top of the queue and add to db
54808c	138	if [[ "$(head -n1 $WAITLIST)" == "$1" ]]; then
3d9186	139	repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) ${pkgs[@]}
9837c8	140	while true; do
54808c	141	if [[ $(cat $WAITLIST_LCK) == 1 ]]; then
e6f0af	142	sleep 1
4ebf3d	143	else
e72b38	144	# Remove self from top of queue
5e843f	145	echo 1 > $WAITLIST_LCK
54808c	146	TEMP=$(mktemp)
C	147	tail -n +2 $WAITLIST > $TEMP
	148	cp $TEMP $WAITLIST
	149	rm $TEMP
	150	unset TEMP
	151	echo 0 > $WAITLIST_LCK
e6f0af	152	break
C	153	fi
	154	done
	155	break
	156	else
54808c	157	if [[ -z "$(grep $1 $WAITLIST)" ]]; then
b2c2b5	158	# Not on waitlist for some reason, need to readd
54808c	159	if [[ $(cat $WAITLIST_LCK) == 1 ]]; then
e74aea	160	sleep 1
C	161	else
54808c	162	echo 1 > $WAITLIST_LCK
C	163	echo $1 >> $WAITLIST
	164	echo 0 > $WAITLIST_LCK
e74aea	165	fi
C	166	fi
e6f0af	167	sleep 10
4ebf3d	168	fi
e6f0af	169	done
8ea4ef	170
JG	171	#Remove old versions of packages
f6854c	172	#TODO: Want to be able to keep multiple versions of old packages, future work
JG	173	#Currently old package versions stay in the repodir indefinately
	174	# while [ $NUM_OLD \< $(find . -name '*.pkg.tar.xz' \| wc -l) ]
	175	# do
7c4087	176	# old=$(newold_matching_file o '*.pkg.tar.xz')
JG	177	# rm $REPODIR/$old $old
f6854c	178	# done
8ea4ef	179	return 0
JG	180	}
	181
	182	#Update packages in BUILDDIR
f6854c	183	# Usage: build_all [-f force]
8ea4ef	184	function build_all {
JG	185	#system update
9837c8	186	if [[ $UPDATE == "Y" ]]; then
8ea4ef	187	sudo pacman -Syu --noconfirm
JG	188	fi
ac333f	189
8ea4ef	190	#update every package currently stored
c3d80e	191	for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -type d)
8ea4ef	192	do
JG	193	cd $d
9837c8	194	if [[ "$PARALLEL" == "Y" ]]; then
C	195	build_pkg $(echo $d \| rev \| cut -d'/' -f1 \| rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" \|\| echo "/dev/tty") &
	196	else
	197	build_pkg $(echo $d \| rev \| cut -d'/' -f1 \| rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" \|\| echo "/dev/tty")
	198	fi
8ea4ef	199	done
e6f0af	200	wait
8ea4ef	201
JG	202	return 0
	203	}
	204
	205	#Add a new package to be built
7c4087	206	#Adding build dependencies is
f6854c	207	# Usage: add [package name]
8ea4ef	208	function add {
37434c	209	local i j k
67ef73	210	for i in $@; do
JG	211	cd $BUILDDIR
5ac326	212	if [ ! -d $i ] && [[ -z $(git ls-remote https://aur.archlinux.org/$i.git) ]]; then
863ad1	213	echo "Not a package: $i"
ca0d8a	214	exit 2
C	215	fi
67ef73	216	git clone https://aur.archlinux.org/$i.git
JG	217	cd $i
37434c	218	unset depends
JG	219	unset makedepends
	220	local makedeps
	221	source PKGBUILD
ca0d8a	222
3d9186	223	#Check for all build dependencies
37434c	224	for j in ${makedepends[@]}; do
JG	225	k=$(echo $j \| sed 's/[>]=.*//g')
863ad1	226	if ! (pacman -Si $k \|\| pacman -Qi $k); then
37434c	227	makedeps+=($k)
ca0d8a	228	fi &>/dev/null
C	229	done
37434c	230	for j in ${depends[@]}; do
JG	231	k=$(echo $j \| sed 's/[>]=.*//g')
863ad1	232	if ! (pacman -Si $k \|\| pacman -Qi $k); then
37434c	233	makedeps+=($k)
JG	234	fi &>/dev/null
ca0d8a	235	done
3d9186	236
JG	237	#Add dependencies and update so overall build can work
37434c	238	for j in ${makedeps[@]}; do
JG	239	add $j
	240	done
	241	if [[ -n "${makedeps[@]}" ]]; then
3b9f12	242	sudo pacman -Sy
C	243	fi
3d9186	244
ca0d8a	245	#Actually build wanted package
37434c	246	cd $BUILDDIR/$i
9837c8	247	build_pkg $i -f
67ef73	248	done
8ea4ef	249	return 0
c3d80e	250	}
C	251
	252	#Remove a package from the build list and repository
e6670f	253	#Usage of -a removes all packages moved to official repos
JG	254	# Usage remove [-a\|package name]
c3d80e	255	function remove {
e6670f	256	if [[ "$1" == "-a" ]]; then
JG	257	rmlist=""
	258	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq core \| sort) \| tr '\n' ' ')"
	259	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq extra \| sort) \| tr '\n' ' ')"
	260	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq community \| sort) \| tr '\n' ' ')"
	261	for i in $rmlist; do
	262	rm -rf $BUILDDIR/$i
	263	repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) $i
	264	rm -f $REPODIR/$i
	265	done
	266	else
	267	for i in $@; do
	268	rm -rf $BUILDDIR/$i
	269	repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) $i
	270	rm -f $REPODIR/$i
	271	done
	272	fi
8ea4ef	273	}
JG	274
7aa510	275	#Check for packages moved to official repos or removed from the AUR
C	276	function check {
	277	rmlist=""
	278	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq core \| sort) \| tr '\n' ' ')"
	279	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq extra \| sort) \| tr '\n' ' ')"
	280	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq community \| sort) \| tr '\n' ' ')"
	281	TMPFILE=$(mktemp)
	282	for i in $(find $BUILDDIR -mindepth 1 -maxdepth 1 -type d); do
	283	check_pkg $TMPFILE "$(echo $i \| rev \| cut -d'/' -f1 \| rev)" &
	284	done
	285	wait
	286	echo "Merged into official repos: $rmlist"
	287	echo "Not in AUR: $(cat $TMPFILE \| tr '\n' ' ')"
	288	rm -f $TMPFILE
	289	}
	290
3d9186	291	#Check helper function
7aa510	292	function check_pkg {
ae3c9b	293	if [[ -z "$(curl -si "https://aur.archlinux.org/packages/$2" \| head -n1 \| grep 200)" ]]; then
7aa510	294	echo "$2" >> $1
C	295	fi
	296	}
	297
	298
8ea4ef	299	#Check config and create build folders
f6854c	300	#Set variables before usage
JG	301	# Usage: init
8ea4ef	302	function init {
9837c8	303	if [[ $uid != 1 ]]; then
4ebf3d	304	echo "This must be run as root"
JG	305	fi
	306
8ea4ef	307	#check for configuration here
9837c8	308	[[ -z $REPODIR ]] && echo "Enter REPODIR" && return 1
C	309	[[ -z $BUILDDIR ]] && echo "Enter BUILDDIR" && return 2
	310	[[ -z $REPONAME ]] && echo "Enter REPONAME" && return 3
8ea4ef	311
JG	312	#make build directories
9837c8	313	[[ ! -d $REPODIR ]] && mkdir -p $REPODIR
C	314	[[ ! -d $BUILDDIR ]] && mkdir -p $BUILDDIR
8ea4ef	315
JG	316	#packages required to build others
4ebf3d	317	pacman -S --noconfirm base-devel git
8ea4ef	318
JG	319	#add repo to pacman.conf so can install own packages
9837c8	320	if [[ -z $(grep "$REPONAME" /etc/pacman.conf) ]]; then
8ea4ef	321	printf "[$REPONAME]\nSigLevel = Optional TrustAll\nServer = file://$REPODIR\n" >> /etc/pacman.conf
JG	322	fi
	323
	324	#create GPG key for package signing
4ebf3d	325	if [[ "$SIGN" == "Y" && "$KEY" == "" ]]; then
8ea4ef	326	(
JG	327	echo "Key-Type: RSA"
	328	echo "Key-Length: 2048"
	329	echo "Subkey-Type: RSA"
	330	echo "Subkey-Length: 2048"
	331	echo "Passphrase: \"\""
	332	echo "Expire-Date: 0"
	333	echo "Name-Real: John Doe"
	334	echo "Name-Comment: Arch buildbot"
	335	echo "Name-Email: $(whoami)@localhost"
	336	echo "%commit"
	337	) \| gpg --batch --generate-key
	338	gpg --export --output $REPONAME.key --armor "John Doe"
	339	gpg --export-secret-keys --output $REPONAME.secret.key --armor "John Doe"
	340	echo "Please change the key information in this file"
	341	fi
	342
	343	return 0
	344	}
	345
896dec	346	function remake-repo {
JG	347	cd $REPODIR
	348	rm -f $REPONAME.db.*
	349	pkgs="$(find . -name '.pkg.' -not -name '*.sig')"
	350	repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) $pkgs
	351	}
	352
86d762	353	function send_email {
ae3c9b	354	# message=(echo "From: $FROM_EMAIL"
JG	355	# echo "To: $TO_EMAIL"
	356	# echo "Subject: Build errors"
	357	# echo ""
	358	# echo "There were build errors for the build of $REPONAME at $(date), please address them soon."
	359	# echo "The errors were: $@"
	360	# )
	361	if [ 1 ]; then
	362	(
	363	echo "From: $FROM_EMAIL"
	364	echo "To: $TO_EMAIL"
	365	echo "Subject: Build errors"
	366	echo ""
	367	echo "There were build errors for the build of $REPONAME at $(date), please address them soon."
	368	echo "The errors were: $@"
	369	) \| sendmail -t
	370	else
6bd4de	371	curl -s --url "smtp://$EMAIL_HOST" --ssl \
d41201	372	--mail-from "$FROM_EMAIL" \
JG	373	--mail-rcpt "$TO_EMAIL" \
	374	--user "$EMAIL_USER" \
	375	-T <(
	376	echo "From: $FROM_EMAIL"
	377	echo "To: $TO_EMAIL"
	378	echo "Subject: Build errors"
	379	echo ""
	380	echo "There were build errors for the build of $REPONAME at $(date), please address them soon."
	381	echo "The errors were: $@"
	382	)
ae3c9b	383	fi
095fd9	384	return $?
86d762	385	}
C	386
8ea4ef	387	case $1 in
JG	388	"init")
	389	init;;
	390	"add")
9837c8	391	add ${@:2};;
8ea4ef	392	"build-all")
9837c8	393	build_all $([[ "$2" == "-f" ]] && echo "-f");;
c3d80e	394	"remove")
9837c8	395	remove ${@:2};;
7aa510	396	"check")
C	397	check;;
896dec	398	"remake")
JG	399	remake-repo;;
f4a6ac	400	"test-mail")
095fd9	401	send_email
JG	402	exit $?;;
8ea4ef	403	*)
dc0641	404	echo -e "\033[0;31mInvalid usage\033[0m"
C	405	echo -e "Usage: $0 init\|check\|add\|remove\|build-all"
	406	echo -e "\033[0;32minit\033[0m - initialise repository for use"
f4a6ac	407	echo -e "\033[0;32mtest-mail\033[m - test email configuration by sending the default email without a message"
dc0641	408	echo -e "\033[0;32mcheck\033[0m - check if packages have been moved into the official repositories or removed from the AUR"
C	409	echo -e "\033[0;32madd package ...\033[0m - add a package to \$BUILDDIR and repository, also used to rebuild failed packages"
	410	echo -e "\033[0;32mremove -a \| package ...\033[0m - remove package from \$BUILDDIR and repository, \"-a\" removes packages added to official repos"
	411	echo -e "\033[0;32mbuild-all [-f]\033[0m - build all packages in \$BUILDDIR, \"-f\" force builds whole repository"
896dec	412	echo -e "\033[-;32mremake\033[0m - Recreate repo"
8ea4ef	413	esac
9837c8	414
C	415	# Error reporting, send email only for build-all as assuming an batch job for that
7aa510	416	if [[ $1 == "build-all" \|\| $1 == "add" ]]; then
54808c	417	if [[ -n $(cat $ERRORFILE) ]]; then
C	418	ERRORS=$(cat $ERRORFILE \| tr '\n' ' ')
7aa510	419	echo "Errors in packages: $ERRORS"
f4a6ac	420	if [[ "$TO_EMAIL" != "" && "$1" == "build-all" ]]; then
7aa510	421	send_email $ERRORS
C	422	fi
	423	else
	424	echo "All packages built successfully"
9837c8	425	fi
C	426	fi
54808c	427
C	428	rm $ERRORFILE $WAITLIST $WAITLIST_LCK