Arch-autobuild-repo.git

Chizi123

2020-11-29 96e462c966cbd3aa1970efcd9304f063ce161006

commit \| author \| age
8ea4ef	1	#!/bin/bash
f6854c	2	#A basic bash script to automate the building of arch packages
dc0641	3	# Usage: main.sh init\|check\|add\|remove\|build_all
8ea4ef	4
e6f0af	5	source $(dirname "$(realpath $0)")/vars.sh
8ea4ef	6
54808c	7	ERRORFILE=$(mktemp)
C	8	WAITLIST=$(mktemp)
	9	WAITLIST_LCK=$(mktemp)
	10
8ea4ef	11	#Helper for finding newest and oldest files
JG	12	#Sourced from stack overflow
f6854c	13	# Usage: newold_matching_file [n/o] [filename]
8ea4ef	14	function newold_matching_file
JG	15	{
7c4087	16	# Use ${1-} instead of $1 in case 'nounset' is set
JG	17	local -r glob_pattern=${2-}
8ea4ef	18
7c4087	19	# To avoid printing garbage if no files match the pattern, set
JG	20	# 'nullglob' if necessary
	21	local -i need_to_unset_nullglob=0
	22	if [[ ":$BASHOPTS:" != :nullglob: ]] ; then
	23	shopt -s nullglob
	24	need_to_unset_nullglob=1
	25	fi
8ea4ef	26
7c4087	27	file=
JG	28	for f in $glob_pattern ; do
8ea4ef	29	if [ $1 == "n" ]; then
JG	30	[[ -z $f \|\| $f -nt $_file ]] && file=$f
	31	elif [ $1 == "o" ]; then
	32	[[ -z $f \|\| $f -ot $_file ]] && file=$f
	33	fi
7c4087	34	done
8ea4ef	35
7c4087	36	# To avoid unexpected behaviour elsewhere, unset nullglob if it was
JG	37	# set by this function
	38	(( need_to_unset_nullglob )) && shopt -u nullglob
8ea4ef	39
7c4087	40	# Use printf instead of echo in case the file name begins with '-'
JG	41	[[ -n $file ]] && printf '%s\n' "$file"
8ea4ef	42
7c4087	43	return 0
8ea4ef	44	}
JG	45
	46	#Build latest version of a package
9837c8	47	# Usage: build_pkg [package name] [-f force]
8ea4ef	48	function build_pkg {
JG	49	#check if PKGBUILD has updated, don't rebuild if hasn't changed
b59712	50	if [[ -n $(git pull \| grep 'Already up to date.') && -z $(grep 'pkgver() {' PKGBUILD) && -z "$2" ]]; then
8ea4ef	51	return 2
JG	52	fi
f6854c	53
JG	54	#remove old versions before build
96e462	55	# rm -f $1.tar.*
f6854c	56
JG	57	#make and force rebuild if is git package
9837c8	58	# Mictosoft fonts have problems with checksums and need a seperate argument
C	59	if [[ "$1" == "ttf-ms-win10" \|\|
	60	"$1" == "ttf-office-2007-fonts" \|\|
	61	"$1" == "ttf-ms-win8" \|\|
	62	"$1" == "ttf-win7-fonts" ]]; then
cb04c8	63	makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) --skipchecksums 2>&1
9837c8	64	else
e74aea	65	makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) 2>&1
9837c8	66	fi
96e462	67	if [[ $? != 0 \|\| $? == 13 ]]; then
8ea4ef	68	#Register error
54808c	69	echo $1 >> $ERRORFILE
8ea4ef	70	return 1
JG	71	fi
	72
5f18f4	73	#Get build artifact names from PKGBUILD and build artifacts
C	74	#Remove duplicates from the list
88a91e	75	source PKGBUILD
96e462	76	srcdir="$(pwd)/src"
C	77	ver=$(pkgver)
	78	find . -mindepth 1 -maxdepth 1 -type f $ -name ".pkg.tar." -o -name ".src.tar." $ -not -name "$ver-$pkgrel"
5f18f4	79	ipkgs=()
88a91e	80	for i in ${pkgname[@]}; do
b2c2b5	81	#pkgs+=("$i-$pkgver-$pkgrel")
5f18f4	82	ipkgs+=($(find . -mindepth 1 -maxdepth 1 -type f $ -name "$i.pkg.tar." -o -name "$i.src.tar." $ -not -name "*.sig" \| sed 's/^\.\///'))
88a91e	83	done
5f18f4	84	while read -r -d '' x; do pkgs+=("$x"); done < <(printf "%s\0" "${ipkgs[@]}" \| sort -uz)
88a91e	85
94852c	86	# Weird exceptions
C	87	if [[ "$1" == "zoom" ]]; then
	88	rm zoom_orig
	89	for i in ${pkgs[@]}; do
	90	if [ -z "${i##_orig}" ]; then
	91	pkgs=(${pkgs[@]/$i})
	92	fi
	93	done
	94	fi
	95
e72b38	96	#Move package to repodir and add to repo db
d4b7a7	97	#Dont change the database if rebuilt the same package at same release and version
5f18f4	98	flag=0
88a91e	99	for i in ${pkgs[@]}; do
e4347d	100	if [[ ! -f $REPODIR/$i ]]; then
5f18f4	101	flag=1
d4b7a7	102	fi
88a91e	103	done
5f18f4	104	if [[ $flag == 1 ]]; then
C	105	rm -f $REPODIR/$1.tar.*
	106	for i in ${pkgs[@]}; do
	107	cp $i $REPODIR/
	108	[[ "$SIGN" == "Y" ]] && cp $i.sig $REPODIR/
	109	done
	110	else
	111	return;
9837c8	112	fi
C	113
e72b38	114	# Add package to waiting list to be added to repo db
9837c8	115	while true; do
54808c	116	if [[ $(cat $WAITLIST_LCK) == 1 ]]; then
e6f0af	117	sleep 1
4ebf3d	118	else
54808c	119	echo 1 > $WAITLIST_LCK
C	120	echo $1 >> $WAITLIST
	121	echo 0 > $WAITLIST_LCK
e6f0af	122	break
C	123	fi
	124	done
9837c8	125	while true; do
e72b38	126	# Wait until package is at the top of the queue and add to db
54808c	127	if [[ "$(head -n1 $WAITLIST)" == "$1" ]]; then
5f18f4	128	# for i in ${pkgs[@]}; do
C	129	repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) ${pkgs[@]}
	130	# done
9837c8	131	while true; do
54808c	132	if [[ $(cat $WAITLIST_LCK) == 1 ]]; then
e6f0af	133	sleep 1
4ebf3d	134	else
e72b38	135	# Remove self from top of queue
5e843f	136	echo 1 > $WAITLIST_LCK
54808c	137	TEMP=$(mktemp)
C	138	tail -n +2 $WAITLIST > $TEMP
	139	cp $TEMP $WAITLIST
	140	rm $TEMP
	141	unset TEMP
	142	echo 0 > $WAITLIST_LCK
e6f0af	143	break
C	144	fi
	145	done
	146	break
	147	else
54808c	148	if [[ -z "$(grep $1 $WAITLIST)" ]]; then
b2c2b5	149	# Not on waitlist for some reason, need to readd
54808c	150	if [[ $(cat $WAITLIST_LCK) == 1 ]]; then
e74aea	151	sleep 1
C	152	else
54808c	153	echo 1 > $WAITLIST_LCK
C	154	echo $1 >> $WAITLIST
	155	echo 0 > $WAITLIST_LCK
e74aea	156	fi
C	157	fi
e6f0af	158	sleep 10
4ebf3d	159	fi
e6f0af	160	done
8ea4ef	161
JG	162	#Remove old versions of packages
f6854c	163	#TODO: Want to be able to keep multiple versions of old packages, future work
JG	164	#Currently old package versions stay in the repodir indefinately
	165	# while [ $NUM_OLD \< $(find . -name '*.pkg.tar.xz' \| wc -l) ]
	166	# do
7c4087	167	# old=$(newold_matching_file o '*.pkg.tar.xz')
JG	168	# rm $REPODIR/$old $old
f6854c	169	# done
8ea4ef	170	return 0
JG	171	}
	172
	173	#Update packages in BUILDDIR
f6854c	174	# Usage: build_all [-f force]
8ea4ef	175	function build_all {
JG	176	#system update
9837c8	177	if [[ $UPDATE == "Y" ]]; then
8ea4ef	178	sudo pacman -Syu --noconfirm
JG	179	fi
ac333f	180
8ea4ef	181	#update every package currently stored
c3d80e	182	for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -type d)
8ea4ef	183	do
JG	184	cd $d
9837c8	185	if [[ "$PARALLEL" == "Y" ]]; then
C	186	build_pkg $(echo $d \| rev \| cut -d'/' -f1 \| rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" \|\| echo "/dev/tty") &
	187	else
	188	build_pkg $(echo $d \| rev \| cut -d'/' -f1 \| rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" \|\| echo "/dev/tty")
	189	fi
8ea4ef	190	done
e6f0af	191	wait
8ea4ef	192
JG	193	return 0
	194	}
	195
	196	#Add a new package to be built
7c4087	197	#Adding build dependencies is
f6854c	198	# Usage: add [package name]
8ea4ef	199	function add {
67ef73	200	for i in $@; do
JG	201	cd $BUILDDIR
ca0d8a	202	if [[ -z $(git ls-remote https://aur.archlinux.org/$i.git) ]]; then
C	203	echo "Not a package"
	204	exit 2
	205	fi
67ef73	206	git clone https://aur.archlinux.org/$i.git
JG	207	cd $i
ca0d8a	208
C	209	#check for all build dependencies
	210	for i in ${makedepends[@]}; do
7c4087	211	if pacman -Si $i; then
ca0d8a	212	makedepends=${makedepends[@]/$delete}
C	213	fi &>/dev/null
	214	done
	215	for i in ${makedepends[@]}; do
	216	add $i
	217	done
3b9f12	218	if [[ -n "${makedepends[@]}" ]]; then
C	219	sudo pacman -Sy
	220	fi
	221
ca0d8a	222	#Actually build wanted package
9837c8	223	build_pkg $i -f
67ef73	224	done
8ea4ef	225	return 0
c3d80e	226	}
C	227
	228	#Remove a package from the build list and repository
e6670f	229	#Usage of -a removes all packages moved to official repos
JG	230	# Usage remove [-a\|package name]
c3d80e	231	function remove {
e6670f	232	if [[ "$1" == "-a" ]]; then
JG	233	rmlist=""
	234	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq core \| sort) \| tr '\n' ' ')"
	235	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq extra \| sort) \| tr '\n' ' ')"
	236	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq community \| sort) \| tr '\n' ' ')"
	237	for i in $rmlist; do
	238	rm -rf $BUILDDIR/$i
	239	repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) $i
	240	rm -f $REPODIR/$i
	241	done
	242	else
	243	for i in $@; do
	244	rm -rf $BUILDDIR/$i
	245	repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) $i
	246	rm -f $REPODIR/$i
	247	done
	248	fi
8ea4ef	249	}
JG	250
7aa510	251	#Check for packages moved to official repos or removed from the AUR
C	252	function check {
	253	rmlist=""
	254	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq core \| sort) \| tr '\n' ' ')"
	255	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq extra \| sort) \| tr '\n' ' ')"
	256	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq community \| sort) \| tr '\n' ' ')"
	257	TMPFILE=$(mktemp)
	258	for i in $(find $BUILDDIR -mindepth 1 -maxdepth 1 -type d); do
	259	check_pkg $TMPFILE "$(echo $i \| rev \| cut -d'/' -f1 \| rev)" &
	260	done
	261	wait
	262	echo "Merged into official repos: $rmlist"
	263	echo "Not in AUR: $(cat $TMPFILE \| tr '\n' ' ')"
	264	rm -f $TMPFILE
	265	}
	266
	267	function check_pkg {
	268	if [[ -z "$(curl -sI "https://aur.archlinux.org/packages/$2" \| head -n1 \| grep 200)" ]]; then
	269	echo "$2" >> $1
	270	fi
	271	}
	272
	273
8ea4ef	274	#Check config and create build folders
f6854c	275	#Set variables before usage
JG	276	# Usage: init
8ea4ef	277	function init {
9837c8	278	if [[ $uid != 1 ]]; then
4ebf3d	279	echo "This must be run as root"
JG	280	fi
	281
8ea4ef	282	#check for configuration here
9837c8	283	[[ -z $REPODIR ]] && echo "Enter REPODIR" && return 1
C	284	[[ -z $BUILDDIR ]] && echo "Enter BUILDDIR" && return 2
	285	[[ -z $REPONAME ]] && echo "Enter REPONAME" && return 3
8ea4ef	286
JG	287	#make build directories
9837c8	288	[[ ! -d $REPODIR ]] && mkdir -p $REPODIR
C	289	[[ ! -d $BUILDDIR ]] && mkdir -p $BUILDDIR
8ea4ef	290
JG	291	#packages required to build others
4ebf3d	292	pacman -S --noconfirm base-devel git
8ea4ef	293
JG	294	#add repo to pacman.conf so can install own packages
9837c8	295	if [[ -z $(grep "$REPONAME" /etc/pacman.conf) ]]; then
8ea4ef	296	printf "[$REPONAME]\nSigLevel = Optional TrustAll\nServer = file://$REPODIR\n" >> /etc/pacman.conf
JG	297	fi
	298
	299	#create GPG key for package signing
4ebf3d	300	if [[ "$SIGN" == "Y" && "$KEY" == "" ]]; then
8ea4ef	301	(
JG	302	echo "Key-Type: RSA"
	303	echo "Key-Length: 2048"
	304	echo "Subkey-Type: RSA"
	305	echo "Subkey-Length: 2048"
	306	echo "Passphrase: \"\""
	307	echo "Expire-Date: 0"
	308	echo "Name-Real: John Doe"
	309	echo "Name-Comment: Arch buildbot"
	310	echo "Name-Email: $(whoami)@localhost"
	311	echo "%commit"
	312	) \| gpg --batch --generate-key
	313	gpg --export --output $REPONAME.key --armor "John Doe"
	314	gpg --export-secret-keys --output $REPONAME.secret.key --armor "John Doe"
	315	echo "Please change the key information in this file"
	316	fi
	317
	318	return 0
	319	}
	320
86d762	321	function send_email {
C	322	(
	323	echo "From: build@localhost"
	324	echo "To: $EMAIL"
	325	echo "Subject: Build errors"
e6f0af	326	echo "There were build errors for the build of $REPONAME at $(date), please address them soon."
be70fd	327	echo "The errors were: $@"
86d762	328	) \| sendmail -t
C	329	}
	330
8ea4ef	331	case $1 in
JG	332	"init")
	333	init;;
	334	"add")
9837c8	335	add ${@:2};;
8ea4ef	336	"build-all")
9837c8	337	build_all $([[ "$2" == "-f" ]] && echo "-f");;
c3d80e	338	"remove")
9837c8	339	remove ${@:2};;
7aa510	340	"check")
C	341	check;;
8ea4ef	342	*)
dc0641	343	echo -e "\033[0;31mInvalid usage\033[0m"
C	344	echo -e "Usage: $0 init\|check\|add\|remove\|build-all"
	345	echo -e "\033[0;32minit\033[0m - initialise repository for use"
	346	echo -e "\033[0;32mcheck\033[0m - check if packages have been moved into the official repositories or removed from the AUR"
	347	echo -e "\033[0;32madd package ...\033[0m - add a package to \$BUILDDIR and repository, also used to rebuild failed packages"
	348	echo -e "\033[0;32mremove -a \| package ...\033[0m - remove package from \$BUILDDIR and repository, \"-a\" removes packages added to official repos"
	349	echo -e "\033[0;32mbuild-all [-f]\033[0m - build all packages in \$BUILDDIR, \"-f\" force builds whole repository"
8ea4ef	350	esac
9837c8	351
C	352	# Error reporting, send email only for build-all as assuming an batch job for that
7aa510	353	if [[ $1 == "build-all" \|\| $1 == "add" ]]; then
54808c	354	if [[ -n $(cat $ERRORFILE) ]]; then
C	355	ERRORS=$(cat $ERRORFILE \| tr '\n' ' ')
7aa510	356	echo "Errors in packages: $ERRORS"
C	357	if [[ "$EMAIL" != "" && "$1" == "build-all" ]]; then
	358	send_email $ERRORS
	359	fi
	360	else
	361	echo "All packages built successfully"
9837c8	362	fi
C	363	fi
54808c	364
C	365	rm $ERRORFILE $WAITLIST $WAITLIST_LCK