Arch-autobuild-repo.git

work with all vc packages formatted to guidelines, remove zoom orig

Chizi123

2020-10-19 dfed5fa5e727233ca02f5711c6b781623644f4af

commit \| author \| age
8ea4ef	1	#!/bin/bash
f6854c	2	#A basic bash script to automate the building of arch packages
JG	3	# Usage: main.sh init\|add\|build_all [-f force]
8ea4ef	4
e6f0af	5	source $(dirname "$(realpath $0)")/vars.sh
8ea4ef	6
JG	7	#Helper for finding newest and oldest files
	8	#Sourced from stack overflow
f6854c	9	# Usage: newold_matching_file [n/o] [filename]
8ea4ef	10	function newold_matching_file
JG	11	{
7c4087	12	# Use ${1-} instead of $1 in case 'nounset' is set
JG	13	local -r glob_pattern=${2-}
8ea4ef	14
7c4087	15	# To avoid printing garbage if no files match the pattern, set
JG	16	# 'nullglob' if necessary
	17	local -i need_to_unset_nullglob=0
	18	if [[ ":$BASHOPTS:" != :nullglob: ]] ; then
	19	shopt -s nullglob
	20	need_to_unset_nullglob=1
	21	fi
8ea4ef	22
7c4087	23	file=
JG	24	for f in $glob_pattern ; do
8ea4ef	25	if [ $1 == "n" ]; then
JG	26	[[ -z $f \|\| $f -nt $_file ]] && file=$f
	27	elif [ $1 == "o" ]; then
	28	[[ -z $f \|\| $f -ot $_file ]] && file=$f
	29	fi
7c4087	30	done
8ea4ef	31
7c4087	32	# To avoid unexpected behaviour elsewhere, unset nullglob if it was
JG	33	# set by this function
	34	(( need_to_unset_nullglob )) && shopt -u nullglob
8ea4ef	35
7c4087	36	# Use printf instead of echo in case the file name begins with '-'
JG	37	[[ -n $file ]] && printf '%s\n' "$file"
8ea4ef	38
7c4087	39	return 0
8ea4ef	40	}
JG	41
	42	#Build latest version of a package
9837c8	43	# Usage: build_pkg [package name] [-f force]
8ea4ef	44	function build_pkg {
JG	45	#check if PKGBUILD has updated, don't rebuild if hasn't changed
dfed5f	46	if [[ ! -z $(git pull \| grep "Already up to date.") && -z $(grep 'pkgver() {' PKGBUILD) && -z $2 ]]; then
8ea4ef	47	return 2
JG	48	fi
f6854c	49
JG	50	#remove old versions before build
3b9f12	51	rm -f $1.tar.*
f6854c	52
JG	53	#make and force rebuild if is git package
9837c8	54	# Mictosoft fonts have problems with checksums and need a seperate argument
C	55	if [[ "$1" == "ttf-ms-win10" \|\|
	56	"$1" == "ttf-office-2007-fonts" \|\|
	57	"$1" == "ttf-ms-win8" \|\|
	58	"$1" == "ttf-win7-fonts" ]]; then
	59	makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) --skipchecksums
	60	else
e74aea	61	makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) 2>&1
9837c8	62	fi
C	63	if [[ $? != 0 ]]; then
8ea4ef	64	#Register error
e6f0af	65	echo $1 >> $REPODIR/.errors
8ea4ef	66	return 1
JG	67	fi
	68
5f18f4	69	#Get build artifact names from PKGBUILD and build artifacts
C	70	#Remove duplicates from the list
88a91e	71	source PKGBUILD
JG	72	pkgs=()
5f18f4	73	ipkgs=()
88a91e	74	for i in ${pkgname[@]}; do
b2c2b5	75	#pkgs+=("$i-$pkgver-$pkgrel")
5f18f4	76	ipkgs+=($(find . -mindepth 1 -maxdepth 1 -type f $ -name "$i.pkg.tar." -o -name "$i.src.tar." $ -not -name "*.sig" \| sed 's/^\.\///'))
88a91e	77	done
5f18f4	78	while read -r -d '' x; do pkgs+=("$x"); done < <(printf "%s\0" "${ipkgs[@]}" \| sort -uz)
88a91e	79
e72b38	80	#Move package to repodir and add to repo db
d4b7a7	81	#Dont change the database if rebuilt the same package at same release and version
5f18f4	82	flag=0
88a91e	83	for i in ${pkgs[@]}; do
e4347d	84	if [[ ! -f $REPODIR/$i ]]; then
5f18f4	85	flag=1
d4b7a7	86	fi
88a91e	87	done
5f18f4	88	if [[ $flag == 1 ]]; then
C	89	rm -f $REPODIR/$1.tar.*
	90	for i in ${pkgs[@]}; do
	91	cp $i $REPODIR/
	92	[[ "$SIGN" == "Y" ]] && cp $i.sig $REPODIR/
	93	done
	94	else
	95	return;
	96	fi
9837c8	97
C	98	# Weird exceptions
	99	if [[ "$1" == "zoom" ]]; then
	100	rm zoom_orig.pkg.tar.xz
dfed5f	101	for i in ${pkgs[@]}; do
C	102	if [ -z "${i##orig}" ]; then
	103	pkgs=${pkgs[@]/$i}
	104	fi
	105	done
9837c8	106	fi
C	107
e72b38	108	# Add package to waiting list to be added to repo db
9837c8	109	while true; do
C	110	if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then
e6f0af	111	sleep 1
4ebf3d	112	else
e6f0af	113	echo 1 > $REPODIR/.waitlist.lck
C	114	echo $1 >> $REPODIR/.waitlist
	115	echo 0 > $REPODIR/.waitlist.lck
	116	break
	117	fi
	118	done
9837c8	119	while true; do
e72b38	120	# Wait until package is at the top of the queue and add to db
9837c8	121	if [[ "$(head -n1 $REPODIR/.waitlist)" == "$1" ]]; then
5f18f4	122	# for i in ${pkgs[@]}; do
C	123	repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) ${pkgs[@]}
	124	# done
9837c8	125	while true; do
C	126	if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then
e6f0af	127	sleep 1
4ebf3d	128	else
e72b38	129	# Remove self from top of queue
e6f0af	130	echo 1 > $REPODIR/.waitlist.lck
C	131	tail -n +2 $REPODIR/.waitlist > $REPODIR/.waitlist.tmp
e72b38	132	mv $REPODIR/.waitlist.tmp $REPODIR/.waitlist
e6f0af	133	echo 0 > $REPODIR/.waitlist.lck
C	134	break
	135	fi
	136	done
	137	break
	138	else
e74aea	139	if [[ -z "$(grep $1 $REPODIR/.waitlist)" ]]; then
b2c2b5	140	# Not on waitlist for some reason, need to readd
e74aea	141	if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then
C	142	sleep 1
	143	else
	144	echo 1 > $REPODIR/.waitlist.lck
	145	echo $1 >> $REPODIR/.waitlist
	146	echo 0 > $REPODIR/.waitlist.lck
	147	fi
	148	fi
e6f0af	149	sleep 10
4ebf3d	150	fi
e6f0af	151	done
8ea4ef	152
JG	153	#Remove old versions of packages
f6854c	154	#TODO: Want to be able to keep multiple versions of old packages, future work
JG	155	#Currently old package versions stay in the repodir indefinately
	156	# while [ $NUM_OLD \< $(find . -name '*.pkg.tar.xz' \| wc -l) ]
	157	# do
7c4087	158	# old=$(newold_matching_file o '*.pkg.tar.xz')
JG	159	# rm $REPODIR/$old $old
f6854c	160	# done
8ea4ef	161	return 0
JG	162	}
	163
	164	#Update packages in BUILDDIR
f6854c	165	# Usage: build_all [-f force]
8ea4ef	166	function build_all {
JG	167	#system update
9837c8	168	if [[ $UPDATE == "Y" ]]; then
8ea4ef	169	sudo pacman -Syu --noconfirm
JG	170	fi
ac333f	171
e74aea	172	#Remove waitlist and errors from old builds
C	173	rm -f $REPODIR/{.waitlist,.errors}
8ea4ef	174	#update every package currently stored
c3d80e	175	for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -type d)
8ea4ef	176	do
JG	177	cd $d
9837c8	178	if [[ "$PARALLEL" == "Y" ]]; then
C	179	build_pkg $(echo $d \| rev \| cut -d'/' -f1 \| rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" \|\| echo "/dev/tty") &
	180	else
	181	build_pkg $(echo $d \| rev \| cut -d'/' -f1 \| rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" \|\| echo "/dev/tty")
	182	fi
8ea4ef	183	done
e6f0af	184	wait
8ea4ef	185
JG	186	return 0
	187	}
	188
	189	#Add a new package to be built
7c4087	190	#Adding build dependencies is
f6854c	191	# Usage: add [package name]
8ea4ef	192	function add {
67ef73	193	for i in $@; do
JG	194	cd $BUILDDIR
ca0d8a	195	if [[ -z $(git ls-remote https://aur.archlinux.org/$i.git) ]]; then
C	196	echo "Not a package"
	197	exit 2
	198	fi
67ef73	199	git clone https://aur.archlinux.org/$i.git
JG	200	cd $i
ca0d8a	201
C	202	#check for all build dependencies
	203	for i in ${makedepends[@]}; do
7c4087	204	if pacman -Si $i; then
ca0d8a	205	makedepends=${makedepends[@]/$delete}
C	206	fi &>/dev/null
	207	done
	208	for i in ${makedepends[@]}; do
	209	add $i
	210	done
3b9f12	211	if [[ -n "${makedepends[@]}" ]]; then
C	212	sudo pacman -Sy
	213	fi
	214
ca0d8a	215	#Actually build wanted package
9837c8	216	build_pkg $i -f
67ef73	217	done
8ea4ef	218	return 0
c3d80e	219	}
C	220
	221	#Remove a package from the build list and repository
e6670f	222	#Usage of -a removes all packages moved to official repos
JG	223	# Usage remove [-a\|package name]
c3d80e	224	function remove {
e6670f	225	if [[ "$1" == "-a" ]]; then
JG	226	rmlist=""
	227	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq core \| sort) \| tr '\n' ' ')"
	228	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq extra \| sort) \| tr '\n' ' ')"
	229	rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME \| sort) <(pacman -Slq community \| sort) \| tr '\n' ' ')"
	230	for i in $rmlist; do
	231	rm -rf $BUILDDIR/$i
	232	repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) $i
	233	rm -f $REPODIR/$i
	234	done
	235	else
	236	for i in $@; do
	237	rm -rf $BUILDDIR/$i
	238	repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] \|\| echo $COMPRESSION && echo zst) $i
	239	rm -f $REPODIR/$i
	240	done
	241	fi
8ea4ef	242	}
JG	243
	244	#Check config and create build folders
f6854c	245	#Set variables before usage
JG	246	# Usage: init
8ea4ef	247	function init {
9837c8	248	if [[ $uid != 1 ]]; then
4ebf3d	249	echo "This must be run as root"
JG	250	fi
	251
8ea4ef	252	#check for configuration here
9837c8	253	[[ -z $REPODIR ]] && echo "Enter REPODIR" && return 1
C	254	[[ -z $BUILDDIR ]] && echo "Enter BUILDDIR" && return 2
	255	[[ -z $REPONAME ]] && echo "Enter REPONAME" && return 3
8ea4ef	256
JG	257	#make build directories
9837c8	258	[[ ! -d $REPODIR ]] && mkdir -p $REPODIR
C	259	[[ ! -d $BUILDDIR ]] && mkdir -p $BUILDDIR
8ea4ef	260
JG	261	#packages required to build others
4ebf3d	262	pacman -S --noconfirm base-devel git
8ea4ef	263
JG	264	#add repo to pacman.conf so can install own packages
9837c8	265	if [[ -z $(grep "$REPONAME" /etc/pacman.conf) ]]; then
8ea4ef	266	printf "[$REPONAME]\nSigLevel = Optional TrustAll\nServer = file://$REPODIR\n" >> /etc/pacman.conf
JG	267	fi
	268
	269	#create GPG key for package signing
4ebf3d	270	if [[ "$SIGN" == "Y" && "$KEY" == "" ]]; then
8ea4ef	271	(
JG	272	echo "Key-Type: RSA"
	273	echo "Key-Length: 2048"
	274	echo "Subkey-Type: RSA"
	275	echo "Subkey-Length: 2048"
	276	echo "Passphrase: \"\""
	277	echo "Expire-Date: 0"
	278	echo "Name-Real: John Doe"
	279	echo "Name-Comment: Arch buildbot"
	280	echo "Name-Email: $(whoami)@localhost"
	281	echo "%commit"
	282	) \| gpg --batch --generate-key
	283	gpg --export --output $REPONAME.key --armor "John Doe"
	284	gpg --export-secret-keys --output $REPONAME.secret.key --armor "John Doe"
	285	echo "Please change the key information in this file"
	286	fi
	287
	288	return 0
	289	}
	290
86d762	291	function send_email {
C	292	(
	293	echo "From: build@localhost"
	294	echo "To: $EMAIL"
	295	echo "Subject: Build errors"
e6f0af	296	echo "There were build errors for the build of $REPONAME at $(date), please address them soon."
be70fd	297	echo "The errors were: $@"
86d762	298	) \| sendmail -t
C	299	}
	300
8ea4ef	301	case $1 in
JG	302	"init")
	303	init;;
	304	"add")
9837c8	305	add ${@:2};;
8ea4ef	306	"build-all")
9837c8	307	build_all $([[ "$2" == "-f" ]] && echo "-f");;
c3d80e	308	"remove")
9837c8	309	remove ${@:2};;
8ea4ef	310	*)
f11bfd	311	printf "Invalid usage\nUsage: $0 init\|add\|build-all\n";;
8ea4ef	312	esac
9837c8	313
C	314	# Error reporting, send email only for build-all as assuming an batch job for that
	315	if [[ -f $REPODIR/.errors ]]; then
	316	ERRORS=$(cat $REPODIR/.errors \| tr '\n' ' ')
	317	rm $REPODIR/.errors
	318	echo "Errors in packages: $ERRORS"
	319	if [[ "$EMAIL" != "" && "$1" == "build-all" ]]; then
	320	send_email $ERRORS
	321	fi
	322	else
	323	echo "All packages built successfully"
	324	fi