From 7ac53faa1544a9c8ace058a2721647f368b059a6 Mon Sep 17 00:00:00 2001 From: Chizi123 <build@jsrv.com> Date: Thu, 15 Oct 2020 02:41:30 +0000 Subject: [PATCH] Merge branch 'master' of https://github.com/Chizi123/Arch-autobuild-repo into master --- main.sh | 217 +++++++++++++++++++++++++++++++++++++++++------------- 1 files changed, 164 insertions(+), 53 deletions(-) diff --git a/main.sh b/main.sh old mode 100644 new mode 100755 index 53a4e0b..83e9c4a --- a/main.sh +++ b/main.sh @@ -1,20 +1,12 @@ #!/bin/bash +#A basic bash script to automate the building of arch packages +# Usage: main.sh init|add|build_all [-f force] -REPODIR=/repo/x86_64 -BUILDDIR=/repo/build -REPONAME= -UPDATE=N -CLEAN=N -SIGN=N -KEY="" -NUM_OLD=5 -export PACKAGER="John Doe <jd@change.me>" -EMAIL="" - -ERRORS="" +source $(dirname "$(realpath $0)")/vars.sh #Helper for finding newest and oldest files #Sourced from stack overflow +# Usage: newold_matching_file [n/o] [filename] function newold_matching_file { # Use ${1-} instead of $1 in case 'nounset' is set @@ -48,80 +40,183 @@ } #Build latest version of a package +# Usage: build_pkg [package name] [-f force] function build_pkg { #check if PKGBUILD has updated, don't rebuild if hasn't changed - #rebuild if git is in filename (git tree may change without PKGBUILD) - #rebuild if forced if [[ ! -z $(git pull | grep "Already up to date.") && -z $(echo $1 | grep git) && -z $2 ]]; then return 2 fi - makepkg -s --noconfirm $([ $CLEAN == "Y" ] && echo "-c") $([ $SIGN == "Y" ] && echo "--sign --key $KEY") $([ "$2" == "-f" ] && echo -f) - if [ $? != 0 ]; then + + #remove old versions before build + rm *$1*.pkg.tar.* + + #make and force rebuild if is git package + # Mictosoft fonts have problems with checksums and need a seperate argument + if [[ "$1" == "ttf-ms-win10" || + "$1" == "ttf-office-2007-fonts" || + "$1" == "ttf-ms-win8" || + "$1" == "ttf-win7-fonts" ]]; then + makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) --skipchecksums + else + makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) 2>&1 + fi + if [[ $? != 0 ]]; then #Register error - ERRORS=$(printf '%s %s' "$ERRORS" "$1") + echo $1 >> $REPODIR/.errors return 1 fi - #copy package to repo directory - latest="$(newold_matching_file n '*.pkg.tar.xz')" - cp $latest $REPODIR/$latest - repo-add $([ "$SIGN" == "Y" ] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.xz $REPODIR/$latest + #Get build artifact names + source PKGBUILD + pkgs=() + for i in ${pkgname[@]}; do + #pkgs+=("$i-$pkgver-$pkgrel") + pkgs+=("$i") + done + + #Move package to repodir and add to repo db + for i in ${pkgs[@]}; do + rm $REPODIR/$i*.pkg.tar.??* + cp $i*.pkg.tar.?? $REPODIR/ + [[ "$SIGN" == "Y" ]] && cp $i*.pkg.tar.??.sig $REPODIR/ + done + + # Weird exceptions + if [[ "$1" == "zoom" ]]; then + rm zoom*_orig*.pkg.tar.xz + fi + + # Add package to waiting list to be added to repo db + while true; do + if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then + sleep 1 + else + echo 1 > $REPODIR/.waitlist.lck + echo $1 >> $REPODIR/.waitlist + echo 0 > $REPODIR/.waitlist.lck + break + fi + done + while true; do + # Wait until package is at the top of the queue and add to db + if [[ "$(head -n1 $REPODIR/.waitlist)" == "$1" ]]; then + for i in ${pkgs[@]}; do + repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.xz $REPODIR/$i*.pkg.tar.?? + done + while true; do + if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then + sleep 1 + else + # Remove self from top of queue + echo 1 > $REPODIR/.waitlist.lck + tail -n +2 $REPODIR/.waitlist > $REPODIR/.waitlist.tmp + mv $REPODIR/.waitlist.tmp $REPODIR/.waitlist + echo 0 > $REPODIR/.waitlist.lck + break + fi + done + break + else + if [[ -z "$(grep $1 $REPODIR/.waitlist)" ]]; then + # Not on waitlist for some reason, need to readd + if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then + sleep 1 + else + echo 1 > $REPODIR/.waitlist.lck + echo $1 >> $REPODIR/.waitlist + echo 0 > $REPODIR/.waitlist.lck + fi + fi + sleep 10 + fi + done #Remove old versions of packages - while [ $NUM_OLD \< $(find . -name '*.pkg.tar.xz' | wc -l) ] - do - old=$(newold_matching_file o '*.pkg.tar.xz') - rm $REPODIR/$old $old - done + #TODO: Want to be able to keep multiple versions of old packages, future work + #Currently old package versions stay in the repodir indefinately + # while [ $NUM_OLD \< $(find . -name '*.pkg.tar.xz' | wc -l) ] + # do + # old=$(newold_matching_file o '*.pkg.tar.xz') + # rm $REPODIR/$old $old + # done return 0 } #Update packages in BUILDDIR +# Usage: build_all [-f force] function build_all { #system update - if [ $UPDATE == "Y" ]; then + if [[ $UPDATE == "Y" ]]; then sudo pacman -Syu --noconfirm fi + + #Remove waitlist and errors from old builds + rm -f $REPODIR/{.waitlist,.errors} #update every package currently stored - for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -not -path '*/\.*' -type d) + for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -type d) do cd $d - build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1 + if [[ "$PARALLEL" == "Y" ]]; then + build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" || echo "/dev/tty") & + else + build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" || echo "/dev/tty") + fi done + wait return 0 } #Add a new package to be built +#There is no name checking so be sure to put in the name correctly +# Usage: add [package name] function add { - cd $BUILDDIR - git clone https://aur.archlinux.org/$1.git - cd $1 - build_pkg $1 new + for i in $@; do + cd $BUILDDIR + git clone https://aur.archlinux.org/$i.git + cd $i + build_pkg $i -f + done return 0 } +#Remove a package from the build list and repository +# Usage remove [package name] +function remove { + for i in $@; do + rm -rf $BUILDDIR/$i* + repo-remove $REPODIR/$REPONAME.db.tar.xz $i + rm $REPODIR/$i* + done +} + #Check config and create build folders +#Set variables before usage +# Usage: init function init { + if [[ $uid != 1 ]]; then + echo "This must be run as root" + fi + #check for configuration here - [ -z $REPODIR ] && echo "Enter REPODIR" && return 1 - [ -z $BUILDDIR ] && echo "Enter BUILDDIR" && return 2 - [ -z $REPONAME ] && echo "Enter REPONAME" && return 3 + [[ -z $REPODIR ]] && echo "Enter REPODIR" && return 1 + [[ -z $BUILDDIR ]] && echo "Enter BUILDDIR" && return 2 + [[ -z $REPONAME ]] && echo "Enter REPONAME" && return 3 #make build directories - [ ! -d $REPODIR ] && mkdir -p $REPODIR - [ ! -d $BUILDDIR ] && mkdir -p $BUILDDIR + [[ ! -d $REPODIR ]] && mkdir -p $REPODIR + [[ ! -d $BUILDDIR ]] && mkdir -p $BUILDDIR #packages required to build others - sudo pacman -S --noconfirm base-devel git + pacman -S --noconfirm base-devel git #add repo to pacman.conf so can install own packages - if [ -z $(grep "$REPONAME" /etc/pacman.conf) ]; then + if [[ -z $(grep "$REPONAME" /etc/pacman.conf) ]]; then printf "[$REPONAME]\nSigLevel = Optional TrustAll\nServer = file://$REPODIR\n" >> /etc/pacman.conf fi #create GPG key for package signing - if [ "$SIGN" == "Y" && "$KEY" == "" ]; then + if [[ "$SIGN" == "Y" && "$KEY" == "" ]]; then ( echo "Key-Type: RSA" echo "Key-Length: 2048" @@ -142,22 +237,38 @@ return 0 } +function send_email { + ( + echo "From: build@localhost" + echo "To: $EMAIL" + echo "Subject: Build errors" + echo "There were build errors for the build of $REPONAME at $(date), please address them soon." + echo "The errors were: $@" + ) | sendmail -t +} + case $1 in "init") init;; "add") - add $2;; + add ${@:2};; "build-all") - build_all $([ "$2" == "-f" ] && echo "-f") - if [ "$ERRORS" != "" ]; then - echo "Errors in packages $ERRORS" - if [ "$EMAIL" != "" ]; then - printf "Build for $(date)\nErrors found in $ERRORS\nPlease address these soon" | sendmail $EMAIL - fi - else - echo "All packages built successfully" - fi - ;; + build_all $([[ "$2" == "-f" ]] && echo "-f");; + "remove") + remove ${@:2};; *) - printf "Invalid usage\nUsage: $0 init|add|build_all\n";; + printf "Invalid usage\nUsage: $0 init|add|build-all\n";; esac + +# Error reporting, send email only for build-all as assuming an batch job for that +if [[ -f $REPODIR/.errors ]]; then + ERRORS=$(cat $REPODIR/.errors | tr '\n' ' ') + rm $REPODIR/.errors + echo "Errors in packages: $ERRORS" + if [[ "$EMAIL" != "" && "$1" == "build-all" ]]; then + send_email $ERRORS + fi +else + echo "All packages built successfully" +fi + -- Gitblit v1.9.3