From 7aa5106e2bd22ce1ccdecd2d442c2a78bd8470d4 Mon Sep 17 00:00:00 2001 From: Chizi123 <build@jsrv.com> Date: Tue, 20 Oct 2020 05:47:21 +0000 Subject: [PATCH] added function to check for packages in aur and in repos, also only print errors on add or build --- main.sh | 286 +++++++++++++++++++++++++++++++++++++++++--------------- 1 files changed, 208 insertions(+), 78 deletions(-) diff --git a/main.sh b/main.sh index 587648e..6a590f4 100755 --- a/main.sh +++ b/main.sh @@ -4,95 +4,150 @@ source $(dirname "$(realpath $0)")/vars.sh -ERRORS="" - #Helper for finding newest and oldest files #Sourced from stack overflow # Usage: newold_matching_file [n/o] [filename] function newold_matching_file { - # Use ${1-} instead of $1 in case 'nounset' is set - local -r glob_pattern=${2-} + # Use ${1-} instead of $1 in case 'nounset' is set + local -r glob_pattern=${2-} - # To avoid printing garbage if no files match the pattern, set - # 'nullglob' if necessary - local -i need_to_unset_nullglob=0 - if [[ ":$BASHOPTS:" != *:nullglob:* ]] ; then - shopt -s nullglob - need_to_unset_nullglob=1 - fi + # To avoid printing garbage if no files match the pattern, set + # 'nullglob' if necessary + local -i need_to_unset_nullglob=0 + if [[ ":$BASHOPTS:" != *:nullglob:* ]] ; then + shopt -s nullglob + need_to_unset_nullglob=1 + fi - file= - for f in $glob_pattern ; do + file= + for f in $glob_pattern ; do if [ $1 == "n" ]; then [[ -z $f || $f -nt $_file ]] && file=$f elif [ $1 == "o" ]; then [[ -z $f || $f -ot $_file ]] && file=$f fi - done + done - # To avoid unexpected behaviour elsewhere, unset nullglob if it was - # set by this function - (( need_to_unset_nullglob )) && shopt -u nullglob + # To avoid unexpected behaviour elsewhere, unset nullglob if it was + # set by this function + (( need_to_unset_nullglob )) && shopt -u nullglob - # Use printf instead of echo in case the file name begins with '-' - [[ -n $file ]] && printf '%s\n' "$file" + # Use printf instead of echo in case the file name begins with '-' + [[ -n $file ]] && printf '%s\n' "$file" - return 0 + return 0 } #Build latest version of a package -# Usage: build_pkg [package name] [new?] [-f force] +# Usage: build_pkg [package name] [-f force] function build_pkg { #check if PKGBUILD has updated, don't rebuild if hasn't changed - if [[ ! -z $(git pull | grep "Already up to date.") && -z $(echo $1 | grep git) && -z $2 ]]; then + if [[ ! -z $(git pull | grep "Already up to date.") && -z $(grep 'pkgver() {' PKGBUILD) && -z $2 ]]; then return 2 fi #remove old versions before build - rm *$1*.pkg.tar.xz* + rm -f *$1*.tar.* #make and force rebuild if is git package - makepkg -s --noconfirm $([ $CLEAN == "Y" ] && echo "-c") $([ $SIGN == "Y" ] && echo "--sign --key $KEY") $([ "$2" == "-f" ] && echo -f) - if [ $? != 0 ]; then + # Mictosoft fonts have problems with checksums and need a seperate argument + if [[ "$1" == "ttf-ms-win10" || + "$1" == "ttf-office-2007-fonts" || + "$1" == "ttf-ms-win8" || + "$1" == "ttf-win7-fonts" ]]; then + makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) --skipchecksums + else + makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) 2>&1 + fi + if [[ $? != 0 ]]; then #Register error echo $1 >> $REPODIR/.errors return 1 fi - rm $REPODIR/*$1*.pkg.tar.xz* - cp *$1*.pkg.tar.xz $REPODIR/ - [ "$SIGN" == "Y" ] && cp *$1*.pkg.tar.xz.sig $REPODIR - while [ 1 ]; do - if [ $(cat $REPODIR/.waitlist.lck) == 1 ]; then + #Get build artifact names from PKGBUILD and build artifacts + #Remove duplicates from the list + source PKGBUILD + pkgs=() + ipkgs=() + for i in ${pkgname[@]}; do + #pkgs+=("$i-$pkgver-$pkgrel") + ipkgs+=($(find . -mindepth 1 -maxdepth 1 -type f \( -name "$i*.pkg.tar.*" -o -name "$i*.src.tar.*" \) -not -name "*.sig" | sed 's/^\.\///')) + done + while read -r -d '' x; do pkgs+=("$x"); done < <(printf "%s\0" "${ipkgs[@]}" | sort -uz) + + #Move package to repodir and add to repo db + #Dont change the database if rebuilt the same package at same release and version + flag=0 + for i in ${pkgs[@]}; do + if [[ ! -f $REPODIR/$i ]]; then + flag=1 + fi + done + if [[ $flag == 1 ]]; then + rm -f $REPODIR/*$1*.tar.* + for i in ${pkgs[@]}; do + cp $i $REPODIR/ + [[ "$SIGN" == "Y" ]] && cp $i.sig $REPODIR/ + done + else + return; + fi + + # Weird exceptions + if [[ "$1" == "zoom" ]]; then + rm zoom*_orig*.pkg.tar.xz + for i in ${pkgs[@]}; do + if [ -z "${i##*orig*}" ]; then + pkgs=${pkgs[@]/$i} + fi + done + fi + + # Add package to waiting list to be added to repo db + while true; do + if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then sleep 1 - else + else echo 1 > $REPODIR/.waitlist.lck echo $1 >> $REPODIR/.waitlist echo 0 > $REPODIR/.waitlist.lck break fi done - while [ 1 ]; do - if [ "$(head -n1 $REPODIR/.waitlist)" == "$1" ]; then - - repo-add $([ "$SIGN" == "Y" ] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.xz $REPODIR/*$1*.pkg.tar.xz - while [ 1 ]; do - if [ $(cat $REPODIR/.waitlist.lck) == 1 ]; then + while true; do + # Wait until package is at the top of the queue and add to db + if [[ "$(head -n1 $REPODIR/.waitlist)" == "$1" ]]; then + # for i in ${pkgs[@]}; do + repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] || echo $COMPRESSION && echo zst) ${pkgs[@]} + # done + while true; do + if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then sleep 1 - else + else + # Remove self from top of queue echo 1 > $REPODIR/.waitlist.lck tail -n +2 $REPODIR/.waitlist > $REPODIR/.waitlist.tmp - cat $REPODIR/.waitlist.tmp > $REPODIR/.waitlist - rm $REPODIR/.waitlist.tmp + mv $REPODIR/.waitlist.tmp $REPODIR/.waitlist echo 0 > $REPODIR/.waitlist.lck break fi done break else + if [[ -z "$(grep $1 $REPODIR/.waitlist)" ]]; then + # Not on waitlist for some reason, need to readd + if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then + sleep 1 + else + echo 1 > $REPODIR/.waitlist.lck + echo $1 >> $REPODIR/.waitlist + echo 0 > $REPODIR/.waitlist.lck + fi + fi sleep 10 - fi + fi done #Remove old versions of packages @@ -100,8 +155,8 @@ #Currently old package versions stay in the repodir indefinately # while [ $NUM_OLD \< $(find . -name '*.pkg.tar.xz' | wc -l) ] # do - # old=$(newold_matching_file o '*.pkg.tar.xz') - # rm $REPODIR/$old $old + # old=$(newold_matching_file o '*.pkg.tar.xz') + # rm $REPODIR/$old $old # done return 0 } @@ -110,15 +165,21 @@ # Usage: build_all [-f force] function build_all { #system update - if [ $UPDATE == "Y" ]; then + if [[ $UPDATE == "Y" ]]; then sudo pacman -Syu --noconfirm fi + #Remove waitlist and errors from old builds + rm -f $REPODIR/{.waitlist,.errors} #update every package currently stored for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -type d) do cd $d - build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1 > /dev/null & + if [[ "$PARALLEL" == "Y" ]]; then + build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" || echo "/dev/tty") & + else + build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1 &> $([[ "$QUIET" == "Y" ]] && echo "/dev/null" || echo "/dev/tty") + fi done wait @@ -126,47 +187,111 @@ } #Add a new package to be built -#There is no name checking so be sure to put in the name correctly +#Adding build dependencies is # Usage: add [package name] function add { - cd $BUILDDIR - git clone https://aur.archlinux.org/$1.git - cd $1 - build_pkg $1 new -f + for i in $@; do + cd $BUILDDIR + if [[ -z $(git ls-remote https://aur.archlinux.org/$i.git) ]]; then + echo "Not a package" + exit 2 + fi + git clone https://aur.archlinux.org/$i.git + cd $i + + #check for all build dependencies + for i in ${makedepends[@]}; do + if pacman -Si $i; then + makedepends=${makedepends[@]/$delete} + fi &>/dev/null + done + for i in ${makedepends[@]}; do + add $i + done + if [[ -n "${makedepends[@]}" ]]; then + sudo pacman -Sy + fi + + #Actually build wanted package + build_pkg $i -f + done return 0 } #Remove a package from the build list and repository -# Usage remove [package name] +#Usage of -a removes all packages moved to official repos +# Usage remove [-a|package name] function remove { - rm -rf $BUILDDIR/$1* - repo-remove $REPODIR/$REPONAME.db.tar.xz $1 - rm $REPODIR/$1* + if [[ "$1" == "-a" ]]; then + rmlist="" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq core | sort) | tr '\n' ' ')" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq extra | sort) | tr '\n' ' ')" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq community | sort) | tr '\n' ' ')" + for i in $rmlist; do + rm -rf $BUILDDIR/$i + repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] || echo $COMPRESSION && echo zst) $i + rm -f $REPODIR/*$i* + done + else + for i in $@; do + rm -rf $BUILDDIR/$i + repo-remove $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] || echo $COMPRESSION && echo zst) $i + rm -f $REPODIR/*$i* + done + fi } + +#Check for packages moved to official repos or removed from the AUR +function check { + rmlist="" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq core | sort) | tr '\n' ' ')" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq extra | sort) | tr '\n' ' ')" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq community | sort) | tr '\n' ' ')" + removed=() + TMPFILE=$(mktemp) + for i in $(find $BUILDDIR -mindepth 1 -maxdepth 1 -type d); do + check_pkg $TMPFILE "$(echo $i | rev | cut -d'/' -f1 | rev)" & + done + wait + echo "Merged into official repos: $rmlist" + echo "Not in AUR: $(cat $TMPFILE | tr '\n' ' ')" + rm -f $TMPFILE +} + +function check_pkg { + if [[ -z "$(curl -sI "https://aur.archlinux.org/packages/$2" | head -n1 | grep 200)" ]]; then + echo "$2" >> $1 + fi +} + #Check config and create build folders #Set variables before usage # Usage: init function init { + if [[ $uid != 1 ]]; then + echo "This must be run as root" + fi + #check for configuration here - [ -z $REPODIR ] && echo "Enter REPODIR" && return 1 - [ -z $BUILDDIR ] && echo "Enter BUILDDIR" && return 2 - [ -z $REPONAME ] && echo "Enter REPONAME" && return 3 + [[ -z $REPODIR ]] && echo "Enter REPODIR" && return 1 + [[ -z $BUILDDIR ]] && echo "Enter BUILDDIR" && return 2 + [[ -z $REPONAME ]] && echo "Enter REPONAME" && return 3 #make build directories - [ ! -d $REPODIR ] && mkdir -p $REPODIR - [ ! -d $BUILDDIR ] && mkdir -p $BUILDDIR + [[ ! -d $REPODIR ]] && mkdir -p $REPODIR + [[ ! -d $BUILDDIR ]] && mkdir -p $BUILDDIR #packages required to build others - sudo pacman -S --noconfirm base-devel git + pacman -S --noconfirm base-devel git #add repo to pacman.conf so can install own packages - if [ -z $(grep "$REPONAME" /etc/pacman.conf) ]; then + if [[ -z $(grep "$REPONAME" /etc/pacman.conf) ]]; then printf "[$REPONAME]\nSigLevel = Optional TrustAll\nServer = file://$REPODIR\n" >> /etc/pacman.conf fi #create GPG key for package signing - if [ "$SIGN" == "Y" && "$KEY" == "" ]; then + if [[ "$SIGN" == "Y" && "$KEY" == "" ]]; then ( echo "Key-Type: RSA" echo "Key-Length: 2048" @@ -193,7 +318,7 @@ echo "To: $EMAIL" echo "Subject: Build errors" echo "There were build errors for the build of $REPONAME at $(date), please address them soon." - echo "The errors were: $1" + echo "The errors were: $@" ) | sendmail -t } @@ -201,22 +326,27 @@ "init") init;; "add") - add $2;; + add ${@:2};; "build-all") - build_all $([ "$2" == "-f" ] && echo "-f") - if [ -f $REPODIR/.errors ]; then - ERRORS=$(cat $REPODIR/.errors | tr '\n' ' ') - rm $REPODIR/.errors - echo "Errors in packages: $ERRORS" - if [ "$EMAIL" != "" ]; then - send_email $ERRORS - fi - else - echo "All packages built successfully" - fi - ;; + build_all $([[ "$2" == "-f" ]] && echo "-f");; "remove") - remove $2;; + remove ${@:2};; + "check") + check;; *) - printf "Invalid usage\nUsage: $0 init|add|build-all\n";; + printf "Invalid usage\nUsage: $0 init|add|build-all|check\n";; esac + +# Error reporting, send email only for build-all as assuming an batch job for that +if [[ $1 == "build-all" || $1 == "add" ]]; then + if [[ -f $REPODIR/.errors ]]; then + ERRORS=$(cat $REPODIR/.errors | tr '\n' ' ') + rm $REPODIR/.errors + echo "Errors in packages: $ERRORS" + if [[ "$EMAIL" != "" && "$1" == "build-all" ]]; then + send_email $ERRORS + fi + else + echo "All packages built successfully" + fi +fi -- Gitblit v1.9.3