From 3d918611a7da82247377abf1f12300387ce33c40 Mon Sep 17 00:00:00 2001 From: Joel Grunbaum <joelgrun@gmail.com> Date: Mon, 30 Nov 2020 12:26:22 +0000 Subject: [PATCH] Removed testing comments and unused spaces. added todo --- main.sh | 191 +++++++++++++++++++++++++++++++++-------------- 1 files changed, 132 insertions(+), 59 deletions(-) diff --git a/main.sh b/main.sh index 4b66d2c..f2a4f4a 100755 --- a/main.sh +++ b/main.sh @@ -1,8 +1,12 @@ #!/bin/bash #A basic bash script to automate the building of arch packages -# Usage: main.sh init|add|build_all [-f force] +# Usage: main.sh init|check|add|remove|build_all source $(dirname "$(realpath $0)")/vars.sh + +ERRORFILE=$(mktemp) +WAITLIST=$(mktemp) +WAITLIST_LCK=$(mktemp) #Helper for finding newest and oldest files #Sourced from stack overflow @@ -43,12 +47,9 @@ # Usage: build_pkg [package name] [-f force] function build_pkg { #check if PKGBUILD has updated, don't rebuild if hasn't changed - if [[ ! -z $(git pull | grep "Already up to date.") && -z $(echo $1 | grep git) && -z $2 ]]; then + if [[ -n $(git pull | grep 'Already up to date.') && -z $(grep 'pkgver() {' PKGBUILD) && -z "$2" ]]; then return 2 fi - - #remove old versions before build - rm -f *$1*.tar.* #make and force rebuild if is git package # Mictosoft fonts have problems with checksums and need a seperate argument @@ -56,80 +57,104 @@ "$1" == "ttf-office-2007-fonts" || "$1" == "ttf-ms-win8" || "$1" == "ttf-win7-fonts" ]]; then - makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) --skipchecksums + makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) --skipchecksums 2>&1 else makepkg -s --noconfirm $([[ $CLEAN == "Y" ]] && echo "-c") $([[ $SIGN == "Y" ]] && echo "--sign --key $KEY") $([[ "$2" == "-f" ]] && echo -f) 2>&1 fi - if [[ $? != 0 ]]; then + if [[ $? != 0 && $? != 13 ]]; then #Register error - echo $1 >> $REPODIR/.errors + echo $1 >> $ERRORFILE return 1 fi - #Get build artifact names + #Remove old packages from build directory source PKGBUILD + srcdir="$(pwd)/src" + if grep -q 'pkgver() {' PKGBUILD; then + ver=$(pkgver) + else + ver=$pkgver + fi + find . -mindepth 1 -maxdepth 1 -type f \( -name "*.pkg.tar.*" -o -name "*.src.tar.*" \) -not -name "*$ver-$pkgrel*" -delete + + #Get build artifact names from PKGBUILD and build artifacts + #Remove duplicates from the list pkgs=() + ipkgs=() for i in ${pkgname[@]}; do #pkgs+=("$i-$pkgver-$pkgrel") - pkgs+=("$(find . -mindepth 1 -maxdepth 1 -type f \( -name "$1*.pkg.tar.*" -o -name "$1*.src.tar.*" \) -not -name "*.sig" | sed 's/^\.\///')") + ipkgs+=($(find . -mindepth 1 -maxdepth 1 -type f \( -name "$i*.pkg.tar.*" -o -name "$i*.src.tar.*" \) -not -name "*.sig" | sed 's/^\.\///')) done - - #Move package to repodir and add to repo db - #Dont change the database if rebuilt the same package at same release and version - for i in ${pkgs[@]}; do - if [[ -f $REPODIR/$i ]]; then - pkgs=${pkgs[@]/$i} - else - rm -f $REPODIR/*$1*.tar.* - cp $i $REPODIR/ - [[ "$SIGN" == "Y" ]] && cp $i.sig $REPODIR/ - fi - done + while read -r -d '' x; do pkgs+=("$x"); done < <(printf "%s\0" "${ipkgs[@]}" | sort -uz) # Weird exceptions if [[ "$1" == "zoom" ]]; then - rm zoom*_orig*.pkg.tar.xz + rm zoom*_orig* + for i in ${pkgs[@]}; do + if [ -z "${i##*_orig*}" ]; then + pkgs=(${pkgs[@]/$i}) + fi + done + fi + + #Move package to repodir and add to repo db + #Dont change the database if rebuilt the same package at same release and version + flag=0 + for i in ${pkgs[@]}; do + if [[ ! -f $REPODIR/$i ]]; then + flag=1 + fi + done + if [[ $flag == 1 ]]; then + rm -f $REPODIR/*$1*.tar.* + for i in ${pkgs[@]}; do + cp $i $REPODIR/ + [[ "$SIGN" == "Y" ]] && cp $i.sig $REPODIR/ + done + else + return; fi # Add package to waiting list to be added to repo db while true; do - if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then + if [[ $(cat $WAITLIST_LCK) == 1 ]]; then sleep 1 else - echo 1 > $REPODIR/.waitlist.lck - echo $1 >> $REPODIR/.waitlist - echo 0 > $REPODIR/.waitlist.lck + echo 1 > $WAITLIST_LCK + echo $1 >> $WAITLIST + echo 0 > $WAITLIST_LCK break fi done while true; do # Wait until package is at the top of the queue and add to db - if [[ "$(head -n1 $REPODIR/.waitlist)" == "$1" ]]; then - for i in ${pkgs[@]}; do - repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] || echo $COMPRESSION && echo zst) $REPODIR/$i - done + if [[ "$(head -n1 $WAITLIST)" == "$1" ]]; then + repo-add $([[ "$SIGN" == "Y" ]] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.$([ -n "$COMPRESSION" ] || echo $COMPRESSION && echo zst) ${pkgs[@]} while true; do - if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then + if [[ $(cat $WAITLIST_LCK) == 1 ]]; then sleep 1 else # Remove self from top of queue - echo 1 > $REPODIR/.waitlist.lck - tail -n +2 $REPODIR/.waitlist > $REPODIR/.waitlist.tmp - mv $REPODIR/.waitlist.tmp $REPODIR/.waitlist - echo 0 > $REPODIR/.waitlist.lck + echo 1 > $WAITLIST_LCK + TEMP=$(mktemp) + tail -n +2 $WAITLIST > $TEMP + cp $TEMP $WAITLIST + rm $TEMP + unset TEMP + echo 0 > $WAITLIST_LCK break fi done break else - if [[ -z "$(grep $1 $REPODIR/.waitlist)" ]]; then + if [[ -z "$(grep $1 $WAITLIST)" ]]; then # Not on waitlist for some reason, need to readd - if [[ $(cat $REPODIR/.waitlist.lck) == 1 ]]; then + if [[ $(cat $WAITLIST_LCK) == 1 ]]; then sleep 1 else - echo 1 > $REPODIR/.waitlist.lck - echo $1 >> $REPODIR/.waitlist - echo 0 > $REPODIR/.waitlist.lck + echo 1 > $WAITLIST_LCK + echo $1 >> $WAITLIST + echo 0 > $WAITLIST_LCK fi fi sleep 10 @@ -155,8 +180,6 @@ sudo pacman -Syu --noconfirm fi - #Remove waitlist and errors from old builds - rm -f $REPODIR/{.waitlist,.errors} #update every package currently stored for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -type d) do @@ -176,6 +199,7 @@ #Adding build dependencies is # Usage: add [package name] function add { + local i j k for i in $@; do cd $BUILDDIR if [[ -z $(git ls-remote https://aur.archlinux.org/$i.git) ]]; then @@ -184,21 +208,35 @@ fi git clone https://aur.archlinux.org/$i.git cd $i + unset depends + unset makedepends + local makedeps + source PKGBUILD - #check for all build dependencies - for i in ${makedepends[@]}; do - if pacman -Si $i; then - makedepends=${makedepends[@]/$delete} + #Check for all build dependencies + for j in ${makedepends[@]}; do + k=$(echo $j | sed 's/[>]=.*//g') + if ! pacman -Si $k; then + makedeps+=($k) fi &>/dev/null done - for i in ${makedepends[@]}; do - add $i + for j in ${depends[@]}; do + k=$(echo $j | sed 's/[>]=.*//g') + if ! pacman -Si $k; then + makedeps+=($k) + fi &>/dev/null done - if [[ -n "${makedepends[@]}" ]]; then + + #Add dependencies and update so overall build can work + for j in ${makedeps[@]}; do + add $j + done + if [[ -n "${makedeps[@]}" ]]; then sudo pacman -Sy fi #Actually build wanted package + cd $BUILDDIR/$i build_pkg $i -f done return 0 @@ -226,6 +264,30 @@ done fi } + +#Check for packages moved to official repos or removed from the AUR +function check { + rmlist="" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq core | sort) | tr '\n' ' ')" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq extra | sort) | tr '\n' ' ')" + rmlist="$rmlist $(comm -12 <(pacman -Slq $REPONAME | sort) <(pacman -Slq community | sort) | tr '\n' ' ')" + TMPFILE=$(mktemp) + for i in $(find $BUILDDIR -mindepth 1 -maxdepth 1 -type d); do + check_pkg $TMPFILE "$(echo $i | rev | cut -d'/' -f1 | rev)" & + done + wait + echo "Merged into official repos: $rmlist" + echo "Not in AUR: $(cat $TMPFILE | tr '\n' ' ')" + rm -f $TMPFILE +} + +#Check helper function +function check_pkg { + if [[ -z "$(curl -sI "https://aur.archlinux.org/packages/$2" | head -n1 | grep 200)" ]]; then + echo "$2" >> $1 + fi +} + #Check config and create build folders #Set variables before usage @@ -293,18 +355,29 @@ build_all $([[ "$2" == "-f" ]] && echo "-f");; "remove") remove ${@:2};; + "check") + check;; *) - printf "Invalid usage\nUsage: $0 init|add|build-all\n";; + echo -e "\033[0;31mInvalid usage\033[0m" + echo -e "Usage: $0 init|check|add|remove|build-all" + echo -e "\033[0;32minit\033[0m - initialise repository for use" + echo -e "\033[0;32mcheck\033[0m - check if packages have been moved into the official repositories or removed from the AUR" + echo -e "\033[0;32madd package ...\033[0m - add a package to \$BUILDDIR and repository, also used to rebuild failed packages" + echo -e "\033[0;32mremove -a | package ...\033[0m - remove package from \$BUILDDIR and repository, \"-a\" removes packages added to official repos" + echo -e "\033[0;32mbuild-all [-f]\033[0m - build all packages in \$BUILDDIR, \"-f\" force builds whole repository" esac # Error reporting, send email only for build-all as assuming an batch job for that -if [[ -f $REPODIR/.errors ]]; then - ERRORS=$(cat $REPODIR/.errors | tr '\n' ' ') - rm $REPODIR/.errors - echo "Errors in packages: $ERRORS" - if [[ "$EMAIL" != "" && "$1" == "build-all" ]]; then - send_email $ERRORS +if [[ $1 == "build-all" || $1 == "add" ]]; then + if [[ -n $(cat $ERRORFILE) ]]; then + ERRORS=$(cat $ERRORFILE | tr '\n' ' ') + echo "Errors in packages: $ERRORS" + if [[ "$EMAIL" != "" && "$1" == "build-all" ]]; then + send_email $ERRORS + fi + else + echo "All packages built successfully" fi -else - echo "All packages built successfully" fi + +rm $ERRORFILE $WAITLIST $WAITLIST_LCK -- Gitblit v1.9.3