From 8ea4ef181e92bfe5f410e365b8ed614f939c8da1 Mon Sep 17 00:00:00 2001
From: Joel Grunbaum <joelgrun@gmail.com>
Date: Sun, 29 Mar 2020 06:40:55 +0000
Subject: [PATCH] Large change to overall structure, moved to one file
---
README.org | 30 ++++---
/dev/null | 7 -
main.sh | 163 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 180 insertions(+), 20 deletions(-)
diff --git a/README.org b/README.org
index 1d4450d..d51eb89 100755
--- a/README.org
+++ b/README.org
@@ -1,28 +1,32 @@
#+TITLE: Auto-build package repository for Arch Linux
* What is this?
-This is a set of files to auto-build a personal repository for Arch.
-It uses a chroot environment to build the packages, so that a build environment can be made on a server running a non-arch operating system.
+This is a package autobuild solution for Arch Linux.
+It supports the automated building of packages from the AUR.
+
+Currently it supports package signing and building clean from source.
+It can help to set itself up and generate GPG keys if need be.
+
+The chroot files are here from an older version of this project and are here as I think they're interesting.
* Set-up
-Edit ~create_chroot.sh~ variables at the top to your details.
-Run ~sudo create_chroot.sh~, and create the git repository in ~/build/repo~ when prompted, with a remote repository.
+Edit the variables at the top of ~main.sh~ to suit your repository details.
+Have a look at the init function near the bottom of the file to decide if you want to run it.
+If you decide to generate a GPG key, edit the name, comment and email so that it fits you.
You may also want to set ~MAKEFLAGS="-j$(nproc)"~ within the chroot to enable parallel compilation.
For xz compression, add the ~--threads=0~ option to COMPRESSXZ such that it reads ~COMPRESSXZ=(xz -c -z - --threads=0)~.
ccache may be useful to decrease the times of git packages, due to the little changes.
* Usage
-All scripts must be run as root.
-To enter the chroot, use the ~enter_chroot.sh~ script.
-To add a package from the AUR, use the ~add_package.sh~ script, with the git URL as the argument.
-To build the packages use the ~build_packages.sh~ script, which will enter the chroot, and build the packages from the PKGBUILDs stored in ~/build/repo~.
-
-The packages are built and put into the directory ~/build/repo/x86_64~, then pushed with git.
-Only the latest packages are stored in ~/build/repo/x86_64~ to avoid multiple older versions building up.
-
-To enable automatic building, create a cronjob for root, something of the sort of ~0 12 * * * /path/to/dir/build_packages.sh~ to build ever day at noon.
+To add a package ~main.sh add [package]~ should be run. Currently I only support adding one package at a time.
+To update all packages, use ~main.sh build-all~. A ~-f~ flag can be used to force a rebuild of all the packages.
+To allow automatic building, use a cronjob or write a systemd unit, there are many guides out there, although I may add examples if I feel like it.
* To Dos
+- Multiple packages from a single PKGBUILD
+ Some PKGBUILDs can create many packages at once, currently I am unable to handle this.
- Error catching for builds which fail.
+ Have emailing, but nothing more advanced. Would be nice to have some basic error handling in the script
- Create a universal variables file
+ Not sure how useful this would be, but could be nice
diff --git a/add_dependency.sh b/add_dependency.sh
deleted file mode 100755
index ad37d51..0000000
--- a/add_dependency.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-CHDIR=$(dirname "$(realpath $0)")/chroot
-RUSER=joelgrun
-RLOC=35.225.177.191
-RPATH=/var/www/joelg.cf/html/
-
-mount --bind $CHDIR $CHDIR
-$CHDIR/bin/arch-chroot $CHDIR su joel -c "cd /build/repo/dependencies;
- git clone https://aur.archlinux.org/$1.git
- cd $1;
- makepkg -si --noconfirm;
- ln $1-*.pkg.tar.xz ../../x86_64/;
- cd ../../x86_64;
- #rm Chizi123.db Chizi123.files;
- repo-add ../Chizi123.db.tar.xz $1-*.pkg.tar.xz;
- #ln ../Chizi123.db.tar.xz Chizi123.db;
- #ln ../Chizi123.files.tar.xz Chizi123.files;
- cd ../;
- rsync -aL --delete x86_64 $RUSER@$RLOC:$RPATH
- git add x86_64/;
- git commit -m \"added $1\";
- git push"
-umount $CHDIR
diff --git a/add_package.sh b/add_package.sh
deleted file mode 100755
index 16e46ca..0000000
--- a/add_package.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-CHDIR=$(dirname "$(realpath $0)")/chroot
-RUSER=joelgrun
-RLOC=35.225.177.191
-RPATH=/var/www/joelg.cf/html/
-
-mount --bind $CHDIR $CHDIR
-$CHDIR/bin/arch-chroot $CHDIR su joel -c "cd /build/repo;
- git clone https://aur.archlinux.org/$1.git
- cd $1;
- makepkg -s --noconfirm;
- ln $1-*.pkg.tar.xz ../x86_64/;
- cd ../x86_64;
- #rm Chizi123.db Chizi123.files;
- repo-add ../Chizi123.db.tar.xz $1-*.pkg.tar.xz;
- #ln ../Chizi123.db.tar.xz Chizi123.db;
- #ln ../Chizi123.files.tar.xz Chizi123.files;
- cd ../;
- rsync -aL --delete x86_64 $RUSER@$RLOC:$RPATH;"
- #git add x86_64/;
- #git commit -m \"added $1\";
- #git push"
-umount $CHDIR
diff --git a/autobuild.service b/autobuild.service
deleted file mode 100644
index e62d848..0000000
--- a/autobuild.service
+++ /dev/null
@@ -1,9 +0,0 @@
-[Unit]
-Description=Autobuild repo service
-
-[Service]
-ExecStart=/data/repo/build_packages.sh
-Type=oneshot
-
-[Install]
-WantedBy=multi-user.target
diff --git a/autobuild.timer b/autobuild.timer
deleted file mode 100644
index 67a4471..0000000
--- a/autobuild.timer
+++ /dev/null
@@ -1,9 +0,0 @@
-[Unit]
-Description=Build repo on timer
-
-[Timer]
-OnCalendar=daily
-Persistent=true
-
-[Install]
-WantedBy=timers.target
diff --git a/build.sh b/build.sh
deleted file mode 100755
index 02e42f0..0000000
--- a/build.sh
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/bin/bash
-
-#Number of old packages to store, should be at least 1
-NUM_BACK=5
-#remote details
-RUSER=joelgrun
-RLOC=35.225.177.191
-RPATH=/var/www/joelg.cf/html/
-
-function newest_matching_file
-{
- # Use ${1-} instead of $1 in case 'nounset' is set
- local -r glob_pattern=${1-}
-
- if (( $# != 1 )) ; then
- echo 'usage: newest_matching_file GLOB_PATTERN' >&2
- return 1
- fi
-
- # To avoid printing garbage if no files match the pattern, set
- # 'nullglob' if necessary
- local -i need_to_unset_nullglob=0
- if [[ ":$BASHOPTS:" != *:nullglob:* ]] ; then
- shopt -s nullglob
- need_to_unset_nullglob=1
- fi
-
- newest_file=
- for file in $glob_pattern ; do
- [[ -z $newest_file || $file -nt $newest_file ]] \
- && newest_file=$file
- done
-
- # To avoid unexpected behaviour elsewhere, unset nullglob if it was
- # set by this function
- (( need_to_unset_nullglob )) && shopt -u nullglob
-
- # Use printf instead of echo in case the file name begins with '-'
- [[ -n $newest_file ]] && printf '%s\n' "$newest_file"
-
- return 0
-}
-
-function oldest_matching_file
-{
- # Use ${1-} instead of $1 in case 'nounset' is set
- local -r glob_pattern=${1-}
-
- if (( $# != 1 )) ; then
- echo 'usage: oldest_matching_file GLOB_PATTERN' >&2
- return 1
- fi
-
- # To avoid printing garbage if no files match the pattern, set
- # 'nullglob' if necessary
- local -i need_to_unset_nullglob=0
- if [[ ":$BASHOPTS:" != *:nullglob:* ]] ; then
- shopt -s nullglob
- need_to_unset_nullglob=1
- fi
-
- oldest_file=
- for file in $glob_pattern ; do
- [[ -z $oldest_file || $file -ot $oldest_file ]] \
- && oldest_file=$file
- done
-
- # To avoid unexpected behaviour elsewhere, unset nullglob if it was
- # set by this function
- (( need_to_unset_nullglob )) && shopt -u nullglob
-
- # Use printf instead of echo in case the file name begins with '-'
- [[ -n $oldest_file ]] && printf '%s\n' "$oldest_file"
-
- return 0
-}
-
-#update system
-sudo pacman -Syu --noconfirm
-
-#go to build directory
-cd $(dirname "$(realpath $0)")
-
-#Remove old packages
-#git rm -r x86_64/*
-#rm -r x86_64
-#mkdir x86_64
-
-#dependencies
-cd dependencies
-for d in `find . -maxdepth 1 -not -path '*/\.*' -type d`
-do
- #Only do package directories
- if [ "$d" = "./x86_64" ] || [ "$d" = "." ]; then
- continue
- fi
- cd $d
- #update package to latest from AUR
- git pull -f
- makepkg -si --noconfirm
- if [ $? = 0 ]; then
- latest=$(newest_matching_file '*.pkg.tar.xz')
- while [ $NUM_BACK \< $(find . -name "*.pkg.tar.xz" | wc -l) ]
- do
- oldest=$(oldest_matching_file '*.pkg.tar.xz')
- rm $oldest
- done
- cd ..
- rm ../x86_64/"$d"*".pkg.tar.xz"
- ln $d/$latest ../x86_64/$latest
- else
- cd ..
- fi
- repo-add ../Chizi123.db.tar.xz ../x86_64/$latest
-done
-cd ..
-
-#main packages
-for d in `find . -maxdepth 1 -not -path '*/\.*' -type d`
-do
- #Only do package directories
- if [ "$d" = "./x86_64" ] || [ "$d" = "." ] || [ "$d" = "./dependencies" ]; then
- continue
- fi
- cd $d
- #update package to latest from AUR
- git pull -f
- makepkg -s --noconfirm
- if [ $? = 0 ]; then
- latest=$(newest_matching_file '*.pkg.tar.xz')
- while [ $NUM_BACK \< $(find . -name "*.pkg.tar.xz" | wc -l) ]
- do
- oldest=$(oldest_matching_file '*.pkg.tar.xz')
- rm $oldest
- done
- cd ..
- rm x86_64/"$d"*".pkg.tar.xz"
- ln $d/$latest x86_64/$latest
- else
- cd ..
- fi
- repo-add ./Chizi123.db.tar.xz x86_64/$latest
-done
-
-#ln Chizi123.db.tar.xz x86_64/Chizi123.db
-#ln Chizi123.files.tar.xz x86_64/Chizi123.files
-#git add x86_64
-#git commit -m "'$(date +%d/%m/%y-%H:%M)'"
-#git push
-rsync -aL --delete x86_64 $RUSER@$RLOC:$RPATH
diff --git a/build_packages.sh b/build_packages.sh
deleted file mode 100755
index bb61675..0000000
--- a/build_packages.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-CHDIR=$(dirname "$(realpath $0)")/chroot
-mount --bind $CHDIR $CHDIR
-$CHDIR/bin/arch-chroot $CHDIR su joel -c "/build/repo/build.sh"
-umount $CHDIR
diff --git a/enter_chroot.sh b/enter_chroot.sh
deleted file mode 100755
index a4f6270..0000000
--- a/enter_chroot.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-CHDIR=$(dirname "$(realpath $0)")/chroot
-mount --bind $CHDIR $CHDIR
-$CHDIR/bin/arch-chroot $CHDIR su joel
-umount $CHDIR
diff --git a/fix_AUR.sh b/fix_AUR.sh
deleted file mode 100755
index 3b57c2a..0000000
--- a/fix_AUR.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-CHDIR=$(dirname "$(realpath $0)")/chroot
-mount --bind $CHDIR $CHDIR
-$CHDIR/bin/arch-chroot $CHDIR su joel -c "find /build/repo/dependencies -maxdepth 1 -not -path '*\/.*' -type d -exec git -C {} fetch \; -exec git -C {} reset --hard origin/master \;
- find /build/repo -maxdepth 1 -not -path '*\/.*' -type d -exec git -C {} fetch \; -exec git -C {} reset --hard origin/master \;"
-umount $CHDIR
diff --git a/main.sh b/main.sh
new file mode 100644
index 0000000..53a4e0b
--- /dev/null
+++ b/main.sh
@@ -0,0 +1,163 @@
+#!/bin/bash
+
+REPODIR=/repo/x86_64
+BUILDDIR=/repo/build
+REPONAME=
+UPDATE=N
+CLEAN=N
+SIGN=N
+KEY=""
+NUM_OLD=5
+export PACKAGER="John Doe <jd@change.me>"
+EMAIL=""
+
+ERRORS=""
+
+#Helper for finding newest and oldest files
+#Sourced from stack overflow
+function newold_matching_file
+{
+ # Use ${1-} instead of $1 in case 'nounset' is set
+ local -r glob_pattern=${2-}
+
+ # To avoid printing garbage if no files match the pattern, set
+ # 'nullglob' if necessary
+ local -i need_to_unset_nullglob=0
+ if [[ ":$BASHOPTS:" != *:nullglob:* ]] ; then
+ shopt -s nullglob
+ need_to_unset_nullglob=1
+ fi
+
+ file=
+ for f in $glob_pattern ; do
+ if [ $1 == "n" ]; then
+ [[ -z $f || $f -nt $_file ]] && file=$f
+ elif [ $1 == "o" ]; then
+ [[ -z $f || $f -ot $_file ]] && file=$f
+ fi
+ done
+
+ # To avoid unexpected behaviour elsewhere, unset nullglob if it was
+ # set by this function
+ (( need_to_unset_nullglob )) && shopt -u nullglob
+
+ # Use printf instead of echo in case the file name begins with '-'
+ [[ -n $file ]] && printf '%s\n' "$file"
+
+ return 0
+}
+
+#Build latest version of a package
+function build_pkg {
+ #check if PKGBUILD has updated, don't rebuild if hasn't changed
+ #rebuild if git is in filename (git tree may change without PKGBUILD)
+ #rebuild if forced
+ if [[ ! -z $(git pull | grep "Already up to date.") && -z $(echo $1 | grep git) && -z $2 ]]; then
+ return 2
+ fi
+ makepkg -s --noconfirm $([ $CLEAN == "Y" ] && echo "-c") $([ $SIGN == "Y" ] && echo "--sign --key $KEY") $([ "$2" == "-f" ] && echo -f)
+ if [ $? != 0 ]; then
+ #Register error
+ ERRORS=$(printf '%s %s' "$ERRORS" "$1")
+ return 1
+ fi
+
+ #copy package to repo directory
+ latest="$(newold_matching_file n '*.pkg.tar.xz')"
+ cp $latest $REPODIR/$latest
+ repo-add $([ "$SIGN" == "Y" ] && echo "--sign --key $KEY") $REPODIR/$REPONAME.db.tar.xz $REPODIR/$latest
+
+ #Remove old versions of packages
+ while [ $NUM_OLD \< $(find . -name '*.pkg.tar.xz' | wc -l) ]
+ do
+ old=$(newold_matching_file o '*.pkg.tar.xz')
+ rm $REPODIR/$old $old
+ done
+ return 0
+}
+
+#Update packages in BUILDDIR
+function build_all {
+ #system update
+ if [ $UPDATE == "Y" ]; then
+ sudo pacman -Syu --noconfirm
+ fi
+ #update every package currently stored
+ for d in $(find $BUILDDIR -maxdepth 1 -mindepth 1 -not -path '*/\.*' -type d)
+ do
+ cd $d
+ build_pkg $(echo $d | rev | cut -d'/' -f1 | rev) $1
+ done
+
+ return 0
+}
+
+#Add a new package to be built
+function add {
+ cd $BUILDDIR
+ git clone https://aur.archlinux.org/$1.git
+ cd $1
+ build_pkg $1 new
+ return 0
+}
+
+#Check config and create build folders
+function init {
+ #check for configuration here
+ [ -z $REPODIR ] && echo "Enter REPODIR" && return 1
+ [ -z $BUILDDIR ] && echo "Enter BUILDDIR" && return 2
+ [ -z $REPONAME ] && echo "Enter REPONAME" && return 3
+
+ #make build directories
+ [ ! -d $REPODIR ] && mkdir -p $REPODIR
+ [ ! -d $BUILDDIR ] && mkdir -p $BUILDDIR
+
+ #packages required to build others
+ sudo pacman -S --noconfirm base-devel git
+
+ #add repo to pacman.conf so can install own packages
+ if [ -z $(grep "$REPONAME" /etc/pacman.conf) ]; then
+ printf "[$REPONAME]\nSigLevel = Optional TrustAll\nServer = file://$REPODIR\n" >> /etc/pacman.conf
+ fi
+
+ #create GPG key for package signing
+ if [ "$SIGN" == "Y" && "$KEY" == "" ]; then
+ (
+ echo "Key-Type: RSA"
+ echo "Key-Length: 2048"
+ echo "Subkey-Type: RSA"
+ echo "Subkey-Length: 2048"
+ echo "Passphrase: \"\""
+ echo "Expire-Date: 0"
+ echo "Name-Real: John Doe"
+ echo "Name-Comment: Arch buildbot"
+ echo "Name-Email: $(whoami)@localhost"
+ echo "%commit"
+ ) | gpg --batch --generate-key
+ gpg --export --output $REPONAME.key --armor "John Doe"
+ gpg --export-secret-keys --output $REPONAME.secret.key --armor "John Doe"
+ echo "Please change the key information in this file"
+ fi
+
+ return 0
+}
+
+case $1 in
+ "init")
+ init;;
+ "add")
+ add $2;;
+ "build-all")
+ build_all $([ "$2" == "-f" ] && echo "-f")
+ if [ "$ERRORS" != "" ]; then
+ echo "Errors in packages $ERRORS"
+ if [ "$EMAIL" != "" ]; then
+ printf "Build for $(date)\nErrors found in $ERRORS\nPlease address these soon" | sendmail $EMAIL
+ fi
+ else
+ echo "All packages built successfully"
+ fi
+ ;;
+ *)
+ printf "Invalid usage\nUsage: $0 init|add|build_all\n";;
+esac
--
Gitblit v1.9.3