From cbc630823e875a56ef24775c064461d7b8f8c720 Mon Sep 17 00:00:00 2001
From: Joel Grunbaum <joelgrun@gmail.com>
Date: Fri, 20 Feb 2026 04:15:07 +0000
Subject: [PATCH] Try and copy all packages built
---
src/archrepobuild/builder.py | 165 +++++++++++++++++++++++++++++++++++++++++++++----------
1 files changed, 135 insertions(+), 30 deletions(-)
diff --git a/src/archrepobuild/builder.py b/src/archrepobuild/builder.py
index a99af5d..84077ca 100644
--- a/src/archrepobuild/builder.py
+++ b/src/archrepobuild/builder.py
@@ -10,7 +10,9 @@
from enum import Enum
from pathlib import Path
from concurrent.futures import ProcessPoolExecutor
-from typing import Any
+from typing import Any, TYPE_CHECKING
+if TYPE_CHECKING:
+ from archrepobuild.repo import RepoManager
from archrepobuild.aur import AURClient
from archrepobuild.config import Config, PackageOverride
@@ -76,6 +78,7 @@
sign: bool = False,
key: str = "",
clean: bool = True,
+ force: bool = False,
skip_checksums: bool = False,
extra_args: list[str] | None = None,
env_overrides: dict[str, str] | None = None,
@@ -102,6 +105,8 @@
cmd.append("-c")
if sign and key:
cmd.extend(["--sign", "--key", key])
+ if force:
+ cmd.append("-f")
if skip_checksums:
cmd.append("--skipchecksums")
if extra_args:
@@ -122,7 +127,14 @@
)
if result.returncode != 0:
- return False, result.stderr or result.stdout, []
+ error = result.stderr or result.stdout
+ if "A package has already been built" in error:
+ logger.info("Package already built, treating as success")
+ # Find built packages anyway
+ artifacts = list(package_dir.glob("*.pkg.tar.*"))
+ artifacts = [a for a in artifacts if not a.name.endswith(".sig")]
+ return True, "", artifacts
+ return False, error, []
# Find built packages
artifacts = list(package_dir.glob("*.pkg.tar.*"))
@@ -143,15 +155,18 @@
self,
config: Config,
aur_client: AURClient,
+ repo: RepoManager | None = None,
):
"""Initialize builder.
Args:
config: Application configuration
aur_client: AUR client for package info
+ repo: Optional repository manager for incremental registration
"""
self.config = config
self.aur_client = aur_client
+ self.repo = repo
self.resolver = DependencyResolver(aur_client)
self._lock_dir = config.repository.build_dir / ".locks"
self._executor: ProcessPoolExecutor | None = None
@@ -241,11 +256,16 @@
raise ValueError(f"Package not found in AUR: {package}")
pkg_dir.parent.mkdir(parents=True, exist_ok=True)
- subprocess.run(
- ["git", "clone", pkg_info.git_url, str(pkg_dir)],
- check=True,
- capture_output=True,
- )
+ try:
+ subprocess.run(
+ ["git", "clone", pkg_info.git_url, str(pkg_dir)],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ except subprocess.CalledProcessError as e:
+ logger.error(f"Failed to clone {package} from {pkg_info.git_url}: {e.stderr}")
+ raise ValueError(f"Failed to clone package from AUR: {e.stderr}")
return True
def _is_vcs_package(self, package_dir: Path) -> bool:
@@ -318,6 +338,7 @@
self.config.signing.enabled,
self.config.signing.key,
self.config.building.clean,
+ force or is_vcs,
override.skip_checksums,
override.extra_args,
override.env,
@@ -407,8 +428,74 @@
return list(results)
+ async def download_package(self, package: str) -> BuildResult:
+ """Download a package from a repository using pacman.
+
+ Args:
+ package: Package name
+
+ Returns:
+ BuildResult with status and artifact path
+ """
+ start_time = datetime.now()
+ logger.info(f"Downloading package from repositories: {package}")
+
+ dest_dir = self.config.repository.build_dir / "downloads"
+ dest_dir.mkdir(parents=True, exist_ok=True)
+
+ try:
+ # Use pacman -Sw to download to a specific directory is not directly possible
+ # But we can use pacman -Sp to get the URL and download it
+ result = subprocess.run(
+ ["pacman", "-Sp", "--noconfirm", package],
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+
+ urls = [line for line in result.stdout.strip().split("\n") if line.startswith("http") or line.startswith("ftp") or line.startswith("file")]
+ if not urls:
+ raise ValueError(f"Could not find download URL for package: {package}")
+
+ artifacts: list[Path] = []
+ import aiohttp
+ async with aiohttp.ClientSession() as session:
+ for url in urls:
+ filename = url.split("/")[-1]
+ dest_path = dest_dir / filename
+
+ logger.debug(f"Downloading {url} to {dest_path}")
+ async with session.get(url) as response:
+ response.raise_for_status()
+ with open(dest_path, "wb") as f:
+ while True:
+ chunk = await response.content.read(8192)
+ if not chunk:
+ break
+ f.write(chunk)
+ artifacts.append(dest_path)
+
+ duration = (datetime.now() - start_time).total_seconds()
+ logger.info(f"Successfully downloaded {package} in {duration:.1f}s")
+ return BuildResult(
+ package=package,
+ status=BuildStatus.SUCCESS,
+ duration=duration,
+ artifacts=artifacts,
+ )
+
+ except Exception as e:
+ duration = (datetime.now() - start_time).total_seconds()
+ logger.error(f"Failed to download {package}: {e}")
+ return BuildResult(
+ package=package,
+ status=BuildStatus.FAILED,
+ duration=duration,
+ error=str(e),
+ )
+
async def add_package(self, package: str) -> BuildResult:
- """Add and build a new package with dependencies.
+ """Add and build (or download) a new package with dependencies.
Args:
package: Package name
@@ -421,31 +508,49 @@
# Resolve dependencies
build_order = await self.resolver.resolve([package])
- if package not in build_order.packages:
- logger.info(f"Package {package} does not need to be built")
- return BuildResult(
- package=package,
- status=BuildStatus.SKIPPED,
- )
+ # Filter build order: skip managed repo, download others, build AUR
+ final_results: list[BuildResult] = []
+ for pkg_name in build_order:
+ repo = self.resolver.is_in_repos(pkg_name)
+
+ if repo == self.config.repository.name:
+ logger.info(f"Package {pkg_name} already in managed repository, skipping")
+ if pkg_name == package:
+ return BuildResult(package=package, status=BuildStatus.SKIPPED)
+ continue
- # Build dependencies first
- results: list[BuildResult] = []
- for dep in build_order:
- if dep != package:
- logger.info(f"Building dependency: {dep}")
- result = await self.build_package(dep, force=True)
- results.append(result)
+ if repo:
+ logger.info(f"Package {pkg_name} found in {repo}, downloading...")
+ result = await self.download_package(pkg_name)
+ else:
+ logger.info(f"Package {pkg_name} only in AUR, building...")
+ result = await self.build_package(pkg_name, force=True)
- if result.status == BuildStatus.FAILED:
- logger.error(f"Dependency {dep} failed, aborting")
- return BuildResult(
- package=package,
- status=BuildStatus.FAILED,
- error=f"Dependency {dep} failed to build",
- )
+ final_results.append(result)
- # Build main package
- return await self.build_package(package, force=True)
+ if result.status == BuildStatus.SUCCESS:
+ if self.repo:
+ added = self.repo.add_packages(result)
+ if added:
+ logger.info(f"Added to repository: {', '.join(added)}")
+ # Refresh resolver cache to recognize the newly added packages
+ self.resolver._refresh_pacman_cache(sync=True)
+ elif result.status == BuildStatus.FAILED:
+ logger.error(f"Failed to process {pkg_name}, aborting")
+ if pkg_name == package:
+ return result
+ return BuildResult(
+ package=package,
+ status=BuildStatus.FAILED,
+ error=f"Dependency {pkg_name} failed: {result.error}",
+ )
+
+ # Return result for the main package
+ for r in final_results:
+ if r.package == package:
+ return r
+
+ return BuildResult(package=package, status=BuildStatus.SKIPPED)
def remove_package(self, package: str) -> bool:
"""Remove a package from the build directory.
--
Gitblit v1.10.0