390 lines
11 KiB
Python
390 lines
11 KiB
Python
import base64
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import select
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from urllib.parse import quote, urlparse, urlunparse
|
|
|
|
from fastapi import HTTPException
|
|
|
|
from app.auth import BuildClaims
|
|
from app.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SAFE_DOCKERFILE = re.compile(r"^[A-Za-z0-9._/-]+$")
|
|
READ_CHUNK_SIZE = 1024
|
|
MAX_DOCKERFILE_BYTES = 1024 * 1024
|
|
|
|
|
|
class BuildError(Exception):
|
|
def __init__(self, message: str, *, status_code: int = 500) -> None:
|
|
super().__init__(message)
|
|
self.message = message
|
|
self.status_code = status_code
|
|
|
|
|
|
def sha256_file(path: str) -> str:
|
|
size = os.path.getsize(path)
|
|
if size > MAX_DOCKERFILE_BYTES:
|
|
raise BuildError(
|
|
f"Dockerfile exceeds 1 MB limit ({size} bytes)"
|
|
)
|
|
digest = hashlib.sha256()
|
|
with open(path, "rb") as handle:
|
|
digest.update(handle.read())
|
|
return digest.hexdigest()
|
|
|
|
|
|
@dataclass
|
|
class PreparedBuild:
|
|
job_dir: Path
|
|
repo_dir: Path
|
|
dockerfile: Path
|
|
dockerfile_name: str
|
|
|
|
|
|
def _run(
|
|
args: list[str],
|
|
*,
|
|
timeout: int,
|
|
cwd: Path | None = None,
|
|
input_text: str | None = None,
|
|
env: dict[str, str] | None = None,
|
|
log_label: str | None = None,
|
|
) -> str:
|
|
display = log_label or " ".join(args)
|
|
logger.info("Running: %s", display)
|
|
try:
|
|
completed = subprocess.run(
|
|
args,
|
|
cwd=cwd,
|
|
input=input_text,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
env=env,
|
|
)
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise BuildError(f"Command timed out: {display}", status_code=504) from exc
|
|
|
|
output = "\n".join(
|
|
part for part in (completed.stdout or "", completed.stderr or "") if part
|
|
)
|
|
if completed.returncode != 0:
|
|
detail = output.strip() or f"exit {completed.returncode}"
|
|
raise BuildError(f"Command failed ({display}): {detail}", status_code=500)
|
|
return output
|
|
|
|
|
|
def _stream_run(
|
|
args: list[str],
|
|
*,
|
|
timeout: int,
|
|
cwd: Path | None = None,
|
|
input_text: str | None = None,
|
|
env: dict[str, str] | None = None,
|
|
) -> Iterator[str]:
|
|
display = " ".join(args)
|
|
logger.info("Streaming: %s", display)
|
|
|
|
try:
|
|
proc = subprocess.Popen(
|
|
args,
|
|
cwd=cwd,
|
|
stdin=subprocess.PIPE if input_text is not None else None,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
bufsize=0,
|
|
env=env,
|
|
)
|
|
except OSError as exc:
|
|
raise BuildError(f"Failed to start command: {display}", status_code=500) from exc
|
|
|
|
assert proc.stdout is not None
|
|
if input_text is not None:
|
|
assert proc.stdin is not None
|
|
proc.stdin.write(input_text)
|
|
proc.stdin.close()
|
|
|
|
deadline = time.monotonic() + timeout
|
|
try:
|
|
while True:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
proc.kill()
|
|
proc.wait(timeout=5)
|
|
raise BuildError(f"Command timed out: {display}", status_code=504)
|
|
|
|
ready, _, _ = select.select([proc.stdout], [], [], min(1.0, remaining))
|
|
if not ready:
|
|
if proc.poll() is not None:
|
|
break
|
|
continue
|
|
|
|
chunk = proc.stdout.read(READ_CHUNK_SIZE)
|
|
if chunk:
|
|
yield chunk
|
|
continue
|
|
|
|
if proc.poll() is not None:
|
|
break
|
|
|
|
while True:
|
|
chunk = proc.stdout.read(READ_CHUNK_SIZE)
|
|
if not chunk:
|
|
break
|
|
yield chunk
|
|
|
|
returncode = proc.wait(timeout=5)
|
|
except BuildError:
|
|
raise
|
|
except Exception:
|
|
proc.kill()
|
|
proc.wait(timeout=5)
|
|
raise
|
|
|
|
if returncode != 0:
|
|
raise BuildError(f"Command failed ({display}): exit {returncode}", status_code=500)
|
|
|
|
|
|
def _resolve_dockerfile(repo_dir: Path, dockerfile: str) -> Path:
|
|
if not SAFE_DOCKERFILE.match(dockerfile) or ".." in dockerfile.split("/"):
|
|
raise BuildError("Dockerfile path is not allowed", status_code=400)
|
|
|
|
path = (repo_dir / dockerfile).resolve()
|
|
try:
|
|
path.relative_to(repo_dir.resolve())
|
|
except ValueError as exc:
|
|
raise BuildError("Dockerfile path escapes repository", status_code=400) from exc
|
|
|
|
if not path.is_file():
|
|
raise BuildError(f"Dockerfile not found: {dockerfile}", status_code=400)
|
|
return path
|
|
|
|
|
|
def _write_registry_auth(
|
|
config_dir: Path,
|
|
*,
|
|
remote: str,
|
|
username: str,
|
|
password: str,
|
|
) -> None:
|
|
config_dir.mkdir(parents=True, exist_ok=True)
|
|
token = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
payload = {"auths": {remote: {"auth": token}}}
|
|
(config_dir / "config.json").write_text(json.dumps(payload), encoding="utf-8")
|
|
|
|
|
|
def _buildctl_env(docker_config_dir: Path) -> dict[str, str]:
|
|
env = os.environ.copy()
|
|
env["BUILDKIT_HOST"] = settings.buildkit_host
|
|
env["DOCKER_CONFIG"] = str(docker_config_dir)
|
|
return env
|
|
|
|
|
|
def _clone_url_with_auth(
|
|
clone_url: str,
|
|
*,
|
|
username: str,
|
|
password: str,
|
|
) -> str:
|
|
parsed = urlparse(clone_url)
|
|
if not parsed.hostname:
|
|
raise BuildError("Clone URL is missing a hostname", status_code=500)
|
|
userinfo = f"{quote(username, safe='')}:{quote(password, safe='')}"
|
|
host = parsed.hostname
|
|
if parsed.port:
|
|
host = f"{host}:{parsed.port}"
|
|
return urlunparse(parsed._replace(netloc=f"{userinfo}@{host}"))
|
|
|
|
|
|
def _clone_at_commit(
|
|
clone_url: str,
|
|
repo_dir: Path,
|
|
commit: str,
|
|
*,
|
|
repo_username: str | None = None,
|
|
repo_password: str | None = None,
|
|
) -> None:
|
|
"""Shallow-fetch a single commit and check it out."""
|
|
origin_url = clone_url
|
|
if repo_username is not None and repo_password is not None:
|
|
origin_url = _clone_url_with_auth(
|
|
clone_url,
|
|
username=repo_username,
|
|
password=repo_password,
|
|
)
|
|
|
|
_run(["git", "init", str(repo_dir)], timeout=30)
|
|
_run(
|
|
["git", "remote", "add", "origin", origin_url],
|
|
timeout=30,
|
|
cwd=repo_dir,
|
|
log_label=f"git remote add origin {clone_url}",
|
|
)
|
|
try:
|
|
_run(
|
|
["git", "fetch", "--depth", "1", "origin", commit],
|
|
timeout=settings.git_clone_timeout_seconds,
|
|
cwd=repo_dir,
|
|
)
|
|
except BuildError as exc:
|
|
raise BuildError(
|
|
f"Failed to fetch commit {commit}: {exc.message}",
|
|
status_code=400,
|
|
) from exc
|
|
_run(["git", "checkout", "--quiet", "FETCH_HEAD"], timeout=60, cwd=repo_dir)
|
|
|
|
|
|
def _verify_dockerfile(claims: BuildClaims, repo_dir: Path) -> Path:
|
|
dockerfile = _resolve_dockerfile(repo_dir, claims.dockerfile)
|
|
actual_sha256 = sha256_file(str(dockerfile))
|
|
if actual_sha256 != claims.dockerfile_sha256:
|
|
raise BuildError(
|
|
"Dockerfile hash does not match dockerfile_sha256 claim",
|
|
status_code=400,
|
|
)
|
|
return dockerfile
|
|
|
|
|
|
def _manifest_mismatch_message(
|
|
expected: set[str],
|
|
actual: set[str],
|
|
) -> str:
|
|
missing = sorted(expected - actual)
|
|
extra = sorted(actual - expected)
|
|
parts: list[str] = []
|
|
if missing:
|
|
parts.append(f"missing uploads: {', '.join(missing)}")
|
|
if extra:
|
|
parts.append(f"unexpected uploads: {', '.join(extra)}")
|
|
return "; ".join(parts) or "upload manifest mismatch"
|
|
|
|
|
|
def _overlay_uploads(
|
|
repo_dir: Path,
|
|
uploads: dict[str, bytes],
|
|
claims: BuildClaims,
|
|
) -> None:
|
|
if claims.files is None:
|
|
raise BuildError("JWT missing files claim for uploads", status_code=400)
|
|
|
|
expected = set(claims.files)
|
|
actual = set(uploads)
|
|
if actual != expected:
|
|
raise BuildError(
|
|
_manifest_mismatch_message(expected, actual),
|
|
status_code=400,
|
|
)
|
|
for path, content in uploads.items():
|
|
target = repo_dir / path
|
|
target.write_bytes(content)
|
|
|
|
|
|
def prepare_build(
|
|
claims: BuildClaims,
|
|
*,
|
|
commit: str | None = None,
|
|
uploads: dict[str, bytes] | None = None,
|
|
repo_username: str | None = None,
|
|
repo_password: str | None = None,
|
|
) -> PreparedBuild:
|
|
if commit is None and not uploads:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="request must include commit and/or authorized file uploads",
|
|
)
|
|
|
|
work_root = Path(settings.work_dir)
|
|
work_root.mkdir(parents=True, exist_ok=True)
|
|
job_dir = work_root / str(uuid.uuid4())
|
|
repo_dir = job_dir / "src"
|
|
clone_url = f"https://{settings.git_host}/{claims.repo}.git"
|
|
|
|
try:
|
|
job_dir.mkdir(parents=True, exist_ok=False)
|
|
repo_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
if commit is not None:
|
|
_clone_at_commit(
|
|
clone_url,
|
|
repo_dir,
|
|
commit,
|
|
repo_username=repo_username,
|
|
repo_password=repo_password,
|
|
)
|
|
if uploads:
|
|
_overlay_uploads(repo_dir, uploads, claims)
|
|
|
|
dockerfile = _verify_dockerfile(claims, repo_dir)
|
|
except BuildError as exc:
|
|
shutil.rmtree(job_dir, ignore_errors=True)
|
|
raise HTTPException(status_code=exc.status_code, detail=exc.message) from exc
|
|
except Exception:
|
|
shutil.rmtree(job_dir, ignore_errors=True)
|
|
raise
|
|
|
|
return PreparedBuild(
|
|
job_dir=job_dir,
|
|
repo_dir=repo_dir,
|
|
dockerfile=dockerfile,
|
|
dockerfile_name=claims.dockerfile,
|
|
)
|
|
|
|
|
|
def cleanup_build(prepared: PreparedBuild) -> None:
|
|
shutil.rmtree(prepared.job_dir, ignore_errors=True)
|
|
|
|
|
|
def stream_build_and_push(
|
|
prepared: PreparedBuild,
|
|
*,
|
|
tags: list[str],
|
|
remote: str,
|
|
username: str,
|
|
password: str,
|
|
) -> Iterator[str]:
|
|
docker_config_dir = prepared.job_dir / "docker-config"
|
|
_write_registry_auth(
|
|
docker_config_dir,
|
|
remote=remote,
|
|
username=username,
|
|
password=password,
|
|
)
|
|
env = _buildctl_env(docker_config_dir)
|
|
|
|
# Quote name= so commas between tags are not treated as output CSV separators.
|
|
output = f'type=image,"name={",".join(tags)}",push=true'
|
|
args = [
|
|
"buildctl",
|
|
"--addr",
|
|
settings.buildkit_host,
|
|
"build",
|
|
"--frontend=dockerfile.v0",
|
|
"--progress=plain",
|
|
"--local",
|
|
f"context={prepared.repo_dir}",
|
|
"--local",
|
|
f"dockerfile={prepared.repo_dir}",
|
|
"--opt",
|
|
f"filename={prepared.dockerfile_name}",
|
|
"--output",
|
|
output,
|
|
]
|
|
yield from _stream_run(
|
|
args,
|
|
timeout=settings.build_timeout_seconds,
|
|
env=env,
|
|
)
|