archive: add gitea generic package registry backend

doc/manual/configuration.rst

@@ -2602,6 +2602,14 @@ file        Use a local directory as binary artifact repository. The directory
             directory. The optional ``fileMode`` and ``directoryMode`` keys
             take the desired access modes as numeric value to override the
             default umask derived modes.
+gitea       Uses a `Gitea generic package registry`_ as binary artifact
+            repository. The base server URL is given in ``url``, the registry
+            owner (user or organization) in ``owner`` and the generic package
+            name in ``package``. Credentials are given explicitly in the
+            ``token`` key (a personal access token) or as ``user``/``password``
+            for HTTP basic authentication. The optional ``sslVerify`` boolean
+            key controls whether to verify the SSL certificate and ``retries``
+            sets the number of retries on transient errors.
 http        Uses a HTTP server as binary artifact repository. The server has to
             support the HEAD, PUT and GET methods. The base URL is given in the
             ``url`` key. The optional ``sslVerify`` boolean key controls
@@ -2680,6 +2688,37 @@ the anonymous access to the container can be used like this::
 The ``flags: [download]`` makes sure that Bob does not try to upload artifacts
 in case other backends are configured too.
 
+The ``gitea`` backend stores the artifacts in a Gitea generic package registry.
+In contrast to the ``http`` backend the credentials are *not* part of the URL
+but are configured explicitly. They therefore become part of the archive
+specification and are available even for uploads that run inside the build
+sandbox::
+
+    archive:
+        -
+            backend: gitea
+            url: "https://gitea.example.com"
+            owner: "bob-artifacts"
+            package: "myproject"
+            token: "<personal access token>"
+            flags: [download, upload]
+
+Instead of a ``token`` a ``user``/``password`` pair may be given for HTTP basic
+authentication. The artifacts are stored below
+``{url}/api/packages/{owner}/generic/{package}/`` with one package version per
+artifact.
+
+.. warning::
+   The token/password will be part of the Jenkins job configuration. Anybody
+   who can read the jobs ``config.xml`` will be able to retrieve it. Keep the
+   credentials in the user configuration (e.g. a git ignored ``user.yaml``).
+
+.. note::
+   The ``gitea`` backend does not implement the managed operations of
+   :ref:`manpage-archive` (scan/clean) yet.
+
+.. _Gitea generic package registry: https://docs.gitea.com/usage/packages/generic
+
 .. _configuration-config-archive-prepend-append:
 
 archive{Prepend,Append}

pym/bob/archive.py

@@ -19,18 +19,22 @@
 concurrent uploads the artifact must appear atomically for unrelated readers.
 """
 
+from . import BOB_VERSION
 from .audit import Audit
 from .errors import BuildError, BobError
 from .tty import stepAction, stepMessage, \
     SKIPPED, EXECUTED, WARNING, INFO, TRACE, ERROR, IMPORTANT
-from .utils import asHexStr, removePath, isWindows, getBashPath, tarfileOpen, binStat
+from .utils import asHexStr, removePath, isWindows, getBashPath, tarfileOpen, \
+    binStat, sslNoVerifyContext
 from .webdav import WebDav, WebdavError, WebdavNotFoundError, WebdavAlreadyExistsError
 from tempfile import mkstemp, NamedTemporaryFile, TemporaryFile, gettempdir
 import asyncio
+import base64
 import concurrent.futures
 import concurrent.futures.process
 import errno
 import gzip
+import http.client
 import io
 import os
 import os.path
@@ -39,7 +43,9 @@
 import struct
 import subprocess
 import tarfile
+import urllib.error
 import urllib.parse
+import urllib.request
 
 ARCHIVE_GENERATION = '-1'
 ARTIFACT_SUFFIX = ".tgz"
@@ -1211,6 +1217,236 @@ def __upload(self):
             raise ArtifactError(str(e))
 
 
+class GiteaArchive(BaseArchive):
+    """Bob artifact backend for the Gitea 'generic' package registry.
+
+    Artifacts are stored using the generic package API:
+
+        {url}/api/packages/{owner}/generic/{package}/{version}/{filename}
+
+    with ``version`` and ``filename`` both derived from the build-id. All
+    artifact types belonging to the same key (``.tgz``, ``.buildid``,
+    ``.fprnt``) share a single package version.
+
+    Credentials are taken from the archive spec (``token`` or
+    ``user``/``password``) - never from the URL or ``~/.netrc``. This is
+    intentional: the credential becomes part of the (picklable) archive object
+    and is therefore available to the upload executor even inside the build
+    sandbox, where a ``~/.netrc`` would not be accessible.
+    """
+
+    def __init__(self, spec):
+        super().__init__(spec)
+        self.__url = urllib.parse.urlparse(spec["url"])
+        self.__owner = spec["owner"]
+        self.__package = spec["package"]
+        self.__sslVerify = spec.get("sslVerify", True)
+        self.__retries = spec.get("retries", 1)
+
+        token = spec.get("token")
+        if token:
+            self.__authHeader = "token " + token
+        elif spec.get("user") is not None:
+            userPass = spec["user"] + ":" + spec.get("password", "")
+            self.__authHeader = "Basic " + base64.b64encode(
+                userPass.encode("utf-8")).decode("ascii")
+        else:
+            self.__authHeader = None
+
+    def getArchiveName(self):
+        name = super().getArchiveName()
+        if name:
+            return name
+        basePath = "/".join([self.__url.path.rstrip("/"), "api", "packages",
+            self.__owner, "generic", self.__package])
+        return urllib.parse.urlunparse((self.__url.scheme, self.__url.netloc,
+            basePath, '', '', ''))
+
+    def _canManage(self):
+        # Managed operations (scan/clean) would require the Gitea package list
+        # API. Not implemented yet.
+        return False
+
+    def __makePath(self, buildId, suffix):
+        packageResultId = buildIdToName(buildId)
+        version = packageResultId
+        filename = packageResultId + suffix
+        return "/".join([self.__url.path.rstrip("/"), "api", "packages",
+            self.__owner, "generic", self.__package, version, filename])
+
+    def _remoteName(self, buildId, suffix):
+        return urllib.parse.urlunparse((self.__url.scheme, self.__url.netloc,
+            self.__makePath(buildId, suffix), '', '', ''))
+
+    def __context(self):
+        return None if self.__sslVerify else sslNoVerifyContext()
+
+    def __headers(self, extra=None):
+        headers = {'User-Agent': 'BobBuildTool/{}'.format(BOB_VERSION)}
+        if self.__authHeader is not None:
+            headers['Authorization'] = self.__authHeader
+        if extra:
+            headers.update(extra)
+        return headers
+
+    def __authError(self, code):
+        return ArtifactError("authentication failed (HTTP {}). Check that the "
+            "'token' (or 'user'/'password') of the gitea archive is valid and "
+            "has write access (personal access token scope 'write:package')."
+            .format(code))
+
+    def __retry(self, request):
+        retries = self.__retries
+        while True:
+            try:
+                return request()
+            except (http.client.HTTPException, urllib.error.URLError) as e:
+                # urllib.error.HTTPError (a URLError subclass) is handled and
+                # translated by the request functions themselves and never
+                # reaches this point. Everything left here is a transient
+                # transport error that may be retried.
+                if retries == 0:
+                    raise ArtifactError(str(e))
+                retries -= 1
+
+    def _openDownloadFile(self, buildId, suffix):
+        url = self._remoteName(buildId, suffix)
+        return self.__retry(lambda: self.__openDownloadFile(url))
+
+    def __openDownloadFile(self, url):
+        req = urllib.request.Request(url, headers=self.__headers(), method="GET")
+        try:
+            rsp = urllib.request.urlopen(req, context=self.__context())
+        except urllib.error.HTTPError as e:
+            e.fp.read()
+            if e.code == 404:
+                raise ArtifactNotFoundError()
+            if e.code >= 500:
+                raise   # transient server error -> retryable
+            raise ArtifactError("GET {} {}".format(e.code, e.reason))
+        return GiteaDownloader(rsp)
+
+    def __exists(self, url):
+        # Cheap existence/authentication probe before uploading a possibly large
+        # artifact. Only the first byte is requested. This surfaces an
+        # authentication problem as a clean 401 instead of an opaque connection
+        # reset that would otherwise occur when the server drops the connection
+        # while we are still sending the body.
+        req = urllib.request.Request(url,
+            headers=self.__headers({'Range': 'bytes=0-0'}), method="GET")
+        try:
+            with urllib.request.urlopen(req, context=self.__context()):
+                return True
+        except urllib.error.HTTPError as e:
+            e.fp.read()
+            if e.code == 404:
+                return False
+            if e.code == 416:
+                # "Range Not Satisfiable" -> the (empty) file does exist.
+                return True
+            if e.code in (401, 403):
+                raise self.__authError(e.code)
+            if e.code >= 500:
+                raise   # transient server error -> retryable
+            raise ArtifactError("GET {} {}".format(e.code, e.reason))
+
+    def _openUploadFile(self, buildId, suffix, overwrite):
+        url = self._remoteName(buildId, suffix)
+        if overwrite:
+            # Gitea's generic registry refuses to overwrite an existing file.
+            # Metadata files (build-id, fingerprint) must be replaced, so delete
+            # a possibly existing file first (a missing one is ignored).
+            self.__retry(lambda: self.__delete(url))
+        elif self.__retry(lambda: self.__exists(url)):
+            raise ArtifactExistsError()
+        return GiteaUploader(self, url, overwrite)
+
+    def _putUploadFile(self, url, tmp, overwrite):
+        try:
+            return self.__retry(lambda: self.__putUploadFile(url, tmp, overwrite))
+        except ArtifactExistsError:
+            raise
+        except ArtifactError as e:
+            msg = str(e)
+            if any(s in msg for s in ("EOF", "violation of protocol",
+                                      "reset", "Broken pipe", "closed")):
+                raise ArtifactError(msg + " -- the server closed the connection "
+                    "during upload. This usually means a reverse-proxy body size "
+                    "limit (e.g. nginx 'client_max_body_size') or an "
+                    "authentication problem.")
+            raise
+
+    def __putUploadFile(self, url, tmp, overwrite):
+        tmp.seek(0, os.SEEK_END)
+        length = str(tmp.tell())
+        tmp.seek(0)
+        headers = self.__headers({
+            'Content-Length': length,
+            'Content-Type': 'application/octet-stream',
+        })
+        req = urllib.request.Request(url, data=tmp, headers=headers, method="PUT")
+        try:
+            with urllib.request.urlopen(req, context=self.__context()) as resp:
+                if resp.status not in (200, 201, 202, 204):
+                    raise ArtifactError("PUT {} {}".format(resp.status, resp.reason))
+        except urllib.error.HTTPError as e:
+            e.fp.read()
+            if e.code == 409:
+                # File already exists in the registry.
+                raise ArtifactExistsError()
+            if e.code in (401, 403):
+                raise self.__authError(e.code)
+            if e.code >= 500:
+                raise   # transient server error -> retryable
+            raise ArtifactError("PUT {} {}".format(e.code, e.reason))
+
+    def __delete(self, url):
+        req = urllib.request.Request(url, headers=self.__headers(), method="DELETE")
+        try:
+            with urllib.request.urlopen(req, context=self.__context()):
+                pass
+        except urllib.error.HTTPError as e:
+            e.fp.read()
+            if e.code == 404:
+                return
+            if e.code in (401, 403):
+                raise self.__authError(e.code)
+            if e.code >= 500:
+                raise   # transient server error -> retryable
+            raise ArtifactError("DELETE {} {}".format(e.code, e.reason))
+
+    def getArchiveUri(self):
+        return self.__url.netloc + self.__url.path
+
+
+class GiteaDownloader:
+    def __init__(self, response):
+        self.response = response
+    def __enter__(self):
+        return (None, self.response)
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.response.close()
+        return False
+
+
+class GiteaUploader:
+    def __init__(self, archiver, url, overwrite):
+        self.archiver = archiver
+        self.tmp = TemporaryFile()
+        self.url = url
+        self.overwrite = overwrite
+    def __enter__(self):
+        return (None, self.tmp)
+    def __exit__(self, exc_type, exc_value, traceback):
+        try:
+            # do actual upload on regular handle close
+            if exc_type is None:
+                self.archiver._putUploadFile(self.url, self.tmp, self.overwrite)
+        finally:
+            self.tmp.close()
+        return False
+
+
 class MultiArchive:
     def __init__(self, archives):
         self.__archives = archives
@@ -1283,6 +1519,8 @@ def getSingleArchiver(recipes, archiveSpec):
         return CustomArchive(archiveSpec, recipes.envWhiteList())
     elif archiveBackend == "azure":
         return AzureArchive(archiveSpec)
+    elif archiveBackend == "gitea":
+        return GiteaArchive(archiveSpec)
     elif archiveBackend == "none":
         return DummyArchive()
     elif archiveBackend == "__jenkins":

pym/bob/input.py

@@ -3171,7 +3171,7 @@ def validate(self, data):
 
 class ArchiveValidator:
     def __init__(self):
-        self.__validTypes = schema.Schema({'backend': schema.Or('none', 'file', 'http', 'shell', 'azure')},
+        self.__validTypes = schema.Schema({'backend': schema.Or('none', 'file', 'http', 'shell', 'azure', 'gitea')},
             ignore_extra_keys=True)
         baseArchive = {
             'backend' : str,
@@ -3199,12 +3199,24 @@ def __init__(self):
             schema.Optional('key') : str,
             schema.Optional('sasToken"') : str,
         })
+        giteaArchive = baseArchive.copy()
+        giteaArchive.update({
+            'url' : HttpUrlValidator(),
+            'owner' : str,
+            'package' : str,
+            schema.Optional('token') : str,
+            schema.Optional('user') : str,
+            schema.Optional('password') : str,
+            schema.Optional('sslVerify') : bool,
+            schema.Optional('retries') : PositiveValidator(),
+        })
         self.__backends = {
             'none' : schema.Schema(baseArchive),
             'file' : schema.Schema(fileArchive),
             'http' : schema.Schema(httpArchive),
             'shell' : schema.Schema(shellArchive),
             'azure' : schema.Schema(azureArchive),
+            'gitea' : schema.Schema(giteaArchive),
         }
 
     def validate(self, data):