diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c3f41fc..6112667 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,60 +13,96 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 - name: Set PY - run: - echo "PY=$(python -c 'import hashlib, - sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" - >> $GITHUB_ENV - - uses: actions/cache@v1 + run: | + echo "PY=$(python -c 'import hashlib,sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_ENV + - uses: actions/cache@v5 with: path: ~/.cache/pre-commit key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - uses: pre-commit/action@v1.0.1 build-test-push: + if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event_name == 'pull_request' runs-on: ubuntu-latest needs: pre-commit + permissions: + contents: read + packages: write + services: + registry: + image: registry:3 + ports: + - 5000:5000 strategy: fail-fast: false matrix: - # Test modern Odoo versions with latest Postgres version pg_version: + - "18" + - "17" + - "16" + - "15" + - "14" - "13" - "12" - "11" - "10" - "9.6" env: + DOCKER_PLATFORM: linux/amd64,linux/arm64 + LOCAL_REGISTRY: localhost:5000 # Indicates what's the equivalent to tecnativa/postgres-autoconf:latest image - LATEST_RELEASE: "14.0" - # Variables found by default in Docker Hub builder - DOCKER_REPO: tecnativa/postgres-autoconf - DOCKER_TAG: ${{ matrix.pg_version }}-alpine + LATEST_RELEASE: "18-alpine" + DOCKER_REPO: ${{ github.repository == 'Tecnativa/docker-postgres-autoconf' && 'tecnativa/postgres-autoconf' || github.repository }} + BASE_TAG: ${{ matrix.pg_version }}-alpine + DOCKER_TAG: ${{ github.event_name == 'pull_request' && format('{0}-test-pr{1}', matrix.pg_version, github.event.number) || format('{0}-alpine', matrix.pg_version) }} GIT_SHA1: ${{ github.sha }} + # Github does not allow evaluating a secret in an if condition, so we need to set them as environment variables + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_LOGIN: ${{ secrets.DOCKERHUB_LOGIN }} + BOT_TOKEN: ${{ secrets.BOT_TOKEN }} + BOT_LOGIN: ${{ secrets.BOT_LOGIN }} steps: - # Prepare - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + # Image repo names have to be lowercase. + - name: Lowercase image repository name + run: | + DOCKER_REPO=${DOCKER_REPO,,} + echo "DOCKER_REPO=$DOCKER_REPO" >> "$GITHUB_ENV" + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 - run: pip install -r tests/ci-requirements.txt + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + driver-opts: network=host # Build images - - run: ./hooks/build + - name: Build images + run: ./hooks/build # Test - - run: ./tests/test.py -v + - name: Test each platform + run: | + IFS=',' read -ra PLATFORMS <<< "$DOCKER_PLATFORM" + for platform in "${PLATFORMS[@]}"; do + echo "Testing platform: $platform" + TEST_PLATFORM="$platform" python -m unittest tests.test -v + done # Push - - name: push to docker hub - if: github.repository == 'Tecnativa/docker-postgres-autoconf' && github.ref == 'refs/heads/master' + - name: Push Docker Image to Docker Hub + if: env.DOCKERHUB_TOKEN && env.DOCKERHUB_LOGIN env: REGISTRY_HOST: docker.io - REGISTRY_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_LOGIN }} + REGISTRY_TOKEN: ${{ env.DOCKERHUB_TOKEN }} + REGISTRY_USERNAME: ${{ env.DOCKERHUB_LOGIN }} run: ./hooks/push - - name: push to github registry - if: github.repository == 'Tecnativa/docker-postgres-autoconf' && github.ref == 'refs/heads/master' + - name: Push Docker Image to GitHub Registry + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository || env.BOT_TOKEN && env.BOT_LOGIN env: REGISTRY_HOST: ghcr.io - REGISTRY_TOKEN: ${{ secrets.BOT_TOKEN }} - REGISTRY_USERNAME: ${{ secrets.BOT_LOGIN }} + REGISTRY_TOKEN: ${{ secrets.BOT_LOGIN && secrets.BOT_TOKEN || secrets.GITHUB_TOKEN }} + REGISTRY_USERNAME: ${{ secrets.BOT_TOKEN && secrets.BOT_LOGIN || github.repository_owner }} run: ./hooks/push diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d58ab6..469c467 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,11 +2,11 @@ default_language_version: python: python3 repos: - repo: https://github.com/psf/black - rev: 19.3b0 + rev: 22.3.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v4.3.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -21,10 +21,10 @@ repos: - id: mixed-line-ending args: ["--fix=lf"] - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.3 + rev: v2.2.0 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.21 + rev: v5.10.1 hooks: - id: isort diff --git a/Dockerfile b/Dockerfile index c62cbda..af49463 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ ARG BASE_TAG FROM docker.io/postgres:${BASE_TAG} ENTRYPOINT [ "/autoconf-entrypoint" ] CMD [] +ARG PGVECTOR_VERSION=0.8.1 ENV CERTS="{}" \ CONF_EXTRA="" \ LAN_AUTH_METHOD=md5 \ @@ -15,18 +16,25 @@ ENV CERTS="{}" \ WAN_DATABASES='["all"]' \ WAN_HBA_TPL="{connection} {db} {user} {cidr} {meth}" \ WAN_TLS=1 \ - WAN_USERS='["all"]' -RUN apk add --no-cache python3 \ - && mkdir -p /etc/postgres \ - && chmod a=rwx /etc/postgres -RUN apk add --no-cache -t .build \ - build-base \ - linux-headers \ - py3-pip \ - python3-dev \ - && pip3 install --no-cache-dir \ - netifaces \ - && apk del .build + WAN_USERS='["all"]' \ + HBA_EXTRA_RULES="" +RUN apk add --no-cache python3 py3-netifaces \ + && if [ "${PG_MAJOR:-0}" -ge 13 ]; then \ + PG_CLANG="$(pg_config --configure | tr "'" "\n" | sed -n 's/^CLANG=clang-//p')"; \ + PG_LLVM="$(pg_config --configure | tr "'" "\n" | sed -n 's#^LLVM_CONFIG=/usr/lib/llvm\([0-9][0-9]*\)/bin/llvm-config#\1#p')"; \ + test -n "${PG_CLANG}" && test -n "${PG_LLVM}" && test "${PG_CLANG}" = "${PG_LLVM}"; \ + apk add --no-cache --virtual .pgvector-build build-base linux-headers ca-certificates "clang${PG_CLANG}" "llvm${PG_LLVM}"; \ + wget -qO- "https://github.com/pgvector/pgvector/archive/refs/tags/v${PGVECTOR_VERSION}.tar.gz" \ + | tar -xz -C /tmp; \ + cd "/tmp/pgvector-${PGVECTOR_VERSION}" \ + && make $(if [ "$(uname -m)" = "x86_64" ]; then echo 'CFLAGS=-march=x86-64-v2'; fi) PG_CONFIG=/usr/local/bin/pg_config \ + && make install PG_CONFIG=/usr/local/bin/pg_config; \ + cd / && rm -rf "/tmp/pgvector-${PGVECTOR_VERSION}"; \ + apk del .pgvector-build; \ + fi \ + && mkdir -p /etc/postgres \ + && chmod a=rwx /etc/postgres + COPY autoconf-entrypoint / # Metadata diff --git a/README.md b/README.md index 8aac63c..a90e964 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,8 @@ # PostgreSQL Auto-Conf -[![Build Status](https://travis-ci.org/Tecnativa/docker-postgres-autoconf.svg?branch=master)](https://travis-ci.org/Tecnativa/docker-postgres-autoconf) +[![Build Status](../../actions/workflows/ci.yaml/badge.svg?branch=master)](../../actions/workflows/ci.yaml) [![Docker Pulls](https://img.shields.io/docker/pulls/tecnativa/postgres-autoconf.svg)](https://hub.docker.com/r/tecnativa/postgres-autoconf) -[![Layers](https://images.microbadger.com/badges/image/tecnativa/postgres-autoconf.svg)](https://microbadger.com/images/tecnativa/postgres-autoconf) -[![Commit](https://images.microbadger.com/badges/commit/tecnativa/postgres-autoconf.svg)](https://microbadger.com/images/tecnativa/postgres-autoconf) -[![License](https://img.shields.io/github/license/Tecnativa/docker-postgres-autoconf.svg)](https://github.com/Tecnativa/docker-postgres-autoconf/blob/master/LICENSE) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](../../blob/master/LICENSE) ## What @@ -105,4 +103,17 @@ Wether to enable or not TLS in WAN connections. Users allowed to connect from WAN. +#### `HBA_EXTRA_RULES` + +JSON array of additional pg_hba.conf rules to append. Each array element should be a string representing a valid pg_hba.conf line. + +Example HBA_EXTRA_RULES format in an .env file: + +HBA_EXTRA_RULES=["host all all 192.168.1.0/24 md5", "hostssl mydb myuser 10.0.0.0/8 scram-sha-256"] + +This adds the following lines to pg_hba.conf: + +host all all 192.168.1.0/24 md5 +hostssl mydb myuser 10.0.0.0/8 scram-sha-256 + [`Dockerfile`]: https://github.com/Tecnativa/docker-postgres-autoconf/blob/master/Dockerfile diff --git a/autoconf-entrypoint b/autoconf-entrypoint index b8e9cad..3b8490e 100755 --- a/autoconf-entrypoint +++ b/autoconf-entrypoint @@ -30,6 +30,10 @@ WAN_DATABASES = json.loads(os.environ["WAN_DATABASES"]) WAN_HBA_TPL = os.environ["WAN_HBA_TPL"] WAN_TLS = json.loads(os.environ["WAN_TLS"]) WAN_USERS = json.loads(os.environ["WAN_USERS"]) +PGSSLCERT = os.environ.get("PGSSLCERT") +PGSSLKEY = os.environ.get("PGSSLKEY") +PGSSLROOTCERT = os.environ.get("PGSSLROOTCERT") +HBA_EXTRA_RULES = os.environ.get("HBA_EXTRA_RULES", "") # Configuration file templates CONF_FOLDER = "/etc/postgres" @@ -47,6 +51,7 @@ local all all trust local replication all trust # LAN/WAN autogenerated configurations +{extra_hba} {extra_conf} """ WAN_CIDRS = ("0.0.0.0/0", "::0/0") @@ -54,29 +59,47 @@ WAN_CIDRS = ("0.0.0.0/0", "::0/0") # Configuration helpers hba_conf = [] ssl_conf = [] +extra_hba = [] -def permissions_fix(filename): +def permissions_fix(filename, client=False): """Make :param:`filename` be owned by root user and postgres group.""" shutil.chown(filename, "root", "postgres") - os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP) + if client: + os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR) + else: + os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP) # Configure TLS -for key, filename in SUPPORTED_CERTS.items(): - full_path = os.path.join(CONF_FOLDER, filename) +for key, filen in SUPPORTED_CERTS.items(): + full_path = os.path.join(CONF_FOLDER, filen) # Write PEM file if it came from env variable - if not os.path.exists(full_path) and CERTS.get(filename): + if not os.path.exists(full_path) and CERTS.get(filen): with open(full_path, "w") as cert_file: - cert_file.write(CERTS[filename]) + cert_file.write(CERTS[filen]) if os.path.exists(full_path): # Enable file in postgres configuration ssl_conf.append("{} = '{}'".format(key, full_path)) permissions_fix(full_path) +for filen in (PGSSLCERT, PGSSLKEY, PGSSLROOTCERT): + if filen and os.path.exists(filen): + permissions_fix(filen, client=True) if ssl_conf: ssl_conf.append("ssl = on") +# Parse extra rules for pg_hba.conf +extra_hba_rules = [] +if HBA_EXTRA_RULES: + try: + extra_hba_rules = json.loads(HBA_EXTRA_RULES) + if not isinstance(extra_hba_rules, list): + raise ValueError("HBA_EXTRA_RULES must be a JSON array") + except json.JSONDecodeError: + print("Invalid JSON in HBA_EXTRA_RULES", file=sys.stderr) + sys.exit(1) + # Generate LAN auth configuration for interface in netifaces.interfaces(): for type_, addresses in netifaces.ifaddresses(interface).items(): @@ -114,6 +137,13 @@ if WAN_CONNECTION != "hostssl" or ssl_conf: ) ) +# Append extra rules to extra_hba +for rule in extra_hba_rules: + if not isinstance(rule, str): + print("Each rule in HBA_EXTRA_RULES must be a string", file=sys.stderr) + sys.exit(1) + extra_hba.append(rule) + # Write postgres configuration files with open(CONF_FILE, "w") as conf_file: conf_file.write( @@ -123,7 +153,9 @@ with open(CONF_FILE, "w") as conf_file: ) permissions_fix(CONF_FILE) with open(HBA_FILE, "w") as conf_file: - conf_file.write(HBA_TPL.format(extra_conf="\n".join(hba_conf))) + conf_file.write( + HBA_TPL.format(extra_hba="\n".join(extra_hba), extra_conf="\n".join(hba_conf)) + ) permissions_fix(HBA_FILE) # Continue normal execution diff --git a/hooks/build b/hooks/build index b74f09f..d205ad7 100755 --- a/hooks/build +++ b/hooks/build @@ -3,21 +3,42 @@ from plumbum import FG, local from plumbum.cmd import date, docker # Check environment variables are present +DOCKER_PLATFORM = local.env.get("DOCKER_PLATFORM", "linux/amd64") DOCKER_TAG = local.env["DOCKER_TAG"] +BASE_TAG = local.env.get("BASE_TAG", DOCKER_TAG) +REPO = local.env["DOCKER_REPO"] COMMIT = local.env.get("GIT_SHA1") DATE = date("--rfc-3339", "ns") +LOCAL_REGISTRY = local.env.get("LOCAL_REGISTRY") +IMAGE = ( + "%s/%s:%s" % (LOCAL_REGISTRY, REPO, DOCKER_TAG) + if LOCAL_REGISTRY + else "%s:%s" % (REPO, DOCKER_TAG) +) +PLATFORMS = [p.strip() for p in DOCKER_PLATFORM.split(",") if p.strip()] -# Build image -docker[ - "image", +build = docker[ + "buildx", "build", + "--platform", + DOCKER_PLATFORM, "--build-arg", "VCS_REF={}".format(COMMIT), "--build-arg", "BUILD_DATE={}".format(DATE), "--build-arg", - "BASE_TAG={}".format(DOCKER_TAG), + "BASE_TAG={}".format(BASE_TAG), "--tag", - "tecnativa/postgres-autoconf:{}".format(DOCKER_TAG), + IMAGE, ".", -] & FG +] +if LOCAL_REGISTRY: + build = build["--push"] +elif len(PLATFORMS) == 1: + build = build["--load"] +else: + raise SystemExit( + "Multi-platform builds require LOCAL_REGISTRY; " + "set DOCKER_PLATFORM to one value for local --load builds." + ) +(build & FG) diff --git a/hooks/push b/hooks/push index 2c6a70b..fe7d927 100755 --- a/hooks/push +++ b/hooks/push @@ -7,10 +7,20 @@ REPO = local.env["DOCKER_REPO"] SUFFIX = local.env.get("DOCKER_REPO_SUFFIX", "") VERSION = local.env["DOCKER_TAG"] -# Log all locally available images; will help to pin images -docker["image", "ls", "--digests", REPO] & FG -# Login in Docker Hub +def image_ref(registry, tag): + return "%s/%s%s:%s" % (registry, REPO, SUFFIX, tag) + + +source = image_ref(local.env["LOCAL_REGISTRY"], VERSION) +dest_tags = [image_ref(REGISTRY, VERSION)] +if VERSION == local.env.get("LATEST_RELEASE"): + latest = "alpine" if VERSION.endswith("-alpine") else "latest" + dest_tags.append(image_ref(REGISTRY, latest)) + +docker["buildx", "imagetools", "inspect", source] & FG + +# Login in Docker Hub or ghcr docker( "login", "--username", @@ -20,13 +30,7 @@ docker( REGISTRY, ) -# Push built images -local_image = "%s:%s" % (REPO, VERSION) -public_image = "%s/%s%s:%s" % (REGISTRY, REPO, SUFFIX, VERSION) -docker["image", "tag", local_image, public_image] & FG -docker["image", "push", public_image] & FG -if VERSION == local.env.get("LATEST_RELEASE"): - latest_version = "alpine" if VERSION.endswith("-alpine") else "latest" - public_image = "%s/%s%s:%s" % (REGISTRY, REPO, SUFFIX, latest_version) - docker["image", "tag", local_image, public_image] & FG - docker["image", "push", public_image] & FG +promote = docker["buildx", "imagetools", "create"] +for dest in dest_tags: + promote = promote["-t", dest] +(promote[source] & FG) diff --git a/tests/test.py b/tests/test.py index 5b1f760..8111807 100755 --- a/tests/test.py +++ b/tests/test.py @@ -22,13 +22,20 @@ class PostgresAutoconfCase(unittest.TestCase): """Test behavior for this docker image""" + @classmethod + def _platform_args(cls): + platform = os.environ.get("TEST_PLATFORM") + if platform: + return ("--platform", platform) + return () + @classmethod def setUpClass(cls): - with local.cwd(local.cwd / ".."): - print("Building image") - local["./hooks/build"] & FG - cls.image = "tecnativa/postgres-autoconf:{}".format(local.env["DOCKER_TAG"]) - cls.cert_files = {"client.ca.cert.pem", "server.cert.pem", "server.key.pem"} + registry = local.env.get("LOCAL_REGISTRY") + repo = local.env.get("DOCKER_REPO", "tecnativa/postgres-autoconf") + tag = local.env["DOCKER_TAG"] + cls.image = f"{registry}/{repo}:{tag}" if registry else f"{repo}:{tag}" + cls.cert_files = ("client.ca.cert.pem", "server.cert.pem", "server.key.pem") return super().setUpClass() def setUp(self): @@ -56,7 +63,7 @@ def _check_local_connection(self): # The 1st test could fail while postgres boots for attempt in range(10): try: - time.sleep(5) + time.sleep(15) # Test local connections via unix socket work self.assertEqual( "1\n", @@ -75,13 +82,13 @@ def _check_local_connection(self): "test_user", ), ) - except AssertionError: + except (AssertionError, ProcessExecutionError): if attempt < 9: print("Failure number {}. Retrying...".format(attempt)) else: raise else: - continue + return def _check_password_auth(self, host=None): """Test connection with password auth work fine.""" @@ -93,6 +100,7 @@ def _check_password_auth(self, host=None): docker( "container", "run", + *self._platform_args(), "--network", "lan", "-e", @@ -126,6 +134,7 @@ def _check_cert_auth(self): docker( "container", "run", + *self._platform_args(), "--network", "wan", "-e", @@ -163,6 +172,7 @@ def test_server_certs_var(self): self.postgres_container = docker( "container", "run", + *self._platform_args(), "-d", "--network", "lan", @@ -198,6 +208,7 @@ def test_server_certs_mount(self): self.postgres_container = docker( "container", "run", + *self._platform_args(), "-d", "--network", "lan", @@ -221,6 +232,7 @@ def test_no_certs_lan(self): self.postgres_container = docker( "container", "run", + *self._platform_args(), "-d", "--network", "lan", @@ -244,6 +256,7 @@ def test_no_certs_wan(self): self.postgres_container = docker( "container", "run", + *self._platform_args(), "-d", "--network", "lan", @@ -271,6 +284,7 @@ def test_certs_falsy_lan(self): self.postgres_container = docker( "container", "run", + *self._platform_args(), "-d", "--network", "lan", @@ -299,6 +313,116 @@ def test_certs_falsy_lan(self): with self.assertRaises(ProcessExecutionError): self._check_password_auth("example.localdomain") + def test_hba_extra_rules_added(self): + """Test that HBA_EXTRA_RULES lines are added to pg_hba.conf.""" + if "9.6" in self.image: + self.skipTest("HBA_EXTRA_RULES not supported in PostgreSQL 9.6") + # Define custom HBA rules + hba_extra_rules = [ + "host test_db custom_user 0.0.0.0/0 trust", + "hostssl all all 192.168.0.0/16 md5", + ] + + # Start the Postgres container with HBA_EXTRA_RULES + self.postgres_container = docker( + "run", + *self._platform_args(), + "-d", + "--name", + "postgres_test_hba_extra_rules", + "--network", + "lan", + "-e", + "POSTGRES_DB=test_db", + "-e", + "POSTGRES_USER=test_user", + "-e", + "POSTGRES_PASSWORD=test_password", + "-e", + "HBA_EXTRA_RULES=" + json.dumps(hba_extra_rules), + CONF_EXTRA, + self.image, + ).strip() + + # Give the container some time to initialize + time.sleep(10) + + # Read the pg_hba.conf file content from the container + hba_conf = docker( + "exec", self.postgres_container, "cat", "/etc/postgres/pg_hba.conf" + ).strip() + + # Check that each rule in hba_extra_rules is present in the file + for rule in hba_extra_rules: + self.assertIn(rule, hba_conf) + + def test_pgvector_extension(self): + """Test that pgvector is installed and works.""" + if float(local.env["DOCKER_TAG"].split("-")[0]) < 13: + self.skipTest("pgvector not built for PostgreSQL < 13") + self.postgres_container = docker( + "container", + "run", + "-d", + "--network", + "lan", + "-e", + "POSTGRES_DB=test_db", + "-e", + "POSTGRES_PASSWORD=test_password", + "-e", + "POSTGRES_USER=test_user", + CONF_EXTRA, + self.image, + ).strip() + self._check_local_connection() + self.assertEqual( + "vector\n", + docker( + "container", + "exec", + self.postgres_container, + "psql", + "--command", + "SELECT name FROM pg_available_extensions WHERE name = 'vector';", + "--dbname", + "test_db", + "--no-align", + "--tuples-only", + "--username", + "test_user", + ), + ) + docker( + "container", + "exec", + self.postgres_container, + "psql", + "--command", + "CREATE EXTENSION vector;", + "--dbname", + "test_db", + "--username", + "test_user", + ) + self.assertEqual( + "1\n", + docker( + "container", + "exec", + self.postgres_container, + "psql", + "--command", + "SELECT ('[1,2,3]'::vector <-> '[1,2,4]'::vector)::int;", + "--dbname", + "test_db", + "--no-align", + "--tuples-only", + "--username", + "test_user", + ), + ) + if __name__ == "__main__": unittest.main()