diff --git a/.github/workflows/collect-images.yml b/.github/workflows/collect-images.yml index cbd2c30..5a0c034 100644 --- a/.github/workflows/collect-images.yml +++ b/.github/workflows/collect-images.yml @@ -41,11 +41,6 @@ jobs: APPSCODE_CLOUD_TAG: ${{ github.event.inputs.appscode_cloud_tag }} run: go run . from-orgs - - name: Collect image lists from external charts - env: - APPSCODE_CLOUD_TAG: ${{ github.event.inputs.appscode_cloud_tag }} - run: go run . externals - - name: Collect kluster-manager images embedded in CR specs env: APPSCODE_CLOUD_TAG: ${{ github.event.inputs.appscode_cloud_tag }} diff --git a/Makefile b/Makefile index 33e7028..0d5bd5e 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,6 @@ .PHONY: collect collect: go run . from-orgs - go run . externals go run . kluster-manager .PHONY: fmt diff --git a/README.md b/README.md index f810c60..6eb0277 100644 --- a/README.md +++ b/README.md @@ -11,22 +11,19 @@ git tag — `appscode_cloud_tag` — and runs, in order: 1. `go run . from-orgs` — clone `appscode-cloud/installer` at that tag, regenerate its catalog via its own `hack/scripts/update-catalog.sh` (which drives `image-packer`), copy `catalog/imagelist.yaml` to - `images/appscode-cloud.yaml` and the catalog chart lists into `charts/`. Then + `images/appscode-cloud.yaml`, `catalog/feature-chart-images.yaml` to + `images/feature-charts.yaml`, and the catalog chart lists into `charts/`. Then derive each component installer's tag from those chart lists (see [Anchor charts](#anchor-charts)) and do the same clone + catalog + copy for each one. -2. `go run . externals` — for each external OCI chart with curated - CI values under `hack/ci/`, `helm template` the chart and write the referenced - images to `images/.yaml`. The chart version is resolved from the - `charts/` lists collected in step 1, so it stays in sync with the release. -3. `go run . kluster-manager` — `helm template` the two kluster-manager charts +2. `go run . kluster-manager` — `helm template` the two kluster-manager charts whose images sit inside CR specs (so `image-packer` does not see them) and merge them into `images/kluster-manager.yaml`. See [CR-embedded images](#cr-embedded-images). -4. `bare-scripts/aggregate-lists.sh` — merge `images/*.yaml` and `charts/*.yaml` +3. `bare-scripts/aggregate-lists.sh` — merge `images/*.yaml` and `charts/*.yaml` into grouped `all-images.yaml` / `all-charts.yaml`, each source file becoming a `# ` section. -5. push the directory to an orphan branch named after the appscode-cloud tag, +4. push the directory to an orphan branch named after the appscode-cloud tag, flattened to the branch root, with `bare-scripts/notes.md` as its `README.md`. `image-packer` (`kmodules.xyz/image-packer`) is built from source by @@ -49,12 +46,10 @@ The **appscode-cloud tag names the output directory and the branch**. │ ├── kluster-manager.yaml │ ├── open-viz.yaml │ ├── opnpulse.yaml -│ ├── kube-prometheus-stack.yaml -│ ├── cert-manager.yaml -│ ├── flux2.yaml -│ ├── keda.yaml -│ ├── keda-add-ons-http.yaml -│ └── snapshot-controller.yaml +│ ├── stashed.yaml +│ ├── voyagermesh.yaml +│ ├── virtual-secrets.yaml +│ └── feature-charts.yaml ├── charts/ │ ├── ace.yaml │ ├── editor-charts.yaml @@ -80,6 +75,9 @@ Installer repos (`go run . from-orgs`) — each cloned at its own tag: | `kluster-manager/installer` | derived | `images/kluster-manager.yaml` | | `open-viz/installer` | derived | `images/open-viz.yaml` | | `opnpulse/installer` | derived | `images/opnpulse.yaml` | +| `stashed/installer` | derived | `images/stashed.yaml` | +| `voyagermesh/installer` | derived | `images/voyagermesh.yaml` | +| `virtual-secrets/installer` | derived | `images/virtual-secrets.yaml` | ### Anchor charts @@ -97,6 +95,9 @@ there, and its pinned version is that repo's tag: | `kluster-manager/installer` | `cluster-profile-manager` | `KLUSTER_MANAGER_TAG` | | `open-viz/installer` | `monitoring-operator` | `OPEN_VIZ_TAG` | | `opnpulse/installer` | `appscode-otel-stack` | `OPNPULSE_TAG` | +| `stashed/installer` | `stash` | `STASH_TAG` | +| `voyagermesh/installer` | `voyager` | `VOYAGER_TAG` | +| `virtual-secrets/installer` | `virtual-secrets-server` | `VIRTUAL_SECRETS_TAG` | A repo's other charts are pinned on their own cadence and are **not** valid tag sources — nor is the repo's latest tag. Choosing a component tag by hand collects @@ -109,16 +110,23 @@ an anchor chart is not found in `charts/*.yaml` — a rename in a newer release run fails rather than falling back to a guess; update `components` in `pkg/collect/orgs.go`. -External OCI charts from `ghcr.io/appscode-charts` (`go run . externals`): +### Feature chart images -| chart | CI values | output | -|-------|-----------|--------| -| `kube-prometheus-stack` | `hack/ci/prometheus-stack-ci-values.yaml` | `images/kube-prometheus-stack.yaml` | -| `cert-manager` | `hack/ci/cert-manager-ci-values.yaml` | `images/cert-manager.yaml` | -| `flux2` | `hack/ci/flux2-ci-values.yaml` | `images/flux2.yaml` | -| `keda` | `hack/ci/keda-ci-values.yaml` | `images/keda.yaml` | -| `keda-add-ons-http` | `hack/ci/keda-add-ons-http-ci-values.yaml` | `images/keda-add-ons-http.yaml` | -| `snapshot-controller` | `hack/ci/snapshot-controller-ci-values.yaml` | `images/snapshot-controller.yaml` | +Every chart an ACE release deploys is pinned in `charts/feature-charts.yaml`, but +most of those charts' **images** are published by the installer repo that owns +them. The rest — `reloader`, `prometheus-adapter`, `kyverno`, `longhorn`, +`opencost`, `cert-manager`, `flux2`, `keda`, `kube-prometheus-stack`, +`snapshot-controller` and friends — belong to no installer repo. + +`appscode-cloud/installer` renders those at their pinned version, using the +values the `Feature` itself carries, into `catalog/feature-chart-images.yaml`; +step 1 copies it to `images/feature-charts.yaml`. Which charts are skipped as +already-published is decided there, in `hack/scripts/update-catalog.sh`. + +`pkg/collect/externals.go` and `hack/ci/*-ci-values.yaml` did this for six of +those charts from hand-maintained values. They are retained but no longer run: +the curated values had drifted from what ACE deploys (they built the flux2 +kustomize and notification controllers, which the ACE `Feature` disables). ### CR-embedded images diff --git a/main.go b/main.go index 7d56631..088475b 100644 --- a/main.go +++ b/main.go @@ -33,8 +33,6 @@ func main() { switch os.Args[1] { case "from-orgs": err = collect.FromOrgs() - case "externals": - err = collect.Externals() case "kluster-manager": err = collect.KlusterManager() default: @@ -51,7 +49,6 @@ func usage() { Commands: from-orgs collect per-repo catalog image lists from the installer orgs - externals collect image lists for external charts with curated CI values kluster-manager collect kluster-manager images embedded in CR specs Required env var: diff --git a/pkg/collect/orgs.go b/pkg/collect/orgs.go index 03b4f43..e220055 100644 --- a/pkg/collect/orgs.go +++ b/pkg/collect/orgs.go @@ -22,7 +22,9 @@ limitations under the License. // // For appscode-cloud/installer only, it also copies the catalog chart lists // (ace.yaml, editor-charts.yaml, feature-charts.yaml, reusable-ui-charts.yaml) -// into /charts/. +// into /charts/, and catalog/feature-chart-images.yaml into +// images/feature-charts.yaml. That last one carries the images of the feature +// charts that belong to no installer repo, so nothing else here publishes them. // // APPSCODE_CLOUD_TAG is the only input; it names the output dir and the release // being collected. Every component repo's tag is derived from it: each @@ -33,7 +35,7 @@ limitations under the License. // // Each derived tag can still be overridden by exporting its env var (KUBEDB_TAG, // KUBESTASH_TAG, KUBEVAULT_TAG, KUBEOPS_TAG, KLUSTER_MANAGER_TAG, OPEN_VIZ_TAG, -// OPNPULSE_TAG), +// OPNPULSE_TAG, STASH_TAG, VOYAGER_TAG, VIRTUAL_SECRETS_TAG), // e.g. to collect an rc ahead of an ACE release; each override is logged. package collect @@ -60,11 +62,17 @@ var components = []component{ {org: "kluster-manager", tagEnv: "KLUSTER_MANAGER_TAG", anchor: "cluster-profile-manager"}, {org: "open-viz", tagEnv: "OPEN_VIZ_TAG", anchor: "monitoring-operator"}, {org: "opnpulse", tagEnv: "OPNPULSE_TAG", anchor: "appscode-otel-stack"}, + {org: "stashed", tagEnv: "STASH_TAG", anchor: "stash"}, + {org: "voyagermesh", tagEnv: "VOYAGER_TAG", anchor: "voyager"}, + {org: "virtual-secrets", tagEnv: "VIRTUAL_SECRETS_TAG", anchor: "virtual-secrets-server"}, } // Chart lists copied from appscode-cloud/installer's catalog/ into charts/. var chartFiles = []string{"ace.yaml", "editor-charts.yaml", "feature-charts.yaml", "reusable-ui-charts.yaml"} +// Image list copied from appscode-cloud/installer's catalog/ into images/. +const featureChartImages = "feature-chart-images.yaml" + var imagePackerVersionRE = regexp.MustCompile(`kmodules\.xyz/image-packer\s+(\S+)`) // A pseudo-version (vX-YYYYMMDDhhmmss-<12-hex-commit>) resolves to its commit; @@ -116,6 +124,19 @@ func FromOrgs() error { fmt.Printf("--> wrote %s\n", dst) } + // The images the feature charts deploy. They belong to no installer repo -- + // reloader, kyverno, longhorn, cert-manager, kube-prometheus-stack and the + // rest -- so nothing else in this collection publishes them. + src := filepath.Join(workDir, "appscode-cloud", "catalog", featureChartImages) + if _, err := os.Stat(src); err != nil { + return fmt.Errorf("%s not found for appscode-cloud/installer", src) + } + dst := filepath.Join(l.images, "feature-charts.yaml") + if err := copyFile(src, dst); err != nil { + return err + } + fmt.Printf("--> wrote %s\n", dst) + fmt.Printf("\n==> deriving component tags from appscode-cloud/installer @ %s\n", l.tag) tags := make(map[string]string, len(components)) for _, c := range components {