From 2eef2c102e37f09a9f460b93dd40be93c2bcaf2d Mon Sep 17 00:00:00 2001 From: Ismail Ibrahim Quwarah Date: Tue, 4 Aug 2026 01:19:05 +0200 Subject: [PATCH 1/3] Fix rclone mover crash on PVCs with symlinks Exclude symlinks from the getfacl pipeline in active.sh. On UBI9's acl-2.4.0, getfacl -P returns ELOOP on symlinks, killing the script under pipefail before rclone sync runs. Symlinks have no meaningful ACLs so skipping them loses no data. Adds e2e test with dangling and relative symlinks on the source PVC. Signed-off-by: Ismail Ibrahim Quwarah --- mover-rclone/active.sh | 2 +- .../compare_pvc_data/templates/job.yml.j2 | 12 ++ test-e2e/test_rclone_priv_with_symlinks.yml | 189 ++++++++++++++++++ 3 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 test-e2e/test_rclone_priv_with_symlinks.yml diff --git a/mover-rclone/active.sh b/mover-rclone/active.sh index e46a69896..012a4ce5c 100755 --- a/mover-rclone/active.sh +++ b/mover-rclone/active.sh @@ -77,7 +77,7 @@ fi START_TIME=$SECONDS case "${DIRECTION}" in source) - find "${MOUNT_PATH}" -path "${MOUNT_PATH}/lost+found" -prune -o -print | getfacl -P - > /tmp/permissions.facl + find "${MOUNT_PATH}" -path "${MOUNT_PATH}/lost+found" -prune -o -not -type l -print | getfacl -P - > /tmp/permissions.facl rclone sync "${RCLONE_FLAGS_SYNC[@]}" --exclude "lost+found/**" "${MOUNT_PATH}" "${RCLONE_CONFIG_SECTION}:${RCLONE_DEST_PATH}" --log-level DEBUG rclone copy "${RCLONE_FLAGS_COPY[@]}" --include permissions.facl /tmp "${RCLONE_CONFIG_SECTION}:${RCLONE_DEST_PATH}" --log-level DEBUG ;; diff --git a/test-e2e/roles/compare_pvc_data/templates/job.yml.j2 b/test-e2e/roles/compare_pvc_data/templates/job.yml.j2 index c11927505..d12d903ac 100644 --- a/test-e2e/roles/compare_pvc_data/templates/job.yml.j2 +++ b/test-e2e/roles/compare_pvc_data/templates/job.yml.j2 @@ -38,6 +38,7 @@ spec: echo "... all file contents matched" +{% if not skip_symlink_check | default(false) %} echo "Validating symlinks: PVC1 -> PVC2" find /mnt -type l | { grep -v lost+found || true; } | sort | while IFS= read -r f; do f2=$(echo "$f" | sed 's|^/mnt/|/mnt2/|') @@ -64,13 +65,24 @@ spec: done echo "... all symlinks matched" +{% else %} + echo "Skipping symlink validation (skip_symlink_check=true)" +{% endif %} echo "File attributes:" +{% if skip_symlink_check | default(false) %} + find /mnt -not -type l -exec stat -c "{{ properties_to_verify }}" {} \; | \ +{% else %} find /mnt -exec stat -c "{{ properties_to_verify }}" {} \; | \ +{% endif %} grep -v lost+found | \ sort | tee /tmp/attributes-mnt sed -ri 's|/mnt|/mnt2|' /tmp/attributes-mnt +{% if skip_symlink_check | default(false) %} + find /mnt2 -not -type l -exec stat -c "{{ properties_to_verify }}" {} \; | \ +{% else %} find /mnt2 -exec stat -c "{{ properties_to_verify }}" {} \; | \ +{% endif %} grep -v lost+found | \ sort > /tmp/attributes-mnt2 diff --git a/test-e2e/test_rclone_priv_with_symlinks.yml b/test-e2e/test_rclone_priv_with_symlinks.yml new file mode 100644 index 000000000..0dea66952 --- /dev/null +++ b/test-e2e/test_rclone_priv_with_symlinks.yml @@ -0,0 +1,189 @@ +--- +# Tests that the rclone mover handles symlinks on the source PVC without +# crashing. Covers dangling absolute symlinks (e.g. mysql.sock) that cause +# getfacl to return ELOOP under UBI9's acl-2.4.0. +- hosts: localhost + tags: + - e2e + - rclone + - symlinks + - privileged + - volumepopulator + vars: + rclone_secret_name: rclone-secret + tasks: + - include_role: + name: create_namespace + + - include_role: + name: gather_cluster_info + + - include_role: + name: enable_privileged_mover + + - include_role: + name: create_rclone_secret + vars: + minio_namespace: minio + + - name: Create source PVC + kubernetes.core.k8s: + state: present + definition: + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data-source + namespace: "{{ namespace }}" + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + + - name: Write data and symlinks into the source PVC + include_role: + name: write_to_pvc + vars: + data: 'data' + path: '/datafile' + pvc_name: 'data-source' + symlinks: + # Dangling absolute symlink — matches what MySQL creates on its + # data volume (mysql.sock -> /var/run/mysqld/mysqld.sock). + # On UBI9 acl-2.4.0, getfacl -P returns ELOOP on these. + - path: '/mysql.sock' + target: '/var/run/mysqld/mysqld.sock' + # Valid relative symlink pointing to the data file + - path: '/link-relative' + target: 'datafile' + + - name: Sync data from source volume + kubernetes.core.k8s: + state: present + definition: + apiVersion: volsync.backube/v1alpha1 + kind: ReplicationSource + metadata: + name: source + namespace: "{{ namespace }}" + spec: + sourcePVC: data-source + trigger: + manual: once + rclone: + rcloneConfigSection: rclone-data-mover + rcloneDestPath: "rclone-{{ namespace }}" + rcloneConfig: "{{ rclone_secret_name }}" + copyMethod: Snapshot + + - name: Wait for sync to MinIO to complete + kubernetes.core.k8s_info: + api_version: volsync.backube/v1alpha1 + kind: ReplicationSource + name: source + namespace: "{{ namespace }}" + register: res + until: > + res.resources | length > 0 and + res.resources[0].status.lastManualSync is defined and + res.resources[0].status.lastManualSync=="once" and + res.resources[0].status.latestMoverStatus is defined and + res.resources[0].status.latestMoverStatus.result == "Successful" and + res.resources[0].status.latestMoverStatus.logs is search("Transferred:.*") and + res.resources[0].status.latestMoverStatus.logs is search("Elapsed time:.*") and + res.resources[0].status.latestMoverStatus.logs is search("Rclone completed in.*") + delay: 1 + retries: 900 + + - name: Sync data to destination + kubernetes.core.k8s: + state: present + definition: + apiVersion: volsync.backube/v1alpha1 + kind: ReplicationDestination + metadata: + name: destination + namespace: "{{ namespace }}" + spec: + trigger: + manual: once + rclone: + rcloneConfigSection: rclone-data-mover + rcloneDestPath: "rclone-{{ namespace }}" + rcloneConfig: "{{ rclone_secret_name }}" + copyMethod: Snapshot + accessModes: [ReadWriteOnce] + capacity: 1Gi + + - name: Wait for sync from MinIO to complete + kubernetes.core.k8s_info: + api_version: volsync.backube/v1alpha1 + kind: ReplicationDestination + name: destination + namespace: "{{ namespace }}" + register: res + until: > + res.resources | length > 0 and + res.resources[0].status.lastManualSync is defined and + res.resources[0].status.lastManualSync=="once" and + res.resources[0].status.latestMoverStatus is defined and + res.resources[0].status.latestMoverStatus.result == "Successful" and + res.resources[0].status.latestMoverStatus.logs is search("Transferred:.*") and + res.resources[0].status.latestMoverStatus.logs is search("Elapsed time:.*") and + res.resources[0].status.latestMoverStatus.logs is search("Rclone completed in.*") + delay: 1 + retries: 300 + + - name: Convert latestImage to PVC using VolumePopulator + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-dest + namespace: "{{ namespace }}" + spec: + accessModes: + - ReadWriteOnce + dataSourceRef: + kind: ReplicationDestination + apiGroup: volsync.backube + name: destination + resources: + requests: + storage: 1Gi + when: cluster_info.volumepopulator_supported + + - name: Convert latestImage to PVC + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-dest + namespace: "{{ namespace }}" + spec: + accessModes: + - ReadWriteOnce + dataSource: + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + name: "{{ res.resources[0].status.latestImage.name }}" + resources: + requests: + storage: 1Gi + when: not cluster_info.volumepopulator_supported + + - name: Verify regular file contents were synced (symlinks are not + replicated by rclone, so only check regular files) + include_role: + name: compare_pvc_data + vars: + pvc1_name: data-source + pvc2_name: data-dest + timeout: 900 + skip_symlink_check: true From aacc6ec7f137dc8a605bc2186c3ff8f12b420847 Mon Sep 17 00:00:00 2001 From: Ismail Ibrahim Quwarah Date: Tue, 4 Aug 2026 01:32:25 +0200 Subject: [PATCH 2/3] Regenerate scorecard config for new rclone symlink test Signed-off-by: Ismail Ibrahim Quwarah --- custom-scorecard-tests/config-downstream.yaml | 10 ++++++++++ custom-scorecard-tests/config.yaml | 10 ++++++++++ .../scorecard/bases/patches/e2e-tests-stage1.yaml | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/custom-scorecard-tests/config-downstream.yaml b/custom-scorecard-tests/config-downstream.yaml index 208c6a5a1..7f8314c44 100644 --- a/custom-scorecard-tests/config-downstream.yaml +++ b/custom-scorecard-tests/config-downstream.yaml @@ -55,6 +55,16 @@ stages: storage: spec: mountPath: {} + - entrypoint: + - volsync-custom-scorecard-tests + - test_rclone_priv_with_symlinks.yml + image: quay.io/backube/volsync-custom-scorecard-tests:latest + labels: + suite: volsync-e2e + test: test_rclone_priv_with_symlinks.yml + storage: + spec: + mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/custom-scorecard-tests/config.yaml b/custom-scorecard-tests/config.yaml index 208c6a5a1..7f8314c44 100644 --- a/custom-scorecard-tests/config.yaml +++ b/custom-scorecard-tests/config.yaml @@ -55,6 +55,16 @@ stages: storage: spec: mountPath: {} + - entrypoint: + - volsync-custom-scorecard-tests + - test_rclone_priv_with_symlinks.yml + image: quay.io/backube/volsync-custom-scorecard-tests:latest + labels: + suite: volsync-e2e + test: test_rclone_priv_with_symlinks.yml + storage: + spec: + mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml index b418793e2..a8ef04f33 100644 --- a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml +++ b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml @@ -41,6 +41,16 @@ storage: spec: mountPath: {} + - entrypoint: + - volsync-custom-scorecard-tests + - test_rclone_priv_with_symlinks.yml + image: quay.io/backube/volsync-custom-scorecard-tests:latest + labels: + suite: volsync-e2e + test: test_rclone_priv_with_symlinks.yml + storage: + spec: + mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml From a60c0347d27bab3db94a46b8cb38ba15961e1b2d Mon Sep 17 00:00:00 2001 From: Ismail Ibrahim Quwarah Date: Tue, 4 Aug 2026 21:20:51 +0200 Subject: [PATCH 3/3] Add symlink coverage to existing rclone e2e tests Signed-off-by: Ismail Ibrahim Quwarah --- custom-scorecard-tests/config-downstream.yaml | 10 - custom-scorecard-tests/config.yaml | 10 - .../bases/patches/e2e-tests-stage1.yaml | 10 - test-e2e/test_rclone_normal.yml | 8 +- test-e2e/test_rclone_priv.yml | 8 +- test-e2e/test_rclone_priv_with_symlinks.yml | 189 ------------------ 6 files changed, 14 insertions(+), 221 deletions(-) delete mode 100644 test-e2e/test_rclone_priv_with_symlinks.yml diff --git a/custom-scorecard-tests/config-downstream.yaml b/custom-scorecard-tests/config-downstream.yaml index 7f8314c44..208c6a5a1 100644 --- a/custom-scorecard-tests/config-downstream.yaml +++ b/custom-scorecard-tests/config-downstream.yaml @@ -55,16 +55,6 @@ stages: storage: spec: mountPath: {} - - entrypoint: - - volsync-custom-scorecard-tests - - test_rclone_priv_with_symlinks.yml - image: quay.io/backube/volsync-custom-scorecard-tests:latest - labels: - suite: volsync-e2e - test: test_rclone_priv_with_symlinks.yml - storage: - spec: - mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/custom-scorecard-tests/config.yaml b/custom-scorecard-tests/config.yaml index 7f8314c44..208c6a5a1 100644 --- a/custom-scorecard-tests/config.yaml +++ b/custom-scorecard-tests/config.yaml @@ -55,16 +55,6 @@ stages: storage: spec: mountPath: {} - - entrypoint: - - volsync-custom-scorecard-tests - - test_rclone_priv_with_symlinks.yml - image: quay.io/backube/volsync-custom-scorecard-tests:latest - labels: - suite: volsync-e2e - test: test_rclone_priv_with_symlinks.yml - storage: - spec: - mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml index a8ef04f33..b418793e2 100644 --- a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml +++ b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml @@ -41,16 +41,6 @@ storage: spec: mountPath: {} - - entrypoint: - - volsync-custom-scorecard-tests - - test_rclone_priv_with_symlinks.yml - image: quay.io/backube/volsync-custom-scorecard-tests:latest - labels: - suite: volsync-e2e - test: test_rclone_priv_with_symlinks.yml - storage: - spec: - mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/test-e2e/test_rclone_normal.yml b/test-e2e/test_rclone_normal.yml index bc501d943..a666b031c 100644 --- a/test-e2e/test_rclone_normal.yml +++ b/test-e2e/test_rclone_normal.yml @@ -47,13 +47,18 @@ requests: storage: 1Gi - - name: Write data into the source PVC + - name: Write data and symlinks into the source PVC include_role: name: write_to_pvc vars: data: 'data' path: '/datafile' pvc_name: 'data-source' + symlinks: + - path: '/mysql.sock' + target: '/var/run/mysqld/mysqld.sock' + - path: '/link-relative' + target: 'datafile' - name: Sync data from source volume (w/ mSC) kubernetes.core.k8s: @@ -242,3 +247,4 @@ pvc1_name: data-source pvc2_name: data-dest timeout: 900 + skip_symlink_check: true diff --git a/test-e2e/test_rclone_priv.yml b/test-e2e/test_rclone_priv.yml index fb7f0e2c0..0c4e30752 100644 --- a/test-e2e/test_rclone_priv.yml +++ b/test-e2e/test_rclone_priv.yml @@ -50,13 +50,18 @@ requests: storage: 1Gi - - name: Write data into the source PVC + - name: Write data and symlinks into the source PVC include_role: name: write_to_pvc vars: data: 'data' path: '/datafile' pvc_name: 'data-source' + symlinks: + - path: '/mysql.sock' + target: '/var/run/mysqld/mysqld.sock' + - path: '/link-relative' + target: 'datafile' - name: Sync data from source volume kubernetes.core.k8s: @@ -184,3 +189,4 @@ pvc1_name: data-source pvc2_name: data-dest timeout: 900 + skip_symlink_check: true diff --git a/test-e2e/test_rclone_priv_with_symlinks.yml b/test-e2e/test_rclone_priv_with_symlinks.yml deleted file mode 100644 index 0dea66952..000000000 --- a/test-e2e/test_rclone_priv_with_symlinks.yml +++ /dev/null @@ -1,189 +0,0 @@ ---- -# Tests that the rclone mover handles symlinks on the source PVC without -# crashing. Covers dangling absolute symlinks (e.g. mysql.sock) that cause -# getfacl to return ELOOP under UBI9's acl-2.4.0. -- hosts: localhost - tags: - - e2e - - rclone - - symlinks - - privileged - - volumepopulator - vars: - rclone_secret_name: rclone-secret - tasks: - - include_role: - name: create_namespace - - - include_role: - name: gather_cluster_info - - - include_role: - name: enable_privileged_mover - - - include_role: - name: create_rclone_secret - vars: - minio_namespace: minio - - - name: Create source PVC - kubernetes.core.k8s: - state: present - definition: - kind: PersistentVolumeClaim - apiVersion: v1 - metadata: - name: data-source - namespace: "{{ namespace }}" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - - - name: Write data and symlinks into the source PVC - include_role: - name: write_to_pvc - vars: - data: 'data' - path: '/datafile' - pvc_name: 'data-source' - symlinks: - # Dangling absolute symlink — matches what MySQL creates on its - # data volume (mysql.sock -> /var/run/mysqld/mysqld.sock). - # On UBI9 acl-2.4.0, getfacl -P returns ELOOP on these. - - path: '/mysql.sock' - target: '/var/run/mysqld/mysqld.sock' - # Valid relative symlink pointing to the data file - - path: '/link-relative' - target: 'datafile' - - - name: Sync data from source volume - kubernetes.core.k8s: - state: present - definition: - apiVersion: volsync.backube/v1alpha1 - kind: ReplicationSource - metadata: - name: source - namespace: "{{ namespace }}" - spec: - sourcePVC: data-source - trigger: - manual: once - rclone: - rcloneConfigSection: rclone-data-mover - rcloneDestPath: "rclone-{{ namespace }}" - rcloneConfig: "{{ rclone_secret_name }}" - copyMethod: Snapshot - - - name: Wait for sync to MinIO to complete - kubernetes.core.k8s_info: - api_version: volsync.backube/v1alpha1 - kind: ReplicationSource - name: source - namespace: "{{ namespace }}" - register: res - until: > - res.resources | length > 0 and - res.resources[0].status.lastManualSync is defined and - res.resources[0].status.lastManualSync=="once" and - res.resources[0].status.latestMoverStatus is defined and - res.resources[0].status.latestMoverStatus.result == "Successful" and - res.resources[0].status.latestMoverStatus.logs is search("Transferred:.*") and - res.resources[0].status.latestMoverStatus.logs is search("Elapsed time:.*") and - res.resources[0].status.latestMoverStatus.logs is search("Rclone completed in.*") - delay: 1 - retries: 900 - - - name: Sync data to destination - kubernetes.core.k8s: - state: present - definition: - apiVersion: volsync.backube/v1alpha1 - kind: ReplicationDestination - metadata: - name: destination - namespace: "{{ namespace }}" - spec: - trigger: - manual: once - rclone: - rcloneConfigSection: rclone-data-mover - rcloneDestPath: "rclone-{{ namespace }}" - rcloneConfig: "{{ rclone_secret_name }}" - copyMethod: Snapshot - accessModes: [ReadWriteOnce] - capacity: 1Gi - - - name: Wait for sync from MinIO to complete - kubernetes.core.k8s_info: - api_version: volsync.backube/v1alpha1 - kind: ReplicationDestination - name: destination - namespace: "{{ namespace }}" - register: res - until: > - res.resources | length > 0 and - res.resources[0].status.lastManualSync is defined and - res.resources[0].status.lastManualSync=="once" and - res.resources[0].status.latestMoverStatus is defined and - res.resources[0].status.latestMoverStatus.result == "Successful" and - res.resources[0].status.latestMoverStatus.logs is search("Transferred:.*") and - res.resources[0].status.latestMoverStatus.logs is search("Elapsed time:.*") and - res.resources[0].status.latestMoverStatus.logs is search("Rclone completed in.*") - delay: 1 - retries: 300 - - - name: Convert latestImage to PVC using VolumePopulator - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: data-dest - namespace: "{{ namespace }}" - spec: - accessModes: - - ReadWriteOnce - dataSourceRef: - kind: ReplicationDestination - apiGroup: volsync.backube - name: destination - resources: - requests: - storage: 1Gi - when: cluster_info.volumepopulator_supported - - - name: Convert latestImage to PVC - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: data-dest - namespace: "{{ namespace }}" - spec: - accessModes: - - ReadWriteOnce - dataSource: - kind: VolumeSnapshot - apiGroup: snapshot.storage.k8s.io - name: "{{ res.resources[0].status.latestImage.name }}" - resources: - requests: - storage: 1Gi - when: not cluster_info.volumepopulator_supported - - - name: Verify regular file contents were synced (symlinks are not - replicated by rclone, so only check regular files) - include_role: - name: compare_pvc_data - vars: - pvc1_name: data-source - pvc2_name: data-dest - timeout: 900 - skip_symlink_check: true