From baddf914035b21fce4bf6eccb4be10258d1ce64d Mon Sep 17 00:00:00 2001 From: Ismail Ibrahim Quwarah Date: Tue, 4 Aug 2026 01:19:05 +0200 Subject: [PATCH 1/3] Fix rclone mover crash on PVCs with symlinks Exclude symlinks from the getfacl pipeline in active.sh. On UBI9's acl-2.4.0, getfacl -P returns ELOOP on symlinks, killing the script under pipefail before rclone sync runs. Symlinks have no meaningful ACLs so skipping them loses no data. Adds e2e test with dangling and relative symlinks on the source PVC. Signed-off-by: Ismail Ibrahim Quwarah --- mover-rclone/active.sh | 2 +- .../compare_pvc_data/templates/job.yml.j2 | 12 ++ test-e2e/test_rclone_priv_with_symlinks.yml | 189 ++++++++++++++++++ 3 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 test-e2e/test_rclone_priv_with_symlinks.yml diff --git a/mover-rclone/active.sh b/mover-rclone/active.sh index e46a69896..012a4ce5c 100755 --- a/mover-rclone/active.sh +++ b/mover-rclone/active.sh @@ -77,7 +77,7 @@ fi START_TIME=$SECONDS case "${DIRECTION}" in source) - find "${MOUNT_PATH}" -path "${MOUNT_PATH}/lost+found" -prune -o -print | getfacl -P - > /tmp/permissions.facl + find "${MOUNT_PATH}" -path "${MOUNT_PATH}/lost+found" -prune -o -not -type l -print | getfacl -P - > /tmp/permissions.facl rclone sync "${RCLONE_FLAGS_SYNC[@]}" --exclude "lost+found/**" "${MOUNT_PATH}" "${RCLONE_CONFIG_SECTION}:${RCLONE_DEST_PATH}" --log-level DEBUG rclone copy "${RCLONE_FLAGS_COPY[@]}" --include permissions.facl /tmp "${RCLONE_CONFIG_SECTION}:${RCLONE_DEST_PATH}" --log-level DEBUG ;; diff --git a/test-e2e/roles/compare_pvc_data/templates/job.yml.j2 b/test-e2e/roles/compare_pvc_data/templates/job.yml.j2 index c11927505..d12d903ac 100644 --- a/test-e2e/roles/compare_pvc_data/templates/job.yml.j2 +++ b/test-e2e/roles/compare_pvc_data/templates/job.yml.j2 @@ -38,6 +38,7 @@ spec: echo "... all file contents matched" +{% if not skip_symlink_check | default(false) %} echo "Validating symlinks: PVC1 -> PVC2" find /mnt -type l | { grep -v lost+found || true; } | sort | while IFS= read -r f; do f2=$(echo "$f" | sed 's|^/mnt/|/mnt2/|') @@ -64,13 +65,24 @@ spec: done echo "... all symlinks matched" +{% else %} + echo "Skipping symlink validation (skip_symlink_check=true)" +{% endif %} echo "File attributes:" +{% if skip_symlink_check | default(false) %} + find /mnt -not -type l -exec stat -c "{{ properties_to_verify }}" {} \; | \ +{% else %} find /mnt -exec stat -c "{{ properties_to_verify }}" {} \; | \ +{% endif %} grep -v lost+found | \ sort | tee /tmp/attributes-mnt sed -ri 's|/mnt|/mnt2|' /tmp/attributes-mnt +{% if skip_symlink_check | default(false) %} + find /mnt2 -not -type l -exec stat -c "{{ properties_to_verify }}" {} \; | \ +{% else %} find /mnt2 -exec stat -c "{{ properties_to_verify }}" {} \; | \ +{% endif %} grep -v lost+found | \ sort > /tmp/attributes-mnt2 diff --git a/test-e2e/test_rclone_priv_with_symlinks.yml b/test-e2e/test_rclone_priv_with_symlinks.yml new file mode 100644 index 000000000..0dea66952 --- /dev/null +++ b/test-e2e/test_rclone_priv_with_symlinks.yml @@ -0,0 +1,189 @@ +--- +# Tests that the rclone mover handles symlinks on the source PVC without +# crashing. Covers dangling absolute symlinks (e.g. mysql.sock) that cause +# getfacl to return ELOOP under UBI9's acl-2.4.0. +- hosts: localhost + tags: + - e2e + - rclone + - symlinks + - privileged + - volumepopulator + vars: + rclone_secret_name: rclone-secret + tasks: + - include_role: + name: create_namespace + + - include_role: + name: gather_cluster_info + + - include_role: + name: enable_privileged_mover + + - include_role: + name: create_rclone_secret + vars: + minio_namespace: minio + + - name: Create source PVC + kubernetes.core.k8s: + state: present + definition: + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data-source + namespace: "{{ namespace }}" + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + + - name: Write data and symlinks into the source PVC + include_role: + name: write_to_pvc + vars: + data: 'data' + path: '/datafile' + pvc_name: 'data-source' + symlinks: + # Dangling absolute symlink — matches what MySQL creates on its + # data volume (mysql.sock -> /var/run/mysqld/mysqld.sock). + # On UBI9 acl-2.4.0, getfacl -P returns ELOOP on these. + - path: '/mysql.sock' + target: '/var/run/mysqld/mysqld.sock' + # Valid relative symlink pointing to the data file + - path: '/link-relative' + target: 'datafile' + + - name: Sync data from source volume + kubernetes.core.k8s: + state: present + definition: + apiVersion: volsync.backube/v1alpha1 + kind: ReplicationSource + metadata: + name: source + namespace: "{{ namespace }}" + spec: + sourcePVC: data-source + trigger: + manual: once + rclone: + rcloneConfigSection: rclone-data-mover + rcloneDestPath: "rclone-{{ namespace }}" + rcloneConfig: "{{ rclone_secret_name }}" + copyMethod: Snapshot + + - name: Wait for sync to MinIO to complete + kubernetes.core.k8s_info: + api_version: volsync.backube/v1alpha1 + kind: ReplicationSource + name: source + namespace: "{{ namespace }}" + register: res + until: > + res.resources | length > 0 and + res.resources[0].status.lastManualSync is defined and + res.resources[0].status.lastManualSync=="once" and + res.resources[0].status.latestMoverStatus is defined and + res.resources[0].status.latestMoverStatus.result == "Successful" and + res.resources[0].status.latestMoverStatus.logs is search("Transferred:.*") and + res.resources[0].status.latestMoverStatus.logs is search("Elapsed time:.*") and + res.resources[0].status.latestMoverStatus.logs is search("Rclone completed in.*") + delay: 1 + retries: 900 + + - name: Sync data to destination + kubernetes.core.k8s: + state: present + definition: + apiVersion: volsync.backube/v1alpha1 + kind: ReplicationDestination + metadata: + name: destination + namespace: "{{ namespace }}" + spec: + trigger: + manual: once + rclone: + rcloneConfigSection: rclone-data-mover + rcloneDestPath: "rclone-{{ namespace }}" + rcloneConfig: "{{ rclone_secret_name }}" + copyMethod: Snapshot + accessModes: [ReadWriteOnce] + capacity: 1Gi + + - name: Wait for sync from MinIO to complete + kubernetes.core.k8s_info: + api_version: volsync.backube/v1alpha1 + kind: ReplicationDestination + name: destination + namespace: "{{ namespace }}" + register: res + until: > + res.resources | length > 0 and + res.resources[0].status.lastManualSync is defined and + res.resources[0].status.lastManualSync=="once" and + res.resources[0].status.latestMoverStatus is defined and + res.resources[0].status.latestMoverStatus.result == "Successful" and + res.resources[0].status.latestMoverStatus.logs is search("Transferred:.*") and + res.resources[0].status.latestMoverStatus.logs is search("Elapsed time:.*") and + res.resources[0].status.latestMoverStatus.logs is search("Rclone completed in.*") + delay: 1 + retries: 300 + + - name: Convert latestImage to PVC using VolumePopulator + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-dest + namespace: "{{ namespace }}" + spec: + accessModes: + - ReadWriteOnce + dataSourceRef: + kind: ReplicationDestination + apiGroup: volsync.backube + name: destination + resources: + requests: + storage: 1Gi + when: cluster_info.volumepopulator_supported + + - name: Convert latestImage to PVC + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-dest + namespace: "{{ namespace }}" + spec: + accessModes: + - ReadWriteOnce + dataSource: + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + name: "{{ res.resources[0].status.latestImage.name }}" + resources: + requests: + storage: 1Gi + when: not cluster_info.volumepopulator_supported + + - name: Verify regular file contents were synced (symlinks are not + replicated by rclone, so only check regular files) + include_role: + name: compare_pvc_data + vars: + pvc1_name: data-source + pvc2_name: data-dest + timeout: 900 + skip_symlink_check: true From d5e4a194129b9d18963a40d76c21010bf497c74d Mon Sep 17 00:00:00 2001 From: Ismail Ibrahim Quwarah Date: Tue, 4 Aug 2026 01:32:25 +0200 Subject: [PATCH 2/3] Regenerate scorecard config for new rclone symlink test Signed-off-by: Ismail Ibrahim Quwarah --- custom-scorecard-tests/config-downstream.yaml | 10 ++++++++++ custom-scorecard-tests/config.yaml | 10 ++++++++++ .../scorecard/bases/patches/e2e-tests-stage1.yaml | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/custom-scorecard-tests/config-downstream.yaml b/custom-scorecard-tests/config-downstream.yaml index 5f35a4b2a..5dd0a0df5 100644 --- a/custom-scorecard-tests/config-downstream.yaml +++ b/custom-scorecard-tests/config-downstream.yaml @@ -55,6 +55,16 @@ stages: storage: spec: mountPath: {} + - entrypoint: + - volsync-custom-scorecard-tests + - test_rclone_priv_with_symlinks.yml + image: quay.io/backube/volsync-custom-scorecard-tests:latest + labels: + suite: volsync-e2e + test: test_rclone_priv_with_symlinks.yml + storage: + spec: + mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/custom-scorecard-tests/config.yaml b/custom-scorecard-tests/config.yaml index 5f35a4b2a..5dd0a0df5 100644 --- a/custom-scorecard-tests/config.yaml +++ b/custom-scorecard-tests/config.yaml @@ -55,6 +55,16 @@ stages: storage: spec: mountPath: {} + - entrypoint: + - volsync-custom-scorecard-tests + - test_rclone_priv_with_symlinks.yml + image: quay.io/backube/volsync-custom-scorecard-tests:latest + labels: + suite: volsync-e2e + test: test_rclone_priv_with_symlinks.yml + storage: + spec: + mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml index 7806f81ee..7a289c9c5 100644 --- a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml +++ b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml @@ -41,6 +41,16 @@ storage: spec: mountPath: {} + - entrypoint: + - volsync-custom-scorecard-tests + - test_rclone_priv_with_symlinks.yml + image: quay.io/backube/volsync-custom-scorecard-tests:latest + labels: + suite: volsync-e2e + test: test_rclone_priv_with_symlinks.yml + storage: + spec: + mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml From e8309785a509623190395752b84415a39b48688b Mon Sep 17 00:00:00 2001 From: Ismail Ibrahim Quwarah Date: Tue, 4 Aug 2026 21:20:51 +0200 Subject: [PATCH 3/3] Add symlink coverage to existing rclone e2e tests Signed-off-by: Ismail Ibrahim Quwarah --- custom-scorecard-tests/config-downstream.yaml | 10 - custom-scorecard-tests/config.yaml | 10 - .../bases/patches/e2e-tests-stage1.yaml | 10 - test-e2e/test_rclone_normal.yml | 8 +- test-e2e/test_rclone_priv.yml | 8 +- test-e2e/test_rclone_priv_with_symlinks.yml | 189 ------------------ 6 files changed, 14 insertions(+), 221 deletions(-) delete mode 100644 test-e2e/test_rclone_priv_with_symlinks.yml diff --git a/custom-scorecard-tests/config-downstream.yaml b/custom-scorecard-tests/config-downstream.yaml index 5dd0a0df5..5f35a4b2a 100644 --- a/custom-scorecard-tests/config-downstream.yaml +++ b/custom-scorecard-tests/config-downstream.yaml @@ -55,16 +55,6 @@ stages: storage: spec: mountPath: {} - - entrypoint: - - volsync-custom-scorecard-tests - - test_rclone_priv_with_symlinks.yml - image: quay.io/backube/volsync-custom-scorecard-tests:latest - labels: - suite: volsync-e2e - test: test_rclone_priv_with_symlinks.yml - storage: - spec: - mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/custom-scorecard-tests/config.yaml b/custom-scorecard-tests/config.yaml index 5dd0a0df5..5f35a4b2a 100644 --- a/custom-scorecard-tests/config.yaml +++ b/custom-scorecard-tests/config.yaml @@ -55,16 +55,6 @@ stages: storage: spec: mountPath: {} - - entrypoint: - - volsync-custom-scorecard-tests - - test_rclone_priv_with_symlinks.yml - image: quay.io/backube/volsync-custom-scorecard-tests:latest - labels: - suite: volsync-e2e - test: test_rclone_priv_with_symlinks.yml - storage: - spec: - mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml index 7a289c9c5..7806f81ee 100644 --- a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml +++ b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml @@ -41,16 +41,6 @@ storage: spec: mountPath: {} - - entrypoint: - - volsync-custom-scorecard-tests - - test_rclone_priv_with_symlinks.yml - image: quay.io/backube/volsync-custom-scorecard-tests:latest - labels: - suite: volsync-e2e - test: test_rclone_priv_with_symlinks.yml - storage: - spec: - mountPath: {} - entrypoint: - volsync-custom-scorecard-tests - test_rclone_with_customca_configmap.yml diff --git a/test-e2e/test_rclone_normal.yml b/test-e2e/test_rclone_normal.yml index bc501d943..a666b031c 100644 --- a/test-e2e/test_rclone_normal.yml +++ b/test-e2e/test_rclone_normal.yml @@ -47,13 +47,18 @@ requests: storage: 1Gi - - name: Write data into the source PVC + - name: Write data and symlinks into the source PVC include_role: name: write_to_pvc vars: data: 'data' path: '/datafile' pvc_name: 'data-source' + symlinks: + - path: '/mysql.sock' + target: '/var/run/mysqld/mysqld.sock' + - path: '/link-relative' + target: 'datafile' - name: Sync data from source volume (w/ mSC) kubernetes.core.k8s: @@ -242,3 +247,4 @@ pvc1_name: data-source pvc2_name: data-dest timeout: 900 + skip_symlink_check: true diff --git a/test-e2e/test_rclone_priv.yml b/test-e2e/test_rclone_priv.yml index fb7f0e2c0..0c4e30752 100644 --- a/test-e2e/test_rclone_priv.yml +++ b/test-e2e/test_rclone_priv.yml @@ -50,13 +50,18 @@ requests: storage: 1Gi - - name: Write data into the source PVC + - name: Write data and symlinks into the source PVC include_role: name: write_to_pvc vars: data: 'data' path: '/datafile' pvc_name: 'data-source' + symlinks: + - path: '/mysql.sock' + target: '/var/run/mysqld/mysqld.sock' + - path: '/link-relative' + target: 'datafile' - name: Sync data from source volume kubernetes.core.k8s: @@ -184,3 +189,4 @@ pvc1_name: data-source pvc2_name: data-dest timeout: 900 + skip_symlink_check: true diff --git a/test-e2e/test_rclone_priv_with_symlinks.yml b/test-e2e/test_rclone_priv_with_symlinks.yml deleted file mode 100644 index 0dea66952..000000000 --- a/test-e2e/test_rclone_priv_with_symlinks.yml +++ /dev/null @@ -1,189 +0,0 @@ ---- -# Tests that the rclone mover handles symlinks on the source PVC without -# crashing. Covers dangling absolute symlinks (e.g. mysql.sock) that cause -# getfacl to return ELOOP under UBI9's acl-2.4.0. -- hosts: localhost - tags: - - e2e - - rclone - - symlinks - - privileged - - volumepopulator - vars: - rclone_secret_name: rclone-secret - tasks: - - include_role: - name: create_namespace - - - include_role: - name: gather_cluster_info - - - include_role: - name: enable_privileged_mover - - - include_role: - name: create_rclone_secret - vars: - minio_namespace: minio - - - name: Create source PVC - kubernetes.core.k8s: - state: present - definition: - kind: PersistentVolumeClaim - apiVersion: v1 - metadata: - name: data-source - namespace: "{{ namespace }}" - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - - - name: Write data and symlinks into the source PVC - include_role: - name: write_to_pvc - vars: - data: 'data' - path: '/datafile' - pvc_name: 'data-source' - symlinks: - # Dangling absolute symlink — matches what MySQL creates on its - # data volume (mysql.sock -> /var/run/mysqld/mysqld.sock). - # On UBI9 acl-2.4.0, getfacl -P returns ELOOP on these. - - path: '/mysql.sock' - target: '/var/run/mysqld/mysqld.sock' - # Valid relative symlink pointing to the data file - - path: '/link-relative' - target: 'datafile' - - - name: Sync data from source volume - kubernetes.core.k8s: - state: present - definition: - apiVersion: volsync.backube/v1alpha1 - kind: ReplicationSource - metadata: - name: source - namespace: "{{ namespace }}" - spec: - sourcePVC: data-source - trigger: - manual: once - rclone: - rcloneConfigSection: rclone-data-mover - rcloneDestPath: "rclone-{{ namespace }}" - rcloneConfig: "{{ rclone_secret_name }}" - copyMethod: Snapshot - - - name: Wait for sync to MinIO to complete - kubernetes.core.k8s_info: - api_version: volsync.backube/v1alpha1 - kind: ReplicationSource - name: source - namespace: "{{ namespace }}" - register: res - until: > - res.resources | length > 0 and - res.resources[0].status.lastManualSync is defined and - res.resources[0].status.lastManualSync=="once" and - res.resources[0].status.latestMoverStatus is defined and - res.resources[0].status.latestMoverStatus.result == "Successful" and - res.resources[0].status.latestMoverStatus.logs is search("Transferred:.*") and - res.resources[0].status.latestMoverStatus.logs is search("Elapsed time:.*") and - res.resources[0].status.latestMoverStatus.logs is search("Rclone completed in.*") - delay: 1 - retries: 900 - - - name: Sync data to destination - kubernetes.core.k8s: - state: present - definition: - apiVersion: volsync.backube/v1alpha1 - kind: ReplicationDestination - metadata: - name: destination - namespace: "{{ namespace }}" - spec: - trigger: - manual: once - rclone: - rcloneConfigSection: rclone-data-mover - rcloneDestPath: "rclone-{{ namespace }}" - rcloneConfig: "{{ rclone_secret_name }}" - copyMethod: Snapshot - accessModes: [ReadWriteOnce] - capacity: 1Gi - - - name: Wait for sync from MinIO to complete - kubernetes.core.k8s_info: - api_version: volsync.backube/v1alpha1 - kind: ReplicationDestination - name: destination - namespace: "{{ namespace }}" - register: res - until: > - res.resources | length > 0 and - res.resources[0].status.lastManualSync is defined and - res.resources[0].status.lastManualSync=="once" and - res.resources[0].status.latestMoverStatus is defined and - res.resources[0].status.latestMoverStatus.result == "Successful" and - res.resources[0].status.latestMoverStatus.logs is search("Transferred:.*") and - res.resources[0].status.latestMoverStatus.logs is search("Elapsed time:.*") and - res.resources[0].status.latestMoverStatus.logs is search("Rclone completed in.*") - delay: 1 - retries: 300 - - - name: Convert latestImage to PVC using VolumePopulator - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: data-dest - namespace: "{{ namespace }}" - spec: - accessModes: - - ReadWriteOnce - dataSourceRef: - kind: ReplicationDestination - apiGroup: volsync.backube - name: destination - resources: - requests: - storage: 1Gi - when: cluster_info.volumepopulator_supported - - - name: Convert latestImage to PVC - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: data-dest - namespace: "{{ namespace }}" - spec: - accessModes: - - ReadWriteOnce - dataSource: - kind: VolumeSnapshot - apiGroup: snapshot.storage.k8s.io - name: "{{ res.resources[0].status.latestImage.name }}" - resources: - requests: - storage: 1Gi - when: not cluster_info.volumepopulator_supported - - - name: Verify regular file contents were synced (symlinks are not - replicated by rclone, so only check regular files) - include_role: - name: compare_pvc_data - vars: - pvc1_name: data-source - pvc2_name: data-dest - timeout: 900 - skip_symlink_check: true