From 4d74980546905f479ef69a30246825d8cd4ecc1a Mon Sep 17 00:00:00 2001 From: Kjeld Schouten Date: Mon, 29 Apr 2024 14:09:31 +0200 Subject: [PATCH] feat: adapt cnpg/volsync Backup pathing (#797) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Description** We want volsync and cnpg to be well integrated when writhing backups to s3. While we also want to prevent potential conflicts **โš™๏ธ Type of change** - [x] โš™๏ธ Feature/App addition - [x] ๐Ÿช› Bugfix - [x] โš ๏ธ Breaking change (fix or feature that would cause existing functionality to not work as expected) - [x] ๐Ÿ”ƒ Refactor of current code **๐Ÿงช How Has This Been Tested?** **๐Ÿ“ƒ Notes:** **โœ”๏ธ Checklist:** - [ ] โš–๏ธ My code follows the style guidelines of this project - [ ] ๐Ÿ‘€ I have performed a self-review of my own code - [ ] #๏ธโƒฃ I have commented my code, particularly in hard-to-understand areas - [ ] ๐Ÿ“„ I have made corresponding changes to the documentation - [ ] โš ๏ธ My changes generate no new warnings - [ ] ๐Ÿงช I have added tests to this description that prove my fix is effective or that my feature works - [ ] โฌ†๏ธ I increased versions for any altered app according to semantic versioning - [ ] I made sure the title starts with `feat(chart-name):`, `fix(chart-name):` or `chore(chart-name):` **โž• App addition** If this PR is an app addition please make sure you have done the following. - [ ] ๐Ÿ–ผ๏ธ I have added an icon in the Chart's root directory called `icon.png` --- _Please don't blindly check all the boxes. Read them and only check those that apply. Those checkboxes are there for the reviewer to see what is this all about and the status of this PR with a quick glance._ --- .../tests/cnpg/cluster_recovery_spec_test.yaml | 12 ++++++------ library/common/Chart.yaml | 2 +- .../templates/lib/cnpg/barmanObjectStore/_azure.tpl | 6 +++++- .../templates/lib/cnpg/barmanObjectStore/_google.tpl | 6 +++++- .../templates/lib/cnpg/barmanObjectStore/_s3.tpl | 6 +++++- library/common/templates/spawner/_pvc.tpl | 9 +++++++-- library/common/values.yaml | 7 +++++-- 7 files changed, 34 insertions(+), 14 deletions(-) diff --git a/library/common-test/tests/cnpg/cluster_recovery_spec_test.yaml b/library/common-test/tests/cnpg/cluster_recovery_spec_test.yaml index ebb69eae..67685b27 100644 --- a/library/common-test/tests/cnpg/cluster_recovery_spec_test.yaml +++ b/library/common-test/tests/cnpg/cluster_recovery_spec_test.yaml @@ -78,6 +78,7 @@ tests: type: google applicationCredentials: some-credentials bucket: some-bucket + path: some-path cnpg: my-pg: enabled: true @@ -85,7 +86,6 @@ tests: database: test-db password: test-password mode: recovery - path: some-path recovery: method: object_store credentials: test @@ -138,7 +138,7 @@ tests: - name: test-release-name-common-test-cnpg-my-pg barmanObjectStore: serverName: test-release-name-common-test-cnpg-my-pg - destinationPath: gs://some-bucket/some-path + destinationPath: gs://some-bucket/some-path/test-release-name/cnpg endpointURL: null googleCredentials: applicationCredentials: @@ -229,6 +229,7 @@ tests: secretKey: some-secret-key bucket: some-bucket region: some-region + path: some-path cnpg: my-pg: enabled: true @@ -236,7 +237,6 @@ tests: database: test-db password: test-password mode: recovery - path: some-path recovery: method: object_store credentials: test @@ -287,7 +287,7 @@ tests: - name: test-release-name-common-test-cnpg-my-pg barmanObjectStore: serverName: test-release-name-common-test-cnpg-my-pg - destinationPath: s3://some-bucket/some-path + destinationPath: s3://some-bucket/some-path/test-release-name/cnpg endpointURL: https://s3.some-region.amazonaws.com s3Credentials: accessKeyId: @@ -382,6 +382,7 @@ tests: storageAccount: some-storage-account serviceName: some-service-name containerName: some-container-name + path: some-path cnpg: my-pg: @@ -390,7 +391,6 @@ tests: database: test-db password: test-password mode: recovery - path: some-path recovery: method: object_store credentials: test @@ -442,7 +442,7 @@ tests: - name: test-release-name-common-test-cnpg-my-pg barmanObjectStore: serverName: test-release-name-common-test-cnpg-my-pg - destinationPath: https://some-storage-account.some-service-name.core.windows.net/some-container-name/some-path + destinationPath: https://some-storage-account.some-service-name.core.windows.net/some-container-name/some-path/test-release-name/cnpg endpointURL: null azureCredentials: connectionString: diff --git a/library/common/Chart.yaml b/library/common/Chart.yaml index 4ad9f07c..f04afc56 100644 --- a/library/common/Chart.yaml +++ b/library/common/Chart.yaml @@ -15,7 +15,7 @@ maintainers: name: common sources: null type: library -version: 22.0.0 +version: 22.0.1 annotations: artifacthub.io/category: "integration-delivery" artifacthub.io/license: "BUSL-1.1" diff --git a/library/common/templates/lib/cnpg/barmanObjectStore/_azure.tpl b/library/common/templates/lib/cnpg/barmanObjectStore/_azure.tpl index 6dc3bb6c..b5d96001 100644 --- a/library/common/templates/lib/cnpg/barmanObjectStore/_azure.tpl +++ b/library/common/templates/lib/cnpg/barmanObjectStore/_azure.tpl @@ -28,7 +28,11 @@ {{- if not $data.containerName -}} {{- fail (printf "CNPG %s - You need to specify [%s.azure.containerName] or [%s.destinationPath]" ($type | camelcase) $key $key) -}} {{- end -}} - {{- $destinationPath = (printf "https://%s.%s.core.windows.net/%s/%s" $data.storageAccount $data.serviceName $data.containerName (($objectData.path | default "/") | trimSuffix "/")) -}} + {{- if $data.path -}} + {{- $destinationPath = (printf "https://%s.%s.core.windows.net/%s/%s/%s/cnpg" $data.storageAccount $data.serviceName $data.containerName ($data.path | trimSuffix "/") $rootCtx.Release.Name) -}} + {{- else -}} + {{- $destinationPath = (printf "https://%s.%s.core.windows.net/%s/%s/cnpg" $data.storageAccount $data.serviceName $data.containerName $rootCtx.Release.Name) -}} + {{- end -}} {{- end }} endpointURL: {{ $endpointURL }} destinationPath: {{ $destinationPath }} diff --git a/library/common/templates/lib/cnpg/barmanObjectStore/_google.tpl b/library/common/templates/lib/cnpg/barmanObjectStore/_google.tpl index 2aa2da95..28cc5a27 100644 --- a/library/common/templates/lib/cnpg/barmanObjectStore/_google.tpl +++ b/library/common/templates/lib/cnpg/barmanObjectStore/_google.tpl @@ -27,7 +27,11 @@ {{- if not $data.bucket -}} {{- fail (printf "CNPG %s - You need to specify [%s.google.bucket] or [%s.destinationPath]" ($type | camelcase) $key $key) -}} {{- end -}} - {{- $destinationPath = (printf "gs://%s/%s" $data.bucket (($objectData.path | default "/") | trimSuffix "/")) -}} + {{- if $data.path -}} + {{- $destinationPath = (printf "gs://%s/%s/%s/cnpg" $data.bucket ($data.path | trimSuffix "/") $rootCtx.Release.Name) -}} + {{- else -}} + {{- $destinationPath = (printf "gs://%s/%s/cnpg" $data.bucket $rootCtx.Release.Name) -}} + {{- end -}} {{- end }} endpointURL: {{ $endpointURL }} destinationPath: {{ $destinationPath }} diff --git a/library/common/templates/lib/cnpg/barmanObjectStore/_s3.tpl b/library/common/templates/lib/cnpg/barmanObjectStore/_s3.tpl index 659fc96e..d193c7c9 100644 --- a/library/common/templates/lib/cnpg/barmanObjectStore/_s3.tpl +++ b/library/common/templates/lib/cnpg/barmanObjectStore/_s3.tpl @@ -22,7 +22,11 @@ {{- if not $data.bucket -}} {{- fail (printf "CNPG %s - You need to specify [%s.s3.bucket] or [%s.destinationPath]" ($type | camelcase) $key $key) -}} {{- end -}} - {{- $destinationPath = (printf "s3://%s/%s" $data.bucket (($objectData.path | default "/") | trimSuffix "/")) -}} + {{- if $data.path -}} + {{- $destinationPath = (printf "s3://%s/%s/%s/cnpg" $data.bucket ($data.path | trimSuffix "/") $rootCtx.Release.Name ) -}} + {{- else -}} + {{- $destinationPath = (printf "s3://%s/%s/cnpg" $data.bucket $rootCtx.Release.Name) -}} + {{- end -}} {{- end -}} {{- if not $endpointURL -}} {{- if not $data.region -}} diff --git a/library/common/templates/spawner/_pvc.tpl b/library/common/templates/spawner/_pvc.tpl index 8823f8d2..a62728af 100644 --- a/library/common/templates/spawner/_pvc.tpl +++ b/library/common/templates/spawner/_pvc.tpl @@ -101,8 +101,13 @@ {{- $_ := set $volsyncData "repository" $volsyncSecretName -}} {{- $credentials := get $.Values.credentials $volsync.credentials -}} - {{- $resticrepository := printf "s3:%s/%s/%s/%s" $credentials.url $credentials.bucket $.Release.Name $volsyncSecretName -}} - {{- $resticpassword := $credentials.encrKey -}} + + {{- $resticrepository := printf "s3:%s/%s/volsync/%s" $credentials.url $credentials.bucket $.Release.Name $volsyncSecretName -}} + {{- if $credentials.path -}} + {{- $resticrepository = printf "s3:%s/%s/%s/volsync/%s" $credentials.url $credentials.bucket ( $credentials.path | trimSuffix "/" ) $.Release.Name $volsyncSecretName -}} + {{- end -}} + + {{- $resticpassword := ( $credentials.encrKey | default $.Release.Name ) -}} {{- $s3id := $credentials.accessKey -}} {{- $s3key := $credentials.secretKey -}} diff --git a/library/common/values.yaml b/library/common/values.yaml index 2535a3de..5ef45e63 100644 --- a/library/common/values.yaml +++ b/library/common/values.yaml @@ -229,14 +229,17 @@ credentials: {} # mys3: # type: s3 # url: "" + # path: "" # region: "" # bucket: "" # accessKey: "" # secretKey: "" + # ## Is used in cases where things are encrypted by a backup utility # encrKey: "" # azure: + # path: "" # connectionString: "" # storageAccount: "" # storageKey: "" @@ -247,6 +250,7 @@ credentials: {} ## TODO: Add support for google: # google: + # path: "" # bucket: "" # gkeEnvironment: false # applicationCredentials: "" @@ -284,6 +288,7 @@ persistence: medium: Memory targetSelectAll: true # backupexample: +# ## the default backup path, is the credential path suffixed by the releasename, volsync and both the pvc and volsync names # enabled: true # type: pvc # mountPath: /backedup @@ -1123,7 +1128,6 @@ cnpg: # Azure: https://..core.windows.net/ # Google: gs:// destinationPath: "" - path: "/" # Database cluster backup configuration backups: @@ -1146,7 +1150,6 @@ cnpg: # name of credentials in .Values.Credentials credentials: "" - path: "/" scheduledBackups: - name: daily-backup # Daily at midnight