Adapt community to allow chart versioning (#434)

* try adding some custom tooling for the job

* polish tooling

* more polish

* process stable

* setup scripting for incubator

* Process incubator

* setup script for dev

* remove dev placeholder app

* remove .test beta folder

* fix some of the workflows and tooling

* fix a testing bug

* update doc generation to not create app-readme.md

* remove an unneeded file

* support multiple charts

* adapt docs

* adapt branch in docs

.github/workflows/apps.release.yaml

@@ -11,8 +11,89 @@ on:
       - '.github/workflows/apps.release.yaml'
 
 jobs:
+  catalog-test:
+    runs-on: ubuntu-latest
+    container:
+      image: ixsystems/catalog_validation:latest
+
+    steps:
+      - uses: actions/checkout@v2
+        name: Checkout
+      - name: Install Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: v3.5.3
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.BOT_TOKEN }}
+          path: master
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          repository: truecharts/catalog
+          token: ${{ secrets.BOT_TOKEN }}
+          path: catalog
+      - name: Remove if release already exists
+        run: |
+          for train in stable incubator dev
+            for chart in master/${train}/*; do
+              if [ -d "${chart}" ]; then
+                maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
+                chartname=$(basename ${chart})
+                echo "Processing... cleanup-existing... Chart: ${chartname} - Version: ${maxchartversion}"
+                rm -Rf catalog/test/${chartname}/${maxchartversion} || echo "Chart release ${chartname} with version ${maxchartversion} does not exist yet... Skipping..."
+                rm -Rf catalog/test/${chartname}/item.yaml || echo "No old catalog item.yaml found for ${chartname}"
+              fi
+            done
+          done
+      - name: fetch dependencies
+        run: |
+          helm dependency update master/.test/*
+      - name: Add Chart release
+        run: |
+          for train in stable incubator dev
+            for chart in master/${train}/*; do
+              if [ -d "${chart}" ]; then
+                maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
+                chartname=$(basename ${chart})
+                echo "Processing... Copying charts... Chart: ${chartname} - Version: ${maxchartversion}"
+                mkdir -p catalog/test/${chartname}/${maxchartversion}
+                cp -Rf ${chart}/* catalog/${train}/${chartname}/${maxchartversion}/
+              fi
+            done
+          done
+      - name: Apply SCALE Patches
+        run: |
+          for train in stable incubator dev
+            for chart in catalog/${train}/*; do
+              if [ -d "${chart}" ]; then
+                maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
+                chartname=$(basename ${chart})
+                path="${chart}/${maxfolderversion}"
+                echo "Processing... Applying SCALE patch... Chart: ${chartname} - Version: ${maxfolderversion}"
+                mv ${path}/SCALE/item.yaml catalog/${train}/${chartname}/
+                mv ${path}/SCALE/ix_values.yaml ${path}/
+                mv ${path}/SCALE/questions.yaml ${path}/
+                cp -rf ${path}/SCALE/templates/* ${path}/templates || echo "ignoring templates directory..."
+                rm -rf ${path}/SCALE/templates
+                mv ${path}/values.yaml ${path}/test_values.yaml
+                touch ${path}/values.yaml
+                description=$(cat ${path}/Chart.yaml | grep "^description: " | sed -r 's/^description: //')
+                echo "${description}" >> ${path}/app-readme.md
+              fi
+            done
+          done
+      - name: Validate catalog format
+        run: |
+          cd catalog
+          /bin/bash -c "PWD=${pwd}; /usr/local/bin/catalog_validate validate --path $PWD"
+
   release:
     runs-on: ubuntu-latest
+    needs: catalog-test
     steps:
       - name: Install Helm
         uses: azure/setup-helm@v1
@@ -33,49 +114,55 @@ jobs:
           path: catalog
       - name: Remove if release already exists
         run: |
-          for chart in master/.test/*; do
-            if [ -d "${chart}" ]; then
-              maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
-              chartname=$(basename ${chart})
-              echo "Processing... cleanup-existing... Chart: ${chartname} - Version: ${maxchartversion}"
-
-              rm -Rf catalog/test/${chartname}/${maxchartversion} || echo "Chart release ${chartname} with version ${maxchartversion} does not exist yet... Skipping..."
-              rm -Rf catalog/test/${chartname}/item.yaml || echo "No old catalog item.yaml found for ${chartname}"
-            fi
+          for train in stable incubator dev
+            for chart in master/${train}/*; do
+              if [ -d "${chart}" ]; then
+                maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
+                chartname=$(basename ${chart})
+                echo "Processing... cleanup-existing... Chart: ${chartname} - Version: ${maxchartversion}"
+                rm -Rf catalog/test/${chartname}/${maxchartversion} || echo "Chart release ${chartname} with version ${maxchartversion} does not exist yet... Skipping..."
+                rm -Rf catalog/test/${chartname}/item.yaml || echo "No old catalog item.yaml found for ${chartname}"
+              fi
+            done
           done
       - name: fetch dependencies
         run: |
           helm dependency update master/.test/*
       - name: Add Chart release
         run: |
-          for chart in master/.test/*; do
-            if [ -d "${chart}" ]; then
-              maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
-              chartname=$(basename ${chart})
-              echo "Processing... Copying charts... Chart: ${chartname} - Version: ${maxchartversion}"
-              mkdir -p catalog/test/${chartname}/${maxchartversion}
-              cp -Rf ${chart}/* catalog/test/${chartname}/${maxchartversion}/
-            fi
+          for train in stable incubator dev
+            for chart in master/${train}/*; do
+              if [ -d "${chart}" ]; then
+                maxchartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
+                chartname=$(basename ${chart})
+                echo "Processing... Copying charts... Chart: ${chartname} - Version: ${maxchartversion}"
+                mkdir -p catalog/test/${chartname}/${maxchartversion}
+                cp -Rf ${chart}/* catalog/${train}/${chartname}/${maxchartversion}/
+              fi
+            done
           done
       - name: Apply SCALE Patches
         run: |
-          for chart in catalog/test/*; do
-            if [ -d "${chart}" ]; then
-              maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
-              chartname=$(basename ${chart})
-              path="${chart}/${maxfolderversion}"
-              echo "Processing... Applying SCALE patch... Chart: ${chartname} - Version: ${maxfolderversion}"
-              mv ${path}/SCALE/item.yaml catalog/test/${chartname}/
-              mv ${path}/SCALE/ix_values.yaml ${path}/
-              mv ${path}/SCALE/questions.yaml ${path}/
-              cp -rf ${path}/SCALE/templates/* ${path}/templates || echo "ignoring templates directory..."
-              rm -rf ${path}/SCALE/templates
-              mv ${path}/values.yaml ${path}/test_values.yaml
-              touch ${path}/values.yaml
-              description=$(cat ${path}/Chart.yaml | grep "^description: " | sed -r 's/^description: //')
-              echo "${description}" >> ${path}/app-readme.md
-            fi
+          for train in stable incubator dev
+            for chart in catalog/${train}/*; do
+              if [ -d "${chart}" ]; then
+                maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
+                chartname=$(basename ${chart})
+                path="${chart}/${maxfolderversion}"
+                echo "Processing... Applying SCALE patch... Chart: ${chartname} - Version: ${maxfolderversion}"
+                mv ${path}/SCALE/item.yaml catalog/${train}/${chartname}/
+                mv ${path}/SCALE/ix_values.yaml ${path}/
+                mv ${path}/SCALE/questions.yaml ${path}/
+                cp -rf ${path}/SCALE/templates/* ${path}/templates || echo "ignoring templates directory..."
+                rm -rf ${path}/SCALE/templates
+                mv ${path}/values.yaml ${path}/test_values.yaml
+                touch ${path}/values.yaml
+                description=$(cat ${path}/Chart.yaml | grep "^description: " | sed -r 's/^description: //')
+                echo "${description}" >> ${path}/app-readme.md
+              fi
+            done
           done
+
       - name: Commit and Push new App releases
         run: |
           cd catalog

.github/workflows/apps.test.yaml

@@ -8,43 +8,31 @@ on:
       - '**'
     paths:
       - 'stable/**'
-      - '!stable/**/*.md'
-      - '!stable/**/README.md'
-      - '!stable/**/README.md.gotmpl'
-      - '!stable/**/app-readme.md'
-      - '!stable/**/app-readme.md.gotmpl'
-      - '!stable/**/docs/*'
+      - '!stable/*.md'
+      - '!stable/README.md'
+      - '!stable/README.md.gotmpl'
+      - '!stable/app-readme.md'
+      - '!stable/app-readme.md.gotmpl'
+      - '!stable/docs/*'
       - 'incubator/**'
-      - '!incubator/**/*.md'
-      - '!incubator/**/README.md'
-      - '!incubator/**/README.md.gotmpl'
-      - '!incubator/**/app-readme.md'
-      - '!incubator/**/app-readme.md.gotmpl'
-      - '!incubator/**/docs/*'
+      - '!incubator/*.md'
+      - '!incubator/README.md'
+      - '!incubator/README.md.gotmpl'
+      - '!incubator/app-readme.md'
+      - '!incubator/app-readme.md.gotmpl'
+      - '!incubator/docs/*'
       - 'dev/**'
-      - '!dev/**/*.md'
-      - '!dev/**/README.md'
-      - '!dev/**/README.md.gotmpl'
-      - '!dev/**/app-readme.md'
-      - '!dev/**/app-readme.md.gotmpl'
-      - '!dev/**/docs/*'
+      - '!dev/*.md'
+      - '!dev/README.md'
+      - '!dev/README.md.gotmpl'
+      - '!dev/app-readme.md'
+      - '!dev/app-readme.md.gotmpl'
+      - '!dev/docs/*'
       - '.github/workflows/apps.test.yaml'
       - '.github/ct-install.yaml'
       - '.github/ct-lint.yaml'
 
 jobs:
-  catalog-test:
-    runs-on: ubuntu-latest
-    container:
-      image: ixsystems/catalog_validation:latest
-
-    steps:
-    - uses: actions/checkout@v2
-      name: Checkout
-    - name: Validate catalog format
-      run: |
-        /bin/bash -c "PWD=${pwd}; /usr/local/bin/catalog_validate validate --path $PWD"
-
   changes:
     name: Get changed Apps
     runs-on: ubuntu-20.04
@@ -69,7 +57,7 @@ jobs:
         echo ::set-output name=apps::${APPS}
 
   app-tests:
-    needs: [catalog-test, changes]
+    needs: [changes]
     name: App Tests
     runs-on: ubuntu-latest
     strategy:
@@ -84,19 +72,14 @@ jobs:
       - name: Prepare-Lint
         id: prep-lint
         run: |
-            if test -f "./stable/${{ matrix.app }}/item.yaml"; then
+            if test -f "./stable/${{ matrix.app }}/Chart.yaml"; then
               train="stable"
-            elif test -f "./incubator/${{ matrix.app }}/item.yaml"; then
+            elif test -f "./incubator/${{ matrix.app }}/Chart.yaml"; then
               train="incubator"
             else
               train="dev"
             fi
             echo ::set-output name=train::${train}
-            maxfolderversion=$(ls -l ${train}/${{ matrix.app }} | grep ^d | awk '{print $9}' | tail -n 1)
-            echo "Detected folder version: ${maxfolderversion}"
-            mkdir ${train}/${{ matrix.app }}/${maxfolderversion}/ci  || exit 0
-            cp ${train}/${{ matrix.app }}/${maxfolderversion}/test_values.yaml ${train}/${{ matrix.app }}/${maxfolderversion}/ci/test-values.yaml
-            echo ::set-output name=maxfolderversion::${maxfolderversion}
 
       - name: Install Helm
         uses: azure/setup-helm@v1
@@ -113,12 +96,7 @@ jobs:
 
       - name: Run chart-testing (lint)
         id: lint
-        run: ct lint --config .github/ct-lint.yaml --charts '${{ steps.prep-lint.outputs.train }}/${{ matrix.app }}/${{ steps.prep-lint.outputs.maxfolderversion }}'
-
-      - name: Prepare-Test
-        run: |
-            mv -f ${{ steps.prep-lint.outputs.train }}/${{ matrix.app }}/${{ steps.prep-lint.outputs.maxfolderversion }}/* ${{ steps.prep-lint.outputs.train }}/${{ matrix.app }}/
-            rm -Rf ${{ steps.prep-lint.outputs.train }}/${{ matrix.app }}/${{ steps.prep-lint.outputs.maxfolderversion }}
+        run: ct lint --config .github/ct-lint.yaml --charts '${{ steps.prep-lint.outputs.train }}/${{ matrix.app }}'
 
       - name: Create k3d cluster
         uses: nolar/setup-k3d-k3s@v1

.github/workflows/charts.renovate.yaml

@@ -1,82 +0,0 @@
-name: "Charts: Renovate"
-
-on:
-  push:
-    branches:
-      - 'renovate/**'
-      - 'update/**'
-    tags-ignore:
-      - '**'
-
-jobs:
-  folderfix:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.BOT_TOKEN }}
-
-      - name: Configure Git
-        run: |
-          git config user.name "TrueCharts-Bot"
-          git config user.email "bot@truecharts.org"
-
-      - name: Install Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.5.2
-
-      - name: update folder names and dependencies for stable
-        run: |
-          for chart in stable/*; do
-            if [ -d "${chart}" ]; then
-                maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
-                maxchartversion=$(cat ${chart}/${maxfolderversion}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
-                chartname=$(basename ${chart})
-                echo "Processing: ${chart} - folder: ${maxfolderversion} - version: ${maxchartversion}"
-                helm dependency update ${chart}/${maxfolderversion}
-                if [ "${maxfolderversion}" != "${maxchartversion}" ]; then
-                    mv -f ${chart}/${maxfolderversion} ${chart}/${maxchartversion}
-                    echo "renamed ${chart}/${maxfolderversion} to ${chart}/${maxchartversion}"
-                fi
-            fi
-          done
-
-      - name: update folder names and dependencies for incubator
-        run: |
-          for chart in incubator/*; do
-            if [ -d "${chart}" ]; then
-                maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
-                maxchartversion=$(cat ${chart}/${maxfolderversion}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
-                chartname=$(basename ${chart})
-                echo "Processing: ${chart} - folder: ${maxfolderversion} - version: ${maxchartversion}"
-                helm dependency update ${chart}/${maxfolderversion}
-                if [ "${maxfolderversion}" != "${maxchartversion}" ]; then
-                    mv -f ${chart}/${maxfolderversion} ${chart}/${maxchartversion}
-                    echo "renamed ${chart}/${maxfolderversion} to ${chart}/${maxchartversion}"
-                fi
-            fi
-          done
-
-      - name: update folder names and dependencies for incubator
-        run: |
-          for chart in dev/*; do
-            if [ -d "${chart}" ]; then
-                maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
-                maxchartversion=$(cat ${chart}/${maxfolderversion}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
-                chartname=$(basename ${chart})
-                echo "Processing: ${chart} - folder: ${maxfolderversion} - version: ${maxchartversion}"
-                helm dependency update ${chart}/${maxfolderversion}
-                if [ "${maxfolderversion}" != "${maxchartversion}" ]; then
-                    mv -f ${chart}/${maxfolderversion} ${chart}/${maxchartversion}
-                    echo "renamed ${chart}/${maxfolderversion} to ${chart}/${maxchartversion}"
-                fi
-            fi
-          done
-
-      - name: Commit and push updated charts
-        run: |
-            git add --all
-            git commit --all -m "Update-Folder-Names-and-Dependencies" || exit 0
-            git push || exit 0

.github/workflows/docs.copy.yaml

@@ -61,13 +61,12 @@ jobs:
         run: |
           for chart in stable/*; do
             if [ -d "${chart}" ]; then
-                maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
                 chartname=$(basename ${chart})
-                echo "Processing: ${chart} - folder: ${maxfolderversion}"
+                echo "Processing: ${chart}"
                 mkdir -p docs/apps/stable/${chartname} || echo "app path already exists, continuing..."
-                yes | cp -rf ${chart}/${maxfolderversion}/README.md docs/apps/stable/${chartname}/index.md || echo "readme copy failed, continuing..."
-                yes | cp -rf ${chart}/${maxfolderversion}/CONFIG.md docs/apps/stable/${chartname}/CONFIG.md || echo "config copy failed, continuing..."
-                yes | cp -rf ${chart}/${maxfolderversion}/LICENSE docs/apps/stable/${chartname}/LICENSE.md || echo "license copy failed, continuing..."
+                yes | cp -rf ${chart}/README.md docs/apps/stable/${chartname}/index.md || echo "readme copy failed, continuing..."
+                yes | cp -rf ${chart}/CONFIG.md docs/apps/stable/${chartname}/CONFIG.md || echo "config copy failed, continuing..."
+                yes | cp -rf ${chart}/LICENSE docs/apps/stable/${chartname}/LICENSE.md || echo "license copy failed, continuing..."
                 sed -i '1s/^/# License<br>\n\n/' docs/apps/stable/${chartname}/LICENSE.md || echo "license edit failed, continuing..."
             fi
           done
@@ -77,13 +76,12 @@ jobs:
         run: |
           for chart in incubator/*; do
             if [ -d "${chart}" ]; then
-                maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
                 chartname=$(basename ${chart})
-                echo "Processing: ${chart} - folder: ${maxfolderversion}"
+                echo "Processing: ${chart}"
                 mkdir -p docs/apps/incubator/${chartname} || echo "app path already exists, continuing..."
-                yes | cp -rf ${chart}/${maxfolderversion}/README.md docs/apps/incubator/${chartname}/index.md || echo "readme copy failed, continuing..."
-                yes | cp -rf ${chart}/${maxfolderversion}/CONFIG.md docs/apps/incubator/${chartname}/CONFIG.md || echo "config copy failed, continuing..."
-                yes | cp -rf ${chart}/${maxfolderversion}/LICENSE docs/apps/incubator/${chartname}/LICENSE.md || echo "license copy failed, continuing..."
+                yes | cp -rf ${chart}/README.md docs/apps/incubator/${chartname}/index.md || echo "readme copy failed, continuing..."
+                yes | cp -rf ${chart}/CONFIG.md docs/apps/incubator/${chartname}/CONFIG.md || echo "config copy failed, continuing..."
+                yes | cp -rf ${chart}/LICENSE docs/apps/incubator/${chartname}/LICENSE.md || echo "license copy failed, continuing..."
                 sed -i '1s/^/# License<br>\n\n/' docs/apps/incubator/${chartname}/LICENSE.md || echo "license edit failed, continuing..."
             fi
           done