From e24e6ca571c40c8910631f45f0503a8128d77f87 Mon Sep 17 00:00:00 2001 From: Craig Rueda Date: Thu, 30 Apr 2020 10:55:43 -0700 Subject: [PATCH] [Helm] - Allow for customization of release name (#9694) * Updated configurations to allow for more customization * Adding gitignore for requirements.lock * Moving Helm chart up a level * Adding bootstrap script / switching image * Adding bootstrap script / switching image --- helm/superset/.gitignore | 4 + {install/helm => helm}/superset/.helmignore | 0 {install/helm => helm}/superset/Chart.yaml | 0 .../helm => helm}/superset/requirements.yaml | 0 .../superset/templates/NOTES.txt | 0 .../superset/templates/_helpers.tpl | 37 +++++--- .../superset/templates/deployment.yaml | 61 +++++-------- .../superset/templates/ingress.yaml | 0 .../superset/templates/init-job.yaml | 61 ++++--------- .../superset/templates/secret-env.yaml | 16 ++-- .../templates/secret-superset-config.yaml | 4 +- .../superset/templates/service.yaml | 0 {install/helm => helm}/superset/values.yaml | 91 +++++++++++++------ 13 files changed, 146 insertions(+), 128 deletions(-) create mode 100644 helm/superset/.gitignore rename {install/helm => helm}/superset/.helmignore (100%) rename {install/helm => helm}/superset/Chart.yaml (100%) rename {install/helm => helm}/superset/requirements.yaml (100%) rename {install/helm => helm}/superset/templates/NOTES.txt (100%) rename {install/helm => helm}/superset/templates/_helpers.tpl (74%) rename {install/helm => helm}/superset/templates/deployment.yaml (62%) rename {install/helm => helm}/superset/templates/ingress.yaml (100%) rename {install/helm => helm}/superset/templates/init-job.yaml (50%) rename install/helm/superset/templates/secret.yaml => helm/superset/templates/secret-env.yaml (67%) rename {install/helm => helm}/superset/templates/secret-superset-config.yaml (84%) rename {install/helm => helm}/superset/templates/service.yaml (100%) rename {install/helm => helm}/superset/values.yaml (68%) diff --git a/helm/superset/.gitignore b/helm/superset/.gitignore new file mode 100644 index 000000000..2e16b9f2f --- /dev/null +++ b/helm/superset/.gitignore @@ -0,0 +1,4 @@ +charts + +# Don't require this to be pushed, as it will require things to be kept in sync and linted +requirements.lock diff --git a/install/helm/superset/.helmignore b/helm/superset/.helmignore similarity index 100% rename from install/helm/superset/.helmignore rename to helm/superset/.helmignore diff --git a/install/helm/superset/Chart.yaml b/helm/superset/Chart.yaml similarity index 100% rename from install/helm/superset/Chart.yaml rename to helm/superset/Chart.yaml diff --git a/install/helm/superset/requirements.yaml b/helm/superset/requirements.yaml similarity index 100% rename from install/helm/superset/requirements.yaml rename to helm/superset/requirements.yaml diff --git a/install/helm/superset/templates/NOTES.txt b/helm/superset/templates/NOTES.txt similarity index 100% rename from install/helm/superset/templates/NOTES.txt rename to helm/superset/templates/NOTES.txt diff --git a/install/helm/superset/templates/_helpers.tpl b/helm/superset/templates/_helpers.tpl similarity index 74% rename from install/helm/superset/templates/_helpers.tpl rename to helm/superset/templates/_helpers.tpl index d5dd26f6a..b56c81743 100644 --- a/install/helm/superset/templates/_helpers.tpl +++ b/helm/superset/templates/_helpers.tpl @@ -49,22 +49,35 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{- define "superset-connections.script" }} +{{- define "superset-bootstrap" }} +#!/bin/sh + +pip install {{ range .Values.additionalRequirements }}{{ . }} {{ end }} + +{{ end -}} + +{{- define "superset-config" }} import os from werkzeug.contrib.cache import RedisCache -MAPBOX_API_KEY = os.getenv('MAPBOX_API_KEY', '') +def env(key, default=None): + return os.getenv(key, default) + +MAPBOX_API_KEY = env('MAPBOX_API_KEY', '') CACHE_CONFIG = { 'CACHE_TYPE': 'redis', 'CACHE_DEFAULT_TIMEOUT': 300, 'CACHE_KEY_PREFIX': 'superset_', - 'CACHE_REDIS_HOST': os.getenv('REDIS_HOST'), - 'CACHE_REDIS_PORT': os.getenv('REDIS_PORT'), + 'CACHE_REDIS_HOST': env('REDIS_HOST'), + 'CACHE_REDIS_PORT': env('REDIS_PORT'), 'CACHE_REDIS_DB': 1, - 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (os.getenv('REDIS_HOST'),os.getenv('REDIS_PORT'))} -SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://%s:%s@%s:%s/%s' % (os.getenv('DB_USER'), os.getenv('DB_PASS'), os.getenv('DB_HOST'), os.getenv('DB_PORT'), os.getenv('DB_NAME')) + 'CACHE_REDIS_URL': f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/1" +} + +SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg2://{env('DB_USER')}:{env('DB_PASS')}@{env('DB_HOST')}:{env('DB_PORT')}/{env('DB_NAME')}" SQLALCHEMY_TRACK_MODIFICATIONS = True -SECRET_KEY = 'thisISaSECRET_1234' +SECRET_KEY = env('SECRET_KEY', 'thisISaSECRET_1234') + # Flask-WTF flag for CSRF WTF_CSRF_ENABLED = True # Add endpoints that need to be exempt from CSRF protection @@ -72,15 +85,15 @@ WTF_CSRF_EXEMPT_LIST = [] # A CSRF token that expires in 1 year WTF_CSRF_TIME_LIMIT = 60 * 60 * 24 * 365 class CeleryConfig(object): - BROKER_URL = 'redis://%s:%s/0' % (os.getenv('REDIS_HOST'),os.getenv('REDIS_PORT')) + BROKER_URL = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0" CELERY_IMPORTS = ('superset.sql_lab', ) - CELERY_RESULT_BACKEND = 'redis://%s:%s/0' % (os.getenv('REDIS_HOST'),os.getenv('REDIS_PORT')) + CELERY_RESULT_BACKEND = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0" CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} CELERY_CONFIG = CeleryConfig RESULTS_BACKEND = RedisCache( - host= os.getenv('REDIS_HOST'), - port= os.getenv('REDIS_PORT'), + host=env('REDIS_HOST'), + port=env('REDIS_PORT'), key_prefix='superset_results' ) -{{- end }} \ No newline at end of file +{{- end }} diff --git a/install/helm/superset/templates/deployment.yaml b/helm/superset/templates/deployment.yaml similarity index 62% rename from install/helm/superset/templates/deployment.yaml rename to helm/superset/templates/deployment.yaml index da2ee8867..063454619 100644 --- a/install/helm/superset/templates/deployment.yaml +++ b/helm/superset/templates/deployment.yaml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ template "superset.fullname" . }} @@ -35,53 +35,36 @@ spec: app: {{ template "superset.name" . }} release: {{ .Release.Name }} spec: + securityContext: + runAsUser: 0 # Needed in order to allow pip install to work in bootstrap + {{- if .Values.supersetNode.initContainers }} + initContainers: + {{- tpl (toYaml .Values.supersetNode.initContainers) . | nindent 6 }} + {{- end }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + command: {{ tpl (toJson .Values.supersetNode.command) . }} env: - - name: REDIS_HOST - valueFrom: - secretKeyRef: - name: superset-secret - key: redis_host - - name: REDIS_PORT - valueFrom: - secretKeyRef: - name: superset-secret - key: redis_port - - name: DB_HOST - valueFrom: - secretKeyRef: - name: superset-secret - key: db_host - - name: DB_PORT - valueFrom: - secretKeyRef: - name: superset-secret - key: db_port - - name: DB_USER - valueFrom: - secretKeyRef: - name: superset-secret - key: db_user - - name: DB_PASS - valueFrom: - secretKeyRef: - name: superset-secret - key: db_pass - - name: DB_NAME - valueFrom: - secretKeyRef: - name: superset-secret - key: db_name + - name: "SUPERSET_PORT" + value: {{ .Values.service.port | quote}} + {{ if .Values.extraEnv }} + {{- range $key, $value := .Values.extraEnv }} + - name: {{ $key | quote}} + value: {{ $value | quote }} + {{- end }} + {{- end }} + envFrom: + - secretRef: + name: {{ tpl .Values.envFromSecret . | quote }} volumeMounts: - name: superset-config - mountPath: "/etc/superset" + mountPath: {{ .Values.configMountPath | quote }} readOnly: true ports: - name: http - containerPort: 8088 + containerPort: {{ .Values.service.port }} protocol: TCP resources: {{ toYaml .Values.resources | indent 12 }} @@ -100,4 +83,4 @@ spec: volumes: - name: superset-config secret: - secretName: superset-config \ No newline at end of file + secretName: {{ tpl .Values.configFromSecret . }} diff --git a/install/helm/superset/templates/ingress.yaml b/helm/superset/templates/ingress.yaml similarity index 100% rename from install/helm/superset/templates/ingress.yaml rename to helm/superset/templates/ingress.yaml diff --git a/install/helm/superset/templates/init-job.yaml b/helm/superset/templates/init-job.yaml similarity index 50% rename from install/helm/superset/templates/init-job.yaml rename to helm/superset/templates/init-job.yaml index 5b287a421..58ab564ac 100644 --- a/install/helm/superset/templates/init-job.yaml +++ b/helm/superset/templates/init-job.yaml @@ -14,67 +14,44 @@ # See the License for the specific language governing permissions and # limitations under the License. # -{{- if and ( .Values.initContainers ) ( .Values.init.enabled ) }} +{{- if .Values.init.enabled }} apiVersion: batch/v1 kind: Job metadata: name: {{ template "superset.name" . }}-init-db - annotations: spec: template: metadata: name: {{ template "superset.name" . }}-init-db spec: + securityContext: + runAsUser: 0 # Needed in order to allow pip install to work in bootstrap + {{- if .Values.init.initContainers }} initContainers: - {{- toYaml .Values.initContainers | nindent 6 }} + {{- tpl (toYaml .Values.init.initContainers) . | nindent 6 }} + {{- end }} containers: - name: {{ template "superset.name" . }}-init-db image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + {{ if .Values.extraEnv }} env: - - name: REDIS_HOST - valueFrom: - secretKeyRef: - name: superset-secret - key: redis_host - - name: REDIS_PORT - valueFrom: - secretKeyRef: - name: superset-secret - key: redis_port - - name: DB_HOST - valueFrom: - secretKeyRef: - name: superset-secret - key: db_host - - name: DB_PORT - valueFrom: - secretKeyRef: - name: superset-secret - key: db_port - - name: DB_USER - valueFrom: - secretKeyRef: - name: superset-secret - key: db_user - - name: DB_PASS - valueFrom: - secretKeyRef: - name: superset-secret - key: db_pass - - name: DB_NAME - valueFrom: - secretKeyRef: - name: superset-secret - key: db_name + {{- range $key, $value := .Values.extraEnv }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + envFrom: + - secretRef: + name: {{ tpl .Values.envFromSecret . }} imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - name: superset-config - mountPath: "/etc/superset" + mountPath: {{ .Values.configMountPath | quote }} readOnly: true - command: [ "/bin/sh", "-c", "{{ .Values.init.initscript }}" ] + command: {{ tpl (toJson .Values.init.command) . }} volumes: - name: superset-config secret: - secretName: superset-config + secretName: {{ tpl .Values.configFromSecret . }} restartPolicy: Never -{{- end }} \ No newline at end of file +{{- end }} diff --git a/install/helm/superset/templates/secret.yaml b/helm/superset/templates/secret-env.yaml similarity index 67% rename from install/helm/superset/templates/secret.yaml rename to helm/superset/templates/secret-env.yaml index 6f326154a..6c8625295 100644 --- a/install/helm/superset/templates/secret.yaml +++ b/helm/superset/templates/secret-env.yaml @@ -17,7 +17,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ template "superset.fullname" . }}-secret + name: {{ template "superset.fullname" . }}-env labels: app: {{ template "superset.fullname" . }} chart: {{ template "superset.chart" . }} @@ -25,10 +25,10 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - redis_host: {{ .Values.supersetNode.connections.redis_host | b64enc | quote }} - redis_port: {{ .Values.supersetNode.connections.redis_port | b64enc | quote }} - db_host: {{ .Values.supersetNode.connections.db_host | b64enc | quote }} - db_port: {{ .Values.supersetNode.connections.db_port | b64enc | quote }} - db_user: {{ .Values.supersetNode.connections.db_user | b64enc | quote }} - db_pass: {{ .Values.supersetNode.connections.db_pass | b64enc | quote }} - db_name: {{ .Values.supersetNode.connections.db_name | b64enc | quote }} \ No newline at end of file + REDIS_HOST: {{ tpl .Values.supersetNode.connections.redis_host . | b64enc | quote }} + REDIS_PORT: {{ .Values.supersetNode.connections.redis_port | b64enc | quote }} + DB_HOST: {{ tpl .Values.supersetNode.connections.db_host . | b64enc | quote }} + DB_PORT: {{ .Values.supersetNode.connections.db_port | b64enc | quote }} + DB_USER: {{ .Values.supersetNode.connections.db_user | b64enc | quote }} + DB_PASS: {{ .Values.supersetNode.connections.db_pass | b64enc | quote }} + DB_NAME: {{ .Values.supersetNode.connections.db_name | b64enc | quote }} diff --git a/install/helm/superset/templates/secret-superset-config.yaml b/helm/superset/templates/secret-superset-config.yaml similarity index 84% rename from install/helm/superset/templates/secret-superset-config.yaml rename to helm/superset/templates/secret-superset-config.yaml index 2886cfad6..604bb2001 100644 --- a/install/helm/superset/templates/secret-superset-config.yaml +++ b/helm/superset/templates/secret-superset-config.yaml @@ -25,4 +25,6 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - superset_config.py: {{ include "superset-connections.script" . | b64enc }} \ No newline at end of file + superset_config.py: {{ include "superset-config" . | b64enc }} + superset_init.sh: {{ tpl .Values.init.initscript . | b64enc }} + superset_bootstrap.sh: {{ include "superset-bootstrap" . | b64enc }} diff --git a/install/helm/superset/templates/service.yaml b/helm/superset/templates/service.yaml similarity index 100% rename from install/helm/superset/templates/service.yaml rename to helm/superset/templates/service.yaml diff --git a/install/helm/superset/values.yaml b/helm/superset/values.yaml similarity index 68% rename from install/helm/superset/values.yaml rename to helm/superset/values.yaml index 4db713e4b..bfeaee1cc 100644 --- a/install/helm/superset/values.yaml +++ b/helm/superset/values.yaml @@ -21,27 +21,33 @@ replicaCount: 1 +## These requirements are used to build a requirements file which is then applied on init +## of superset containers +additionalRequirements: + - "psycopg2==2.8.3" + - "redis==3.2.1" + +## The name of the secret which we will use to generate a superset_config.py file +## Note: this secret must have the key superset_config.py in it and can include other files as well +## +configFromSecret: '{{ template "superset.fullname" . }}-config' + +## The name of the secret which we will use to populate env vars in deployed pods +## This can be useful for secret keys, etc. +## +envFromSecret: '{{ template "superset.fullname" . }}-env' + +## Extra environment variables that will be passed into pods +## +extraEnv: {} + +configMountPath: "/app/pythonpath" + image: - repository: amancevice/superset + repository: preset/superset tag: latest pullPolicy: IfNotPresent -initContainers: - - name: wait-for-postgres - image: busybox:latest - imagePullPolicy: IfNotPresent - env: - - name: DB_HOST - valueFrom: - secretKeyRef: - name: superset-secret - key: db_host - - name: DB_PORT - valueFrom: - secretKeyRef: - name: superset-secret - key: db_port - command: [ "/bin/sh", "-c", "until nc -zv $DB_HOST $DB_PORT -w1; do echo 'waiting for db'; sleep 1; done" ] service: type: NodePort port: 8088 @@ -70,32 +76,65 @@ resources: {} # requests: # cpu: 100m # memory: 128Mi -#Superset node configuration + +## +## Superset node configuration supersetNode: + command: + - "/bin/sh" + - "-c" + - ". {{ .Values.configMountPath }}/superset_bootstrap.sh; /usr/bin/docker-entrypoint.sh" connections: - redis_host: superset-redis-headless + redis_host: '{{ template "superset.fullname" . }}-redis-headless' redis_port: "6379" - db_host: superset-postgresql + db_host: '{{ template "superset.fullname" . }}-postgresql' db_port: "5432" db_user: superset db_pass: superset db_name: superset + initContainers: + - name: wait-for-postgres + image: busybox:latest + imagePullPolicy: IfNotPresent + envFrom: + - secretRef: + name: '{{ tpl .Values.envFromSecret . }}' + command: [ "/bin/sh", "-c", "until nc -zv $DB_HOST $DB_PORT -w1; do echo 'waiting for db'; sleep 1; done" ] -# ----------------------------------------------------------------------------- -# Miscellaneous parameters -# ----------------------------------------------------------------------------- - +## +## Init job configuration init: + command: + - "/bin/sh" + - "-c" + - ". {{ .Values.configMountPath }}/superset_bootstrap.sh; . {{ .Values.configMountPath }}/superset_init.sh" enabled: true + loadExamples: false + initContainers: + - name: wait-for-postgres + image: busybox:latest + imagePullPolicy: IfNotPresent + envFrom: + - secretRef: + name: '{{ tpl .Values.envFromSecret . }}' + command: [ "/bin/sh", "-c", "until nc -zv $DB_HOST $DB_PORT -w1; do echo 'waiting for db'; sleep 1; done" ] initscript: |- - superset db upgrade && \ - superset init && \ + #!/bin/sh + echo "Upgrading DB schema..." + superset db upgrade + echo "Initializing roles..." + superset init + echo "Creating admin user..." superset fab create-admin \ --username admin \ --firstname Superset \ --lastname Admin \ --email admin@superset.com \ --password admin || true + {{ if .Values.init.loadExamples }} + echo "Loading examples..." + superset load_examples + {{- end }} ## ## Configuration values for the postgresql dependency. ## ref: https://github.com/kubernetes/charts/blob/master/stable/postgresql/README.md