Add CronJob that unseals OpenBao automatically after restarts
OpenBao comes back sealed whenever its pod restarts (containerd/MicroK8s restarts took every pod down 6-11 times in three days), and while sealed External Secrets cannot refresh any secret. The job checks every minute and, if sealed, applies the keys from the existing openbao-unseal-keys Secret with the same steps as scripts/unseal.sh. It talks to the pod's headless-service address and verifies the cert-manager CA. Disable with autoUnseal.enabled=false. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
@@ -0,0 +1,120 @@
|
|||||||
|
{{- if .Values.autoUnseal.enabled }}
|
||||||
|
# Unseals OpenBao automatically after it (re)starts sealed. Runs the same steps as
|
||||||
|
# scripts/unseal.sh, but from a CronJob and over the network:
|
||||||
|
# status -> (if sealed) reset progress -> send the keys on stdin -> verify.
|
||||||
|
#
|
||||||
|
# SECURITY MODEL: the unseal keys are read from the Secret `{{ .Values.autoUnseal.keysSecretName }}`,
|
||||||
|
# i.e. they live in the same cluster as OpenBao - exactly like the manual procedure that
|
||||||
|
# already reads them from there. This removes the manual step, not the trust assumption:
|
||||||
|
# anyone who can read Secrets in this namespace can unseal OpenBao. Set
|
||||||
|
# autoUnseal.enabled=false to go back to unsealing by hand.
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: openbao-auto-unseal
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: openbao-auto-unseal
|
||||||
|
spec:
|
||||||
|
schedule: {{ .Values.autoUnseal.schedule | quote }}
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
startingDeadlineSeconds: 60
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 3
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 0
|
||||||
|
activeDeadlineSeconds: 55
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: openbao-auto-unseal
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 100
|
||||||
|
runAsGroup: 1000
|
||||||
|
fsGroup: 1000
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: unseal
|
||||||
|
image: {{ .Values.autoUnseal.image }}
|
||||||
|
env:
|
||||||
|
# The pod itself (headless Service), because the regular Service only lists
|
||||||
|
# READY pods and a sealed OpenBao is never ready.
|
||||||
|
- name: BAO_ADDR
|
||||||
|
value: "https://{{ include "openbao-gitops.openbaoFullname" . }}-0.{{ include "openbao-gitops.openbaoFullname" . }}-internal.{{ .Release.Namespace }}.svc:8200"
|
||||||
|
# Verify against the cert-manager CA; the certificate names the Service, not the pod.
|
||||||
|
- name: BAO_CACERT
|
||||||
|
value: /tls/ca.crt
|
||||||
|
- name: BAO_TLS_SERVER_NAME
|
||||||
|
value: {{ include "openbao-gitops.openbaoFullname" . }}.{{ .Release.Namespace }}.svc
|
||||||
|
- name: HOME
|
||||||
|
value: /tmp
|
||||||
|
- name: KEYS_FILE
|
||||||
|
value: /keys/{{ .Values.autoUnseal.keysSecretKey }}
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
set -u
|
||||||
|
is_true() { grep -Eq "\"$1\":[[:space:]]*true"; }
|
||||||
|
json_int() { sed -nE "s/.*\"$1\":[[:space:]]*([0-9]+).*/\1/p" | head -n1; }
|
||||||
|
|
||||||
|
# `bao status` exits 2 while sealed but still prints the JSON.
|
||||||
|
STATUS=$(bao status -format=json 2>/dev/null </dev/null || true)
|
||||||
|
if [ -z "$STATUS" ]; then echo "OpenBao not reachable (starting?) - next run will retry"; exit 0; fi
|
||||||
|
if ! printf '%s' "$STATUS" | is_true initialized; then echo "OpenBao is not initialized - nothing to unseal"; exit 0; fi
|
||||||
|
if ! printf '%s' "$STATUS" | is_true sealed; then exit 0; fi # unsealed: stay quiet
|
||||||
|
|
||||||
|
THRESHOLD=$(printf '%s' "$STATUS" | json_int t); THRESHOLD=${THRESHOLD:-3}
|
||||||
|
echo "OpenBao is sealed (threshold $THRESHOLD) - unsealing"
|
||||||
|
[ -s "$KEYS_FILE" ] || { echo "ERROR: no keys in $KEYS_FILE"; exit 1; }
|
||||||
|
|
||||||
|
bao operator unseal -reset >/dev/null </dev/null # drop half-entered keys
|
||||||
|
count=0
|
||||||
|
while IFS= read -r key; do
|
||||||
|
key=$(printf '%s' "$key" | tr -d '\r'); [ -n "$key" ] || continue
|
||||||
|
count=$((count + 1))
|
||||||
|
RESP=$(printf '%s' "$key" | bao write -format=json sys/unseal key=- 2>&1) \
|
||||||
|
|| { echo "ERROR: unseal request failed at key $count"; exit 1; }
|
||||||
|
if printf '%s' "$RESP" | is_true sealed; then
|
||||||
|
echo " key $count accepted (progress $(printf '%s' "$RESP" | json_int progress)/$THRESHOLD)"
|
||||||
|
else
|
||||||
|
echo " key $count accepted"; break
|
||||||
|
fi
|
||||||
|
done < "$KEYS_FILE"
|
||||||
|
|
||||||
|
if bao status -format=json 2>/dev/null </dev/null | is_true sealed; then
|
||||||
|
echo "ERROR: still sealed after $count key(s) - check the keys in the Secret"; exit 1
|
||||||
|
fi
|
||||||
|
echo "OpenBao is unsealed."
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop: ["ALL"]
|
||||||
|
resources:
|
||||||
|
requests: { cpu: 10m, memory: 32Mi }
|
||||||
|
limits: { memory: 128Mi }
|
||||||
|
volumeMounts:
|
||||||
|
- { name: keys, mountPath: /keys, readOnly: true }
|
||||||
|
- { name: tls, mountPath: /tls, readOnly: true }
|
||||||
|
- { name: tmp, mountPath: /tmp }
|
||||||
|
volumes:
|
||||||
|
- name: keys
|
||||||
|
secret:
|
||||||
|
secretName: {{ .Values.autoUnseal.keysSecretName }}
|
||||||
|
defaultMode: 0440
|
||||||
|
- name: tls
|
||||||
|
secret:
|
||||||
|
secretName: {{ .Values.tls.secretName }}
|
||||||
|
items:
|
||||||
|
- key: ca.crt
|
||||||
|
path: ca.crt
|
||||||
|
- name: tmp
|
||||||
|
emptyDir: {}
|
||||||
|
{{- end }}
|
||||||
+14
@@ -102,6 +102,20 @@ eso:
|
|||||||
# KV v2 mount path that ClusterSecretStore(s) will read from.
|
# KV v2 mount path that ClusterSecretStore(s) will read from.
|
||||||
kvMountPath: secret
|
kvMountPath: secret
|
||||||
|
|
||||||
|
# Automatic unseal after (re)starts. OpenBao comes back SEALED whenever its pod restarts
|
||||||
|
# (which happens whenever containerd/MicroK8s restarts), and while it is sealed External
|
||||||
|
# Secrets cannot refresh any secret. This CronJob checks every minute and, if sealed, applies
|
||||||
|
# the keys from the Secret below - the same procedure as scripts/unseal.sh.
|
||||||
|
# Trade-off: the keys stay in the cluster (as they do for the manual procedure). To go back
|
||||||
|
# to unsealing by hand set `enabled: false`.
|
||||||
|
autoUnseal:
|
||||||
|
enabled: true
|
||||||
|
schedule: "* * * * *"
|
||||||
|
image: openbao/openbao:2.6.2
|
||||||
|
# Secret holding the unseal keys, one per line.
|
||||||
|
keysSecretName: openbao-unseal-keys
|
||||||
|
keysSecretKey: keys
|
||||||
|
|
||||||
# One-time bootstrap Job that enables the Kubernetes auth method, creates the
|
# One-time bootstrap Job that enables the Kubernetes auth method, creates the
|
||||||
# "eso-read" policy and the "eso-role" role bound to the ESO ServiceAccount.
|
# "eso-read" policy and the "eso-role" role bound to the ESO ServiceAccount.
|
||||||
#
|
#
|
||||||
|
|||||||
Reference in New Issue
Block a user