Add CronJob that unseals OpenBao automatically after restarts
OpenBao comes back sealed whenever its pod restarts (containerd/MicroK8s restarts took every pod down 6-11 times in three days), and while sealed External Secrets cannot refresh any secret. The job checks every minute and, if sealed, applies the keys from the existing openbao-unseal-keys Secret with the same steps as scripts/unseal.sh. It talks to the pod's headless-service address and verifies the cert-manager CA. Disable with autoUnseal.enabled=false. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
{{- if .Values.autoUnseal.enabled }}
|
||||
# Unseals OpenBao automatically after it (re)starts sealed. Runs the same steps as
|
||||
# scripts/unseal.sh, but from a CronJob and over the network:
|
||||
# status -> (if sealed) reset progress -> send the keys on stdin -> verify.
|
||||
#
|
||||
# SECURITY MODEL: the unseal keys are read from the Secret `{{ .Values.autoUnseal.keysSecretName }}`,
|
||||
# i.e. they live in the same cluster as OpenBao - exactly like the manual procedure that
|
||||
# already reads them from there. This removes the manual step, not the trust assumption:
|
||||
# anyone who can read Secrets in this namespace can unseal OpenBao. Set
|
||||
# autoUnseal.enabled=false to go back to unsealing by hand.
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: openbao-auto-unseal
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: openbao-auto-unseal
|
||||
spec:
|
||||
schedule: {{ .Values.autoUnseal.schedule | quote }}
|
||||
concurrencyPolicy: Forbid
|
||||
startingDeadlineSeconds: 60
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 55
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: openbao-auto-unseal
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 100
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: unseal
|
||||
image: {{ .Values.autoUnseal.image }}
|
||||
env:
|
||||
# The pod itself (headless Service), because the regular Service only lists
|
||||
# READY pods and a sealed OpenBao is never ready.
|
||||
- name: BAO_ADDR
|
||||
value: "https://{{ include "openbao-gitops.openbaoFullname" . }}-0.{{ include "openbao-gitops.openbaoFullname" . }}-internal.{{ .Release.Namespace }}.svc:8200"
|
||||
# Verify against the cert-manager CA; the certificate names the Service, not the pod.
|
||||
- name: BAO_CACERT
|
||||
value: /tls/ca.crt
|
||||
- name: BAO_TLS_SERVER_NAME
|
||||
value: {{ include "openbao-gitops.openbaoFullname" . }}.{{ .Release.Namespace }}.svc
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
- name: KEYS_FILE
|
||||
value: /keys/{{ .Values.autoUnseal.keysSecretKey }}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -u
|
||||
is_true() { grep -Eq "\"$1\":[[:space:]]*true"; }
|
||||
json_int() { sed -nE "s/.*\"$1\":[[:space:]]*([0-9]+).*/\1/p" | head -n1; }
|
||||
|
||||
# `bao status` exits 2 while sealed but still prints the JSON.
|
||||
STATUS=$(bao status -format=json 2>/dev/null </dev/null || true)
|
||||
if [ -z "$STATUS" ]; then echo "OpenBao not reachable (starting?) - next run will retry"; exit 0; fi
|
||||
if ! printf '%s' "$STATUS" | is_true initialized; then echo "OpenBao is not initialized - nothing to unseal"; exit 0; fi
|
||||
if ! printf '%s' "$STATUS" | is_true sealed; then exit 0; fi # unsealed: stay quiet
|
||||
|
||||
THRESHOLD=$(printf '%s' "$STATUS" | json_int t); THRESHOLD=${THRESHOLD:-3}
|
||||
echo "OpenBao is sealed (threshold $THRESHOLD) - unsealing"
|
||||
[ -s "$KEYS_FILE" ] || { echo "ERROR: no keys in $KEYS_FILE"; exit 1; }
|
||||
|
||||
bao operator unseal -reset >/dev/null </dev/null # drop half-entered keys
|
||||
count=0
|
||||
while IFS= read -r key; do
|
||||
key=$(printf '%s' "$key" | tr -d '\r'); [ -n "$key" ] || continue
|
||||
count=$((count + 1))
|
||||
RESP=$(printf '%s' "$key" | bao write -format=json sys/unseal key=- 2>&1) \
|
||||
|| { echo "ERROR: unseal request failed at key $count"; exit 1; }
|
||||
if printf '%s' "$RESP" | is_true sealed; then
|
||||
echo " key $count accepted (progress $(printf '%s' "$RESP" | json_int progress)/$THRESHOLD)"
|
||||
else
|
||||
echo " key $count accepted"; break
|
||||
fi
|
||||
done < "$KEYS_FILE"
|
||||
|
||||
if bao status -format=json 2>/dev/null </dev/null | is_true sealed; then
|
||||
echo "ERROR: still sealed after $count key(s) - check the keys in the Secret"; exit 1
|
||||
fi
|
||||
echo "OpenBao is unsealed."
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
resources:
|
||||
requests: { cpu: 10m, memory: 32Mi }
|
||||
limits: { memory: 128Mi }
|
||||
volumeMounts:
|
||||
- { name: keys, mountPath: /keys, readOnly: true }
|
||||
- { name: tls, mountPath: /tls, readOnly: true }
|
||||
- { name: tmp, mountPath: /tmp }
|
||||
volumes:
|
||||
- name: keys
|
||||
secret:
|
||||
secretName: {{ .Values.autoUnseal.keysSecretName }}
|
||||
defaultMode: 0440
|
||||
- name: tls
|
||||
secret:
|
||||
secretName: {{ .Values.tls.secretName }}
|
||||
items:
|
||||
- key: ca.crt
|
||||
path: ca.crt
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
+14
@@ -102,6 +102,20 @@ eso:
|
||||
# KV v2 mount path that ClusterSecretStore(s) will read from.
|
||||
kvMountPath: secret
|
||||
|
||||
# Automatic unseal after (re)starts. OpenBao comes back SEALED whenever its pod restarts
|
||||
# (which happens whenever containerd/MicroK8s restarts), and while it is sealed External
|
||||
# Secrets cannot refresh any secret. This CronJob checks every minute and, if sealed, applies
|
||||
# the keys from the Secret below - the same procedure as scripts/unseal.sh.
|
||||
# Trade-off: the keys stay in the cluster (as they do for the manual procedure). To go back
|
||||
# to unsealing by hand set `enabled: false`.
|
||||
autoUnseal:
|
||||
enabled: true
|
||||
schedule: "* * * * *"
|
||||
image: openbao/openbao:2.6.2
|
||||
# Secret holding the unseal keys, one per line.
|
||||
keysSecretName: openbao-unseal-keys
|
||||
keysSecretKey: keys
|
||||
|
||||
# One-time bootstrap Job that enables the Kubernetes auth method, creates the
|
||||
# "eso-read" policy and the "eso-role" role bound to the ESO ServiceAccount.
|
||||
#
|
||||
|
||||
Reference in New Issue
Block a user