Here’s a handy set of commands for removing namespaces which are stuck in “Terminating” status for whatever reason.
for ns in $(kubectl get ns --field-selector status.phase=Terminating -o jsonpath='{.items[*].metadata.name}')
do
kubectl get ns $ns -ojson | \
jq '.spec.finalizers = []' | \
kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f -
done
for ns in $(kubectl get ns --field-selector status.phase=Terminating -o jsonpath='{.items[*].metadata.name}')
do
kubectl get ns $ns -ojson | \
jq '.metadata.finalizers = []' | \
kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f -
done
It should go without saying that just yolo’ing code off the internet into your k8s environment is a bad idea, but this worked for me when trying to excise a stuck namespace.
Thanks to @rajendra off StackOverflow for this comment.
Looks like it was some sub-resources that just wouldn’t die when I was removing Longhorn from my environment (which was its own exercise in pain…)
{
"kind": "Namespace",
"apiVersion": "v1",
"metadata": {
"name": "longhorn-system",
"uid": "e01c9b0d-d95f-4f2a-aa39-e76577effa99",
"resourceVersion": "9641340",
"creationTimestamp": "2023-08-30T02:05:40Z",
"deletionTimestamp": "2023-10-31T04:43:37Z",
"labels": {
"kubernetes.io/metadata.name": "longhorn-system"
},
"managedFields": [
{
"manager": "HashiCorp",
"operation": "Update",
"apiVersion": "v1",
"time": "2023-08-30T02:05:40Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:labels": {
".": {},
"f:kubernetes.io/metadata.name": {}
}
}
}
},
{
"manager": "k3s",
"operation": "Update",
"apiVersion": "v1",
"time": "2023-10-31T04:43:43Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:status": {
"f:conditions": {
".": {},
"k:{\"type\":\"NamespaceContentRemaining\"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
},
"k:{\"type\":\"NamespaceDeletionContentFailure\"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
},
"k:{\"type\":\"NamespaceDeletionDiscoveryFailure\"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
},
"k:{\"type\":\"NamespaceDeletionGroupVersionParsingFailure\"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
},
"k:{\"type\":\"NamespaceFinalizersRemaining\"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
}
}
}
},
"subresource": "status"
}
]
},
"spec": {},
"status": {
"phase": "Terminating",
"conditions": [
{
"type": "NamespaceDeletionDiscoveryFailure",
"status": "True",
"lastTransitionTime": "2023-10-31T04:43:42Z",
"reason": "DiscoveryFailed",
"message": "Discovery failed for some groups, 1 failing: unable to retrieve the complete list of server APIs: metrics.k8s.io/v1beta1: stale GroupVersion discovery: metrics.k8s.io/v1beta1"
},
{
"type": "NamespaceDeletionGroupVersionParsingFailure",
"status": "False",
"lastTransitionTime": "2023-10-31T04:43:43Z",
"reason": "ParsedGroupVersions",
"message": "All legacy kube types successfully parsed"
},
{
"type": "NamespaceDeletionContentFailure",
"status": "False",
"lastTransitionTime": "2023-10-31T04:43:43Z",
"reason": "ContentDeleted",
"message": "All content successfully deleted, may be waiting on finalization"
},
{
"type": "NamespaceContentRemaining",
"status": "True",
"lastTransitionTime": "2023-10-31T04:43:43Z",
"reason": "SomeResourcesRemain",
"message": "Some resources are remaining: engineimages.longhorn.io has 1 resource instances, nodes.longhorn.io has 4 resource instances"
},
{
"type": "NamespaceFinalizersRemaining",
"status": "True",
"lastTransitionTime": "2023-10-31T04:43:43Z",
"reason": "SomeFinalizersRemain",
"message": "Some content in the namespace has finalizers remaining: longhorn.io in 5 resource instances"
}
]
}
}