Fix update helper health for ready failed release (#361)

src/server/update-helper/service.test.ts

@@ -351,4 +351,78 @@ describe('update helper service', () => {
       ],
     });
   });
+
+  it('treats ready workloads as healthy even when the latest helm revision is marked failed', async () => {
+    const status = await getUpdateHelperStatus(
+      {
+        namespace: 'ai',
+        releaseName: 'metapi',
+      },
+      {
+        runCommand: async ({ command, args }) => {
+          if (command === 'helm' && args[0] === 'status') {
+            return {
+              stdout: JSON.stringify({
+                version: 22,
+                info: {
+                  status: 'failed',
+                  description: 'Upgrade "metapi" failed: context deadline exceeded',
+                },
+              }),
+            };
+          }
+          if (command === 'helm' && args[0] === 'get') {
+            return {
+              stdout: JSON.stringify({
+                image: {
+                  repository: '1467078763/metapi',
+                  tag: 'latest',
+                  digest: 'sha256:1523dfcbaca146b8a64a3a022878165027900ee4cb396a45e474aa7a969dacb1',
+                },
+              }),
+            };
+          }
+          if (command === 'helm' && args[0] === 'history') {
+            return {
+              stdout: JSON.stringify([
+                { revision: '21', updated: '2026-03-31T14:07:40.115551823Z', status: 'deployed', description: 'Rollback to 17' },
+                { revision: '22', updated: '2026-03-31T14:13:24.726024175Z', status: 'failed', description: 'Upgrade "metapi" failed: context deadline exceeded' },
+              ]),
+            };
+          }
+          if (command === 'kubectl' && args[0] === 'get') {
+            return {
+              stdout: JSON.stringify({
+                items: [
+                  {
+                    status: {
+                      phase: 'Running',
+                      containerStatuses: [
+                        {
+                          ready: true,
+                          imageID: 'docker-pullable://1467078763/metapi@sha256:1523dfcbaca146b8a64a3a022878165027900ee4cb396a45e474aa7a969dacb1',
+                        },
+                      ],
+                    },
+                  },
+                ],
+              }),
+            };
+          }
+          throw new Error('unexpected command');
+        },
+      },
+    );
+
+    expect(status).toMatchObject({
+      ok: true,
+      releaseName: 'metapi',
+      namespace: 'ai',
+      revision: '22',
+      imageRepository: '1467078763/metapi',
+      imageTag: 'latest',
+      imageDigest: 'sha256:1523dfcbaca146b8a64a3a022878165027900ee4cb396a45e474aa7a969dacb1',
+      healthy: true,
+    });
+  });
 });

src/server/update-helper/service.ts

@@ -82,6 +82,11 @@ type ParsedReleaseValues = {
   imageDigest: string | null;
 };
 
+type ParsedRuntimePodState = {
+  imageDigest: string | null;
+  allPodsReady: boolean;
+};
+
 async function runCommand(input: RunCommandInput): Promise<RunCommandResult> {
   return await new Promise<RunCommandResult>((resolve, reject) => {
     const child = spawn(input.command, input.args, {
@@ -172,12 +177,14 @@ function parseReleaseValues(stdout: string): ParsedReleaseValues {
   }
 }
 
-function parseRuntimeImageDigest(stdout: string): string | null {
+function parseRuntimePodState(stdout: string): ParsedRuntimePodState {
   try {
     const payload = JSON.parse(stdout || '{}') as {
       items?: Array<{
         status?: {
+          phase?: string | null;
           containerStatuses?: Array<{
+            ready?: boolean | null;
             imageID?: string | null;
             image?: string | null;
           }>;
@@ -185,18 +192,42 @@ function parseRuntimeImageDigest(stdout: string): string | null {
       }>;
     };
 
+    let imageDigest: string | null = null;
+    const items = payload.items || [];
+    if (items.length <= 0) {
+      return {
+        imageDigest: null,
+        allPodsReady: false,
+      };
+    }
+
+    let allPodsReady = true;
+
     for (const item of payload.items || []) {
-      for (const status of item.status?.containerStatuses || []) {
+      const phase = String(item.status?.phase || '').trim().toLowerCase();
+      const containerStatuses = item.status?.containerStatuses || [];
+      if (phase !== 'running' || containerStatuses.length <= 0 || containerStatuses.some((status) => status.ready !== true)) {
+        allPodsReady = false;
+      }
+
+      for (const status of containerStatuses) {
         const candidate = `${status.imageID || ''} ${status.image || ''}`;
         const match = candidate.match(/(sha256:[a-f0-9]{64})/i);
-        if (match?.[1]) {
-          return match[1].toLowerCase();
+        if (!imageDigest && match?.[1]) {
+          imageDigest = match[1].toLowerCase();
         }
       }
     }
-    return null;
+
+    return {
+      imageDigest,
+      allPodsReady,
+    };
   } catch {
-    return null;
+    return {
+      imageDigest: null,
+      allPodsReady: false,
+    };
   }
 }
 
@@ -388,8 +419,9 @@ export async function getUpdateHelperStatus(
     };
   };
   const currentValues = parseReleaseValues(values.stdout);
-  const runtimeDigest = parseRuntimeImageDigest(pods.stdout);
+  const runtimeState = parseRuntimePodState(pods.stdout);
   const historyRows = parseHistoryRows(history.stdout).slice(0, 5);
+  const releaseStatus = String(statusPayload.info?.status || '').trim().toLowerCase();
 
   const historyEntries: UpdateHelperHistoryEntry[] = [];
   for (const row of historyRows) {
@@ -426,8 +458,8 @@ export async function getUpdateHelperStatus(
     revision: statusPayload.version === undefined || statusPayload.version === null ? null : String(statusPayload.version),
     imageRepository: currentValues.imageRepository,
     imageTag: currentValues.imageTag,
-    imageDigest: runtimeDigest || currentValues.imageDigest,
-    healthy: String(statusPayload.info?.status || '').toLowerCase() === 'deployed',
+    imageDigest: runtimeState.imageDigest || currentValues.imageDigest,
+    healthy: releaseStatus === 'deployed' || (releaseStatus === 'failed' && runtimeState.allPodsReady),
     history: historyEntries,
   };
 }