fix: helm operation in progress

src/cmd/apply.test.ts

@@ -11,6 +11,7 @@ jest.mock('fs', () => ({
 }))
 
 jest.mock('src/common/k8s', () => ({
+  checkOperationsInProgress: jest.fn(),
   getDeploymentState: jest.fn(),
   setDeploymentState: jest.fn(),
   restartOtomiApiDeployment: jest.fn(),
@@ -93,6 +94,7 @@ describe('Apply command', () => {
       applyAsApps: require('./apply-as-apps').applyAsApps,
       commit: require('./commit').commit,
       runtimeUpgrade: require('src/common/runtime-upgrade').runtimeUpgrade,
+      checkOperationsInProgress: require('src/common/k8s').checkOperationsInProgress,
       cd: require('zx').cd,
       getParsedArgs: require('src/common/yargs').getParsedArgs,
     }
@@ -104,6 +106,7 @@ describe('Apply command', () => {
     mockDeps.applyAsApps.mockResolvedValue(true)
     mockDeps.commit.mockResolvedValue(undefined)
     mockDeps.runtimeUpgrade.mockResolvedValue(undefined)
+    mockDeps.checkOperationsInProgress.mockResolvedValue(undefined)
     mockDeps.getParsedArgs.mockReturnValue({})
   })
 

src/cmd/apply.ts

@@ -4,7 +4,7 @@ import { cloneDeep } from 'lodash'
 import { cleanupHandler, prepareEnvironment } from 'src/common/cli'
 import { terminal } from 'src/common/debug'
 import { env } from 'src/common/envalid'
-import { getDeploymentState, setDeploymentState } from 'src/common/k8s'
+import { checkOperationsInProgress, getDeploymentState, setDeploymentState } from 'src/common/k8s'
 import { getFilename, rootDir } from 'src/common/utils'
 import { getImageTagFromValues, getPackageVersion } from 'src/common/values'
 import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs'
@@ -91,6 +91,7 @@ export const apply = async (): Promise<void> => {
           d.error('Failed to collect traces:', traceError)
         }
         d.info(`Retrying in ${retryOptions.maxTimeout} ms`)
+        await checkOperationsInProgress()
         throw e
       }
     }, retryOptions)

src/common/hf.ts

@@ -1,17 +1,17 @@
+import { existsSync, mkdirSync, writeFileSync } from 'fs'
 import { readFile } from 'fs/promises'
 import { glob } from 'glob'
 import { has, set } from 'lodash'
+import { resolve } from 'path'
 import { parse } from 'yaml'
 import { $, ProcessPromise } from 'zx'
 import { logLevels, terminal } from './debug'
 import { env } from './envalid'
+import { applyServerSide } from './k8s'
 import { getFileMaps, setValuesFile } from './repo'
 import { asArray, extract, flattenObject, getValuesSchema, isCore, rootDir } from './utils'
 import { getParsedArgs, HelmArguments } from './yargs'
 import { ProcessOutputTrimmed, Streams } from './zx-enhance'
-import { resolve } from 'path'
-import { existsSync, mkdirSync, writeFileSync } from 'fs'
-import { applyServerSide } from './k8s'
 
 const replaceHFPaths = (output: string, envDir = env.ENV_DIR): string => output.replaceAll('../env', envDir)
 export const HF_DEFAULT_SYNC_ARGS = [

src/common/k8s.test.ts

@@ -13,6 +13,10 @@ import {
   V1StatefulSet,
   V1Status,
 } from '@kubernetes/client-node'
+import retry from 'async-retry'
+import { ARGOCD_APP_PARAMS } from './constants'
+import { terminal } from './debug'
+import { env } from './envalid'
 import * as k8s from './k8s'
 import {
   appRevisionMatches,
@@ -22,10 +26,6 @@ import {
   patchContainerResourcesOfSts,
   patchStatefulSetResources,
 } from './k8s'
-import { terminal } from './debug'
-import retry from 'async-retry'
-import { env } from './envalid'
-import { ARGOCD_APP_PARAMS } from './constants'
 
 class MockApiException<T> extends ApiException<T> {
   code: number
@@ -613,6 +613,44 @@ describe('patchArgoCdApp', () => {
   })
 })
 
+describe('helm operations in progress check', () => {
+  it('should get pending helm releases', async () => {
+    const mockGetK8sHelmReleases = jest.spyOn(k8s, 'getK8sHelmReleases').mockResolvedValue({
+      'release-1:ns-1': { name: 'release-1', namespace: 'ns-1', revision: 2, status: 'pending-upgrade' },
+      'release-2:ns-2': { name: 'release-2', namespace: 'ns-2', revision: 1, status: 'deployed' },
+    })
+
+    const pendingReleases = await k8s.getPendingHelmReleases()
+    expect(mockGetK8sHelmReleases).toHaveBeenCalled()
+    expect(pendingReleases).toEqual([{ name: 'release-1', namespace: 'ns-1', revision: 2, status: 'pending-upgrade' }])
+  })
+
+  it('should delete secrets for helm releases', async () => {
+    const mockDeleteSecretForHelmRelease = jest.spyOn(k8s, 'deleteSecretForHelmRelease').mockResolvedValue()
+    await k8s.deleteSecretForHelmRelease('release-1', 'ns-1')
+    expect(mockDeleteSecretForHelmRelease).toHaveBeenCalledWith('release-1', 'ns-1')
+  })
+
+  it('should delete secrets for pending releases', async () => {
+    const mockDeleteSecretForHelmRelease = jest.spyOn(k8s, 'deleteSecretForHelmRelease').mockResolvedValue()
+    const mockGetPendingHelmReleases = jest.spyOn(k8s, 'getPendingHelmReleases').mockResolvedValue([
+      {
+        name: 'release-1',
+        namespace: 'ns-1',
+        revision: 2,
+        status: 'pending-upgrade',
+        app_version: '',
+      },
+      { name: 'release-2', namespace: 'ns-2', revision: 1, status: 'pending-install', app_version: '' },
+    ])
+
+    await k8s.checkOperationsInProgress()
+    expect(mockGetPendingHelmReleases).toHaveBeenCalled()
+    expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(1, 'release-1', 'ns-1')
+    expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(3, 'release-2', 'ns-2')
+  })
+})
+
 describe('restartOtomiApiDeployment', () => {
   let mockAppApi: jest.Mocked<AppsV1Api>
 

src/common/k8s.ts

@@ -148,6 +148,19 @@ export const getK8sSecret = async (name: string, namespace: string): Promise<Rec
   }
 }
 
+export const deleteSecretForHelmRelease = async (releaseName: string, namespace: string) => {
+  const d = terminal('common:k8s:deleteSecretForHelmRelease')
+  d.info(`Deleting secret for Helm release ${releaseName} in namespace ${namespace}`)
+  try {
+    await coreClient.deleteNamespacedSecret({ name: `sh.helm.release.v1.${releaseName}.v1`, namespace })
+    d.debug(`Deleted secret for Helm release ${releaseName} in namespace ${namespace}`)
+  } catch (error) {
+    if (error?.response?.statusCode !== 404) {
+      throw error
+    }
+  }
+}
+
 export interface DeploymentState {
   status?: 'deploying' | 'deployed'
   tag?: string
@@ -164,6 +177,46 @@ export const getDeploymentState = async (): Promise<DeploymentState> => {
   const result = await $`kubectl get cm -n otomi ${DEPLOYMENT_STATUS_CONFIGMAP} -o jsonpath='{.data}'`.nothrow().quiet()
   return JSON.parse(result.stdout || '{}')
 }
+interface HelmRelease {
+  name: string
+  namespace: string
+  status: string
+  app_version: string
+  revision: number
+  first_deployed?: string
+  last_deployed?: string
+  chart?: string
+}
+
+export const checkOperationsInProgress = async (): Promise<void> => {
+  const d = terminal(`common:k8s:checkOperationsInProgress`)
+  const pendingHelmReleases = await getPendingHelmReleases()
+  if (pendingHelmReleases.length > 0) {
+    d.info(`Pending Helm operations detected for releases: ${pendingHelmReleases.join(', ')}. removing secrets...`)
+    for (const release of pendingHelmReleases) {
+      await deleteSecretForHelmRelease(release.name, release.namespace)
+    }
+  }
+}
+
+export const getPendingHelmReleases = async (): Promise<HelmRelease[]> => {
+  const d = terminal('common:k8s:getPendingHelmReleases')
+  d.info('Checking for pending Helm operations')
+  const releases = await getK8sHelmReleases()
+  const pendingReleases: HelmRelease[] = []
+  Object.keys(releases).forEach((key) => {
+    const release = releases[key]
+    if (release.labelName === 'apl' || release.labelName === 'otomi') return
+    if (
+      release.status === 'pending-upgrade' ||
+      release.status === 'pending-install' ||
+      release.status === 'pending-rollback'
+    ) {
+      pendingReleases.push(release)
+    }
+  })
+  return pendingReleases
+}
 
 /**
  * Result of command execution
@@ -241,6 +294,49 @@ export const getHelmReleases = async (): Promise<Record<string, any>> => {
   return status
 }
 
+export const getK8sHelmReleases = async (): Promise<Record<string, any>> => {
+  const coreApi = k8s.core()
+
+  try {
+    const secretsResponse = await coreApi.listSecretForAllNamespaces({
+      labelSelector: 'owner=helm,status',
+    })
+
+    const releases: Record<string, any> = {}
+
+    for (const secret of secretsResponse.items) {
+      if (!secret.metadata?.name || !secret.metadata?.namespace || !secret.metadata?.labels) continue
+
+      const match = secret.metadata.name.match(/^sh\.helm\.release\.v1\.(.+)\.v(\d+)$/)
+      if (!match) continue
+
+      const [, releaseName, revision] = match
+      const releaseKey = `${secret.metadata.namespace}/${releaseName}`
+
+      const release = {
+        name: releaseName,
+        labelName: secret.metadata.labels.name,
+        namespace: secret.metadata.namespace,
+        revision: parseInt(revision),
+        status: secret.metadata.labels.status,
+        app_version: secret.metadata.labels.version || secret.metadata.labels.app_version,
+        chart: secret.metadata.labels.chart,
+        first_deployed: secret.metadata?.creationTimestamp,
+        last_deployed: secret.metadata.labels.modifiedAt,
+      }
+
+      // Keep only the latest revision for each release
+      if (!releases[releaseKey] || releases[releaseKey].revision < release.revision) {
+        releases[releaseKey] = release
+      }
+    }
+
+    return releases
+  } catch (error) {
+    throw new Error(`Failed to get Helm releases from Kubernetes: ${error.message}`)
+  }
+}
+
 export const setDeploymentState = async (state: Record<string, any>): Promise<void> => {
   if (env.isDev && env.DISABLE_SYNC) return
   const d = terminal('common:k8s:setDeploymentState')

src/operator/installer.test.ts

@@ -1,7 +1,7 @@
-import { Installer } from './installer'
-import { AplOperations } from './apl-operations'
-import * as k8s from '../common/k8s'
 import { hfValues } from '../common/hf'
+import * as k8s from '../common/k8s'
+import { AplOperations } from './apl-operations'
+import { Installer } from './installer'
 
 jest.mock('../common/debug', () => ({
   terminal: jest.fn().mockImplementation(() => ({
@@ -13,6 +13,7 @@ jest.mock('../common/debug', () => ({
 }))
 
 jest.mock('../common/k8s', () => ({
+  checkOperationsInProgress: jest.fn(),
   getK8sConfigMap: jest.fn(),
   getK8sSecret: jest.fn(),
   createUpdateConfigMap: jest.fn(),

src/operator/installer.ts

@@ -1,9 +1,16 @@
+import * as process from 'node:process'
 import { terminal } from '../common/debug'
-import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s'
 import { hfValues } from '../common/hf'
+import {
+  checkOperationsInProgress,
+  createUpdateConfigMap,
+  createUpdateGenericSecret,
+  getK8sConfigMap,
+  getK8sSecret,
+  k8s,
+} from '../common/k8s'
 import { AplOperations } from './apl-operations'
 import { getErrorMessage } from './utils'
-import * as process from 'node:process'
 
 export interface GitCredentials {
   username: string
@@ -48,6 +55,7 @@ export class Installer {
         this.d.error(`Installation attempt ${attemptNumber} failed:`, errorMessage)
         await this.updateInstallationStatus('failed', attemptNumber, errorMessage)
         this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`, getErrorMessage(error))
+        await checkOperationsInProgress()
 
         // Wait 1 second before retrying
         await new Promise((resolve) => setTimeout(resolve, 1000))