feat: [CI-7817]: CI output vars as env (#49008)

* feat: [CI-7817]: CI output vars as env

* added ff

Author: devkimittal

332-ci-manager/build.properties

@@ -1,4 +1,4 @@
 build.majorVersion=1
 build.minorVersion=0
-build.number=4101
+build.number=4102
 build.patch=000

332-ci-manager/service/src/main/java/io/harness/ci/execution/execution/PipelineExecutionUpdateEventHandler.java

@@ -46,6 +46,7 @@
 import io.harness.pms.sdk.core.resolver.RefObjectUtils;
 import io.harness.pms.sdk.core.resolver.outcome.OutcomeService;
 import io.harness.repositories.CIAccountExecutionMetadataRepository;
+import io.harness.repositories.CIStageOutputRepository;
 import io.harness.repositories.CITaskDetailsRepository;
 import io.harness.service.DelegateGrpcClientWrapper;
 import io.harness.steps.StepUtils;
@@ -90,6 +91,7 @@ public class PipelineExecutionUpdateEventHandler implements OrchestrationEventHa
   @Inject @Named("ciEventHandlerExecutor") private ExecutorService executorService;
   @Inject @Named("ciRatelimitHandlerExecutor") private ExecutorService ciRatelimitHandlerExecutor;
   @Inject private DelegateGrpcClientWrapper delegateGrpcClientWrapper;
+  @Inject CIStageOutputRepository ciStageOutputRepository;
 
   @Override
   public void handleEvent(OrchestrationEvent event) {
@@ -105,6 +107,15 @@ public void handleEvent(OrchestrationEvent event) {
     });
   }
 
+  private void deleteCIStageOutputs(Ambiance ambiance) {
+    String stageExecutionId = ambiance.getStageExecutionId();
+    try {
+      ciStageOutputRepository.deleteFirstByStageExecutionId(stageExecutionId);
+    } catch (Exception e) {
+      log.error("Error while deleting CI outputs for stageExecutionId " + stageExecutionId, e);
+    }
+  }
+
   private void sendCleanupRequest(Level level, Ambiance ambiance, Status status, String accountId) {
     try {
       RetryPolicy<Object> retryPolicy = getRetryPolicy(format("[Retrying failed call to clean pod attempt: {}"),
@@ -131,6 +142,7 @@ private void sendCleanupRequest(Level level, Ambiance ambiance, Status status, S
             log.info("failed to remove execution record from db", ex);
           }
 
+          deleteCIStageOutputs(ambiance);
           CICleanupTaskParams ciCleanupTaskParams = stageCleanupUtility.buildAndfetchCleanUpParameters(ambiance);
 
           log.info("Received event with status {} to clean planExecutionId {}, stage {}", status,

332-ci-manager/service/src/main/java/io/harness/ci/execution/states/AbstractStepExecutable.java

@@ -17,13 +17,15 @@
 import static java.util.Collections.singletonList;
 
 import io.harness.annotations.dev.OwnedBy;
+import io.harness.beans.FeatureName;
 import io.harness.beans.outcomes.VmDetailsOutcome;
 import io.harness.beans.plugin.compatible.PluginCompatibleStep;
 import io.harness.beans.steps.CIStepInfo;
 import io.harness.beans.steps.CIStepInfoType;
 import io.harness.beans.steps.outcome.CIStepArtifactOutcome;
 import io.harness.beans.steps.outcome.CIStepOutcome;
 import io.harness.beans.steps.outcome.StepArtifacts;
+import io.harness.beans.steps.output.CIStageOutput;
 import io.harness.beans.steps.stepinfo.BackgroundStepInfo;
 import io.harness.beans.steps.stepinfo.PluginStepInfo;
 import io.harness.beans.steps.stepinfo.RunStepInfo;
@@ -57,6 +59,10 @@
 import io.harness.delegate.beans.ci.vm.CIVmInitializeTaskParams;
 import io.harness.delegate.beans.ci.vm.VmTaskExecutionResponse;
 import io.harness.delegate.beans.ci.vm.dlite.DliteVmExecuteStepTaskParams;
+import io.harness.delegate.beans.ci.vm.steps.VmBackgroundStep;
+import io.harness.delegate.beans.ci.vm.steps.VmPluginStep;
+import io.harness.delegate.beans.ci.vm.steps.VmRunStep;
+import io.harness.delegate.beans.ci.vm.steps.VmRunTestStep;
 import io.harness.delegate.beans.ci.vm.steps.VmStepInfo;
 import io.harness.delegate.task.HDelegateTask;
 import io.harness.delegate.task.stepstatus.StepStatusTaskResponseData;
@@ -101,6 +107,8 @@
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 
@@ -168,11 +176,12 @@ public AsyncExecutableResponse executeVmAsyncAfterRbac(Ambiance ambiance, String
     VmStepInfo vmStepInfo = vmStepSerializer.serialize(ambiance, ciStepInfo, stageInfraDetails, stepIdentifier,
         ParameterField.createValueField(Timeout.fromString(stringTimeout)), stageDetails.getRegistries(),
         stageDetails.getExecutionSource(), vmDetailsOutcome.getDelegateId());
+    injectOutputVarsAsEnvVars(vmStepInfo, accountId, ambiance.getStageExecutionId());
+
     Set<String> secrets = vmStepSerializer.getStepSecrets(vmStepInfo, ambiance);
     secrets.addAll(stepPreProcessSecrets);
     CIExecuteStepTaskParams params = getVmTaskParams(ambiance, vmStepInfo, secrets, stageInfraDetails, stageDetails,
         vmDetailsOutcome, runtimeId, stepIdentifier, logKey);
-
     List<String> eligibleToExecuteDelegateIds = new ArrayList<>();
     if (isNotEmpty(vmDetailsOutcome.getDelegateId())) {
       eligibleToExecuteDelegateIds.add(vmDetailsOutcome.getDelegateId());
@@ -188,6 +197,47 @@ public AsyncExecutableResponse executeVmAsyncAfterRbac(Ambiance ambiance, String
         .build();
   }
 
+  public void injectOutputVarsAsEnvVars(VmStepInfo stepInfo, String accountId, String stageExecutionId) {
+    if (!featureFlagService.isEnabled(FeatureName.CI_OUTPUT_VARIABLES_AS_ENV, accountId)) {
+      return;
+    }
+
+    Optional<CIStageOutput> ciStageOutputResponse =
+        ciStageOutputRepository.findFirstByStageExecutionId(stageExecutionId);
+    if (ciStageOutputResponse.isPresent()) {
+      CIStageOutput ciStageOutput = ciStageOutputResponse.get();
+      Map<String, String> outputs = ciStageOutput.getOutputs();
+      if (Objects.isNull(outputs) || outputs.isEmpty()) {
+        return;
+      }
+
+      switch (stepInfo.getType()) {
+        case RUN:
+          VmRunStep runStep = (VmRunStep) stepInfo;
+          outputs.putAll(runStep.getEnvVariables());
+          runStep.getEnvVariables().putAll(outputs);
+          break;
+        case PLUGIN:
+          VmPluginStep pluginStep = (VmPluginStep) stepInfo;
+          outputs.putAll(pluginStep.getEnvVariables());
+          pluginStep.getEnvVariables().putAll(outputs);
+          break;
+        case RUN_TEST:
+          VmRunTestStep runTestStep = (VmRunTestStep) stepInfo;
+          outputs.putAll(runTestStep.getEnvVariables());
+          runTestStep.getEnvVariables().putAll(outputs);
+          break;
+        case BACKGROUND:
+          VmBackgroundStep vmBackgroundStep = (VmBackgroundStep) stepInfo;
+          outputs.putAll(vmBackgroundStep.getEnvVariables());
+          vmBackgroundStep.getEnvVariables().putAll(outputs);
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
   private CIExecuteStepTaskParams getVmTaskParams(Ambiance ambiance, VmStepInfo vmStepInfo, Set<String> secrets,
       StageInfraDetails stageInfraDetails, StageDetails stageDetails, VmDetailsOutcome vmDetailsOutcome,
       String runtimeId, String stepIdentifier, String logKey) {
@@ -300,6 +350,8 @@ public StepResponse handleVmStepResponse(Ambiance ambiance, String stepIdentifie
     }
 
     if (taskResponse.getCommandExecutionStatus() == CommandExecutionStatus.SUCCESS) {
+      populateCIStageOutputs(
+          taskResponse.getOutputVars(), AmbianceUtils.getAccountId(ambiance), ambiance.getStageExecutionId());
       if (isNotEmpty(taskResponse.getOutputVars())) {
         StepResponse.StepOutcome stepOutcome =
             StepResponse.StepOutcome.builder()

332-ci-manager/service/src/main/java/io/harness/ci/serializer/morphia/CIBeansMorphiaRegistrar.java

@@ -12,6 +12,7 @@
 import io.harness.app.beans.entities.PluginMetadataConfig;
 import io.harness.app.beans.entities.PluginMetadataStatus;
 import io.harness.beans.outcomes.VmDetailsOutcome;
+import io.harness.beans.steps.output.CIStageOutput;
 import io.harness.beans.sweepingoutputs.ContextElement;
 import io.harness.beans.sweepingoutputs.DliteVmStageInfraDetails;
 import io.harness.beans.sweepingoutputs.K8PodDetails;
@@ -54,6 +55,7 @@ public void registerClasses(Set<Class> set) {
     set.add(CIExecutionMetadata.class);
     set.add(PluginMetadataConfig.class);
     set.add(PluginMetadataStatus.class);
+    set.add(CIStageOutput.class);
   }
 
   @Override

332-ci-manager/service/src/test/java/io/harness/ci/states/BackgroundStepTest.java

@@ -41,6 +41,7 @@
 import io.harness.ci.buildstate.ConnectorUtils;
 import io.harness.ci.config.CIExecutionServiceConfig;
 import io.harness.ci.executionplan.CIExecutionTestBase;
+import io.harness.ci.ff.CIFeatureFlagService;
 import io.harness.ci.serializer.BackgroundStepProtobufSerializer;
 import io.harness.ci.serializer.vm.VmStepSerializer;
 import io.harness.delegate.beans.ci.vm.steps.VmBackgroundStep;
@@ -60,6 +61,7 @@
 import io.harness.pms.sdk.core.steps.io.StepInputPackage;
 import io.harness.pms.yaml.ParameterField;
 import io.harness.product.ci.engine.proto.UnitStep;
+import io.harness.repositories.CIStageOutputRepository;
 import io.harness.rule.Owner;
 
 import java.util.ArrayList;
@@ -87,6 +89,8 @@ public class BackgroundStepTest extends CIExecutionTestBase {
   @Mock private OutcomeService outcomeService;
   @Mock private ConnectorUtils connectorUtils;
   @Mock private VmStepSerializer vmStepSerializer;
+  @Mock protected CIFeatureFlagService featureFlagService;
+  @Mock protected CIStageOutputRepository ciStageOutputRepository;
   @InjectMocks BackgroundStep backgroundStep;
   private Ambiance ambiance;
   private BackgroundStepInfo stepInfo;

332-ci-manager/service/src/test/java/io/harness/ci/states/DockerStepTest.java

@@ -25,6 +25,7 @@
 import io.harness.beans.sweepingoutputs.K8StageInfraDetails;
 import io.harness.category.element.UnitTests;
 import io.harness.ci.executionplan.CIExecutionTestBase;
+import io.harness.ci.ff.CIFeatureFlagService;
 import io.harness.ci.integrationstage.K8InitializeStepUtilsHelper;
 import io.harness.delegate.beans.ci.vm.VmTaskExecutionResponse;
 import io.harness.delegate.task.stepstatus.StepExecutionStatus;
@@ -48,6 +49,7 @@
 import io.harness.pms.sdk.core.resolver.outputs.ExecutionSweepingOutputService;
 import io.harness.pms.sdk.core.steps.io.StepResponse;
 import io.harness.pms.yaml.ParameterField;
+import io.harness.repositories.CIStageOutputRepository;
 import io.harness.rule.Owner;
 import io.harness.tasks.ResponseData;
 
@@ -72,6 +74,8 @@ public class DockerStepTest extends CIExecutionTestBase {
   @InjectMocks DockerStep dockerStep;
   @Mock SerializedResponseDataHelper serializedResponseDataHelper;
   @Mock private ExecutionSweepingOutputService executionSweepingOutputResolver;
+  @Mock protected CIFeatureFlagService featureFlagService;
+  @Mock protected CIStageOutputRepository ciStageOutputRepository;
   private Ambiance ambiance;
   private StepElementParameters stepElementParameters;
   private HashMap<String, String> setupAbstractions = new HashMap<>();

332-ci-manager/service/src/test/java/io/harness/ci/states/ECRStepTest.java

@@ -23,6 +23,7 @@
 import io.harness.beans.sweepingoutputs.DliteVmStageInfraDetails;
 import io.harness.category.element.UnitTests;
 import io.harness.ci.executionplan.CIExecutionTestBase;
+import io.harness.ci.ff.CIFeatureFlagService;
 import io.harness.ci.integrationstage.K8InitializeStepUtilsHelper;
 import io.harness.delegate.beans.ci.vm.VmTaskExecutionResponse;
 import io.harness.delegate.task.stepstatus.artifact.ArtifactMetadata;
@@ -42,6 +43,7 @@
 import io.harness.pms.sdk.core.resolver.outputs.ExecutionSweepingOutputService;
 import io.harness.pms.sdk.core.steps.io.StepResponse;
 import io.harness.pms.yaml.ParameterField;
+import io.harness.repositories.CIStageOutputRepository;
 import io.harness.rule.Owner;
 import io.harness.tasks.ResponseData;
 
@@ -60,6 +62,8 @@ public class ECRStepTest extends CIExecutionTestBase {
   @InjectMocks ECRStep ecrStep;
   @Mock SerializedResponseDataHelper serializedResponseDataHelper;
   @Mock private ExecutionSweepingOutputService executionSweepingOutputResolver;
+  @Mock protected CIFeatureFlagService featureFlagService;
+  @Mock protected CIStageOutputRepository ciStageOutputRepository;
 
   public static final String STEP_ID = "ecrStepID";
   private Ambiance ambiance;

332-ci-manager/service/src/test/java/io/harness/ci/states/GCRStepTest.java

@@ -23,6 +23,7 @@
 import io.harness.beans.sweepingoutputs.DliteVmStageInfraDetails;
 import io.harness.category.element.UnitTests;
 import io.harness.ci.executionplan.CIExecutionTestBase;
+import io.harness.ci.ff.CIFeatureFlagService;
 import io.harness.ci.integrationstage.K8InitializeStepUtilsHelper;
 import io.harness.delegate.beans.ci.vm.VmTaskExecutionResponse;
 import io.harness.delegate.task.stepstatus.artifact.ArtifactMetadata;
@@ -42,6 +43,7 @@
 import io.harness.pms.sdk.core.resolver.outputs.ExecutionSweepingOutputService;
 import io.harness.pms.sdk.core.steps.io.StepResponse;
 import io.harness.pms.yaml.ParameterField;
+import io.harness.repositories.CIStageOutputRepository;
 import io.harness.rule.Owner;
 import io.harness.tasks.ResponseData;
 
@@ -60,6 +62,8 @@ public class GCRStepTest extends CIExecutionTestBase {
   @InjectMocks GCRStep gcrStep;
   @Mock SerializedResponseDataHelper serializedResponseDataHelper;
   @Mock private ExecutionSweepingOutputService executionSweepingOutputResolver;
+  @Mock protected CIFeatureFlagService featureFlagService;
+  @Mock protected CIStageOutputRepository ciStageOutputRepository;
 
   public static final String STEP_ID = "gcrStepID";
   private Ambiance ambiance;