From f0f9530a3d73ed63b832c97942ff7fa7e585e1ce Mon Sep 17 00:00:00 2001 From: Faye Date: Tue, 10 Jun 2025 17:12:18 +0200 Subject: [PATCH 01/18] Implement optional flag and loop to poll SonarQube and wait for scan to complete --- cmd/kosli/attestSonar.go | 2 ++ cmd/kosli/root.go | 1 + internal/sonar/sonar.go | 49 +++++++++++++++++++++++++++++++++------- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/cmd/kosli/attestSonar.go b/cmd/kosli/attestSonar.go index b6a149c07..c1c6edf75 100644 --- a/cmd/kosli/attestSonar.go +++ b/cmd/kosli/attestSonar.go @@ -24,6 +24,7 @@ type attestSonarOptions struct { projectKey string serverURL string revision string + allowWait bool payload SonarAttestationPayload } @@ -156,6 +157,7 @@ func newAttestSonarCmd(out io.Writer) *cobra.Command { cmd.Flags().StringVar(&o.projectKey, "sonar-project-key", "", sonarProjectKeyFlag) cmd.Flags().StringVar(&o.serverURL, "sonar-server-url", "https://sonarcloud.io", sonarServerURLFlag) cmd.Flags().StringVar(&o.revision, "sonar-revision", o.commitSHA, sonarRevisionFlag) + cmd.Flags().BoolVar(&o.allowWait, "allow-wait", false, sonarWaitFlag) err := RequireFlags(cmd, []string{"flow", "trail", "name", "sonar-api-token"}) if err != nil { diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index 28af62cbb..be0ddbe1d 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -241,6 +241,7 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, sonarProjectKeyFlag = "[conditional] The project key of the SonarCloud/SonarQube project. Only required if you want to use the project key/revision to get the scan results rather than using Sonar's metadata file." sonarServerURLFlag = "[conditional] The URL of your SonarQube server. Only required if you are using SonarQube and not using SonarQube's metadata file to get scan results." sonarRevisionFlag = "[conditional] The revision of the SonarCloud/SonarQube project. Only required if you want to use the project key/revision to get the scan results rather than using Sonar's metadata file and you have overridden the default revision, or you aren't using a CI. Defaults to the value of the git commit flag." + sonarWaitFlag = "[optional] Allow the command to check and wait for the SonarQube scan to complete, if necessary. Useful for scans that take a long time to complete. Defaults to false." logicalEnvFlag = "[required] The logical environment." physicalEnvFlag = "[required] The physical environment." attestationTypeDescriptionFlag = "[optional] The attestation type description." diff --git a/internal/sonar/sonar.go b/internal/sonar/sonar.go index 242348eaf..d70ee7382 100644 --- a/internal/sonar/sonar.go +++ b/internal/sonar/sonar.go @@ -8,6 +8,9 @@ import ( "os" "path/filepath" "strings" + "time" + + log "github.com/kosli-dev/cli/internal/logger" ) type SonarConfig struct { @@ -17,6 +20,7 @@ type SonarConfig struct { revision string projectKey string serverURL string + allowWait bool } // Structs to build the JSON for our attestation payload @@ -109,7 +113,7 @@ type Error struct { Msg string `json:"msg"` } -func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revision string) *SonarConfig { +func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revision string, allowWait bool) *SonarConfig { return &SonarConfig{ APIToken: apiToken, WorkingDir: workingDir, @@ -117,6 +121,7 @@ func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revi revision: revision, projectKey: projectKey, serverURL: serverURL, + allowWait: allowWait, } } @@ -159,7 +164,7 @@ func (sc *SonarConfig) GetSonarResults() (*SonarResults, error) { if analysisID == "" { //Get the analysis ID, status, project name and branch data from the ceTaskURL (ce API) - analysisID, err = GetCETaskData(httpClient, project, sonarResults, sc.CETaskUrl, tokenHeader) + analysisID, err = GetCETaskData(httpClient, project, sonarResults, sc.CETaskUrl, tokenHeader, sc.allowWait) if err != nil { return nil, err } @@ -210,7 +215,7 @@ func (sc *SonarConfig) readFile(project *Project, results *SonarResults) error { return nil } -func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *SonarResults, ceTaskURL, tokenHeader string) (string, error) { +func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *SonarResults, ceTaskURL, tokenHeader string, allowWait bool) (string, error) { taskRequest, err := http.NewRequest("GET", ceTaskURL, nil) taskRequest.Header.Add("Authorization", tokenHeader) if err != nil { @@ -228,11 +233,39 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona return "", fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube") } - project.Name = taskResponseData.Task.ComponentName - project.Key = taskResponseData.Task.ComponentKey - sonarResults.TaskID = taskResponseData.Task.TaskID - analysisId := taskResponseData.Task.AnalysisID - sonarResults.Status = taskResponseData.Task.Status + if allowWait { + logger := log.NewStandardLogger() + wait := 10 // seconds + maxWait := 20 * 60 // 20 minutes + + for wait < maxWait { + taskResponse, err := httpClient.Do(taskRequest) + if err != nil { + return "", err + } + + err = json.NewDecoder(taskResponse.Body).Decode(taskResponseData) + if err != nil { + return "", fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube") + } + + if taskResponseData.Task.Status == "PENDING" { + logger.Info("waiting for SonarQube scan to complete...") + time.Sleep(time.Duration(wait) * time.Second) + wait *= 2 + } else { + break + } + } + } + + task := taskResponseData.Task + + project.Name = task.ComponentName + project.Key = task.ComponentKey + sonarResults.TaskID = task.TaskID + analysisId := task.AnalysisID + sonarResults.Status = task.Status if analysisId == "" { return "", fmt.Errorf("analysis ID not found on %s", sonarResults.ServerUrl) // This should never happen From 308ada35ad6d5ea57a9e50c4864c9e51389c8a84 Mon Sep 17 00:00:00 2001 From: Faye Date: Tue, 10 Jun 2025 17:16:19 +0200 Subject: [PATCH 02/18] Add missing argument to NewSonarConfig --- cmd/kosli/attestSonar.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kosli/attestSonar.go b/cmd/kosli/attestSonar.go index c1c6edf75..5d2692043 100644 --- a/cmd/kosli/attestSonar.go +++ b/cmd/kosli/attestSonar.go @@ -175,7 +175,7 @@ func (o *attestSonarOptions) run(args []string) error { return err } - sc := sonar.NewSonarConfig(o.apiToken, o.workingDir, o.ceTaskURL, o.projectKey, o.serverURL, o.revision) + sc := sonar.NewSonarConfig(o.apiToken, o.workingDir, o.ceTaskURL, o.projectKey, o.serverURL, o.revision, o.allowWait) o.payload.SonarResults, err = sc.GetSonarResults() if err != nil { From f29001b11987841da489889c500e90b89818feeb Mon Sep 17 00:00:00 2001 From: Faye Date: Wed, 11 Jun 2025 17:21:21 +0200 Subject: [PATCH 03/18] Add 'IN_PROGRESS' status to sleep loop --- internal/sonar/sonar.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/sonar/sonar.go b/internal/sonar/sonar.go index d70ee7382..7e26a511f 100644 --- a/internal/sonar/sonar.go +++ b/internal/sonar/sonar.go @@ -249,7 +249,7 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona return "", fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube") } - if taskResponseData.Task.Status == "PENDING" { + if taskResponseData.Task.Status == "PENDING" || taskResponseData.Task.Status == "IN_PROGRESS" { logger.Info("waiting for SonarQube scan to complete...") time.Sleep(time.Duration(wait) * time.Second) wait *= 2 @@ -267,8 +267,10 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona analysisId := task.AnalysisID sonarResults.Status = task.Status + // This should only happen if the task is pending - either because the project is large and the scan takes a long time + // to process, or because SonarQube is experiencing delays for some reason. if analysisId == "" { - return "", fmt.Errorf("analysis ID not found on %s", sonarResults.ServerUrl) // This should never happen + return "", fmt.Errorf("analysis ID not found on %s. This is because either ", sonarResults.ServerUrl) } if project.Url == "" { From 385c266441725d06860f76a28033e64484698984 Mon Sep 17 00:00:00 2001 From: Faye Date: Wed, 11 Jun 2025 17:45:03 +0200 Subject: [PATCH 04/18] Add more thorough error message to attest sonar command --- internal/sonar/sonar.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sonar/sonar.go b/internal/sonar/sonar.go index 7e26a511f..321b01cc9 100644 --- a/internal/sonar/sonar.go +++ b/internal/sonar/sonar.go @@ -270,7 +270,7 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona // This should only happen if the task is pending - either because the project is large and the scan takes a long time // to process, or because SonarQube is experiencing delays for some reason. if analysisId == "" { - return "", fmt.Errorf("analysis ID not found on %s. This is because either ", sonarResults.ServerUrl) + return "", fmt.Errorf("analysis ID not found on %s. The scan results are not yet available, likely due to: \n1. Your project being particularly large and the scan taking time to process, or \n2. SonarQube is experiencing delays in processing scans. \nTry rerunning the command with the --allow-wait flag.", sonarResults.ServerUrl) } if project.Url == "" { From d4dfd60b8ea6a9bcaf799550fd404eaf73578f85 Mon Sep 17 00:00:00 2001 From: Faye Date: Fri, 13 Jun 2025 17:11:14 +0200 Subject: [PATCH 05/18] Change --allow-wait flag to take number of seconds to wait as parameter --- cmd/kosli/attestSonar.go | 20 ++++++----- cmd/kosli/root.go | 2 +- internal/sonar/sonar.go | 71 ++++++++++++++++++++++------------------ 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/cmd/kosli/attestSonar.go b/cmd/kosli/attestSonar.go index 5d2692043..b6d4bcd56 100644 --- a/cmd/kosli/attestSonar.go +++ b/cmd/kosli/attestSonar.go @@ -24,7 +24,7 @@ type attestSonarOptions struct { projectKey string serverURL string revision string - allowWait bool + allowWait int payload SonarAttestationPayload } @@ -37,7 +37,9 @@ The results are parsed to find the status of the project's quality gate which is The scan to be retrieved can be specified in two ways: 1. (Default) Using metadata created by the Sonar scanner. By default this is located within a temporary .scannerwork folder in the repo base directory. If you have overriden the location of this folder by passing parameters to the Sonar scanner, or are running Kosli's CLI locally outside the repo's base directory, -you can provide the correct path using the --sonar-working-dir flag. This metadata is generated by a specific scan, allowing Kosli to retrieve the results of that scan. +you can provide the correct path using the --sonar-working-dir flag. This metadata is generated by a specific scan, allowing Kosli to retrieve the results of that scan. +If there are delays in the scan processing (either because the scanned project is very large, or because SonarQube is experiencing delays), you can use the --allow-wait flag to +wait for the scan to be processed before trying to attest the results. 2. Providing the Sonar project key and the revision of the scan (plus the SonarQube server URL if relevant). If running the Kosli CLI in some CI/CD pipeline, the revision is defaulted to the commit SHA. If you are running the command locally, or have overriden the revision in SonarQube via parameters to the Sonar scanner, you can provide the correct revision using the --sonar-revision flag. Kosli then finds the scan results for the specified project key and revision. @@ -47,7 +49,7 @@ In this case, we recommend using Kosli's Sonar webhook integration ( https://doc ` + attestationBindingDesc const attestSonarExample = ` -# report a SonarQube Cloud attestation about a trail using Sonar's metadata: +# report a SonarQube Cloud attestation about a trail using SonarQube's metadata: kosli attest sonar \ --name yourAttestationName \ --flow yourFlowName \ @@ -57,7 +59,7 @@ kosli attest sonar \ --api-token yourAPIToken \ --org yourOrgName \ -# report a SonarQube Server attestation about a trail using Sonar's metadata: +# report a SonarQube Server attestation about a trail using SonarWube's metadata, waiting for up to 10 minutes for scan processing: kosli attest sonar \ --name yourAttestationName \ --flow yourFlowName \ @@ -66,6 +68,7 @@ kosli attest sonar \ --sonar-working-dir yourSonarWorkingDirPath \ --api-token yourAPIToken \ --org yourOrgName \ + --allow-wait 600 # report a SonarQube Cloud attestation for a specific branch about a trail using key/revision: kosli attest sonar \ @@ -92,7 +95,7 @@ kosli attest sonar \ --api-token yourAPIToken \ --org yourOrgName \ -# report a SonarQube Cloud attestation about a trail with an attachment using Sonar's metadata: +# report a SonarQube Cloud attestation about a trail with an attachment using Sonar's metadata, waiting for up to 10 minutes for scan processing: kosli attest sonar \ --name yourAttestationName \ --flow yourFlowName \ @@ -101,7 +104,8 @@ kosli attest sonar \ --sonar-working-dir yourSonarWorkingDirPath \ --attachment yourAttachmentPath \ --api-token yourAPIToken \ - --org yourOrgName + --org yourOrgName \ + --allow-wait 600 ` func newAttestSonarCmd(out io.Writer) *cobra.Command { @@ -157,7 +161,7 @@ func newAttestSonarCmd(out io.Writer) *cobra.Command { cmd.Flags().StringVar(&o.projectKey, "sonar-project-key", "", sonarProjectKeyFlag) cmd.Flags().StringVar(&o.serverURL, "sonar-server-url", "https://sonarcloud.io", sonarServerURLFlag) cmd.Flags().StringVar(&o.revision, "sonar-revision", o.commitSHA, sonarRevisionFlag) - cmd.Flags().BoolVar(&o.allowWait, "allow-wait", false, sonarWaitFlag) + cmd.Flags().IntVar(&o.allowWait, "allow-wait", 0, sonarWaitFlag) err := RequireFlags(cmd, []string{"flow", "trail", "name", "sonar-api-token"}) if err != nil { @@ -177,7 +181,7 @@ func (o *attestSonarOptions) run(args []string) error { sc := sonar.NewSonarConfig(o.apiToken, o.workingDir, o.ceTaskURL, o.projectKey, o.serverURL, o.revision, o.allowWait) - o.payload.SonarResults, err = sc.GetSonarResults() + o.payload.SonarResults, err = sc.GetSonarResults(logger) if err != nil { return err } diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index be0ddbe1d..1748438a6 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -241,7 +241,7 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, sonarProjectKeyFlag = "[conditional] The project key of the SonarCloud/SonarQube project. Only required if you want to use the project key/revision to get the scan results rather than using Sonar's metadata file." sonarServerURLFlag = "[conditional] The URL of your SonarQube server. Only required if you are using SonarQube and not using SonarQube's metadata file to get scan results." sonarRevisionFlag = "[conditional] The revision of the SonarCloud/SonarQube project. Only required if you want to use the project key/revision to get the scan results rather than using Sonar's metadata file and you have overridden the default revision, or you aren't using a CI. Defaults to the value of the git commit flag." - sonarWaitFlag = "[optional] Allow the command to check and wait for the SonarQube scan to complete, if necessary. Useful for scans that take a long time to complete. Defaults to false." + sonarWaitFlag = "[optional] Allow the command to wait for the SonarQube scan to be processed, up to the number of seconds provided. Useful when using SonarQube's metadata file to retrieve and attest scans that take a long time to process . Defaults to 0." logicalEnvFlag = "[required] The logical environment." physicalEnvFlag = "[required] The physical environment." attestationTypeDescriptionFlag = "[optional] The attestation type description." diff --git a/internal/sonar/sonar.go b/internal/sonar/sonar.go index 321b01cc9..929f1e6ba 100644 --- a/internal/sonar/sonar.go +++ b/internal/sonar/sonar.go @@ -20,7 +20,7 @@ type SonarConfig struct { revision string projectKey string serverURL string - allowWait bool + allowWait int } // Structs to build the JSON for our attestation payload @@ -113,7 +113,7 @@ type Error struct { Msg string `json:"msg"` } -func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revision string, allowWait bool) *SonarConfig { +func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revision string, allowWait int) *SonarConfig { return &SonarConfig{ APIToken: apiToken, WorkingDir: workingDir, @@ -125,7 +125,7 @@ func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revi } } -func (sc *SonarConfig) GetSonarResults() (*SonarResults, error) { +func (sc *SonarConfig) GetSonarResults(logger *log.Logger) (*SonarResults, error) { httpClient := &http.Client{} var analysisID, tokenHeader string var err error @@ -164,7 +164,7 @@ func (sc *SonarConfig) GetSonarResults() (*SonarResults, error) { if analysisID == "" { //Get the analysis ID, status, project name and branch data from the ceTaskURL (ce API) - analysisID, err = GetCETaskData(httpClient, project, sonarResults, sc.CETaskUrl, tokenHeader, sc.allowWait) + analysisID, err = GetCETaskData(httpClient, project, sonarResults, sc.CETaskUrl, tokenHeader, sc.allowWait, logger) if err != nil { return nil, err } @@ -215,50 +215,58 @@ func (sc *SonarConfig) readFile(project *Project, results *SonarResults) error { return nil } -func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *SonarResults, ceTaskURL, tokenHeader string, allowWait bool) (string, error) { +func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *SonarResults, ceTaskURL, tokenHeader string, allowWait int, logger *log.Logger) (string, error) { taskRequest, err := http.NewRequest("GET", ceTaskURL, nil) taskRequest.Header.Add("Authorization", tokenHeader) if err != nil { return "", err } - taskResponse, err := httpClient.Do(taskRequest) - if err != nil { - return "", err + wait := 10 // seconds to sleep between checks + if allowWait < wait { + wait = allowWait } - + maxWait := allowWait // 20 minutes + elapsed := 0 // seconds elapsed taskResponseData := &TaskResponse{} - err = json.NewDecoder(taskResponse.Body).Decode(taskResponseData) - if err != nil { - return "", fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube") - } - if allowWait { - logger := log.NewStandardLogger() - wait := 10 // seconds - maxWait := 20 * 60 // 20 minutes + for elapsed < maxWait || maxWait == 0 { + taskResponse, err := httpClient.Do(taskRequest) + if err != nil { + return "", err + } - for wait < maxWait { - taskResponse, err := httpClient.Do(taskRequest) - if err != nil { - return "", err - } + err = json.NewDecoder(taskResponse.Body).Decode(taskResponseData) + if err != nil { + return "", fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube") + } - err = json.NewDecoder(taskResponse.Body).Decode(taskResponseData) - if err != nil { - return "", fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube") - } + if maxWait == 0 { + taskResponse.Body.Close() + break + } - if taskResponseData.Task.Status == "PENDING" || taskResponseData.Task.Status == "IN_PROGRESS" { - logger.Info("waiting for SonarQube scan to complete...") - time.Sleep(time.Duration(wait) * time.Second) + //taskResponseData.Task.Status = "PENDING" + if taskResponseData.Task.Status == "PENDING" || taskResponseData.Task.Status == "IN_PROGRESS" { + logger.Info("waiting %ds for SonarQube scan to be processed... \n%d seconds elapsed", wait, elapsed) + time.Sleep(time.Duration(wait) * time.Second) + if elapsed > 300 { // If we've waited 5 minutes, we'll wait 300 seconds (5 minutes) before checking again. This is so that we don't end up with extremely long waiting intervals with the doubling approach. + elapsed += wait + wait += 300 + } else { // Otherwise, we'll double the wait time each time + elapsed += wait wait *= 2 - } else { - break } + } else { + taskResponse.Body.Close() + break } } + if elapsed != 0 { + logger.Info("Waited for %d seconds for SonarQube scan to be processed", elapsed) + } + task := taskResponseData.Task project.Name = task.ComponentName @@ -285,7 +293,6 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona sonarResults.Branch = nil } - taskResponse.Body.Close() return analysisId, nil } From 063f211a7007b54a0404952ab0e4301fef3d728b Mon Sep 17 00:00:00 2001 From: Faye Date: Fri, 13 Jun 2025 17:12:39 +0200 Subject: [PATCH 06/18] Clean up leftover comment --- internal/sonar/sonar.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/sonar/sonar.go b/internal/sonar/sonar.go index 929f1e6ba..e2c949529 100644 --- a/internal/sonar/sonar.go +++ b/internal/sonar/sonar.go @@ -246,7 +246,6 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona break } - //taskResponseData.Task.Status = "PENDING" if taskResponseData.Task.Status == "PENDING" || taskResponseData.Task.Status == "IN_PROGRESS" { logger.Info("waiting %ds for SonarQube scan to be processed... \n%d seconds elapsed", wait, elapsed) time.Sleep(time.Duration(wait) * time.Second) From ea13f4e831bc5d4a3243fe5dc13c0452bb742d0f Mon Sep 17 00:00:00 2001 From: Faye Date: Mon, 16 Jun 2025 13:41:03 +0200 Subject: [PATCH 07/18] Add more details on new flag to attest sonar description --- cmd/kosli/attestSonar.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/kosli/attestSonar.go b/cmd/kosli/attestSonar.go index b6d4bcd56..a2b485302 100644 --- a/cmd/kosli/attestSonar.go +++ b/cmd/kosli/attestSonar.go @@ -35,14 +35,17 @@ Retrieves results for the specified scan from SonarQube Cloud or SonarQube Serve The results are parsed to find the status of the project's quality gate which is used to determine the attestation's compliance status. The scan to be retrieved can be specified in two ways: -1. (Default) Using metadata created by the Sonar scanner. By default this is located within a temporary .scannerwork folder in the repo base directory. +1. (Default) Using metadata created by the Sonar scanner. By default this is located within a temporary ^.scannerwork^ folder in the repo base directory. If you have overriden the location of this folder by passing parameters to the Sonar scanner, or are running Kosli's CLI locally outside the repo's base directory, -you can provide the correct path using the --sonar-working-dir flag. This metadata is generated by a specific scan, allowing Kosli to retrieve the results of that scan. -If there are delays in the scan processing (either because the scanned project is very large, or because SonarQube is experiencing delays), you can use the --allow-wait flag to -wait for the scan to be processed before trying to attest the results. +you can provide the correct path using the ^--sonar-working-dir^ flag. This metadata is generated by a specific scan, allowing Kosli to retrieve the results of that scan. +If there are delays in the scan processing (either because the scanned project is very large, or because SonarQube is experiencing processing delays), it may happen that +the scan results are not available by the time the attest sonar command is executed. In this case you can use the ^--allow-wait^ flag to wait for the scan to be processed. +This flag takes the maximum number of seconds to wait, and allows the Kosli CLI to periodically poll SonarQube to check if the scan results are available. Once the results +are available they are attested to Kosli as usual. + 2. Providing the Sonar project key and the revision of the scan (plus the SonarQube server URL if relevant). If running the Kosli CLI in some CI/CD pipeline, the revision is defaulted to the commit SHA. If you are running the command locally, or have overriden the revision in SonarQube via parameters to the Sonar scanner, you can -provide the correct revision using the --sonar-revision flag. Kosli then finds the scan results for the specified project key and revision. +provide the correct revision using the ^--sonar-revision^ flag. Kosli then finds the scan results for the specified project key and revision. Note that if your project is very large and you are using SonarQube Cloud's automatic analysis, it is possible for the attest sonar command to run before the SonarQube Cloud scan is completed. In this case, we recommend using Kosli's Sonar webhook integration ( https://docs.kosli.com/integrations/sonar/ ) rather than the CLI to attest the scan results. From 092feafa8c454312ccaab5d1f3293d502c70bcb1 Mon Sep 17 00:00:00 2001 From: Faye Date: Tue, 24 Jun 2025 15:56:08 +0200 Subject: [PATCH 08/18] Change --allow-wait flag to --max-retries --- cmd/kosli/attestSonar.go | 10 +++++----- cmd/kosli/root.go | 2 +- internal/sonar/sonar.go | 31 +++++++++++++++---------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/cmd/kosli/attestSonar.go b/cmd/kosli/attestSonar.go index a2b485302..93c3102e8 100644 --- a/cmd/kosli/attestSonar.go +++ b/cmd/kosli/attestSonar.go @@ -24,7 +24,7 @@ type attestSonarOptions struct { projectKey string serverURL string revision string - allowWait int + maxRetries int payload SonarAttestationPayload } @@ -39,8 +39,8 @@ The scan to be retrieved can be specified in two ways: If you have overriden the location of this folder by passing parameters to the Sonar scanner, or are running Kosli's CLI locally outside the repo's base directory, you can provide the correct path using the ^--sonar-working-dir^ flag. This metadata is generated by a specific scan, allowing Kosli to retrieve the results of that scan. If there are delays in the scan processing (either because the scanned project is very large, or because SonarQube is experiencing processing delays), it may happen that -the scan results are not available by the time the attest sonar command is executed. In this case you can use the ^--allow-wait^ flag to wait for the scan to be processed. -This flag takes the maximum number of seconds to wait, and allows the Kosli CLI to periodically poll SonarQube to check if the scan results are available. Once the results +the scan results are not available by the time the attest sonar command is executed. In this case you can use the ^--max-retries^ flag to retry the command while waiting for the scan to be processed. +This flag takes the maximum number of retries for the Kosli CLI to attempt to retrieve the scan results, with exponential backoff. Once the results are available they are attested to Kosli as usual. 2. Providing the Sonar project key and the revision of the scan (plus the SonarQube server URL if relevant). If running the Kosli CLI in some CI/CD pipeline, the revision @@ -164,7 +164,7 @@ func newAttestSonarCmd(out io.Writer) *cobra.Command { cmd.Flags().StringVar(&o.projectKey, "sonar-project-key", "", sonarProjectKeyFlag) cmd.Flags().StringVar(&o.serverURL, "sonar-server-url", "https://sonarcloud.io", sonarServerURLFlag) cmd.Flags().StringVar(&o.revision, "sonar-revision", o.commitSHA, sonarRevisionFlag) - cmd.Flags().IntVar(&o.allowWait, "allow-wait", 0, sonarWaitFlag) + cmd.Flags().IntVar(&o.maxRetries, "max-retries", 0, sonarMaxRetriesFlag) err := RequireFlags(cmd, []string{"flow", "trail", "name", "sonar-api-token"}) if err != nil { @@ -182,7 +182,7 @@ func (o *attestSonarOptions) run(args []string) error { return err } - sc := sonar.NewSonarConfig(o.apiToken, o.workingDir, o.ceTaskURL, o.projectKey, o.serverURL, o.revision, o.allowWait) + sc := sonar.NewSonarConfig(o.apiToken, o.workingDir, o.ceTaskURL, o.projectKey, o.serverURL, o.revision, o.maxRetries) o.payload.SonarResults, err = sc.GetSonarResults(logger) if err != nil { diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index 1748438a6..9ef49d19e 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -241,7 +241,7 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file, sonarProjectKeyFlag = "[conditional] The project key of the SonarCloud/SonarQube project. Only required if you want to use the project key/revision to get the scan results rather than using Sonar's metadata file." sonarServerURLFlag = "[conditional] The URL of your SonarQube server. Only required if you are using SonarQube and not using SonarQube's metadata file to get scan results." sonarRevisionFlag = "[conditional] The revision of the SonarCloud/SonarQube project. Only required if you want to use the project key/revision to get the scan results rather than using Sonar's metadata file and you have overridden the default revision, or you aren't using a CI. Defaults to the value of the git commit flag." - sonarWaitFlag = "[optional] Allow the command to wait for the SonarQube scan to be processed, up to the number of seconds provided. Useful when using SonarQube's metadata file to retrieve and attest scans that take a long time to process . Defaults to 0." + sonarMaxRetriesFlag = "[optional] Allow the command to retry fetching the scan results from SonarQube, up to the maximum number of retries provided, with exponential backoff. Useful when using SonarQube's metadata file to retrieve and attest scans that take a long time to process . Defaults to 0." logicalEnvFlag = "[required] The logical environment." physicalEnvFlag = "[required] The physical environment." attestationTypeDescriptionFlag = "[optional] The attestation type description." diff --git a/internal/sonar/sonar.go b/internal/sonar/sonar.go index e2c949529..3ef4ee79f 100644 --- a/internal/sonar/sonar.go +++ b/internal/sonar/sonar.go @@ -20,7 +20,7 @@ type SonarConfig struct { revision string projectKey string serverURL string - allowWait int + maxRetries int } // Structs to build the JSON for our attestation payload @@ -113,7 +113,7 @@ type Error struct { Msg string `json:"msg"` } -func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revision string, allowWait int) *SonarConfig { +func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revision string, maxRetries int) *SonarConfig { return &SonarConfig{ APIToken: apiToken, WorkingDir: workingDir, @@ -121,7 +121,7 @@ func NewSonarConfig(apiToken, workingDir, ceTaskUrl, projectKey, serverURL, revi revision: revision, projectKey: projectKey, serverURL: serverURL, - allowWait: allowWait, + maxRetries: maxRetries, } } @@ -164,7 +164,7 @@ func (sc *SonarConfig) GetSonarResults(logger *log.Logger) (*SonarResults, error if analysisID == "" { //Get the analysis ID, status, project name and branch data from the ceTaskURL (ce API) - analysisID, err = GetCETaskData(httpClient, project, sonarResults, sc.CETaskUrl, tokenHeader, sc.allowWait, logger) + analysisID, err = GetCETaskData(httpClient, project, sonarResults, sc.CETaskUrl, tokenHeader, sc.maxRetries, logger) if err != nil { return nil, err } @@ -215,22 +215,19 @@ func (sc *SonarConfig) readFile(project *Project, results *SonarResults) error { return nil } -func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *SonarResults, ceTaskURL, tokenHeader string, allowWait int, logger *log.Logger) (string, error) { +func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *SonarResults, ceTaskURL, tokenHeader string, maxRetries int, logger *log.Logger) (string, error) { taskRequest, err := http.NewRequest("GET", ceTaskURL, nil) taskRequest.Header.Add("Authorization", tokenHeader) if err != nil { return "", err } - wait := 10 // seconds to sleep between checks - if allowWait < wait { - wait = allowWait - } - maxWait := allowWait // 20 minutes - elapsed := 0 // seconds elapsed + wait := 10 // start wait period + retries := 0 // number of retries so far + elapsed := 0 // seconds elapsed taskResponseData := &TaskResponse{} - for elapsed < maxWait || maxWait == 0 { + for retries < maxRetries || maxRetries == 0 { taskResponse, err := httpClient.Do(taskRequest) if err != nil { return "", err @@ -241,19 +238,21 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona return "", fmt.Errorf("please check your API token is correct and you have the correct permissions in SonarQube") } - if maxWait == 0 { + if maxRetries == 0 { taskResponse.Body.Close() break } if taskResponseData.Task.Status == "PENDING" || taskResponseData.Task.Status == "IN_PROGRESS" { - logger.Info("waiting %ds for SonarQube scan to be processed... \n%d seconds elapsed", wait, elapsed) + logger.Info("retry %d: waiting %ds for SonarQube scan to be processed...", retries+1, wait) time.Sleep(time.Duration(wait) * time.Second) if elapsed > 300 { // If we've waited 5 minutes, we'll wait 300 seconds (5 minutes) before checking again. This is so that we don't end up with extremely long waiting intervals with the doubling approach. elapsed += wait + retries++ wait += 300 } else { // Otherwise, we'll double the wait time each time elapsed += wait + retries++ wait *= 2 } } else { @@ -263,7 +262,7 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona } if elapsed != 0 { - logger.Info("Waited for %d seconds for SonarQube scan to be processed", elapsed) + logger.Info("Waited for %d seconds for SonarQube scan to be processed. %d retries.\n", elapsed, retries) } task := taskResponseData.Task @@ -277,7 +276,7 @@ func GetCETaskData(httpClient *http.Client, project *Project, sonarResults *Sona // This should only happen if the task is pending - either because the project is large and the scan takes a long time // to process, or because SonarQube is experiencing delays for some reason. if analysisId == "" { - return "", fmt.Errorf("analysis ID not found on %s. The scan results are not yet available, likely due to: \n1. Your project being particularly large and the scan taking time to process, or \n2. SonarQube is experiencing delays in processing scans. \nTry rerunning the command with the --allow-wait flag.", sonarResults.ServerUrl) + return "", fmt.Errorf("analysis ID not found on %s. The scan results are not yet available, likely due to: \n1. Your project being particularly large and the scan taking time to process, or \n2. SonarQube is experiencing delays in processing scans. \nTry rerunning the command with the --max-retries flag.", sonarResults.ServerUrl) } if project.Url == "" { From 3ca60482e26df05eb3c474be15bdf7b428d24923 Mon Sep 17 00:00:00 2001 From: Faye Date: Tue, 24 Jun 2025 16:11:12 +0200 Subject: [PATCH 09/18] Change --allow-wait flag in example commands for docs --- cmd/kosli/attestSonar.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/kosli/attestSonar.go b/cmd/kosli/attestSonar.go index 93c3102e8..0a56a5ec8 100644 --- a/cmd/kosli/attestSonar.go +++ b/cmd/kosli/attestSonar.go @@ -62,7 +62,7 @@ kosli attest sonar \ --api-token yourAPIToken \ --org yourOrgName \ -# report a SonarQube Server attestation about a trail using SonarWube's metadata, waiting for up to 10 minutes for scan processing: +# report a SonarQube Server attestation about a trail using SonarWube's metadata, allowing for up to 10 retries: kosli attest sonar \ --name yourAttestationName \ --flow yourFlowName \ @@ -71,7 +71,7 @@ kosli attest sonar \ --sonar-working-dir yourSonarWorkingDirPath \ --api-token yourAPIToken \ --org yourOrgName \ - --allow-wait 600 + --max-retries 10 # report a SonarQube Cloud attestation for a specific branch about a trail using key/revision: kosli attest sonar \ @@ -98,7 +98,7 @@ kosli attest sonar \ --api-token yourAPIToken \ --org yourOrgName \ -# report a SonarQube Cloud attestation about a trail with an attachment using Sonar's metadata, waiting for up to 10 minutes for scan processing: +# report a SonarQube Cloud attestation about a trail with an attachment using Sonar's metadata, allowing for up to 5 retries: kosli attest sonar \ --name yourAttestationName \ --flow yourFlowName \ @@ -108,7 +108,7 @@ kosli attest sonar \ --attachment yourAttachmentPath \ --api-token yourAPIToken \ --org yourOrgName \ - --allow-wait 600 + --max-retries 5 ` func newAttestSonarCmd(out io.Writer) *cobra.Command { From c502b015d33653dbe58dbae0a485fe9bc315f428 Mon Sep 17 00:00:00 2001 From: Faye Date: Tue, 24 Jun 2025 16:35:31 +0200 Subject: [PATCH 10/18] Fix typo --- cmd/kosli/attestSonar.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/kosli/attestSonar.go b/cmd/kosli/attestSonar.go index 0a56a5ec8..46a034f2d 100644 --- a/cmd/kosli/attestSonar.go +++ b/cmd/kosli/attestSonar.go @@ -52,7 +52,7 @@ In this case, we recommend using Kosli's Sonar webhook integration ( https://doc ` + attestationBindingDesc const attestSonarExample = ` -# report a SonarQube Cloud attestation about a trail using SonarQube's metadata: +# report a SonarQube Cloud attestation about a trail using SonarQube's metadata, with no retries: kosli attest sonar \ --name yourAttestationName \ --flow yourFlowName \ @@ -62,7 +62,7 @@ kosli attest sonar \ --api-token yourAPIToken \ --org yourOrgName \ -# report a SonarQube Server attestation about a trail using SonarWube's metadata, allowing for up to 10 retries: +# report a SonarQube Server attestation about a trail using SonarQube's metadata, allowing for up to 10 retries: kosli attest sonar \ --name yourAttestationName \ --flow yourFlowName \ @@ -98,7 +98,7 @@ kosli attest sonar \ --api-token yourAPIToken \ --org yourOrgName \ -# report a SonarQube Cloud attestation about a trail with an attachment using Sonar's metadata, allowing for up to 5 retries: +# report a SonarQube Cloud attestation about a trail with an attachment using SonarQube's metadata, allowing for up to 5 retries: kosli attest sonar \ --name yourAttestationName \ --flow yourFlowName \ From 4c2d22aca9c0040b9c0c165fead74d69faed2e3a Mon Sep 17 00:00:00 2001 From: Faye Date: Tue, 1 Jul 2025 16:54:00 +0200 Subject: [PATCH 11/18] Start implementing GraphQL for Github PRs --- go.mod | 1 + go.sum | 2 + internal/github/github.go | 187 ++++++++++++++++++++++++++++++++++++-- internal/types/types.go | 28 +++++- 4 files changed, 208 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index cca88cda7..82deebde3 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/owenrumney/go-sarif/v2 v2.3.3 github.com/pkg/errors v0.9.1 github.com/rjeczalik/notify v0.9.3 + github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 github.com/spf13/viper v1.20.1 diff --git a/go.sum b/go.sum index e5b8d2394..1ff6f58bb 100644 --- a/go.sum +++ b/go.sum @@ -611,6 +611,8 @@ github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPr github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/internal/github/github.go b/internal/github/github.go index 50a5a0550..34167719b 100644 --- a/internal/github/github.go +++ b/internal/github/github.go @@ -2,10 +2,14 @@ package github import ( "context" + "fmt" + "io" "strings" + "time" gh "github.com/google/go-github/v42/github" "github.com/kosli-dev/cli/internal/types" + "github.com/shurcooL/graphql" "golang.org/x/oauth2" ) @@ -59,7 +63,7 @@ func NewGithubClientFromToken(ctx context.Context, ghToken string, baseURL strin return gh.NewClient(tc), nil } -func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, error) { +func (c *GithubConfig) PREvidenceForCommit_old(commit string) ([]*types.PREvidence, error) { pullRequestsEvidence := []*types.PREvidence{} prs, err := c.PullRequestsForCommit(commit) if err != nil { @@ -75,17 +79,188 @@ func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, return pullRequestsEvidence, nil } +func graphqlEndpoint(baseURL string) string { + if baseURL == "" || baseURL == "https://api.github.com" { + return "https://api.github.com/graphql" + } + return strings.TrimSuffix(baseURL, "/") + "/api/graphql" +} + +func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, error) { + ctx := context.Background() + ghClient, err := NewGithubClientFromToken(ctx, c.Token, c.BaseURL) + httpClient := ghClient.Client() + + client := graphql.NewClient(graphqlEndpoint(c.BaseURL), httpClient) + + pullRequestsEvidence := []*types.PREvidence{} + + var query struct { + Repository struct { + Object struct { + Commit struct { + AssociatedPullRequests struct { + Nodes []struct { + Number graphql.Int + Title graphql.String + State graphql.String + HeadRefName graphql.String + URL graphql.String + CreatedAt graphql.String + MergedAt graphql.String + + Author struct { + Login graphql.String + } + + Commits struct { + Nodes []struct { + Commit struct { + Oid graphql.String + MessageHeadline graphql.String + CommittedDate graphql.String + Committer struct { + Name graphql.String + Email graphql.String + Date graphql.String + User *struct { + Login graphql.String + } + } + } + } + PageInfo struct { + HasNextPage graphql.Boolean + EndCursor graphql.String + } + } `graphql:"commits(first: 100, after: $commitCursor)"` + + Reviews struct { + Nodes []struct { + Author struct { + Login graphql.String + } + State graphql.String + SubmittedAt graphql.String + } + PageInfo struct { + HasNextPage graphql.Boolean + EndCursor graphql.String + } + } `graphql:"reviews(first: 100, states: APPROVED, after: $reviewCursor)"` + } + PageInfo struct { + HasNextPage graphql.Boolean + EndCursor graphql.String + } + } `graphql:"associatedPullRequests(first: 100, after: $prCursor)"` + } `graphql:"... on Commit"` + } `graphql:"object(oid: $commitSHA)"` + } `graphql:"repository(owner: $owner, name: $repo)"` + } + + variables := map[string]interface{}{ + "owner": graphql.String(c.Org), + "repo": graphql.String(c.Repository), + "commitSHA": GitObjectID(commit), + "prCursor": (*graphql.String)(nil), + "commitCursor": (*graphql.String)(nil), + "reviewCursor": (*graphql.String)(nil), + } + + fmt.Printf("variables: %v\n", variables) + err = client.Query(context.Background(), &query, variables) + fmt.Printf("query: %v\n", query) + if err != nil { + return pullRequestsEvidence, err + } + + // Print results for demonstration + for _, pr := range query.Repository.Object.Commit.AssociatedPullRequests.Nodes { + createdAt, err := time.Parse(time.RFC3339, string(pr.CreatedAt)) + if err != nil { + return pullRequestsEvidence, err + } + mergedAt, err := time.Parse(time.RFC3339, string(pr.MergedAt)) + if err != nil { + return pullRequestsEvidence, err + } + + evidence := &types.PREvidence{ + URL: string(pr.URL), + MergeCommit: commit, + State: string(pr.State), + Author: string(pr.Author.Login), + CreatedAt: createdAt.Unix(), + MergedAt: mergedAt.Unix(), + Title: string(pr.Title), + HeadBranch: string(pr.HeadRefName), + Approvers2: []types.PRApprovals{}, + Commits: []types.Commit{}, + } + + for _, c := range pr.Commits.Nodes { + timestamp, err := time.Parse(time.RFC3339, string(c.Commit.CommittedDate)) + if err != nil { + return pullRequestsEvidence, err + } + + evidence.Commits = append(evidence.Commits, types.Commit{ + SHA: string(c.Commit.Oid), + Message: string(c.Commit.MessageHeadline), + Committer: string(c.Commit.Committer.User.Login), + Timestamp: timestamp.Unix(), + }) + } + + for _, r := range pr.Reviews.Nodes { + submittedAt, err := time.Parse(time.RFC3339, string(r.SubmittedAt)) + if err != nil { + return pullRequestsEvidence, err + } + + evidence.Approvers2 = append(evidence.Approvers2, types.PRApprovals{ + Author: string(r.Author.Login), + State: string(r.State), + Timestamp: submittedAt.Unix(), + }) + } + + // fmt.Printf("PR #%d: %s (State: %s, Branch: %s)\n", pr.Number, pr.Title, pr.State, pr.HeadRefName) + // fmt.Printf("URL: %s\n", pr.URL) + // fmt.Printf("Author: %s\n", pr.Author.Login) + // fmt.Printf("Commits:\n") + // for _, c := range pr.Commits.Nodes { + // fmt.Printf("- %s %s by %s\n", c.Commit.Oid, c.Commit.MessageHeadline, c.Commit.Committer.Name) + // } + // fmt.Printf("Approvals:\n") + // for _, r := range pr.Reviews.Nodes { + // fmt.Printf("- %s at %s\n", r.Author.Login, r.SubmittedAt) + // } + // fmt.Println() + pullRequestsEvidence = append(pullRequestsEvidence, evidence) + } + fmt.Printf("pullRequestsEvidence: %v\n", pullRequestsEvidence) + return pullRequestsEvidence, nil +} + +type GitObjectID string + +func (v GitObjectID) MarshalGQL(w io.Writer) { + fmt.Fprintf(w, `"%s"`, string(v)) +} + func (c *GithubConfig) newPRGithubEvidence(pr *gh.PullRequest) (*types.PREvidence, error) { evidence := &types.PREvidence{ URL: pr.GetHTMLURL(), MergeCommit: pr.GetMergeCommitSHA(), State: pr.GetState(), } - approvers, err := c.GetPullRequestApprovers(pr.GetNumber()) - if err != nil { - return evidence, err - } - evidence.Approvers = approvers + // approvers, err := c.GetPullRequestApprovers(pr.GetNumber()) + // if err != nil { + // return evidence, err + // } + //evidence.Approvers = approvers return evidence, nil } diff --git a/internal/types/types.go b/internal/types/types.go index 772ff3b44..89239b131 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1,15 +1,35 @@ package types type PREvidence struct { - MergeCommit string `json:"merge_commit"` - URL string `json:"url"` - State string `json:"state"` - Approvers []string `json:"approvers"` + MergeCommit string `json:"merge_commit"` + URL string `json:"url"` + State string `json:"state"` + Approvers []string `json:"approvers"` + Approvers2 []PRApprovals `json:"approvers2"` + Author string `json:"author"` + CreatedAt int64 `json:"created_at"` + MergedAt int64 `json:"merged_at"` + Title string `json:"title"` + HeadBranch string `json:"head_branch"` + Commits []Commit `json:"commits"` // LastCommit string `json:"lastCommit"` // LastCommitter string `json:"lastCommitter"` // SelfApproved bool `json:"selfApproved"` } +type PRApprovals struct { + Author string `json:"author"` + State string `json:"state"` + Timestamp int64 `json:"timestamp"` +} + +type Commit struct { + SHA string `json:"sha"` + Message string `json:"message"` + Committer string `json:"committer"` + Timestamp int64 `json:"timestamp"` +} + type PRRetriever interface { PREvidenceForCommit(string) ([]*PREvidence, error) } From 5510cbd328b39e28870e71ac6191ce3c4a39baa4 Mon Sep 17 00:00:00 2001 From: Faye Date: Wed, 2 Jul 2025 16:51:20 +0200 Subject: [PATCH 12/18] Update variable names for PR structs to match API --- internal/github/github.go | 2 +- internal/types/types.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/github/github.go b/internal/github/github.go index 34167719b..36f7614cc 100644 --- a/internal/github/github.go +++ b/internal/github/github.go @@ -194,7 +194,7 @@ func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, CreatedAt: createdAt.Unix(), MergedAt: mergedAt.Unix(), Title: string(pr.Title), - HeadBranch: string(pr.HeadRefName), + HeadRef: string(pr.HeadRefName), Approvers2: []types.PRApprovals{}, Commits: []types.Commit{}, } diff --git a/internal/types/types.go b/internal/types/types.go index 89239b131..064b40bc2 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -10,7 +10,7 @@ type PREvidence struct { CreatedAt int64 `json:"created_at"` MergedAt int64 `json:"merged_at"` Title string `json:"title"` - HeadBranch string `json:"head_branch"` + HeadRef string `json:"head_ref"` Commits []Commit `json:"commits"` // LastCommit string `json:"lastCommit"` // LastCommitter string `json:"lastCommitter"` @@ -24,9 +24,9 @@ type PRApprovals struct { } type Commit struct { - SHA string `json:"sha"` + SHA string `json:"sha1"` Message string `json:"message"` - Committer string `json:"committer"` + Committer string `json:"author"` Timestamp int64 `json:"timestamp"` } From c4e0cad602e120b974e6424e1e9a590a9f690d25 Mon Sep 17 00:00:00 2001 From: Faye Date: Thu, 3 Jul 2025 10:32:21 +0200 Subject: [PATCH 13/18] Update PR struct to work with both old and new versions of PR attestation --- internal/azure/azure.go | 3 ++- internal/bitbucket/bitbucket.go | 3 ++- internal/github/github.go | 46 ++++++++++++++++++--------------- internal/gitlab/gitlab.go | 3 ++- internal/types/types.go | 18 ++++++------- internal/utils/utils.go | 8 ++++++ 6 files changed, 48 insertions(+), 33 deletions(-) diff --git a/internal/azure/azure.go b/internal/azure/azure.go index 557f8cf2a..686ba8eb6 100644 --- a/internal/azure/azure.go +++ b/internal/azure/azure.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/kosli-dev/cli/internal/types" + "github.com/kosli-dev/cli/internal/utils" "github.com/microsoft/azure-devops-go-api/azuredevops" "github.com/microsoft/azure-devops-go-api/azuredevops/git" ) @@ -85,7 +86,7 @@ func (c *AzureConfig) newPRAzureEvidence(pr git.GitPullRequest) (*types.PREviden if err != nil { return evidence, err } - evidence.Approvers = approvers + evidence.Approvers = utils.ConvertStringListToInterfaceList(approvers) return evidence, nil } diff --git a/internal/bitbucket/bitbucket.go b/internal/bitbucket/bitbucket.go index f3b0a85e6..481390505 100644 --- a/internal/bitbucket/bitbucket.go +++ b/internal/bitbucket/bitbucket.go @@ -8,6 +8,7 @@ import ( "github.com/kosli-dev/cli/internal/logger" "github.com/kosli-dev/cli/internal/requests" "github.com/kosli-dev/cli/internal/types" + "github.com/kosli-dev/cli/internal/utils" ) type Config struct { @@ -122,7 +123,7 @@ func (c *Config) getPullRequestDetailsFromBitbucket(prApiUrl, prHtmlLink, commit } else { c.Logger.Debug("no approvers found") } - evidence.Approvers = approvers + evidence.Approvers = utils.ConvertStringListToInterfaceList(approvers) // prID := int(responseData["id"].(float64)) // evidence.LastCommit, evidence.LastCommitter, err = getBitbucketPRLastCommit(workspace, repository, username, password, prID) // if err != nil { diff --git a/internal/github/github.go b/internal/github/github.go index 36f7614cc..40fc1c1c2 100644 --- a/internal/github/github.go +++ b/internal/github/github.go @@ -63,21 +63,21 @@ func NewGithubClientFromToken(ctx context.Context, ghToken string, baseURL strin return gh.NewClient(tc), nil } -func (c *GithubConfig) PREvidenceForCommit_old(commit string) ([]*types.PREvidence, error) { - pullRequestsEvidence := []*types.PREvidence{} - prs, err := c.PullRequestsForCommit(commit) - if err != nil { - return pullRequestsEvidence, err - } - for _, pr := range prs { - evidence, err := c.newPRGithubEvidence(pr) - if err != nil { - return pullRequestsEvidence, err - } - pullRequestsEvidence = append(pullRequestsEvidence, evidence) - } - return pullRequestsEvidence, nil -} +// func (c *GithubConfig) PREvidenceForCommit_old(commit string) ([]*types.PREvidence, error) { +// pullRequestsEvidence := []*types.PREvidence{} +// prs, err := c.PullRequestsForCommit(commit) +// if err != nil { +// return pullRequestsEvidence, err +// } +// for _, pr := range prs { +// evidence, err := c.newPRGithubEvidence(pr) +// if err != nil { +// return pullRequestsEvidence, err +// } +// pullRequestsEvidence = append(pullRequestsEvidence, evidence) +// } +// return pullRequestsEvidence, nil +// } func graphqlEndpoint(baseURL string) string { if baseURL == "" || baseURL == "https://api.github.com" { @@ -181,9 +181,13 @@ func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, if err != nil { return pullRequestsEvidence, err } - mergedAt, err := time.Parse(time.RFC3339, string(pr.MergedAt)) - if err != nil { - return pullRequestsEvidence, err + mergedAt := int64(0) + if pr.MergedAt != "" { + mergedAtTime, err := time.Parse(time.RFC3339, string(pr.MergedAt)) + if err != nil { + return pullRequestsEvidence, err + } + mergedAt = mergedAtTime.Unix() } evidence := &types.PREvidence{ @@ -192,10 +196,10 @@ func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, State: string(pr.State), Author: string(pr.Author.Login), CreatedAt: createdAt.Unix(), - MergedAt: mergedAt.Unix(), + MergedAt: mergedAt, Title: string(pr.Title), HeadRef: string(pr.HeadRefName), - Approvers2: []types.PRApprovals{}, + Approvers: []interface{}{}, Commits: []types.Commit{}, } @@ -219,7 +223,7 @@ func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, return pullRequestsEvidence, err } - evidence.Approvers2 = append(evidence.Approvers2, types.PRApprovals{ + evidence.Approvers = append(evidence.Approvers, types.PRApprovals{ Author: string(r.Author.Login), State: string(r.State), Timestamp: submittedAt.Unix(), diff --git a/internal/gitlab/gitlab.go b/internal/gitlab/gitlab.go index fd8307c57..e8333203f 100644 --- a/internal/gitlab/gitlab.go +++ b/internal/gitlab/gitlab.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/kosli-dev/cli/internal/types" + "github.com/kosli-dev/cli/internal/utils" gitlab "gitlab.com/gitlab-org/api/client-go" ) @@ -64,7 +65,7 @@ func (c *GitlabConfig) newPRGitlabEvidence(mr *gitlab.BasicMergeRequest) (*types if err != nil { return evidence, err } - evidence.Approvers = approvers + evidence.Approvers = utils.ConvertStringListToInterfaceList(approvers) return evidence, nil } diff --git a/internal/types/types.go b/internal/types/types.go index 064b40bc2..9734cc338 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -4,21 +4,21 @@ type PREvidence struct { MergeCommit string `json:"merge_commit"` URL string `json:"url"` State string `json:"state"` - Approvers []string `json:"approvers"` - Approvers2 []PRApprovals `json:"approvers2"` - Author string `json:"author"` - CreatedAt int64 `json:"created_at"` - MergedAt int64 `json:"merged_at"` - Title string `json:"title"` - HeadRef string `json:"head_ref"` - Commits []Commit `json:"commits"` + Approvers []interface{} `json:"approvers"` + //Approvers2 []PRApprovals `json:"approvers2"` + Author string `json:"author"` + CreatedAt int64 `json:"created_at"` + MergedAt int64 `json:"merged_at"` + Title string `json:"title"` + HeadRef string `json:"head_ref"` + Commits []Commit `json:"commits"` // LastCommit string `json:"lastCommit"` // LastCommitter string `json:"lastCommitter"` // SelfApproved bool `json:"selfApproved"` } type PRApprovals struct { - Author string `json:"author"` + Author string `json:"username"` State string `json:"state"` Timestamp int64 `json:"timestamp"` } diff --git a/internal/utils/utils.go b/internal/utils/utils.go index d6e9d069c..e104f593e 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -150,3 +150,11 @@ func CreateFileWithContent(path, content string) error { _, err = file.Write([]byte(content)) return err } + +func ConvertStringListToInterfaceList(approversList []string) []interface{} { + approversIface := make([]interface{}, len(approversList)) + for i, v := range approversList { + approversIface[i] = v + } + return approversIface +} From 837984b0a20f1e70ac4f4da6b0c278cfce290d96 Mon Sep 17 00:00:00 2001 From: Faye Date: Thu, 3 Jul 2025 10:54:35 +0200 Subject: [PATCH 14/18] Fix payload for non-github PR attestations and tidy up comments --- internal/github/github.go | 52 ++++----------------------------------- internal/types/types.go | 16 +++++------- 2 files changed, 11 insertions(+), 57 deletions(-) diff --git a/internal/github/github.go b/internal/github/github.go index 40fc1c1c2..2a29a706e 100644 --- a/internal/github/github.go +++ b/internal/github/github.go @@ -63,22 +63,6 @@ func NewGithubClientFromToken(ctx context.Context, ghToken string, baseURL strin return gh.NewClient(tc), nil } -// func (c *GithubConfig) PREvidenceForCommit_old(commit string) ([]*types.PREvidence, error) { -// pullRequestsEvidence := []*types.PREvidence{} -// prs, err := c.PullRequestsForCommit(commit) -// if err != nil { -// return pullRequestsEvidence, err -// } -// for _, pr := range prs { -// evidence, err := c.newPRGithubEvidence(pr) -// if err != nil { -// return pullRequestsEvidence, err -// } -// pullRequestsEvidence = append(pullRequestsEvidence, evidence) -// } -// return pullRequestsEvidence, nil -// } - func graphqlEndpoint(baseURL string) string { if baseURL == "" || baseURL == "https://api.github.com" { return "https://api.github.com/graphql" @@ -88,13 +72,16 @@ func graphqlEndpoint(baseURL string) string { func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, error) { ctx := context.Background() + pullRequestsEvidence := []*types.PREvidence{} + ghClient, err := NewGithubClientFromToken(ctx, c.Token, c.BaseURL) + if err != nil { + return pullRequestsEvidence, err + } httpClient := ghClient.Client() client := graphql.NewClient(graphqlEndpoint(c.BaseURL), httpClient) - pullRequestsEvidence := []*types.PREvidence{} - var query struct { Repository struct { Object struct { @@ -168,9 +155,7 @@ func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, "reviewCursor": (*graphql.String)(nil), } - fmt.Printf("variables: %v\n", variables) err = client.Query(context.Background(), &query, variables) - fmt.Printf("query: %v\n", query) if err != nil { return pullRequestsEvidence, err } @@ -230,21 +215,8 @@ func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, }) } - // fmt.Printf("PR #%d: %s (State: %s, Branch: %s)\n", pr.Number, pr.Title, pr.State, pr.HeadRefName) - // fmt.Printf("URL: %s\n", pr.URL) - // fmt.Printf("Author: %s\n", pr.Author.Login) - // fmt.Printf("Commits:\n") - // for _, c := range pr.Commits.Nodes { - // fmt.Printf("- %s %s by %s\n", c.Commit.Oid, c.Commit.MessageHeadline, c.Commit.Committer.Name) - // } - // fmt.Printf("Approvals:\n") - // for _, r := range pr.Reviews.Nodes { - // fmt.Printf("- %s at %s\n", r.Author.Login, r.SubmittedAt) - // } - // fmt.Println() pullRequestsEvidence = append(pullRequestsEvidence, evidence) } - fmt.Printf("pullRequestsEvidence: %v\n", pullRequestsEvidence) return pullRequestsEvidence, nil } @@ -254,20 +226,6 @@ func (v GitObjectID) MarshalGQL(w io.Writer) { fmt.Fprintf(w, `"%s"`, string(v)) } -func (c *GithubConfig) newPRGithubEvidence(pr *gh.PullRequest) (*types.PREvidence, error) { - evidence := &types.PREvidence{ - URL: pr.GetHTMLURL(), - MergeCommit: pr.GetMergeCommitSHA(), - State: pr.GetState(), - } - // approvers, err := c.GetPullRequestApprovers(pr.GetNumber()) - // if err != nil { - // return evidence, err - // } - //evidence.Approvers = approvers - return evidence, nil -} - // PullRequestsForCommit returns a list of pull requests for a specific commit func (c *GithubConfig) PullRequestsForCommit(commit string) ([]*gh.PullRequest, error) { ctx := context.Background() diff --git a/internal/types/types.go b/internal/types/types.go index 9734cc338..838da8639 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -5,16 +5,12 @@ type PREvidence struct { URL string `json:"url"` State string `json:"state"` Approvers []interface{} `json:"approvers"` - //Approvers2 []PRApprovals `json:"approvers2"` - Author string `json:"author"` - CreatedAt int64 `json:"created_at"` - MergedAt int64 `json:"merged_at"` - Title string `json:"title"` - HeadRef string `json:"head_ref"` - Commits []Commit `json:"commits"` - // LastCommit string `json:"lastCommit"` - // LastCommitter string `json:"lastCommitter"` - // SelfApproved bool `json:"selfApproved"` + Author string `json:"author,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` + MergedAt int64 `json:"merged_at,omitempty"` + Title string `json:"title,omitempty"` + HeadRef string `json:"head_ref,omitempty"` + Commits []Commit `json:"commits,omitempty"` } type PRApprovals struct { From 72fcbaa8f0ab06396b18bb3a2fd4dff9e832d38c Mon Sep 17 00:00:00 2001 From: Faye Date: Fri, 4 Jul 2025 10:15:26 +0200 Subject: [PATCH 15/18] Fix payload issues for non-Github-attest commands --- cmd/kosli/assertPRAzure.go | 2 +- cmd/kosli/assertPRBitbucket.go | 2 +- cmd/kosli/assertPRGithub.go | 2 +- cmd/kosli/assertPRGithub_test.go | 2 +- cmd/kosli/assertPRGitlab.go | 2 +- cmd/kosli/pullrequest.go | 6 +++--- internal/azure/azure.go | 8 +++++++- internal/azure/azure_test.go | 2 +- internal/bitbucket/bitbucket.go | 8 +++++++- internal/github/github.go | 33 +++++++++++++++++++++++++++++++- internal/github/github_test.go | 2 +- internal/gitlab/gitlab.go | 8 +++++++- internal/gitlab/gitlab_test.go | 2 +- internal/types/types.go | 3 ++- 14 files changed, 66 insertions(+), 16 deletions(-) diff --git a/cmd/kosli/assertPRAzure.go b/cmd/kosli/assertPRAzure.go index 29513ba57..cf4031677 100644 --- a/cmd/kosli/assertPRAzure.go +++ b/cmd/kosli/assertPRAzure.go @@ -59,7 +59,7 @@ func newAssertPullRequestAzureCmd(out io.Writer) *cobra.Command { } func (o *assertPullRequestAzureOptions) run(args []string) error { - pullRequestsEvidence, err := o.azureConfig.PREvidenceForCommit(o.commit) + pullRequestsEvidence, err := o.azureConfig.PREvidenceForCommitV2(o.commit) if err != nil { return err } diff --git a/cmd/kosli/assertPRBitbucket.go b/cmd/kosli/assertPRBitbucket.go index 69f7d1b74..79bb9ebb9 100644 --- a/cmd/kosli/assertPRBitbucket.go +++ b/cmd/kosli/assertPRBitbucket.go @@ -81,7 +81,7 @@ func newAssertPullRequestBitbucketCmd(out io.Writer) *cobra.Command { } func (o *assertPullRequestBitbucketOptions) run(args []string) error { - pullRequestsEvidence, err := o.bbConfig.PREvidenceForCommit(o.commit) + pullRequestsEvidence, err := o.bbConfig.PREvidenceForCommitV2(o.commit) if err != nil { return err } diff --git a/cmd/kosli/assertPRGithub.go b/cmd/kosli/assertPRGithub.go index 83e60c518..a74243128 100644 --- a/cmd/kosli/assertPRGithub.go +++ b/cmd/kosli/assertPRGithub.go @@ -58,7 +58,7 @@ func newAssertPullRequestGithubCmd(out io.Writer) *cobra.Command { } func (o *assertPullRequestGithubOptions) run(args []string) error { - pullRequestsEvidence, err := o.githubConfig.PREvidenceForCommit(o.commit) + pullRequestsEvidence, err := o.githubConfig.PREvidenceForCommitV2(o.commit) if err != nil { return err } diff --git a/cmd/kosli/assertPRGithub_test.go b/cmd/kosli/assertPRGithub_test.go index 1d442ac28..65cc63f10 100644 --- a/cmd/kosli/assertPRGithub_test.go +++ b/cmd/kosli/assertPRGithub_test.go @@ -47,7 +47,7 @@ func (suite *AssertPRGithubCommandTestSuite) TestAssertPRGithubCmd() { name: "assert Github PR evidence fails when commit does not exist", cmd: `assert pullrequest github --github-org kosli-dev --repository cli --commit 19aab7f063147614451c88969602a10afba123ab` + suite.defaultKosliArguments, - golden: "Error: GET https://api.github.com/repos/kosli-dev/cli/commits/19aab7f063147614451c88969602a10afba123ab/pulls: 422 No commit found for SHA: 19aab7f063147614451c88969602a10afba123ab []\n", + golden: "Error: assert failed: found no pull request(s) in Github for commit: 19aab7f063147614451c88969602a10afba123ab\n", }, } diff --git a/cmd/kosli/assertPRGitlab.go b/cmd/kosli/assertPRGitlab.go index f906eaf83..019f5912e 100644 --- a/cmd/kosli/assertPRGitlab.go +++ b/cmd/kosli/assertPRGitlab.go @@ -58,7 +58,7 @@ func newAssertPullRequestGitlabCmd(out io.Writer) *cobra.Command { } func (o *assertPullRequestGitlabOptions) run(args []string) error { - pullRequestsEvidence, err := o.gitlabConfig.PREvidenceForCommit(o.commit) + pullRequestsEvidence, err := o.gitlabConfig.PREvidenceForCommitV2(o.commit) if err != nil { return err } diff --git a/cmd/kosli/pullrequest.go b/cmd/kosli/pullrequest.go index 4b352d0f2..938d37def 100644 --- a/cmd/kosli/pullrequest.go +++ b/cmd/kosli/pullrequest.go @@ -49,7 +49,7 @@ func (o *pullRequestArtifactOptions) run(out io.Writer, args []string) error { } url := fmt.Sprintf("%s/api/v2/evidence/%s/artifact/%s/pull_request", global.Host, global.Org, o.flowName) - pullRequestsEvidence, err := o.getRetriever().PREvidenceForCommit(o.commit) + pullRequestsEvidence, err := o.getRetriever().PREvidenceForCommitV1(o.commit) if err != nil { return err } @@ -119,7 +119,7 @@ func (o *attestPROptions) run(args []string) error { return err } - pullRequestsEvidence, err := o.getRetriever().PREvidenceForCommit(o.payload.Commit.Sha1) + pullRequestsEvidence, err := o.getRetriever().PREvidenceForCommitV2(o.payload.Commit.Sha1) if err != nil { return err } @@ -175,7 +175,7 @@ func (o *pullRequestCommitOptions) run(args []string) error { return err } - pullRequestsEvidence, err := o.getRetriever().PREvidenceForCommit(o.payload.CommitSHA) + pullRequestsEvidence, err := o.getRetriever().PREvidenceForCommitV1(o.payload.CommitSHA) if err != nil { return err } diff --git a/internal/azure/azure.go b/internal/azure/azure.go index 686ba8eb6..5cc269daa 100644 --- a/internal/azure/azure.go +++ b/internal/azure/azure.go @@ -55,7 +55,8 @@ func NewAzureClientFromToken(ctx context.Context, azToken, orgURL string) (git.C return gitClient, nil } -func (c *AzureConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, error) { +// This is the old implementation, it will be removed after the PR payload is enhanced for Azure +func (c *AzureConfig) PREvidenceForCommitV2(commit string) ([]*types.PREvidence, error) { pullRequestsEvidence := []*types.PREvidence{} prs, err := c.PullRequestsForCommit(commit) if err != nil { @@ -71,6 +72,11 @@ func (c *AzureConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, e return pullRequestsEvidence, nil } +// This is the new implementation, it will be used for Azure +func (c *AzureConfig) PREvidenceForCommitV1(commit string) ([]*types.PREvidence, error) { + return []*types.PREvidence{}, nil +} + func (c *AzureConfig) newPRAzureEvidence(pr git.GitPullRequest) (*types.PREvidence, error) { prID := strconv.Itoa(*pr.PullRequestId) url, err := url.JoinPath(c.OrgURL, c.Project, "_git", c.Repository, "pullrequest", prID) diff --git a/internal/azure/azure_test.go b/internal/azure/azure_test.go index 155c051e2..4a75eab4e 100644 --- a/internal/azure/azure_test.go +++ b/internal/azure/azure_test.go @@ -66,7 +66,7 @@ func (suite *AzureTestSuite) TestPREvidenceForCommit() { testHelpers.SkipIfEnvVarUnset(suite.Suite.T(), []string{"KOSLI_AZURE_TOKEN"}) t.config.Token = os.Getenv("KOSLI_AZURE_TOKEN") } - prs, err := t.config.PREvidenceForCommit(t.commit) + prs, err := t.config.PREvidenceForCommitV2(t.commit) if t.result.wantError { require.Errorf(suite.Suite.T(), err, "expected an error but got: %s", err) } else { diff --git a/internal/bitbucket/bitbucket.go b/internal/bitbucket/bitbucket.go index 481390505..33a57faad 100644 --- a/internal/bitbucket/bitbucket.go +++ b/internal/bitbucket/bitbucket.go @@ -22,10 +22,16 @@ type Config struct { Assert bool } -func (c *Config) PREvidenceForCommit(commit string) ([]*types.PREvidence, error) { +// This is the old implementation, it will be removed after the PR payload is enhanced for Bitbucket +func (c *Config) PREvidenceForCommitV2(commit string) ([]*types.PREvidence, error) { return c.getPullRequestsFromBitbucketApi(commit) } +// This is the new implementation, it will be used for Bitbucket +func (c *Config) PREvidenceForCommitV1(commit string) ([]*types.PREvidence, error) { + return []*types.PREvidence{}, nil +} + func (c *Config) getPullRequestsFromBitbucketApi(commit string) ([]*types.PREvidence, error) { pullRequestsEvidence := []*types.PREvidence{} diff --git a/internal/github/github.go b/internal/github/github.go index 2a29a706e..892fecb6b 100644 --- a/internal/github/github.go +++ b/internal/github/github.go @@ -9,6 +9,7 @@ import ( gh "github.com/google/go-github/v42/github" "github.com/kosli-dev/cli/internal/types" + "github.com/kosli-dev/cli/internal/utils" "github.com/shurcooL/graphql" "golang.org/x/oauth2" @@ -70,7 +71,7 @@ func graphqlEndpoint(baseURL string) string { return strings.TrimSuffix(baseURL, "/") + "/api/graphql" } -func (c *GithubConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, error) { +func (c *GithubConfig) PREvidenceForCommitV2(commit string) ([]*types.PREvidence, error) { ctx := context.Background() pullRequestsEvidence := []*types.PREvidence{} @@ -226,6 +227,22 @@ func (v GitObjectID) MarshalGQL(w io.Writer) { fmt.Fprintf(w, `"%s"`, string(v)) } +func (c *GithubConfig) PREvidenceForCommitV1(commit string) ([]*types.PREvidence, error) { + pullRequestsEvidence := []*types.PREvidence{} + prs, err := c.PullRequestsForCommit(commit) + if err != nil { + return pullRequestsEvidence, err + } + for _, pr := range prs { + evidence, err := c.newPRGithubEvidence(pr) + if err != nil { + return pullRequestsEvidence, err + } + pullRequestsEvidence = append(pullRequestsEvidence, evidence) + } + return pullRequestsEvidence, nil +} + // PullRequestsForCommit returns a list of pull requests for a specific commit func (c *GithubConfig) PullRequestsForCommit(commit string) ([]*gh.PullRequest, error) { ctx := context.Background() @@ -258,3 +275,17 @@ func (c *GithubConfig) GetPullRequestApprovers(number int) ([]string, error) { } return approvers, nil } + +func (c *GithubConfig) newPRGithubEvidence(pr *gh.PullRequest) (*types.PREvidence, error) { + evidence := &types.PREvidence{ + URL: pr.GetHTMLURL(), + MergeCommit: pr.GetMergeCommitSHA(), + State: pr.GetState(), + } + approvers, err := c.GetPullRequestApprovers(pr.GetNumber()) + if err != nil { + return evidence, err + } + evidence.Approvers = utils.ConvertStringListToInterfaceList(approvers) + return evidence, nil +} diff --git a/internal/github/github_test.go b/internal/github/github_test.go index 5977cb1c4..7845e8a69 100644 --- a/internal/github/github_test.go +++ b/internal/github/github_test.go @@ -95,7 +95,7 @@ func (suite *GithubTestSuite) TestPREvidenceForCommit() { if t.commit == "" { t.commit = testHelpers.GithubCommitWithPR() } - prs, err := t.config.PREvidenceForCommit(t.commit) + prs, err := t.config.PREvidenceForCommitV2(t.commit) if t.result.wantError { require.Errorf(suite.Suite.T(), err, "expected an error but got: %s", err) } else { diff --git a/internal/gitlab/gitlab.go b/internal/gitlab/gitlab.go index e8333203f..ec4545521 100644 --- a/internal/gitlab/gitlab.go +++ b/internal/gitlab/gitlab.go @@ -39,7 +39,8 @@ func (c *GitlabConfig) ProjectID() string { return fmt.Sprintf("%s/%s", c.Org, c.Repository) } -func (c *GitlabConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, error) { +// This is the old implementation, it will be removed after the PR payload is enhanced for Gitlab +func (c *GitlabConfig) PREvidenceForCommitV2(commit string) ([]*types.PREvidence, error) { pullRequestsEvidence := []*types.PREvidence{} mrs, err := c.MergeRequestsForCommit(commit) if err != nil { @@ -55,6 +56,11 @@ func (c *GitlabConfig) PREvidenceForCommit(commit string) ([]*types.PREvidence, return pullRequestsEvidence, nil } +// This is the new implementation, it will be used for Gitlab +func (c *GitlabConfig) PREvidenceForCommitV1(commit string) ([]*types.PREvidence, error) { + return []*types.PREvidence{}, nil +} + func (c *GitlabConfig) newPRGitlabEvidence(mr *gitlab.BasicMergeRequest) (*types.PREvidence, error) { evidence := &types.PREvidence{ URL: mr.WebURL, diff --git a/internal/gitlab/gitlab_test.go b/internal/gitlab/gitlab_test.go index 4786106e4..40bc3af33 100644 --- a/internal/gitlab/gitlab_test.go +++ b/internal/gitlab/gitlab_test.go @@ -214,7 +214,7 @@ func (suite *GitlabTestSuite) TestPREvidenceForCommit() { testHelpers.SkipIfEnvVarUnset(suite.Suite.T(), []string{"KOSLI_GITLAB_TOKEN"}) t.gitlabConfig.Token = os.Getenv("KOSLI_GITLAB_TOKEN") } - prs, err := t.gitlabConfig.PREvidenceForCommit(t.commit) + prs, err := t.gitlabConfig.PREvidenceForCommitV2(t.commit) if t.result.wantError { require.Error(suite.Suite.T(), err) } else { diff --git a/internal/types/types.go b/internal/types/types.go index 838da8639..e3b5649d7 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -27,5 +27,6 @@ type Commit struct { } type PRRetriever interface { - PREvidenceForCommit(string) ([]*PREvidence, error) + PREvidenceForCommitV2(string) ([]*PREvidence, error) + PREvidenceForCommitV1(string) ([]*PREvidence, error) } From 8a7d3fb8a9c7c20405500286232b01ab90de9450 Mon Sep 17 00:00:00 2001 From: Faye Date: Fri, 4 Jul 2025 10:31:24 +0200 Subject: [PATCH 16/18] Update report evidence command payloads --- cmd/kosli/pullrequest.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cmd/kosli/pullrequest.go b/cmd/kosli/pullrequest.go index 938d37def..8c6f556b9 100644 --- a/cmd/kosli/pullrequest.go +++ b/cmd/kosli/pullrequest.go @@ -48,8 +48,19 @@ func (o *pullRequestArtifactOptions) run(out io.Writer, args []string) error { } } + label := "" + o.payload.GitProvider, label = getGitProviderAndLabel(o.retriever) + url := fmt.Sprintf("%s/api/v2/evidence/%s/artifact/%s/pull_request", global.Host, global.Org, o.flowName) - pullRequestsEvidence, err := o.getRetriever().PREvidenceForCommitV1(o.commit) + + // TODO: after the PR payload is enhanced for all git providers they will all use the same method + var pullRequestsEvidence []*types.PREvidence + if o.payload.GitProvider == "github" { + pullRequestsEvidence, err = o.getRetriever().PREvidenceForCommitV1(o.commit) + } else { + pullRequestsEvidence, err = o.getRetriever().PREvidenceForCommitV2(o.commit) + } + if err != nil { return err } @@ -60,9 +71,6 @@ func (o *pullRequestArtifactOptions) run(out io.Writer, args []string) error { return err } - label := "" - o.payload.GitProvider, label = getGitProviderAndLabel(o.retriever) - // PR evidence does not have files to upload form, cleanupNeeded, evidencePath, err := newEvidenceForm(o.payload, []string{}) // if we created a tar package, remove it after uploading it @@ -175,7 +183,13 @@ func (o *pullRequestCommitOptions) run(args []string) error { return err } - pullRequestsEvidence, err := o.getRetriever().PREvidenceForCommitV1(o.payload.CommitSHA) + // TODO: after the PR payload is enhanced for all git providers they will all use the same method + var pullRequestsEvidence []*types.PREvidence + if o.payload.GitProvider == "github" { + pullRequestsEvidence, err = o.getRetriever().PREvidenceForCommitV1(o.payload.CommitSHA) + } else { + pullRequestsEvidence, err = o.getRetriever().PREvidenceForCommitV2(o.payload.CommitSHA) + } if err != nil { return err } From 1415dd72a6ad19c5570f6cd02a00b7ea5652594e Mon Sep 17 00:00:00 2001 From: Faye Date: Fri, 4 Jul 2025 10:41:59 +0200 Subject: [PATCH 17/18] Get git provider for reporting commit evidence properly --- cmd/kosli/pullrequest.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/kosli/pullrequest.go b/cmd/kosli/pullrequest.go index 8c6f556b9..897c45338 100644 --- a/cmd/kosli/pullrequest.go +++ b/cmd/kosli/pullrequest.go @@ -183,6 +183,9 @@ func (o *pullRequestCommitOptions) run(args []string) error { return err } + label := "" + o.payload.GitProvider, label = getGitProviderAndLabel(o.retriever) + // TODO: after the PR payload is enhanced for all git providers they will all use the same method var pullRequestsEvidence []*types.PREvidence if o.payload.GitProvider == "github" { @@ -195,8 +198,6 @@ func (o *pullRequestCommitOptions) run(args []string) error { } o.payload.PullRequests = pullRequestsEvidence - label := "" - o.payload.GitProvider, label = getGitProviderAndLabel(o.retriever) // PR evidence does not have files to upload form, cleanupNeeded, evidencePath, err := newEvidenceForm(o.payload, []string{}) From d7cf3f88d39c2159e916598d42e0491ad49760b9 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Fri, 4 Jul 2025 12:02:43 +0200 Subject: [PATCH 18/18] add author_username to github pr attestation commits --- cmd/kosli/docs.go | 4 ++-- internal/github/github.go | 14 +++++++++----- internal/types/types.go | 13 ++++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/cmd/kosli/docs.go b/cmd/kosli/docs.go index 491b4df85..1ba802983 100644 --- a/cmd/kosli/docs.go +++ b/cmd/kosli/docs.go @@ -340,6 +340,6 @@ var liveCliMap = map[string]string{ "kosli list flows": "kosli list flows --output=json", "kosli get flow": "kosli get flow dashboard-ci --output=json", //"kosli list trails": "kosli list trails dashboard-ci --output=json", // Produces too much output - "kosli get trail": "kosli get trail dashboard-ci 1159a6f1193150681b8484545150334e89de6c1c --output=json", - "kosli get attestation": "kosli get attestation snyk-container-scan --flow=differ-ci --fingerprint=0cbbe3a6e73e733e8ca4b8813738d68e824badad0508ff20842832b5143b48c0 --output=json", + "kosli get trail": "kosli get trail dashboard-ci 1159a6f1193150681b8484545150334e89de6c1c --output=json", + "kosli get attestation": "kosli get attestation snyk-container-scan --flow=differ-ci --fingerprint=0cbbe3a6e73e733e8ca4b8813738d68e824badad0508ff20842832b5143b48c0 --output=json", } diff --git a/internal/github/github.go b/internal/github/github.go index 892fecb6b..6c2f380ff 100644 --- a/internal/github/github.go +++ b/internal/github/github.go @@ -107,6 +107,7 @@ func (c *GithubConfig) PREvidenceForCommitV2(commit string) ([]*types.PREvidence Oid graphql.String MessageHeadline graphql.String CommittedDate graphql.String + URL graphql.String Committer struct { Name graphql.String Email graphql.String @@ -196,10 +197,13 @@ func (c *GithubConfig) PREvidenceForCommitV2(commit string) ([]*types.PREvidence } evidence.Commits = append(evidence.Commits, types.Commit{ - SHA: string(c.Commit.Oid), - Message: string(c.Commit.MessageHeadline), - Committer: string(c.Commit.Committer.User.Login), - Timestamp: timestamp.Unix(), + SHA: string(c.Commit.Oid), + Message: string(c.Commit.MessageHeadline), + Committer: fmt.Sprintf("%s <%s>", string(c.Commit.Committer.Name), string(c.Commit.Committer.Email)), + CommitterUsername: string(c.Commit.Committer.User.Login), + Timestamp: timestamp.Unix(), + Branch: string(pr.HeadRefName), + URL: string(c.Commit.URL), }) } @@ -210,7 +214,7 @@ func (c *GithubConfig) PREvidenceForCommitV2(commit string) ([]*types.PREvidence } evidence.Approvers = append(evidence.Approvers, types.PRApprovals{ - Author: string(r.Author.Login), + Username: string(r.Author.Login), State: string(r.State), Timestamp: submittedAt.Unix(), }) diff --git a/internal/types/types.go b/internal/types/types.go index e3b5649d7..303182729 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -14,16 +14,19 @@ type PREvidence struct { } type PRApprovals struct { - Author string `json:"username"` + Username string `json:"username"` State string `json:"state"` Timestamp int64 `json:"timestamp"` } type Commit struct { - SHA string `json:"sha1"` - Message string `json:"message"` - Committer string `json:"author"` - Timestamp int64 `json:"timestamp"` + SHA string `json:"sha1"` + Message string `json:"message"` + Committer string `json:"author"` + CommitterUsername string `json:"author_username"` + Timestamp int64 `json:"timestamp"` + Branch string `json:"branch"` + URL string `json:"url,omitempty"` } type PRRetriever interface {