Skip to content

Commit 14a8f52

Browse files
frjcompfrjcomp
authored
Azure DevOps (#149)
* Azure DevOps Support
1 parent fc320db commit 14a8f52

File tree

11 files changed

+773
-11
lines changed

11 files changed

+773
-11
lines changed

‎readme.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ It supports the following platforms:
1414
* GitLab
1515
* GitHub
1616
* BitBucket
17+
* Azure DevOps
1718

1819
## Getting Started
1920

@@ -141,6 +142,31 @@ Scan all public repositories
141142
pipeleak bb scan --token xxxxxxxxxxx --username auser --public --maxPipelines 5 --after 2025-03-01T15:00:00+00:00
142143
```
143144

145+
## Azure DevOps
146+
147+
Scan all pipelines the current user has access to
148+
```bash
149+
pipeleak ad scan --token xxxxxxxxxxx --username auser --artifacts
150+
```
151+
152+
Scan all pipelines of an organization
153+
```bash
154+
pipeleak ad scan --token xxxxxxxxxxx --username auser --artifacts --organization myOrganization
155+
```
156+
157+
Scan all pipelines of a project e.g. https://dev.azure.com/PowerShell/PowerShell
158+
```bash
159+
pipeleak ad scan --token xxxxxxxxxxx --username auser --artifacts --organization powershell --project PowerShell
160+
```
161+
162+
### Authentication
163+
Create your PAT here: https://dev.azure.com/{yourproject}/_usersSettings/tokens
164+
165+
> In the top right corner you can choose the scope (Global, Project etc.).
166+
> Global in that case means per tenant. If you have access to multiple tentants you need to run a scan per tenant.
167+
> Get you username from an HTTPS git clone url from the UI.
168+
169+
144170
# ELK Integration
145171

146172
To easily analyze the results you can [redirect the pipeleak](https://github.com/deviantony/docker-elk?tab=readme-ov-file#injecting-data) output using `nc` into Logstash.

‎src/pipeleak/cmd/bitbucket/api.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ func NewClient(username string, password string) BitBucketApiClient {
2323
if 429 == res.StatusCode() {
2424
log.Info().Int("status", res.StatusCode()).Msg("Retrying request, we are rate limited")
2525
} else {
26-
log.Debug().Int("status", res.StatusCode()).Msg("Retrying request, not due to rate limit")
26+
log.Info().Int("status", res.StatusCode()).Msg("Retrying request, not due to rate limit")
2727
}
2828
},
2929
)

‎src/pipeleak/cmd/bitbucket/scan.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,12 @@ func getSteplog(client BitBucketApiClient, workspaceSlug string, repoSlug string
258258
log.Error().Err(err).Msg("Failed fetching pipeline steps")
259259
}
260260

261-
findings := scanner.DetectHits(logBytes, options.MaxScanGoRoutines, options.TruffleHogVerification)
261+
findings, err := scanner.DetectHits(logBytes, options.MaxScanGoRoutines, options.TruffleHogVerification)
262+
if err != nil {
263+
log.Debug().Err(err).Str("stepUUid", stepUUID).Msg("Failed detecting secrets")
264+
return
265+
}
266+
262267
for _, finding := range findings {
263268
log.Warn().Str("confidence", finding.Pattern.Pattern.Confidence).Str("ruleName", finding.Pattern.Pattern.Name).Str("value", finding.Text).Str("Run", "https://bitbucket.org/"+workspaceSlug+"/"+repoSlug+"/pipelines/results/"+pipelineUuid+"/steps/"+stepUUID).Msg("HIT")
264269
}
@@ -278,5 +283,5 @@ func getDownloadArtifact(client BitBucketApiClient, downloadUrl string, webUrl s
278283
}
279284

280285
func scanStatus() *zerolog.Event {
281-
return log.Info().Str("debug", "test")
286+
return log.Info().Str("debug", "nothing to show ✔️")
282287
}

‎src/pipeleak/cmd/devops/api.go‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
package devops
2+
3+
import (
4+
"net/url"
5+
"path"
6+
"strconv"
7+
8+
"github.com/rs/zerolog/log"
9+
10+
"resty.dev/v3"
11+
)
12+
13+
// https://learn.microsoft.com/en-us/rest/api/azure/devops/
14+
type AzureDevOpsApiClient struct {
15+
Client resty.Client
16+
}
17+
18+
func NewClient(username string, password string) AzureDevOpsApiClient {
19+
bbClient := AzureDevOpsApiClient{Client: *resty.New().SetBasicAuth(username, password).SetRedirectPolicy(resty.FlexibleRedirectPolicy(5))}
20+
bbClient.Client.AddRetryHooks(
21+
func(res *resty.Response, err error) {
22+
if 429 == res.StatusCode() {
23+
log.Info().Int("status", res.StatusCode()).Msg("Retrying request, we are rate limited")
24+
} else {
25+
log.Info().Int("status", res.StatusCode()).Msg("Retrying request, not due to rate limit")
26+
}
27+
},
28+
)
29+
return bbClient
30+
}
31+
32+
// https://learn.microsoft.com/en-us/rest/api/azure/devops/profile/profiles/get?view=azure-devops-rest-7.2&tabs=HTTP
33+
func (a AzureDevOpsApiClient) GetAuthenticatedUser() (*AuthenticatedUser, *resty.Response, error) {
34+
u, err := url.Parse("https://app.vssps.visualstudio.com/_apis/profile/profiles/me")
35+
if err != nil {
36+
log.Fatal().Err(err).Msg("Unable to parse GetAuthenticatedUser url")
37+
}
38+
reqUrl := u.String()
39+
40+
user := &AuthenticatedUser{}
41+
res, err := a.Client.R().
42+
SetQueryParam("api-version", "7.2-preview.3").
43+
SetResult(user).
44+
Get(reqUrl)
45+
46+
if res.StatusCode() > 400 {
47+
log.Fatal().Int("status", res.StatusCode()).Str("response", res.String()).Msg("Failed fetching authenticated user")
48+
}
49+
50+
return user, res, err
51+
}
52+
53+
// https://learn.microsoft.com/en-us/rest/api/azure/devops/account/accounts/list?view=azure-devops-rest-7.2&tabs=HTTP
54+
func (a AzureDevOpsApiClient) ListAccounts(ownerId string) ([]Account, *resty.Response, error) {
55+
u, err := url.Parse("https://app.vssps.visualstudio.com/_apis/accounts")
56+
if err != nil {
57+
log.Fatal().Err(err).Msg("Unable to parse ListAccounts url")
58+
}
59+
reqUrl := u.String()
60+
61+
resp := &PaginatedResponse[Account]{}
62+
res, err := a.Client.R().
63+
SetQueryParam("api-version", "7.2-preview.1").
64+
SetQueryParam("ownerId", ownerId).
65+
SetResult(resp).
66+
Get(reqUrl)
67+
68+
if res.StatusCode() > 400 {
69+
log.Fatal().Int("status", res.StatusCode()).Str("ownerId", ownerId).Msg("Fetching accounts failed")
70+
}
71+
72+
return resp.Value, res, err
73+
}
74+
75+
// https://learn.microsoft.com/en-us/rest/api/azure/devops/core/projects/list?view=azure-devops-rest-7.2&tabs=HTTP
76+
func (a AzureDevOpsApiClient) ListProjects(continuationToken string, organization string) ([]Project, *resty.Response, string, error) {
77+
reqUrl := ""
78+
u, err := url.Parse("https://dev.azure.com/")
79+
if err != nil {
80+
log.Fatal().Err(err).Msg("Unable to parse ListProjects url")
81+
}
82+
u.Path = path.Join(u.Path, organization, "_apis", "projects")
83+
reqUrl = u.String()
84+
85+
resp := &PaginatedResponse[Project]{}
86+
res, err := a.Client.R().
87+
SetQueryParam("api-version", "7.2-preview.4").
88+
SetQueryParam("$top", "100").
89+
SetQueryParam("continuationtoken", continuationToken).
90+
SetResult(resp).
91+
Get(reqUrl)
92+
93+
if res.StatusCode() == 404 || res.StatusCode() == 401 {
94+
log.Fatal().Int("status", res.StatusCode()).Str("organization", organization).Msg("Projects list does not exist or you do not have access")
95+
}
96+
97+
return resp.Value, res, res.Header().Get("x-ms-continuationtoken"), err
98+
}
99+
100+
// https://learn.microsoft.com/en-us/rest/api/azure/devops/build/builds/list?view=azure-devops-rest-7.2
101+
func (a AzureDevOpsApiClient) ListBuilds(continuationToken string, organization string, project string) ([]Build, *resty.Response, string, error) {
102+
reqUrl := ""
103+
u, err := url.Parse("https://dev.azure.com/")
104+
if err != nil {
105+
log.Fatal().Err(err).Msg("Unable to parse ListBuilds url")
106+
}
107+
108+
u.Path = path.Join(u.Path, organization, project, "_apis", "build", "builds")
109+
reqUrl = u.String()
110+
111+
resp := &PaginatedResponse[Build]{}
112+
res, err := a.Client.R().
113+
SetQueryParam("api-version", "7.2-preview.7").
114+
SetQueryParam("$top", "100").
115+
SetQueryParam("continuationtoken", continuationToken).
116+
SetResult(resp).
117+
Get(reqUrl)
118+
119+
if res.StatusCode() == 404 || res.StatusCode() == 401 {
120+
log.Fatal().Int("status", res.StatusCode()).Str("project", project).Str("organization", organization).Msg("Build list does not exist or you do not have access")
121+
}
122+
123+
return resp.Value, res, res.Header().Get("x-ms-continuationtoken"), err
124+
}
125+
126+
// https://learn.microsoft.com/en-us/rest/api/azure/devops/build/builds/get-build-logs?view=azure-devops-rest-7.2
127+
// this endpoint is NOT paged
128+
func (a AzureDevOpsApiClient) ListBuildLogs(organization string, project string, buildId int) ([]BuildLog, *resty.Response, error) {
129+
reqUrl := ""
130+
u, err := url.Parse("https://dev.azure.com/")
131+
if err != nil {
132+
log.Fatal().Err(err).Msg("Unable to parse ListBuilds url")
133+
}
134+
135+
u.Path = path.Join(u.Path, organization, project, "_apis", "build", "builds", strconv.Itoa(buildId), "logs")
136+
reqUrl = u.String()
137+
138+
resp := &PaginatedResponse[BuildLog]{}
139+
res, err := a.Client.R().
140+
SetQueryParam("api-version", "7.2-preview.2").
141+
SetResult(resp).
142+
Get(reqUrl)
143+
144+
if res.StatusCode() == 404 || res.StatusCode() == 401 {
145+
log.Fatal().Int("status", res.StatusCode()).Str("project", project).Str("organization", organization).Msg("Build log list does not exist or you do not have access")
146+
}
147+
148+
return resp.Value, res, err
149+
}
150+
151+
// https://learn.microsoft.com/en-us/rest/api/azure/devops/build/builds/get-build-log?view=azure-devops-rest-7.2
152+
func (a AzureDevOpsApiClient) GetLog(organization string, project string, buildId int, logId int) ([]byte, *resty.Response, error) {
153+
reqUrl := ""
154+
u, err := url.Parse("https://dev.azure.com/")
155+
if err != nil {
156+
log.Fatal().Err(err).Msg("Unable to parse ListBuilds url")
157+
}
158+
159+
u.Path = path.Join(u.Path, organization, project, "_apis", "build", "builds", strconv.Itoa(buildId), "logs", strconv.Itoa(logId))
160+
reqUrl = u.String()
161+
162+
res, err := a.Client.R().
163+
SetQueryParam("api-version", "7.2-preview.2").
164+
Get(reqUrl)
165+
166+
if res.StatusCode() == 404 || res.StatusCode() == 401 {
167+
log.Error().Int("status", res.StatusCode()).Str("project", project).Str("organization", organization).Msg("Log does not exist or you do not have access")
168+
}
169+
170+
return res.Bytes(), res, err
171+
}
172+
173+
func (a AzureDevOpsApiClient) DownloadArtifactZip(url string) ([]byte, *resty.Response, error) {
174+
res, err := a.Client.R().
175+
Get(url)
176+
177+
if res.StatusCode() == 404 || res.StatusCode() == 401 {
178+
log.Error().Int("status", res.StatusCode()).Str("url", url).Msg("Failed downloading artifact zip")
179+
}
180+
181+
return res.Bytes(), res, err
182+
}
183+
184+
// https://learn.microsoft.com/en-us/rest/api/azure/devops/build/artifacts/list?view=azure-devops-rest-7.1
185+
// this endpoint is NOT paged
186+
func (a AzureDevOpsApiClient) ListBuildArtifacts(continuationToken string, organization string, project string, buildId int) ([]Artifact, *resty.Response, string, error) {
187+
reqUrl := ""
188+
u, err := url.Parse("https://dev.azure.com/")
189+
if err != nil {
190+
log.Fatal().Err(err).Msg("Unable to parse ListBuildArtifacts url")
191+
}
192+
193+
u.Path = path.Join(u.Path, organization, project, "_apis", "build", "builds", strconv.Itoa(buildId), "artifacts")
194+
reqUrl = u.String()
195+
196+
resp := &PaginatedResponse[Artifact]{}
197+
res, err := a.Client.R().
198+
SetQueryParam("api-version", "7.1").
199+
SetQueryParam("continuationtoken", continuationToken).
200+
SetResult(resp).
201+
Get(reqUrl)
202+
203+
if res.StatusCode() == 404 || res.StatusCode() == 401 {
204+
log.Fatal().Int("status", res.StatusCode()).Str("project", project).Str("organization", organization).Msg("Build artifacts list does not exist or you do not have access")
205+
}
206+
207+
return resp.Value, res, res.Header().Get("x-ms-continuationtoken"), err
208+
}

‎src/pipeleak/cmd/devops/devops.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package devops
2+
3+
import (
4+
"github.com/spf13/cobra"
5+
)
6+
7+
func NewAzureDevOpsRootCmd() *cobra.Command {
8+
dvoCmd := &cobra.Command{
9+
Use: "ad [command]",
10+
Short: "Azure DevOps related commands",
11+
}
12+
13+
dvoCmd.AddCommand(NewScanCmd())
14+
15+
return dvoCmd
16+
}

0 commit comments

Comments
 (0)