diff --git a/.github/workflows/apps/latest_major_version.go b/.github/workflows/apps/latest_major_version.go deleted file mode 100644 index 45f26aa523..0000000000 --- a/.github/workflows/apps/latest_major_version.go +++ /dev/null @@ -1,154 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024 Datadog, Inc. - -package main - -import ( - "encoding/json" - "fmt" - "net/http" - "os" - "regexp" - "sort" - "strings" - - "github.com/Masterminds/semver/v3" - "golang.org/x/mod/modfile" -) - -type Tag struct { - Name string -} - -func getLatestMajorVersion(repo string) (string, error) { - // Get latest major version available for repo from github. - const apiURL = "https://api.github.com/repos/%s/tags" - url := fmt.Sprintf(apiURL, repo) - - resp, err := http.Get(url) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to fetch tags: %s", resp.Status) - } - - var tags []struct { - Name string `json:"name"` - } - - if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil { - return "", err - } - latestByMajor := make(map[int]*semver.Version) - - for _, tag := range tags { - v, err := semver.NewVersion(tag.Name) - if err != nil { - continue // Skip invalid versions - } - - if v.Prerelease() != "" { - continue // Ignore pre-release versions - } - - major := int(v.Major()) - if current, exists := latestByMajor[major]; !exists || v.GreaterThan(current) { - latestByMajor[major] = v - } - } - - var latestMajor *semver.Version - for _, v := range latestByMajor { - if latestMajor == nil || v.Major() > latestMajor.Major() { - latestMajor = v - } - } - - if latestMajor != nil { - return fmt.Sprintf("v%d", latestMajor.Major()), nil - } - - return "", fmt.Errorf("no valid versions found") - -} - -func main() { - - data, err := os.ReadFile("integration_go.mod") - if err != nil { - fmt.Println("Error reading integration_go.mod:", err) - return - } - - modFile, err := modfile.Parse("integration_go.mod", data, nil) - if err != nil { - fmt.Println("Error parsing integration_go.mod:", err) - return - } - - latestMajor := make(map[string]*semver.Version) - - // Match on versions with /v{major} - versionRegex := regexp.MustCompile(`^(?P.+?)/v(\d+)$`) - - // Iterate over the required modules and update latest major version if necessary - for _, req := range modFile.Require { - module := req.Mod.Path - - if match := versionRegex.FindStringSubmatch(module); match != nil { - url := match[1] // base module URL (e.g., github.com/foo) - majorVersionStr := "v" + match[2] // Create semantic version string (e.g., "v2") - - moduleName := strings.TrimPrefix(strings.TrimSpace(url), "github.com/") - - // Parse the semantic version - majorVersion, err := semver.NewVersion(majorVersionStr) - if err != nil { - fmt.Printf("Skip invalid version for module %s: %v\n", module, err) - continue - } - - if existing, ok := latestMajor[moduleName]; !ok || majorVersion.GreaterThan(existing) { - latestMajor[moduleName] = majorVersion - } - } - } - - // Output latest major version that we support. - // Check if a new major version in Github is available that we don't support. - // If so, output that a new latest is available. - - // Sort the output - modules := make([]string, 0, len(latestMajor)) - for module := range latestMajor { - modules = append(modules, module) - } - sort.Strings(modules) - - for _, module := range modules { - major := latestMajor[module] - - latestVersion, err := getLatestMajorVersion(module) // latest version available - if err != nil { - fmt.Printf("Error fetching latest version for module '%s': %v\n", module, err) - continue - } - - latestVersionParsed, err := semver.NewVersion(latestVersion) - if err != nil { - fmt.Printf("Error parsing latest version '%s' for module '%s': %v\n", latestVersion, module, err) - continue - } - - fmt.Printf("Latest DD major version of %s: %d\n", module, major.Major()) - if major.LessThan(latestVersionParsed) { - fmt.Printf("New latest major version of %s available: %d\n", module, latestVersionParsed.Major()) - } - - } -} diff --git a/.github/workflows/apps/latest_major_versions.go b/.github/workflows/apps/latest_major_versions.go new file mode 100644 index 0000000000..8c5f647eb8 --- /dev/null +++ b/.github/workflows/apps/latest_major_versions.go @@ -0,0 +1,310 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "regexp" + "sort" + "strings" + "time" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/semver" + + "github.com/DataDog/dd-trace-go/v2/instrumentation" +) + +type ModuleInfo struct { + Origin struct { + URL string `json:"url"` + } `json:"Origin"` +} + +type GithubLatests struct { + Version string + Module string // the base name of the module +} + +func main() { + + // Find latest major + githubLatests := map[string]GithubLatests{} // map module (base name) => latest on github + contribLatests := map[string]string{} // map module (base name) => latest on go.mod + + for pkg, packageInfo := range instrumentation.GetPackages() { + + // 1. Get package and version from the module go.mod + fmt.Printf("package: %v\n", pkg) + repository := packageInfo.TracedPackage + + if packageInfo.IsStdLib { + continue + } + + base := getBaseVersion(string(pkg)) + repo, version, err := getGoModVersion(repository, string(pkg)) + if err != nil { + fmt.Printf("%v go.mod not found.", pkg) + continue + } + + versionMajorContrib := getMajorVersion(version) + + if currentLatest, ok := contribLatests[base]; ok { + if semver.Compare(versionMajorContrib, currentLatest) > 0 { + contribLatests[base] = versionMajorContrib + } + } else { + contribLatests[base] = versionMajorContrib + } + + // 2. Create the string repository@{version} and run command go list -m -json @. + // This should return a JSON: extract Origin[URL] if the JSON contains it, otherwise continue + + origin, err := getModuleOrigin(repo, version) + if err != nil { + log.Printf("failed to get module origin: %v\n", err) + continue + } + + // 3. From the VCS url, do `git ls-remote --tags ` to get the tags + // Parse the tags, and extract all the majors from them (ex v2, v3, v4) + tags, err := getTags(origin) + if err != nil { + log.Printf("error fetching tags from origin: %v", err) + continue + } + + majors := groupByMajor(tags) + + // 4. Get the latest version of each major. For each latest version of each major: + // curl https://raw.githubusercontent.com//refs/tags/v4.18.3/go.mod + // 5a. If request returns 404, module is not a go module. This means version belongs to the module without /v at the end. + // 5b. If request returns a `go.mod`, parse the modfile and extract the mod name + // Get the latest version for each major + + for major, versions := range majors { + latest := getLatestVersion(versions) + + log.Printf("fetching go.mod for %s@%s\n", origin, latest) + f, err := fetchGoMod(origin, latest) + if err != nil { + log.Printf("failed to fetch go.mod for %s@%s: %+v\n", origin, latest, err) + continue + } + if latestGithub, ok := githubLatests[base]; ok { + if semver.Compare(major, latestGithub.Version) > 0 { + githubLatests[base] = GithubLatests{major, base} + } + } else { + githubLatests[base] = GithubLatests{major, base} + } + } + } + + // 6. Check if there are any outdated majors + // Output if there is a new major package we do not support + for base, contribMajor := range contribLatests { + if latestGithub, ok := githubLatests[base]; ok { + if semver.Compare(latestGithub.Version, contribMajor) > 0 { + if (base == "go-redis/redis" && latestGithub.Version == "v9") || (base == "k8s.io/client-go") { + // go-redis/redis => redis/go-redis in v9 + // for k8s.io we provide a generic http client middleware that can be plugged with any of the versions + continue + } + fmt.Printf("New latest major %s of repository %s on Github at module: %s\n", latestGithub.Version, base, latestGithub.Module) + fmt.Printf("latest contrib major: %v\n", contribMajor) + fmt.Printf("latest github major: %v\n", latestGithub.Version) + } + } + } + +} + +func fetchGoMod(origin, tag string) (*modfile.File, error) { + // Parse and process the URL + repoPath := strings.TrimPrefix(origin, "https://github.com/") + repoPath = strings.TrimPrefix(repoPath, "https://gopkg.in/") + repoPath = strings.TrimSuffix(repoPath, ".git") + repoPath = regexp.MustCompile(`\.v\d+$`).ReplaceAllString(repoPath, "") + + url := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", repoPath, tag) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return modfile.Parse("go.mod", b, nil) +} + +func truncateVersionSuffix(repository string) string { + parts := strings.Split(repository, "/") + lastPart := parts[len(parts)-1] + + if len(lastPart) > 1 && strings.HasPrefix(lastPart, "v") && semver.IsValid(lastPart) { + return strings.Join(parts[:len(parts)-1], "/") + } + + return repository +} + +func getModuleOrigin(repository, version string) (string, error) { + cmd := exec.Command("go", "list", "-m", "-json", fmt.Sprintf("%s@%s", repository, version)) + cmd.Env = append(os.Environ(), "GOPROXY=direct") + output, err := cmd.Output() + + if err != nil { + return "", fmt.Errorf("failed to execute command: %w", err) + } + + var moduleInfo ModuleInfo + if err := json.Unmarshal(output, &moduleInfo); err != nil { + return "", fmt.Errorf("failed to parse JSON output: %w", err) + } + + if moduleInfo.Origin.URL == "" { + // If Origin.URL is not found, return an error + return "", fmt.Errorf("Origin.URL not found in JSON for %s@%s", repository, version) + } + + return moduleInfo.Origin.URL, nil +} + +// Run `git ls-remote` and fetch tags +func getTags(vcsURL string) ([]string, error) { + cmd := exec.Command("git", "ls-remote", "--tags", vcsURL) + output, err := cmd.Output() + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewReader(output)) + tags := []string{} + tagRegex := regexp.MustCompile(`refs/tags/(v[\d.]+)$`) + for scanner.Scan() { + line := scanner.Text() + if matches := tagRegex.FindStringSubmatch(line); matches != nil { + tags = append(tags, matches[1]) + } + } + return tags, nil +} + +// Group tags by major version +func groupByMajor(tags []string) map[string][]string { + majors := make(map[string][]string) + for _, tag := range tags { + if semver.IsValid(tag) { + major := semver.Major(tag) + majors[major] = append(majors[major], tag) + } + } + return majors +} + +// Get the latest version for a list of versions +func getLatestVersion(versions []string) string { + + // Filter out pre-release versions and non-valid versions + validVersions := make([]string, 0) + for _, v := range versions { + if semver.IsValid(v) && semver.Prerelease(v) == "" { + validVersions = append(validVersions, v) + } + } + sort.Slice(validVersions, func(i, j int) bool { + return semver.Compare(validVersions[i], validVersions[j]) < 0 + }) + return validVersions[len(validVersions)-1] +} + +func getBaseVersion(pkg string) string { + // Match on the pattern ".v{X}" or "/v{X}" where {X} is a number + // Replace the matched pattern with an empty string + re := regexp.MustCompile(`(\.v\d+|/v\d+)$`) + return re.ReplaceAllString(pkg, "") +} + +func getMajorVersion(version string) string { + parts := strings.Split(version, ".") + return parts[0] +} + +func getGoModVersion(pkg string, instrumentationName string) (string, string, error) { + // Look for go.mod in contrib/{package} + // If go.mod exists, look for repository name in the go.mod + // Parse the version associated with repository + // Define the path to the go.mod file within the contrib/{pkg} directory + pkg = truncateVersionSuffix(pkg) + + goModPath := fmt.Sprintf("contrib/%s/go.mod", instrumentationName) + + // Read the go.mod file + content, err := os.ReadFile(goModPath) + if err != nil { + return "", "", fmt.Errorf("failed to read go.mod file at %s: %w", goModPath, err) + } + + // Parse the go.mod file + modFile, err := modfile.Parse(goModPath, content, nil) + if err != nil { + return "", "", fmt.Errorf("failed to parse go.mod file at %s: %w", goModPath, err) + } + + // Keep track of largest version from go.mod + var largestVersion string + var largestVersionRepo string + + // Search for the module matching the repository + for _, req := range modFile.Require { + if strings.HasPrefix(req.Mod.Path, pkg) { + version := req.Mod.Version + + if !semver.IsValid(version) { + return "", "", fmt.Errorf("invalid semantic version %s in go.mod", version) + } + + if largestVersion == "" || semver.Compare(version, largestVersion) > 0 { + largestVersion = version + largestVersionRepo = req.Mod.Path + } + } + } + + if largestVersion == "" { + // If the repository is not found in the dependencies, return + return "", "", fmt.Errorf("package %s not found in go.mod file", pkg) + } + return largestVersionRepo, largestVersion, nil +} diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml index 695ef9b570..021880d523 100644 --- a/.github/workflows/outdated-integrations.yml +++ b/.github/workflows/outdated-integrations.yml @@ -22,20 +22,13 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - run: go get github.com/Masterminds/semver/v3 - - - run: go run .github/workflows/apps/latest_major_version.go > latests.txt - - - run: git diff + - run: go clean -modcache + + - name: Add dependencies + run: | + go get github.com/go-git/go-billy/v5/memfs + go get github.com/go-git/go-git/v5 + go get github.com/go-git/go-git/v5/plumbing + go get github.com/go-git/go-git/v5/storage/memory - - name: Create Pull Request - id: pr - uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: "upgrade-latest-major-version" - commit-message: "Update latests file" - base: main - title: "chore: update latest majors" - labels: changelog/no-changelog - body: "Auto-generated PR from Outdated Integrations workflow to update latests major versions" + - run: go run .github/workflows/apps/latest_major_versions.go diff --git a/go.work.sum b/go.work.sum index a0f0705876..31fe816f4e 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1658,6 +1658,7 @@ github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1ls github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= diff --git a/instrumentation/packages.go b/instrumentation/packages.go index e2ef520c52..2fefcda3cd 100644 --- a/instrumentation/packages.go +++ b/instrumentation/packages.go @@ -103,6 +103,7 @@ type PackageInfo struct { external bool TracedPackage string + IsStdLib bool EnvVarPrefix string naming map[Component]componentNames @@ -228,6 +229,7 @@ var packages = map[Package]PackageInfo{ }, PackageDatabaseSQL: { TracedPackage: "database/sql", + IsStdLib: true, EnvVarPrefix: "SQL", naming: map[Component]componentNames{ ComponentDefault: { @@ -459,6 +461,7 @@ var packages = map[Package]PackageInfo{ PackageNetHTTP: { TracedPackage: "net/http", + IsStdLib: true, EnvVarPrefix: "HTTP", naming: map[Component]componentNames{ ComponentServer: { @@ -743,6 +746,7 @@ var packages = map[Package]PackageInfo{ }, PackageLogSlog: { TracedPackage: "log/slog", + IsStdLib: true, }, PackageEnvoyProxyGoControlPlane: { TracedPackage: "github.com/envoyproxy/go-control-plane", @@ -794,10 +798,10 @@ func isAWSMessagingSendOp(awsService, awsOperation string) bool { } // GetPackages returns a map of Package to the corresponding instrumented module. -func GetPackages() map[Package]string { - cp := make(map[Package]string) +func GetPackages() map[Package]PackageInfo { + cp := make(map[Package]PackageInfo) for pkg, info := range packages { - cp[pkg] = info.TracedPackage + cp[pkg] = info } return cp }