package main import ( "bufio" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "net" "net/http" "net/url" "os" "regexp" "strings" "time" "golang.org/x/mod/semver" ) var downloadRetryLog func(string) var errRemoteAssetSkip = errors.New("skip remote asset sha256") type downloadCandidate struct { source string url string } func repoRefForVersion(version string) string { v := strings.TrimSpace(version) if !strings.HasPrefix(v, "v") { v = "v" + v } if canonical := semver.Canonical(v); canonical != "" { return canonical } return v } func specVersionForVersion(version string) string { return strings.TrimPrefix(repoRefForVersion(version), "v") } func verifyDownload(ctx context.Context, spec *specInfo) error { if spec.GoImportPath == "" { return fmt.Errorf("no go_import_path") } if spec.LatestVersion == "" { return fmt.Errorf("no latest version") } info, err := getPkgsiteInfo(ctx, spec.GoImportPath) if err != nil { return fmt.Errorf("get pkgsite info: %w", err) } candidates, archiveErr, err := downloadCandidatesForSpec(spec, info) if err != nil { return err } if archiveErr != nil { reportDownloadRetry(" FALLBACK: %s repo archive unavailable (%v), trying module proxy", spec.GoImportPath, archiveErr) } return verifyDownloadCandidates(ctx, spec.GoImportPath, candidates) } func downloadCandidatesForSpec(spec *specInfo, info *pkgsiteInfo) ([]downloadCandidate, error, error) { repoURL := gitCloneURLFromRepoURL(info.Module.RepoURL) if repoURL == "" { return nil, nil, fmt.Errorf("no repository URL") } version := spec.LatestVersion if !strings.HasPrefix(version, "v") { version = "v" + version } repoRef := repoRefForVersion(version) sourceModulePath := sourceImportPathForPackage(spec.GoImportPath, info) proxyURL := fmt.Sprintf("https://proxy.golang.org/%s/@v/%s.zip", moduleProxyEscapedPath(sourceModulePath), moduleProxyVersionForSpec(version)) if moduleUsesRepoSubdir(sourceModulePath, repoURL) { return []downloadCandidate{ {source: "proxy", url: proxyURL}, }, nil, nil } tarURL, err := (&upstream{repoURL: repoURL}).tarballURLForRef(repoRef, "gz") if err != nil { return []downloadCandidate{ {source: "proxy", url: proxyURL}, }, err, nil } return []downloadCandidate{ {source: "repo", url: tarURL}, {source: "proxy", url: proxyURL}, }, nil, nil } func remoteAssetSHA256(ctx context.Context, spec *specInfo, newVersion string) (string, error) { sourceURL, err := resolvedSource0URL(spec.FilePath, sourceVersionForSpec(newVersion)) if err != nil { return "", err } releaseURL, tag := releasePageURLForSource(sourceURL) if releaseURL == "" { reportDownloadRetry(" HASH: %s no release checksum page, downloading Source0", spec.GoImportPath) return downloadAssetSHA256(ctx, sourceURL) } content, err := fetchTextContext(ctx, releaseURL) if err == nil { if sum := extractSHA256FromHTML(content, tag); sum != "" { return sum, nil } } reportDownloadRetry(" HASH: %s release checksum unavailable, downloading Source0", spec.GoImportPath) return downloadAssetSHA256(ctx, sourceURL) } func sourceVersionForSpec(version string) string { return specVersionForVersion(version) } func verifyDownloadCandidates(ctx context.Context, importPath string, candidates []downloadCandidate) error { var errs []string for i, candidate := range candidates { err := verifyHTTPRequest(ctx, candidate.url) if err == nil { if i > 0 { reportDownloadRetry(" OK: %s verified via %s after fallback", importPath, candidate.source) } return nil } errs = append(errs, fmt.Sprintf("%s: %v", candidate.source, err)) if i+1 < len(candidates) { reportDownloadRetry(" FALLBACK: %s %s probe failed (%v), trying %s", importPath, candidate.source, err, candidates[i+1].source) } } return errors.New(strings.Join(errs, "; ")) } func verifyHTTPRequest(ctx context.Context, downloadURL string) error { var lastErr error for attempt := 1; attempt <= 3; attempt++ { if err := verifyHTTPHead(ctx, downloadURL); err == nil { return nil } else { lastErr = err } if err := verifyHTTPRangeGet(ctx, downloadURL); err == nil { return nil } else { lastErr = err } if attempt < 3 && isRetryableDownloadError(lastErr) { backoff := time.Duration(attempt*2) * time.Second reportDownloadRetry(" RETRY: download probe %s failed (%v), retry %d/3 in %s", downloadURL, lastErr, attempt+1, backoff) if err := sleepWithContext(ctx, backoff); err != nil { return err } continue } break } return lastErr } func verifyHTTPHead(ctx context.Context, downloadURL string) error { req, err := http.NewRequestWithContext(ctx, http.MethodHead, downloadURL, nil) if err != nil { return fmt.Errorf("create HEAD request: %w", err) } client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(req) if err != nil { return fmt.Errorf("HEAD request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 400 { return fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL) } if resp.ContentLength == 0 { return fmt.Errorf("HEAD response has no content length from %s", downloadURL) } return nil } func verifyHTTPRangeGet(ctx context.Context, downloadURL string) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { return fmt.Errorf("create GET request: %w", err) } req.Header.Set("Range", "bytes=0-0") client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(req) if err != nil { return fmt.Errorf("GET request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 400 { return fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL) } buf := make([]byte, 1) n, err := resp.Body.Read(buf) if err != nil && err != io.EOF { return fmt.Errorf("read download probe: %w", err) } if n == 0 { return fmt.Errorf("empty download from %s", downloadURL) } return nil } func isRetryableDownloadError(err error) bool { if err == nil { return false } var netErr net.Error if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) { return true } msg := err.Error() return strings.Contains(msg, "context deadline exceeded") || strings.Contains(msg, "Client.Timeout exceeded") || strings.Contains(msg, "connection reset by peer") || strings.Contains(msg, "EOF") || strings.Contains(msg, "HTTP 408") || strings.Contains(msg, "HTTP 425") || strings.Contains(msg, "HTTP 429") || strings.Contains(msg, "HTTP 500") || strings.Contains(msg, "HTTP 502") || strings.Contains(msg, "HTTP 503") || strings.Contains(msg, "HTTP 504") } func reportDownloadRetry(format string, args ...any) { if downloadRetryLog != nil { downloadRetryLog(fmt.Sprintf(format, args...)) } } func readSpecLines(path string) ([]string, error) { f, err := os.Open(path) if err != nil { return nil, err } defer f.Close() var lines []string scanner := bufio.NewScanner(f) for scanner.Scan() { lines = append(lines, scanner.Text()) } return lines, scanner.Err() } func writeSpecLines(path string, lines []string) error { f, err := os.Create(path) if err != nil { return err } defer f.Close() w := bufio.NewWriter(f) for i, line := range lines { if _, err := w.WriteString(line); err != nil { return err } if i < len(lines)-1 { if _, err := w.WriteString("\n"); err != nil { return err } } } if len(lines) > 0 { if _, err := w.WriteString("\n"); err != nil { return err } } return w.Flush() } func updateSpecVersion(spec *specInfo, newVersion, remoteSHA256 string, updateRemoteAsset bool) error { lines, err := readSpecLines(spec.FilePath) if err != nil { return err } versionLineRe := regexp.MustCompile(`^(\s*Version:\s*)(\S+)(.*)$`) autoChangelogRe := regexp.MustCompile(`^(\s*)%\{\?autochangelog\}(\s*)$`) remoteAssetHashRe := regexp.MustCompile(`^(\s*)#!RemoteAsset(?::\s*sha256:[0-9a-fA-F]+)?(\s*)$`) source0LineRe := regexp.MustCompile(`^(\s*Source0:\s*)(\S+)(.*)$`) updated := false source0Index := -1 source0RemoteAssetIndex := -1 for i, line := range lines { matches := versionLineRe.FindStringSubmatch(line) if len(matches) == 4 { lines[i] = matches[1] + newVersion + matches[3] updated = true } matches = autoChangelogRe.FindStringSubmatch(line) if len(matches) == 3 { lines[i] = matches[1] + "%autochangelog" + matches[2] } if source0Index < 0 && source0LineRe.MatchString(line) { source0Index = i if i > 0 && remoteAssetHashRe.MatchString(lines[i-1]) { source0RemoteAssetIndex = i - 1 } } } if !updated { return fmt.Errorf("Version line not found in %s", spec.FilePath) } if !updateRemoteAsset { return writeSpecLines(spec.FilePath, lines) } if source0RemoteAssetIndex >= 0 { matches := remoteAssetHashRe.FindStringSubmatch(lines[source0RemoteAssetIndex]) if remoteSHA256 != "" { lines[source0RemoteAssetIndex] = matches[1] + "#!RemoteAsset: sha256:" + remoteSHA256 + matches[2] } else { lines[source0RemoteAssetIndex] = matches[1] + "#!RemoteAsset" + matches[2] } } else if source0Index >= 0 { insert := "#!RemoteAsset" if remoteSHA256 != "" { insert = "#!RemoteAsset: sha256:" + remoteSHA256 } lines = append(lines[:source0Index], append([]string{insert}, lines[source0Index:]...)...) } return writeSpecLines(spec.FilePath, lines) } type downloadResult struct { spec *specInfo err error } func verifyAndUpdateSpec(ctx context.Context, spec *specInfo, dryRun bool) error { err := verifyDownload(ctx, spec) if err != nil { return fmt.Errorf("download verification failed: %w", err) } if !dryRun { newVer := specVersionForVersion(spec.LatestVersion) remoteSHA256, err := remoteAssetSHA256(ctx, spec, newVer) updateRemoteAsset := true if errors.Is(err, errRemoteAssetSkip) { updateRemoteAsset = false err = nil } if err != nil { return fmt.Errorf("compute remote asset sha256: %w", err) } if err := updateSpecVersion(spec, newVer, remoteSHA256, updateRemoteAsset); err != nil { return fmt.Errorf("update spec file: %w", err) } } return nil } func processUpdate(ctx context.Context, spec *specInfo, dryRun bool) error { err := verifyDownload(ctx, spec) if err != nil { return err } if !dryRun { newVer := specVersionForVersion(spec.LatestVersion) remoteSHA256, err := remoteAssetSHA256(ctx, spec, newVer) updateRemoteAsset := true if errors.Is(err, errRemoteAssetSkip) { updateRemoteAsset = false err = nil } if err != nil { return fmt.Errorf("compute remote asset sha256: %w", err) } if err := updateSpecVersion(spec, newVer, remoteSHA256, updateRemoteAsset); err != nil { return fmt.Errorf("update spec file: %w", err) } } return nil } func resolvedSource0URL(path, newVersion string) (string, error) { lines, err := readSpecLines(path) if err != nil { return "", err } macros := make(map[string]string) defineRe := regexp.MustCompile(`^%(?:define|global)\s+(\S+)\s+(.+)$`) fieldRe := regexp.MustCompile(`^(Name|Version):\s+(\S+)`) source0Re := regexp.MustCompile(`^Source0:\s+(\S+)`) for _, line := range lines { trimmed := strings.TrimSpace(line) if matches := defineRe.FindStringSubmatch(trimmed); len(matches) == 3 { macros[matches[1]] = strings.TrimSpace(matches[2]) continue } if matches := fieldRe.FindStringSubmatch(trimmed); len(matches) == 3 { switch matches[1] { case "Name": macros["name"] = matches[2] case "Version": macros["version"] = newVersion } continue } if matches := source0Re.FindStringSubmatch(trimmed); len(matches) == 2 { if source0UsesPinnedCommit(matches[1]) { return "", fmt.Errorf("%w: Source0 uses pinned commit_id", errRemoteAssetSkip) } macros["version"] = newVersion resolved := expandRPMMacros(matches[1], macros, 16) if strings.Contains(resolved, "%{") { return "", fmt.Errorf("unresolved Source0 macros in %s: %s", path, resolved) } return strings.SplitN(resolved, "#", 2)[0], nil } } return "", fmt.Errorf("Source0 line not found in %s", path) } func source0UsesPinnedCommit(source string) bool { return strings.Contains(source, "%{commit_id}") || strings.Contains(source, "%{?commit_id}") } func expandRPMMacros(s string, macros map[string]string, limit int) string { re := regexp.MustCompile(`%\{(\??)([^}]+)\}`) expanded := s for i := 0; i < limit; i++ { changed := false next := re.ReplaceAllStringFunc(expanded, func(match string) string { parts := re.FindStringSubmatch(match) if len(parts) != 3 { return match } optional := parts[1] == "?" name := parts[2] value, ok := macros[name] if !ok { if optional { changed = true return "" } return match } changed = true return value }) expanded = next if !changed { break } } return expanded } func releasePageURLForSource(sourceURL string) (releaseURL, tag string) { u, err := url.Parse(sourceURL) if err != nil { return "", "" } tag = sourceTagFromURLPath(u.Path) if tag == "" { return "", "" } parts := strings.Split(strings.Trim(u.Path, "/"), "/") switch u.Host { case "github.com": if len(parts) < 2 { return "", "" } repo := fmt.Sprintf("%s://%s/%s/%s", u.Scheme, u.Host, parts[0], parts[1]) return fmt.Sprintf("%s/releases/tag/%s", repo, tag), tag case "gitlab.com", "salsa.debian.org": dashArchive := -1 for i := 0; i+1 < len(parts); i++ { if parts[i] == "-" && parts[i+1] == "archive" { dashArchive = i break } } if dashArchive <= 0 { return "", "" } repo := fmt.Sprintf("%s://%s/%s", u.Scheme, u.Host, strings.Join(parts[:dashArchive], "/")) return fmt.Sprintf("%s/-/releases/%s", repo, tag), tag default: return "", "" } } func sourceTagFromURLPath(path string) string { path, err := url.PathUnescape(path) if err != nil { return "" } const refsPrefix = "refs/tags/" parts := strings.Split(strings.Trim(path, "/"), "/") for i, part := range parts { if part != "archive" || i+1 >= len(parts) { continue } rest := strings.Join(parts[i+1:], "/") if strings.HasPrefix(rest, refsPrefix) { return trimArchiveSuffix(strings.TrimPrefix(rest, refsPrefix)) } if idx := strings.Index(rest, "/"); idx >= 0 { rest = rest[:idx] } return trimArchiveSuffix(rest) } return "" } func trimArchiveSuffix(s string) string { for _, suffix := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tgz", ".zip"} { if strings.HasSuffix(s, suffix) { return strings.TrimSuffix(s, suffix) } } return s } func fetchTextContext(ctx context.Context, fetchURL string) (string, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL, nil) if err != nil { return "", fmt.Errorf("create GET request: %w", err) } client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(req) if err != nil { return "", fmt.Errorf("GET request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP %d from %s", resp.StatusCode, fetchURL) } body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { return "", fmt.Errorf("read response: %w", err) } return string(body), nil } func downloadAssetSHA256(ctx context.Context, downloadURL string) (string, error) { var lastErr error for attempt := 1; attempt <= 3; attempt++ { sum, err := fetchSHA256(ctx, downloadURL) if err == nil { return sum, nil } lastErr = err if attempt < 3 && isRetryableDownloadError(lastErr) { backoff := time.Duration(attempt*2) * time.Second reportDownloadRetry(" RETRY: asset hash %s failed (%v), retry %d/3 in %s", downloadURL, lastErr, attempt+1, backoff) if err := sleepWithContext(ctx, backoff); err != nil { return "", err } continue } break } return "", lastErr } func fetchSHA256(ctx context.Context, downloadURL string) (string, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { return "", fmt.Errorf("create GET request: %w", err) } client := &http.Client{Timeout: 120 * time.Second} resp, err := client.Do(req) if err != nil { return "", fmt.Errorf("GET request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 400 { return "", fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL) } sum := sha256.New() if _, err := io.Copy(sum, resp.Body); err != nil { return "", fmt.Errorf("read download for sha256: %w", err) } return hex.EncodeToString(sum.Sum(nil)), nil }