feat: support packing multiple packages #13

Open
Jvlegod wants to merge 4 commits from Jvlegod/go2spec:feature/pack-multiple-packages into master
3 changed files with 482 additions and 34 deletions
+34 -4
View File
@@ -2,6 +2,7 @@ package main
import (
"fmt"
"golang.org/x/mod/module"
"log"
"os/exec"
"regexp"
@@ -36,6 +37,18 @@ func shortCommitHash(hash string) string {
return hash
}
func preferredRevisionCandidates(preferredRev string) []string {
preferredRev = strings.TrimSpace(preferredRev)
if preferredRev == "" {
return nil
}
candidates := []string{preferredRev}
if !strings.HasPrefix(preferredRev, "v") && semverRegexp.MatchString("v"+preferredRev) {
candidates = append(candidates, "v"+preferredRev)
}
return candidates
}
// pkgVersionFromGit determines the actual version to be packaged
// from the git repository status and user preference.
// Besides returning the upstream version, the "upstream" struct
@@ -48,13 +61,30 @@ func pkgVersionFromGit(gitdir string, u *upstream, preferredRev string, forcePre
var commitsAhead int
var cmd *exec.Cmd // the temporary shell commands we execute
preferredRev = strings.TrimSpace(preferredRev)
if module.IsPseudoVersion(preferredRev) {
commit, err := module.PseudoVersionRev(preferredRev)
if err != nil {
return "", fmt.Errorf("parse pseudo-version %q: %w", preferredRev, err)
}
cmd = exec.Command("git", "rev-parse", "--verify", commit+"^{commit}")
cmd.Dir = gitdir
if _, err := cmd.Output(); err != nil {
return "", fmt.Errorf("git revision %q: %w", commit, err)
}
u.commitIsh = commit
u.version = strings.TrimPrefix(preferredRev, "v")
return u.version, nil
}
// If the user specifies a valid tag as the preferred revision, that tag should be used without additional heuristics.
if preferredRev != "" {
cmd = exec.Command("git", "tag", "--list", preferredRev)
for _, candidate := range preferredRevisionCandidates(preferredRev) {
cmd = exec.Command("git", "tag", "--list", candidate)
cmd.Dir = gitdir
if out, err := cmd.Output(); err == nil && slices.Contains(strings.Fields(string(out)), preferredRev) {
latestTag = preferredRev
if out, err := cmd.Output(); err == nil && slices.Contains(strings.Fields(string(out)), candidate) {
latestTag = candidate
break
}
}
+197 -30
View File
@@ -8,6 +8,8 @@ import (
"errors"
"flag"
"fmt"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
"golang.org/x/net/publicsuffix"
"io"
"log"
@@ -55,6 +57,12 @@ type upstream struct {
var errUnsupportedHoster = errors.New("unsupported hoster")
type packRequest struct {
arg string
gopkg string
revision string
}
func passthroughEnv() []string {
var relevantVariables = []string{
"HOME",
@@ -177,13 +185,20 @@ func (u *upstream) get(gopath, sourceRepo, requestedPath, rev string) error {
}
if rev != "" {
cmd = exec.Command("git", "-c", "advice.detachedHead=false", "checkout", rev)
checkoutRev := rev
if module.IsPseudoVersion(rev) {
checkoutRev, err = module.PseudoVersionRev(rev)
if err != nil {
return fmt.Errorf("parse pseudo-version %q: %w", rev, err)
}
}
cmd = exec.Command("git", "-c", "advice.detachedHead=false", "checkout", checkoutRev)
cmd.Dir = dir
cmd.Env = passthroughEnv()
cmd.Stderr = os.Stderr
log.Println("get: Running", cmd, "in", cmd.Dir)
if err := cmd.Run(); err != nil {
return fmt.Errorf("git checkout %q: %w", rev, err)
return fmt.Errorf("git checkout %q: %w", checkoutRev, err)
}
}
return nil
@@ -959,10 +974,12 @@ func mainPack(args []string, usage func()) {
flagSet.Usage = usage
} else {
flagSet.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [pack] [FLAG]... <go-package-importpath>\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Example: %s pack golang.org/x/oauth2\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Usage: %s [pack] [FLAG]... <go-package-importpath>[@revision]...\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s [pack] [FLAG]... -list <go.mod>\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Example: %s pack golang.org/x/oauth2@v0.30.0\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Example: %s pack -list packages.go.mod\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "\"%s pack\" downloads the specified Go package from the Internet,\nand creates new files and directories in the current working directory.\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\"%s pack\" downloads the specified Go packages from the Internet,\nand creates new files and directories in the current working directory.\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\n")
fmt.Fprintf(os.Stderr, "Flags:\n")
flagSet.PrintDefaults()
@@ -973,10 +990,16 @@ func mainPack(args []string, usage func()) {
flagSet.StringVar(&gitRevision,
"git_revision",
"",
"git revision (see gitrevisions(7)) of the specified Go package\n"+
"git revision (see gitrevisions(7)) for package arguments without @revision\n"+
"to check out, defaulting to the default behavior of git clone.\n"+
"Useful in case you do not want to package e.g. current HEAD.")
var listFile string
flagSet.StringVar(&listFile,
"list",
"",
"Read packages to pack from the require directives of a go.mod file")
var forcePrerelease bool
flagSet.BoolVar(&forcePrerelease,
"force_prerelease",
@@ -1019,35 +1042,13 @@ func mainPack(args []string, usage func()) {
log.Fatalf("parse args: %v", err)
}
// Check for required positional argument
if flagSet.NArg() < 1 {
// A package argument or a package-list file is required.
if flagSet.NArg() < 1 && listFile == "" {
flagSet.Usage()
os.Exit(1)
}
gitRevision = strings.TrimSpace(gitRevision)
gopkg := flagSet.Arg(0)
// Remove URL scheme if present (https://, http://, git://, etc.)
gopkg = strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(gopkg, "https://"), "http://"), "git://")
// Verify the provided argument using the official pkg.go.dev API. Keep the
// requested path for naming and go_import_path; use the module path only for
// source checkout and tarball generation.
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
log.Fatalf("Verifying arguments: %v — did you specify a Go package import path?", err)
}
sourceGopkg := sourceImportPathForPackage(gopkg, info)
if sourceGopkg != gopkg {
log.Printf("Using module root %q as source checkout for specified import path %q", sourceGopkg, gopkg)
}
// Set default source and binary package names.
openRuyiSrc := nameFromGopkg(gopkg, typeLibrary, customProgPkgName, allowUnknownHoster)
openRuyiLib := openRuyiSrc
openRuyiProgram := nameFromGopkg(gopkg, typeProgram, customProgPkgName, allowUnknownHoster)
var pkgType packageType
switch strings.TrimSpace(pkgTypeString) {
@@ -1068,6 +1069,172 @@ func mainPack(args []string, usage func()) {
log.Fatalf("-type=%q not recognized, aborting\n", pkgTypeString)
}
// Now we collect all package's info into packRequest structs
requests, err := collectPackRequests(flagSet.Args(), listFile, gitRevision)
if err != nil {
log.Fatalf("parse package list: %v", err)
}
// Now we verify that all packages are valid
if err := preflightPackRequests(context.TODO(), requests, pkgType, customProgPkgName, allowUnknownHoster); err != nil {
log.Fatalf("preflight package list: %v", err)
}
for _, req := range requests {
packPackage(req.gopkg, req.revision, forcePrerelease, pkgType, customProgPkgName, allowUnknownHoster)
}
}
func collectPackRequests(args []string, listFile, defaultRevision string) ([]packRequest, error) {
// If a list file is specified
// for example, "pack -list packages.go.mod"
if listFile != "" {
if len(args) != 0 {
return nil, fmt.Errorf("-list cannot be combined with package arguments")
}
return parsePackListFile(listFile)
}
// if no list file is specified
// for example, "pack golang.org/x/a@v0.30.0 golang.org/x/b@v0.7.0"
var requests []packRequest
for _, arg := range args {
gopkg, revision, err := parsePackArg(arg, defaultRevision)
if err != nil {
return nil, fmt.Errorf("parse package argument %q: %w", arg, err)
}
requests = append(requests, packRequest{arg: arg, gopkg: gopkg, revision: revision})
}
return requests, nil
}
func parsePackListFile(filename string) ([]packRequest, error) {
data, err := os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("read %q: %w", filename, err)
}
f, err := modfile.Parse(filename, data, nil)
if err != nil {
return nil, fmt.Errorf("parse %q: %w", filename, err)
}
// TODO: Support go.mod directives that affect module version selection,
// such as replace, exclude, and retract, before accepting them in pack lists.
if len(f.Replace) > 0 || len(f.Exclude) > 0 || len(f.Retract) > 0 {
return nil, fmt.Errorf("%q contains unsupported directives", filename)
}
if len(f.Require) == 0 {
return nil, fmt.Errorf("%q contains no require directives", filename)
}
requests := make([]packRequest, 0, len(f.Require))
for _, require := range f.Require {
// TODO: Decide whether indirect requirements should be included in pack lists.
if err := module.Check(require.Mod.Path, require.Mod.Version); err != nil {
return nil, fmt.Errorf("invalid require %q %q: %w",
require.Mod.Path, require.Mod.Version, err)
}
requests = append(requests, packRequest{
arg: require.Mod.Path + "@" + require.Mod.Version,
gopkg: require.Mod.Path,
revision: require.Mod.Version,
})
}
return requests, nil
}
func preflightPackRequests(ctx context.Context, requests []packRequest, pkgType packageType, customProgPkgName string, allowUnknownHoster bool) error {
seenOutputDirs := make(map[string]string)
for _, req := range requests {
gopkg := strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(req.gopkg, "https://"), "http://"), "git://")
info, err := getPkgsiteInfo(ctx, gopkg)
if err != nil {
return fmt.Errorf("%q: %w", req.arg, err)
}
if repoURL := gitCloneURLFromRepoURL(info.Module.RepoURL); repoURL == "" {
return fmt.Errorf("%q: no repository URL", req.arg)
}
if req.revision != "" && strings.HasPrefix(req.revision, "v") && !module.IsPseudoVersion(req.revision) {
if _, err := getPkgsiteModule(ctx, info.Module.Path, req.revision); err != nil {
return fmt.Errorf("%q@%s: %w", req.arg, req.revision, err)
}
}
effectivePkgType := pkgType
if effectivePkgType == typeGuess {
effectivePkgType = typeLibrary
if info.Package.Name == "main" {
effectivePkgType = typeProgram
}
}
dir := nameFromGopkg(gopkg, effectivePkgType, customProgPkgName, allowUnknownHoster)
if previous, ok := seenOutputDirs[dir]; ok {
return fmt.Errorf("%q and %q use %q", previous, req.arg, dir)
}
seenOutputDirs[dir] = req.arg
dirInfo, err := os.Stat(dir)
if err == nil {
if !dirInfo.IsDir() {
return fmt.Errorf("%q: %q is not a directory", req.arg, dir)
}
entries, err := os.ReadDir(dir)
if err != nil {
return fmt.Errorf("%q: read %q: %w", req.arg, dir, err)
}
if len(entries) != 0 {
return fmt.Errorf("%q: %q is non empty", req.arg, dir)
}
} else if !os.IsNotExist(err) {
return fmt.Errorf("%q: stat %q: %w", req.arg, dir, err)
}
}
return nil
}
func parsePackArg(arg, defaultRevision string) (string, string, error) {
arg = strings.TrimSpace(arg)
defaultRevision = strings.TrimSpace(defaultRevision)
if arg == "" {
return "", "", fmt.Errorf("package import path is empty")
}
if strings.HasPrefix(arg, "-") {
return "", "", fmt.Errorf("-git_revision must be specified before package arguments")
}
if at := strings.LastIndex(arg, "@"); at >= 0 {
revision := strings.TrimSpace(arg[at+1:])
if revision == "" {
return "", "", fmt.Errorf("package revision is empty")
}
path := strings.TrimSpace(arg[:at])
if path == "" {
return "", "", fmt.Errorf("package import path is empty")
}
return path, revision, nil
}
return arg, defaultRevision, nil
}
func packPackage(gopkg, gitRevision string, forcePrerelease bool, pkgType packageType, customProgPkgName string, allowUnknownHoster bool) {
// Remove URL scheme if present (https://, http://, git://, etc.)
gopkg = strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(gopkg, "https://"), "http://"), "git://")
// Verify the provided argument using the official pkg.go.dev API. Keep the
// requested path for naming and go_import_path; use the module path only for
// source checkout and tarball generation.
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
log.Fatalf("Verifying arguments: %v — did you specify a Go package import path?", err)
}
sourceGopkg := sourceImportPathForPackage(gopkg, info)
if sourceGopkg != gopkg {
log.Printf("Using module root %q as source checkout for specified import path %q", sourceGopkg, gopkg)
}
// Set default source and binary package names.
openRuyiSrc := nameFromGopkg(gopkg, typeLibrary, customProgPkgName, allowUnknownHoster)
openRuyiLib := openRuyiSrc
openRuyiProgram := nameFromGopkg(gopkg, typeProgram, customProgPkgName, allowUnknownHoster)
if pkgType != typeGuess {
openRuyiSrc = nameFromGopkg(gopkg, pkgType, customProgPkgName, allowUnknownHoster)
}
+251
View File
@@ -547,6 +547,257 @@ func TestPkgVersionFromGitUsesPackagingDateAndSevenCharHash(t *testing.T) {
}
}
func TestPkgVersionFromGitAcceptsUnprefixedSemverPreferredRevision(t *testing.T) {
dir := t.TempDir()
runGit(t, dir, nil, "init")
if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module example.com/project\n"), 0644); err != nil {
t.Fatalf("write go.mod: %v", err)
}
runGit(t, dir, nil, "add", "go.mod")
runGit(t, dir, []string{
"GIT_AUTHOR_NAME=Test",
"GIT_AUTHOR_EMAIL=test@example.invalid",
"GIT_COMMITTER_NAME=Test",
"GIT_COMMITTER_EMAIL=test@example.invalid",
}, "commit", "-m", "initial")
runGit(t, dir, nil, "tag", "v1.2.0")
u := upstream{}
got, err := pkgVersionFromGit(dir, &u, "1.2.0", false)
if err != nil {
t.Fatalf("pkgVersionFromGit() returned error: %v", err)
}
if got != "1.2.0" {
t.Fatalf("pkgVersionFromGit() = %q, want %q", got, "1.2.0")
}
if u.tag != "v1.2.0" || !u.isRelease {
t.Fatalf("tag = %q, isRelease = %v, want v1.2.0 release", u.tag, u.isRelease)
}
}
func TestPkgVersionFromGitAcceptsPseudoVersion(t *testing.T) {
dir := t.TempDir()
runGit(t, dir, nil, "init")
if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module example.com/project\n"), 0644); err != nil {
t.Fatalf("write go.mod: %v", err)
}
runGit(t, dir, nil, "add", "go.mod")
runGit(t, dir, []string{
"GIT_AUTHOR_NAME=Test",
"GIT_AUTHOR_EMAIL=test@example.invalid",
"GIT_COMMITTER_NAME=Test",
"GIT_COMMITTER_EMAIL=test@example.invalid",
}, "commit", "-m", "initial")
fullHash := strings.TrimSpace(runGit(t, dir, nil, "rev-parse", "HEAD"))
pseudoVersion := "v0.0.0-20260722051018-" + fullHash[:12]
u := upstream{}
got, err := pkgVersionFromGit(dir, &u, pseudoVersion, false)
if err != nil {
t.Fatalf("pkgVersionFromGit() returned error: %v", err)
}
if got != strings.TrimPrefix(pseudoVersion, "v") {
t.Fatalf("pkgVersionFromGit() = %q, want %q", got, strings.TrimPrefix(pseudoVersion, "v"))
}
if u.commitIsh != fullHash[:12] || u.isRelease {
t.Fatalf("commitIsh = %q, isRelease = %v, want %q and false", u.commitIsh, u.isRelease, fullHash[:12])
}
}
func TestParsePackArg(t *testing.T) {
tests := []struct {
name string
arg string
defaultRevision string
wantPath string
wantRevision string
wantErr bool
}{
{
name: "explicit revision",
arg: "github.com/a/pkg@v1.2.3",
wantPath: "github.com/a/pkg",
wantRevision: "v1.2.3",
},
{
name: "explicit unprefixed semver revision",
arg: "github.com/a/pkg@1.2.3",
wantPath: "github.com/a/pkg",
wantRevision: "1.2.3",
},
{
name: "uses default revision",
arg: "github.com/a/pkg",
defaultRevision: "v0.9.0",
wantPath: "github.com/a/pkg",
wantRevision: "v0.9.0",
},
{
name: "keeps empty default revision",
arg: "github.com/a/pkg",
wantPath: "github.com/a/pkg",
},
{
name: "semantic import version with explicit revision",
arg: "github.com/a/pkg/v2@v2.1.0",
wantPath: "github.com/a/pkg/v2",
wantRevision: "v2.1.0",
},
{
name: "rejects empty explicit revision",
arg: "github.com/a/pkg@",
wantErr: true,
},
{
name: "rejects empty import path",
arg: "@v1.2.3",
wantErr: true,
},
{
name: "rejects misplaced flag",
arg: "-list",
wantErr: true,
},
{
name: "rejects misplaced git revision assignment",
arg: "-git_revision=v1.2.3",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotPath, gotRevision, err := parsePackArg(tt.arg, tt.defaultRevision)
if tt.wantErr {
if err == nil {
t.Fatal("expected error")
}
return
}
if err != nil {
t.Fatalf("parsePackArg() returned error: %v", err)
}
if gotPath != tt.wantPath || gotRevision != tt.wantRevision {
t.Fatalf("parsePackArg() = (%q, %q), want (%q, %q)", gotPath, gotRevision, tt.wantPath, tt.wantRevision)
}
})
}
}
func TestParsePackListFile(t *testing.T) {
dir := t.TempDir()
listPath := filepath.Join(dir, "packages.go.mod")
content := strings.Join([]string{
"module example.com/package-list",
"",
"go 1.25",
"",
"require (",
" github.com/a/pkg v1.2.3",
" github.com/b/tool v0.4.5 // indirect",
")",
"",
"require golang.org/x/text v0.31.0",
}, "\n")
if err := os.WriteFile(listPath, []byte(content), 0o644); err != nil {
t.Fatalf("write list: %v", err)
}
got, err := parsePackListFile(listPath)
if err != nil {
t.Fatalf("parsePackListFile() returned error: %v", err)
}
want := []packRequest{
{arg: "github.com/a/pkg@v1.2.3", gopkg: "github.com/a/pkg", revision: "v1.2.3"},
{arg: "github.com/b/tool@v0.4.5", gopkg: "github.com/b/tool", revision: "v0.4.5"},
{arg: "golang.org/x/text@v0.31.0", gopkg: "golang.org/x/text", revision: "v0.31.0"},
}
if len(got) != len(want) {
t.Fatalf("len(parsePackListFile()) = %d, want %d", len(got), len(want))
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("parsePackListFile()[%d] = %#v, want %#v", i, got[i], want[i])
}
}
}
func TestParsePackListFileRejectsUnsupportedDirectives(t *testing.T) {
dir := t.TempDir()
listPath := filepath.Join(dir, "packages.go.mod")
content := strings.Join([]string{
"module example.com/package-list",
"",
"go 1.25",
"require github.com/a/pkg v1.2.3",
"replace github.com/a/pkg => ./pkg",
}, "\n")
if err := os.WriteFile(listPath, []byte(content), 0o644); err != nil {
t.Fatalf("write list: %v", err)
}
_, err := parsePackListFile(listPath)
if err == nil {
t.Fatal("expected parsePackListFile() to reject replace directives")
}
if !strings.Contains(err.Error(), "unsupported directives") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParsePackListFileRejectsEmptyRequireList(t *testing.T) {
dir := t.TempDir()
listPath := filepath.Join(dir, "packages.go.mod")
content := strings.Join([]string{
"module example.com/package-list",
"",
"go 1.25",
}, "\n")
if err := os.WriteFile(listPath, []byte(content), 0o644); err != nil {
t.Fatalf("write list: %v", err)
}
_, err := parsePackListFile(listPath)
if err == nil {
t.Fatal("expected parsePackListFile() to reject an empty require list")
}
if !strings.Contains(err.Error(), "no require directives") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestCollectPackRequestsUsesListFile(t *testing.T) {
dir := t.TempDir()
listPath := filepath.Join(dir, "packages.go.mod")
content := strings.Join([]string{
"module example.com/package-list",
"",
"go 1.25",
"require github.com/a/pkg v1.2.3",
}, "\n")
if err := os.WriteFile(listPath, []byte(content), 0o644); err != nil {
t.Fatalf("write list: %v", err)
}
got, err := collectPackRequests(nil, listPath, "v9.9.9")
if err != nil {
t.Fatalf("collectPackRequests() returned error: %v", err)
}
if len(got) != 1 || got[0].gopkg != "github.com/a/pkg" || got[0].revision != "v1.2.3" {
t.Fatalf("collectPackRequests() = %#v, want github.com/a/pkg@v1.2.3", got)
}
}
func TestCollectPackRequestsRejectsListAndPackages(t *testing.T) {
_, err := collectPackRequests([]string{"github.com/a/pkg@v1.2.3"}, "packages.go.mod", "")
if err == nil {
t.Fatal("expected collectPackRequests() to reject -list combined with package arguments")
}
if !strings.Contains(err.Error(), "-list cannot be combined") {
t.Fatalf("unexpected error: %v", err)
}
}
func runGit(t *testing.T, dir string, env []string, args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)