modfile: support retract directive and version intervals · golang/mod@c0d644d ·...

This CL adds support for parsing and programmatically adding and
removing a new directive, "retract", as described in golang/go#24031.

The "retract" directive comes in two forms:

    retract v1.0.0           // single version
    retract [v1.1.0, v1.2.0] // closed interval

Updates golang/go#24031

Change-Id: I1236c7d89e7674abf694e49e9b4869b14a59fac0
Reviewed-on: https://go-review.googlesource.com/c/mod/+/228039
Run-TryBot: Jay Conrod <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Michael Matloob <[email protected]>

@@ -30,6 +30,7 @@ import (

"golang.org/x/mod/internal/lazyregexp"

"golang.org/x/mod/module"

"golang.org/x/mod/semver"

)

// A File is the parsed, interpreted form of a go.mod file.

@@ -39,6 +40,7 @@ type File struct {

Require []*Require

Exclude []*Exclude

Replace []*Replace

Retract []*Retract

Syntax *FileSyntax

}

@@ -75,6 +77,21 @@ type Replace struct {

Syntax *Line

}

// A Retract is a single retract statement.

type Retract struct {

VersionInterval

Rationale string

Syntax *Line

}

// A VersionInterval represents a range of versions with upper and lower bounds.

// Intervals are closed: both bounds are included. When Low is equal to High,

// the interval may refer to a single version ('v1.2.3') or an interval

// ('[v1.2.3, v1.2.3]'); both have the same representation.

type VersionInterval struct {

Low, High string

}

func (f *File) AddModuleStmt(path string) error {

if f.Syntax == nil {

f.Syntax = new(FileSyntax)

@@ -138,7 +155,7 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File

for _, x := range fs.Stmt {

switch x := x.(type) {

case *Line:

f.add(&errs, x, x.Token[0], x.Token[1:], fix, strict)

f.add(&errs, nil, x, x.Token[0], x.Token[1:], fix, strict)

case *LineBlock:

if len(x.Token) > 1 {

@@ -161,9 +178,9 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File

})

}

continue

case "module", "require", "exclude", "replace":

case "module", "require", "exclude", "replace", "retract":

for _, l := range x.Line {

f.add(&errs, l, x.Token[0], l.Token, fix, strict)

f.add(&errs, x, l, x.Token[0], l.Token, fix, strict)

}

}

}

@@ -177,7 +194,7 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File

var GoVersionRE = lazyregexp.New(`^([1-9][0-9]*)\.(0|[1-9][0-9]*)$`)

func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix VersionFixer, strict bool) {

func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, args []string, fix VersionFixer, strict bool) {

// If strict is false, this module is a dependency.

// We ignore all unknown directives as well as main-module-only

// directives like replace and exclude. It will work better for

@@ -186,7 +203,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix

// and simply ignore those statements.

if !strict {

switch verb {

case "module", "require", "go":

case "go", "module", "retract", "require":

// want these even for dependency go.mods

default:

return

@@ -232,6 +249,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix

f.Go = &Go{Syntax: line}

f.Go.Version = args[0]

case "module":

if f.Module != nil {

errorf("repeated module statement")

@@ -248,6 +266,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix

return

}

f.Module.Mod = module.Version{Path: s}

case "require", "exclude":

if len(args) != 2 {

errorf("usage: %s module/path v1.2.3", verb)

@@ -284,6 +303,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix

Syntax: line,

})

}

case "replace":

arrow := 2

if len(args) >= 2 && args[1] == "=>" {

@@ -347,6 +367,33 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix

New: module.Version{Path: ns, Version: nv},

Syntax: line,

})

case "retract":

rationale := parseRetractRationale(block, line)

vi, err := parseVersionInterval(verb, &args, fix)

if err != nil {

if strict {

wrapError(err)

return

} else {

// Only report errors parsing intervals in the main module. We may

// support additional syntax in the future, such as open and half-open

// intervals. Those can't be supported now, because they break the

// go.mod parser, even in lax mode.

return

}

}

if len(args) > 0 && strict {

// In the future, there may be additional information after the version.

errorf("unexpected token after version: %q", args[0])

return

}

retract := &Retract{

VersionInterval: vi,

Rationale: rationale,

Syntax: line,

}

f.Retract = append(f.Retract, retract)

}

}

@@ -444,6 +491,53 @@ func AutoQuote(s string) string {

return s

}

func parseVersionInterval(verb string, args *[]string, fix VersionFixer) (VersionInterval, error) {

toks := *args

if len(toks) == 0 || toks[0] == "(" {

return VersionInterval{}, fmt.Errorf("expected '[' or version")

}

if toks[0] != "[" {

v, err := parseVersion(verb, "", &toks[0], fix)

if err != nil {

return VersionInterval{}, err

}

*args = toks[1:]

return VersionInterval{Low: v, High: v}, nil

}

toks = toks[1:]

if len(toks) == 0 {

return VersionInterval{}, fmt.Errorf("expected version after '['")

}

low, err := parseVersion(verb, "", &toks[0], fix)

if err != nil {

return VersionInterval{}, err

}

toks = toks[1:]

if len(toks) == 0 || toks[0] != "," {

return VersionInterval{}, fmt.Errorf("expected ',' after version")

}

toks = toks[1:]

if len(toks) == 0 {

return VersionInterval{}, fmt.Errorf("expected version after ','")

}

high, err := parseVersion(verb, "", &toks[0], fix)

if err != nil {

return VersionInterval{}, err

}

toks = toks[1:]

if len(toks) == 0 || toks[0] != "]" {

return VersionInterval{}, fmt.Errorf("expected ']' after version")

}

toks = toks[1:]

*args = toks

return VersionInterval{Low: low, High: high}, nil

}

func parseString(s *string) (string, error) {

t := *s

if strings.HasPrefix(t, `"`) {

@@ -461,6 +555,27 @@ func parseString(s *string) (string, error) {

return t, nil

}

// parseRetractRationale extracts the rationale for a retract directive from the

// surrounding comments. If the line does not have comments and is part of a

// block that does have comments, the block's comments are used.

func parseRetractRationale(block *LineBlock, line *Line) string {

comments := line.Comment()

if block != nil && len(comments.Before) == 0 && len(comments.Suffix) == 0 {

comments = block.Comment()

}

groups := [][]Comment{comments.Before, comments.Suffix}

var lines []string

for _, g := range groups {

for _, c := range g {

if !strings.HasPrefix(c.Token, "//") {

continue // blank line

}

lines = append(lines, strings.TrimSpace(strings.TrimPrefix(c.Token, "//")))

}

}

return strings.Join(lines, "\n")

}

type ErrorList []Error

func (e ErrorList) Error() string {

@@ -494,6 +609,8 @@ func (e *Error) Error() string {

var directive string

if e.ModPath != "" {

directive = fmt.Sprintf("%s %s: ", e.Verb, e.ModPath)

} else if e.Verb != "" {

directive = fmt.Sprintf("%s: ", e.Verb)

}

return pos + directive + e.Err.Error()

@@ -585,6 +702,15 @@ func (f *File) Cleanup() {

}

f.Replace = f.Replace[:w]

w = 0

for _, r := range f.Retract {

if r.Low != "" || r.High != "" {

f.Retract[w] = r

w++

}

}

f.Retract = f.Retract[:w]

f.Syntax.Cleanup()

}

@@ -778,6 +904,34 @@ func (f *File) DropReplace(oldPath, oldVers string) error {

return nil

}

func (f *File) AddRetract(vi VersionInterval, rationale string) error {

r := &Retract{

VersionInterval: vi,

}

if vi.Low == vi.High {

r.Syntax = f.Syntax.addLine(nil, "retract", AutoQuote(vi.Low))

} else {

r.Syntax = f.Syntax.addLine(nil, "retract", "[", AutoQuote(vi.Low), ",", AutoQuote(vi.High), "]")

}

if rationale != "" {

for _, line := range strings.Split(rationale, "\n") {

com := Comment{Token: "// " + line}

r.Syntax.Comment().Before = append(r.Syntax.Comment().Before, com)

}

}

return nil

}

func (f *File) DropRetract(vi VersionInterval) error {

for _, r := range f.Retract {

if r.VersionInterval == vi {

f.Syntax.removeLine(r.Syntax)

*r = Retract{}

}

}

return nil

}

func (f *File) SortBlocks() {

f.removeDups() // otherwise sorting is unsafe

@@ -786,28 +940,38 @@ func (f *File) SortBlocks() {

if !ok {

continue

}

sort.Slice(block.Line, func(i, j int) bool {

li := block.Line[i]

lj := block.Line[j]

for k := 0; k < len(li.Token) && k < len(lj.Token); k++ {

if li.Token[k] != lj.Token[k] {

return li.Token[k] < lj.Token[k]

}

}

return len(li.Token) < len(lj.Token)

less := lineLess

if block.Token[0] == "retract" {

less = lineRetractLess

}

sort.SliceStable(block.Line, func(i, j int) bool {

return less(block.Line[i], block.Line[j])

})

}

}

// removeDups removes duplicate exclude and replace directives.

//

// Earlier exclude directives take priority.

//

// Later replace directives take priority.

//

// require directives are not de-duplicated. That's left up to higher-level

// logic (MVS).

//

// retract directives are not de-duplicated since comments are

// meaningful, and versions may be retracted multiple times.

func (f *File) removeDups() {

have := make(map[module.Version]bool)

kill := make(map[*Line]bool)

// Remove duplicate excludes.

haveExclude := make(map[module.Version]bool)

for _, x := range f.Exclude {

if have[x.Mod] {

if haveExclude[x.Mod] {

kill[x.Syntax] = true

continue

}

have[x.Mod] = true

haveExclude[x.Mod] = true

}

var excl []*Exclude

for _, x := range f.Exclude {

@@ -817,15 +981,16 @@ func (f *File) removeDups() {

}

f.Exclude = excl

have = make(map[module.Version]bool)

// Remove duplicate replacements.

// Later replacements take priority over earlier ones.

haveReplace := make(map[module.Version]bool)

for i := len(f.Replace) - 1; i >= 0; i-- {

x := f.Replace[i]

if have[x.Old] {

if haveReplace[x.Old] {

kill[x.Syntax] = true

continue

}

have[x.Old] = true

haveReplace[x.Old] = true

}

var repl []*Replace

for _, x := range f.Replace {

@@ -835,6 +1000,9 @@ func (f *File) removeDups() {

}

f.Replace = repl

// Duplicate require and retract directives are not removed.

// Drop killed statements from the syntax tree.

var stmts []Expr

for _, stmt := range f.Syntax.Stmt {

switch stmt := stmt.(type) {

@@ -858,3 +1026,38 @@ func (f *File) removeDups() {

}

f.Syntax.Stmt = stmts

}

// lineLess returns whether li should be sorted before lj. It sorts

// lexicographically without assigning any special meaning to tokens.

func lineLess(li, lj *Line) bool {

for k := 0; k < len(li.Token) && k < len(lj.Token); k++ {

if li.Token[k] != lj.Token[k] {

return li.Token[k] < lj.Token[k]

}

}

return len(li.Token) < len(lj.Token)

}

// lineRetractLess returns whether li should be sorted before lj for lines in

// a "retract" block. It treats each line as a version interval. Single versions

// are compared as if they were intervals with the same low and high version.

// Intervals are sorted in descending order, first by low version, then by

// high version, using semver.Compare.

func lineRetractLess(li, lj *Line) bool {

interval := func(l *Line) VersionInterval {

if len(l.Token) == 1 {

return VersionInterval{Low: l.Token[0], High: l.Token[0]}

} else if len(l.Token) == 5 && l.Token[0] == "[" && l.Token[2] == "," && l.Token[4] == "]" {

return VersionInterval{Low: l.Token[1], High: l.Token[3]}

} else {

// Line in unknown format. Treat as an invalid version.

return VersionInterval{}

}

}

vii := interval(li)

vij := interval(lj)

if cmp := semver.Compare(vii.Low, vij.Low); cmp != 0 {

return cmp > 0

}

return semver.Compare(vii.High, vij.High) > 0

}