Move builder shell parser into subpackage

Moves builder/shell_parser and into its own subpackage at builder/shell since it
has no dependencies other than the standard library. This will make it
much easier to vendor for downstream libraries, without pulling all the
dependencies of builder/.

Fixes #36154

Signed-off-by: Matt Rickard <mrick@google.com>

builder/dockerfile/builder.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/builder/remotecontext"
 	"github.com/docker/docker/errdefs"
@@ -256,8 +257,8 @@ func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error
 	return aux.Emit(types.BuildResult{ID: state.imageID})
 }
 
-func processMetaArg(meta instructions.ArgCommand, shlex *ShellLex, args *buildArgs) error {
-	// ShellLex currently only support the concatenated string format
+func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *buildArgs) error {
+	// shell.Lex currently only support the concatenated string format
 	envs := convertMapToEnvList(args.GetAllAllowed())
 	if err := meta.Expand(func(word string) (string, error) {
 		return shlex.ProcessWord(word, envs)
@@ -283,7 +284,7 @@ func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.
 	for _, stage := range parseResult {
 		totalCommands += len(stage.Commands)
 	}
-	shlex := NewShellLex(escapeToken)
+	shlex := shell.NewLex(escapeToken)
 	for _, meta := range metaArgs {
 		currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta)
 

builder/dockerfile/dispatchers.go

@@ -20,6 +20,7 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/jsonmessage"
@@ -47,7 +48,7 @@ func dispatchEnv(d dispatchRequest, c *instructions.EnvCommand) error {
 		for i, envVar := range runConfig.Env {
 			envParts := strings.SplitN(envVar, "=", 2)
 			compareFrom := envParts[0]
-			if equalEnvKeys(compareFrom, name) {
+			if shell.EqualEnvKeys(compareFrom, name) {
 				runConfig.Env[i] = newVar
 				gotOne = true
 				break
@@ -197,7 +198,7 @@ func dispatchTriggeredOnBuild(d dispatchRequest, triggers []string) error {
 	return nil
 }
 
-func (d *dispatchRequest) getExpandedImageName(shlex *ShellLex, name string) (string, error) {
+func (d *dispatchRequest) getExpandedImageName(shlex *shell.Lex, name string) (string, error) {
 	substitutionArgs := []string{}
 	for key, value := range d.state.buildArgs.GetAllMeta() {
 		substitutionArgs = append(substitutionArgs, key+"="+value)
@@ -242,7 +243,7 @@ func (d *dispatchRequest) getImageOrStage(name string) (builder.Image, error) {
 	}
 	return imageMount.Image(), nil
 }
-func (d *dispatchRequest) getFromImage(shlex *ShellLex, name string) (builder.Image, error) {
+func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string) (builder.Image, error) {
 	name, err := d.getExpandedImageName(shlex, name)
 	if err != nil {
 		return nil, err

builder/dockerfile/dispatchers_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/docker/api/types/strslice"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/go-connections/nat"
 	"github.com/stretchr/testify/assert"
@@ -141,7 +142,7 @@ func TestFromWithArg(t *testing.T) {
 	cmd := &instructions.Stage{
 		BaseName: "alpine:${THETAG}",
 	}
-	err := processMetaArg(metaArg, NewShellLex('\\'), args)
+	err := processMetaArg(metaArg, shell.NewLex('\\'), args)
 
 	sb := newDispatchRequest(b, '\\', nil, args, newStagesBuildResults())
 	require.NoError(t, err)

builder/dockerfile/dispatchers_unix.go

@@ -21,9 +21,3 @@ func normalizeWorkdir(_ string, current string, requested string) (string, error
 	}
 	return requested, nil
 }
-
-// equalEnvKeys compare two strings and returns true if they are equal. On
-// Windows this comparison is case insensitive.
-func equalEnvKeys(from, to string) bool {
-	return from == to
-}

builder/dockerfile/dispatchers_windows.go

@@ -93,9 +93,3 @@ func normalizeWorkdirWindows(current string, requested string) (string, error) {
 	// Upper-case drive letter
 	return (strings.ToUpper(string(requested[0])) + requested[1:]), nil
 }
-
-// equalEnvKeys compare two strings and returns true if they are equal. On
-// Windows this comparison is case insensitive.
-func equalEnvKeys(from, to string) bool {
-	return strings.ToUpper(from) == strings.ToUpper(to)
-}

builder/dockerfile/evaluator.go

@@ -28,6 +28,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/instructions"
+	"github.com/docker/docker/builder/dockerfile/shell"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/runconfig/opts"
@@ -187,7 +188,7 @@ func commitStage(state *dispatchState, stages *stagesBuildResults) error {
 
 type dispatchRequest struct {
 	state   *dispatchState
-	shlex   *ShellLex
+	shlex   *shell.Lex
 	builder *Builder
 	source  builder.Source
 	stages  *stagesBuildResults
@@ -196,7 +197,7 @@ type dispatchRequest struct {
 func newDispatchRequest(builder *Builder, escapeToken rune, source builder.Source, buildArgs *buildArgs, stages *stagesBuildResults) dispatchRequest {
 	return dispatchRequest{
 		state:   newDispatchState(buildArgs),
-		shlex:   NewShellLex(escapeToken),
+		shlex:   shell.NewLex(escapeToken),
 		builder: builder,
 		source:  source,
 		stages:  stages,

builder/dockerfile/shell/equal_env_unix.go

@@ -0,0 +1,9 @@
+// +build !windows
+
+package shell
+
+// EqualEnvKeys compare two strings and returns true if they are equal. On
+// Windows this comparison is case insensitive.
+func EqualEnvKeys(from, to string) bool {
+	return from == to
+}

builder/dockerfile/shell/equal_env_windows.go

@@ -0,0 +1,9 @@
+package shell
+
+import "strings"
+
+// EqualEnvKeys compare two strings and returns true if they are equal. On
+// Windows this comparison is case insensitive.
+func EqualEnvKeys(from, to string) bool {
+	return strings.ToUpper(from) == strings.ToUpper(to)
+}

builder/dockerfile/shell/lex.go

@@ -1,4 +1,4 @@
-package dockerfile
+package shell
 
 import (
 	"bytes"
@@ -9,25 +9,25 @@ import (
 	"github.com/pkg/errors"
 )
 
-// ShellLex performs shell word splitting and variable expansion.
+// Lex performs shell word splitting and variable expansion.
 //
-// ShellLex takes a string and an array of env variables and
+// Lex takes a string and an array of env variables and
 // process all quotes (" and ') as well as $xxx and ${xxx} env variable
 // tokens.  Tries to mimic bash shell process.
 // It doesn't support all flavors of ${xx:...} formats but new ones can
 // be added by adding code to the "special ${} format processing" section
-type ShellLex struct {
+type Lex struct {
 	escapeToken rune
 }
 
-// NewShellLex creates a new ShellLex which uses escapeToken to escape quotes.
-func NewShellLex(escapeToken rune) *ShellLex {
-	return &ShellLex{escapeToken: escapeToken}
+// NewLex creates a new Lex which uses escapeToken to escape quotes.
+func NewLex(escapeToken rune) *Lex {
+	return &Lex{escapeToken: escapeToken}
 }
 
 // ProcessWord will use the 'env' list of environment variables,
 // and replace any env var references in 'word'.
-func (s *ShellLex) ProcessWord(word string, env []string) (string, error) {
+func (s *Lex) ProcessWord(word string, env []string) (string, error) {
 	word, _, err := s.process(word, env)
 	return word, err
 }
@@ -39,12 +39,12 @@ func (s *ShellLex) ProcessWord(word string, env []string) (string, error) {
 // this splitting is done **after** the env var substitutions are done.
 // Note, each one is trimmed to remove leading and trailing spaces (unless
 // they are quoted", but ProcessWord retains spaces between words.
-func (s *ShellLex) ProcessWords(word string, env []string) ([]string, error) {
+func (s *Lex) ProcessWords(word string, env []string) ([]string, error) {
 	_, words, err := s.process(word, env)
 	return words, err
 }
 
-func (s *ShellLex) process(word string, env []string) (string, []string, error) {
+func (s *Lex) process(word string, env []string) (string, []string, error) {
 	sw := &shellWord{
 		envs:        env,
 		escapeToken: s.escapeToken,
@@ -327,7 +327,7 @@ func (sw *shellWord) getEnv(name string) string {
 	for _, env := range sw.envs {
 		i := strings.Index(env, "=")
 		if i < 0 {
-			if equalEnvKeys(name, env) {
+			if EqualEnvKeys(name, env) {
 				// Should probably never get here, but just in case treat
 				// it like "var" and "var=" are the same
 				return ""
@@ -335,7 +335,7 @@ func (sw *shellWord) getEnv(name string) string {
 			continue
 		}
 		compareName := env[:i]
-		if !equalEnvKeys(name, compareName) {
+		if !EqualEnvKeys(name, compareName) {
 			continue
 		}
 		return env[i+1:]

builder/dockerfile/shell/lex_test.go

@@ -1,4 +1,4 @@
-package dockerfile
+package shell
 
 import (
 	"bufio"
@@ -18,7 +18,7 @@ func TestShellParser4EnvVars(t *testing.T) {
 	assert.NoError(t, err)
 	defer file.Close()
 
-	shlex := NewShellLex('\\')
+	shlex := NewLex('\\')
 	scanner := bufio.NewScanner(file)
 	envs := []string{"PWD=/home", "SHELL=bash", "KOREAN=한국어"}
 	for scanner.Scan() {
@@ -70,7 +70,7 @@ func TestShellParser4Words(t *testing.T) {
 	}
 	defer file.Close()
 
-	shlex := NewShellLex('\\')
+	shlex := NewLex('\\')
 	envs := []string{}
 	scanner := bufio.NewScanner(file)
 	lineNum := 0