tools/builders/embedcfg.go

// Copyright 2021 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
)

// buildEmbedcfgFile writes an embedcfg file to be read by the compiler.
// An embedcfg file can be used in Go 1.16 or higher if the "embed" package
// is imported and there are one or more //go:embed comments in .go files.
// The embedcfg file maps //go:embed patterns to actual file names.
//
// The embedcfg file will be created in workDir, and its name is returned.
// The caller is responsible for deleting it. If no embedcfg file is needed,
// "" is returned with no error.
//
// All source files listed in goSrcs with //go:embed comments must be in one
// of the directories in embedRootDirs (not in a subdirectory). Embed patterns
// are evaluated relative to the source directory. Embed sources (embedSrcs)
// outside those directories are ignored, since they can't be matched by any
// valid pattern.
func buildEmbedcfgFile(goSrcs []fileInfo, embedSrcs, embedRootDirs []string, workDir string) (string, error) {
	// Check whether this package uses embedding and whether the toolchain
	// supports it (Go 1.16+). With Go 1.15 and lower, we'll try to compile
	// without an embedcfg file, and the compiler will complain the "embed"
	// package is missing.
	var major, minor int
	if n, err := fmt.Sscanf(runtime.Version(), "go%d.%d", &major, &minor); n != 2 || err != nil {
		// Can't parse go version. Maybe it's a development version; fall through.
	} else if major < 1 || (major == 1 && minor < 16) {
		return "", nil
	}
	importEmbed := false
	haveEmbed := false
	for _, src := range goSrcs {
		if len(src.embeds) > 0 {
			haveEmbed = true
			rootDir := findInRootDirs(src.filename, embedRootDirs)
			if rootDir == "" || strings.Contains(src.filename[len(rootDir)+1:], string(filepath.Separator)) {
				// Report an error if a source files appears in a subdirectory of
				// another source directory. In this situation, the same file could be
				// referenced with different paths.
				return "", fmt.Errorf("%s: source files with //go:embed should be in same directory. Allowed directories are:\n\t%s",
					src.filename,
					strings.Join(embedRootDirs, "\n\t"))
			}
		}
		for _, imp := range src.imports {
			if imp.path == "embed" {
				importEmbed = true
			}
		}
	}
	if !importEmbed || !haveEmbed {
		return "", nil
	}

	// Build a tree of embeddable files. This includes paths listed with
	// -embedsrc. If one of those paths is a directory, the tree includes
	// its files and subdirectories. Paths in the tree are relative to the
	// path in embedRootDirs that contains them.
	root, err := buildEmbedTree(embedSrcs, embedRootDirs)
	if err != nil {
		return "", err
	}

	// Resolve patterns to sets of files.
	var embedcfg struct {
		Patterns map[string][]string
		Files    map[string]string
	}
	embedcfg.Patterns = make(map[string][]string)
	embedcfg.Files = make(map[string]string)
	for _, src := range goSrcs {
		for _, embed := range src.embeds {
			matchedPaths, matchedFiles, err := resolveEmbed(embed, root)
			if err != nil {
				return "", err
			}
			embedcfg.Patterns[embed.pattern] = matchedPaths
			for i, rel := range matchedPaths {
				embedcfg.Files[rel] = matchedFiles[i]
			}
		}
	}

	// Write the configuration to a JSON file.
	embedcfgData, err := json.MarshalIndent(&embedcfg, "", "\t")
	if err != nil {
		return "", err
	}
	embedcfgName := filepath.Join(workDir, "embedcfg")
	if err := ioutil.WriteFile(embedcfgName, embedcfgData, 0o666); err != nil {
		return "", err
	}
	return embedcfgName, nil
}

// findInRootDirs returns a string from rootDirs which is a parent of the
// file path p. If there is no such string, findInRootDirs returns "".
func findInRootDirs(p string, rootDirs []string) string {
	dir := filepath.Dir(p)
	for _, rootDir := range rootDirs {
		if rootDir == dir ||
			(strings.HasPrefix(dir, rootDir) && len(dir) > len(rootDir)+1 && dir[len(rootDir)] == filepath.Separator) {
			return rootDir
		}
	}
	return ""
}

// embedNode represents an embeddable file or directory in a tree.
type embedNode struct {
	name       string                // base name
	path       string                // absolute file path
	children   map[string]*embedNode // non-nil for directory
	childNames []string              // sorted
}

// add inserts file nodes into the tree rooted at f for the slash-separated
// path src, relative to the absolute file path rootDir. If src points to a
// directory, add recursively inserts nodes for its contents. If a node already
// exists (for example, if a source file and a generated file have the same
// name), add leaves the existing node in place.
func (n *embedNode) add(rootDir, src string) error {
	// Create nodes for parents of src.
	parent := n
	parts := strings.Split(src, "/")
	for _, p := range parts[:len(parts)-1] {
		if parent.children[p] == nil {
			parent.children[p] = &embedNode{
				name:     p,
				children: make(map[string]*embedNode),
			}
		}
		parent = parent.children[p]
	}

	// Create a node for src. If src is a directory, recursively create nodes for
	// its contents. Go embedding ignores symbolic links, but Bazel may use links
	// for generated files and directories, so we follow them here.
	var visit func(*embedNode, string, os.FileInfo) error
	visit = func(parent *embedNode, path string, fi os.FileInfo) error {
		base := filepath.Base(path)
		if parent.children[base] == nil {
			parent.children[base] = &embedNode{name: base, path: path}
		}
		if !fi.IsDir() {
			return nil
		}
		node := parent.children[base]
		node.children = make(map[string]*embedNode)
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		names, err := f.Readdirnames(0)
		f.Close()
		if err != nil {
			return err
		}
		for _, name := range names {
			cPath := filepath.Join(path, name)
			cfi, err := os.Stat(cPath)
			if err != nil {
				return err
			}
			if err := visit(node, cPath, cfi); err != nil {
				return err
			}
		}
		return nil
	}

	path := filepath.Join(rootDir, src)
	fi, err := os.Stat(path)
	if err != nil {
		return err
	}
	return visit(parent, path, fi)
}

func (n *embedNode) isDir() bool {
	return n.children != nil
}

// get returns a tree node, given a slash-separated path relative to the
// receiver. get returns nil if no node exists with that path.
func (n *embedNode) get(path string) *embedNode {
	if path == "." || path == "" {
		return n
	}
	for _, part := range strings.Split(path, "/") {
		n = n.children[part]
		if n == nil {
			return nil
		}
	}
	return n
}

var errSkip = errors.New("skip")

// walk calls fn on each node in the tree rooted at n in depth-first pre-order.
func (n *embedNode) walk(fn func(rel string, n *embedNode) error) error {
	var visit func(string, *embedNode) error
	visit = func(rel string, node *embedNode) error {
		err := fn(rel, node)
		if err == errSkip {
			return nil
		} else if err != nil {
			return err
		}
		for _, name := range node.childNames {
			if err := visit(path.Join(rel, name), node.children[name]); err != nil && err != errSkip {
				return err
			}
		}
		return nil
	}
	err := visit("", n)
	if err == errSkip {
		return nil
	}
	return err
}

// buildEmbedTree constructs a logical directory tree of embeddable files.
// The tree may contain a mix of static and generated files from multiple
// root directories. Directory artifacts are recursively expanded.
func buildEmbedTree(embedSrcs, embedRootDirs []string) (root *embedNode, err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("building tree of embeddable files in directories %s: %v", strings.Join(embedRootDirs, string(filepath.ListSeparator)), err)
		}
	}()

	// Add each path to the tree.
	root = &embedNode{name: "", children: make(map[string]*embedNode)}
	for _, src := range embedSrcs {
		rootDir := findInRootDirs(src, embedRootDirs)
		if rootDir == "" {
			// Embedded path cannot be matched by any valid pattern. Ignore.
			continue
		}
		rel := filepath.ToSlash(src[len(rootDir)+1:])
		if err := root.add(rootDir, rel); err != nil {
			return nil, err
		}
	}

	// Sort children in each directory node.
	var visit func(*embedNode)
	visit = func(node *embedNode) {
		node.childNames = make([]string, 0, len(node.children))
		for name, child := range node.children {
			node.childNames = append(node.childNames, name)
			visit(child)
		}
		sort.Strings(node.childNames)
	}
	visit(root)

	return root, nil
}

// resolveEmbed matches a //go:embed pattern in a source file to a set of
// embeddable files in the given tree.
func resolveEmbed(embed fileEmbed, root *embedNode) (matchedPaths, matchedFiles []string, err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("%v: could not embed %s: %v", embed.pos, embed.pattern, err)
		}
	}()

	// Remove optional "all:" prefix from pattern and set matchAll flag if present.
	// See https://pkg.go.dev/embed#hdr-Directives for details.
	pattern := embed.pattern
	var matchAll bool
	if strings.HasPrefix(pattern, "all:") {
		matchAll = true
		pattern = pattern[4:]
	}

	// Check that the pattern has valid syntax.
	if _, err := path.Match(pattern, ""); err != nil || !validEmbedPattern(pattern) {
		return nil, nil, fmt.Errorf("invalid pattern syntax")
	}

	// Search for matching files.
	err = root.walk(func(matchRel string, matchNode *embedNode) error {
		if ok, _ := path.Match(pattern, matchRel); !ok {
			// Non-matching file or directory.
			return nil
		}

		// TODO: Should check that directories along path do not begin a new module
		// (do not contain a go.mod).
		// https://cs.opensource.google/go/go/+/master:src/cmd/go/internal/load/pkg.go;l=2158;drc=261fe25c83a94fc3defe064baed3944cd3d16959
		for dir := matchRel; len(dir) > 1; dir = filepath.Dir(dir) {
			if base := path.Base(matchRel); isBadEmbedName(base) {
				what := "file"
				if matchNode.isDir() {
					what = "directory"
				}
				if dir == matchRel {
					return fmt.Errorf("cannot embed %s %s: invalid name %s", what, matchRel, base)
				} else {
					return fmt.Errorf("cannot embed %s %s: in invalid directory %s", what, matchRel, base)
				}
			}
		}

		if !matchNode.isDir() {
			// Matching file. Add to list.
			matchedPaths = append(matchedPaths, matchRel)
			matchedFiles = append(matchedFiles, matchNode.path)
			return nil
		}

		// Matching directory. Recursively add all files in subdirectories.
		// Don't add hidden files or directories (starting with "." or "_"),
		// unless "all:" prefix was set.
		// See golang/go#42328.
		matchTreeErr := matchNode.walk(func(childRel string, childNode *embedNode) error {
			// TODO: Should check that directories along path do not begin a new module
			// https://cs.opensource.google/go/go/+/master:src/cmd/go/internal/load/pkg.go;l=2158;drc=261fe25c83a94fc3defe064baed3944cd3d16959
			if childRel != "" {
				base := path.Base(childRel)
				if isBadEmbedName(base) || (!matchAll && (strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_"))) {
					if childNode.isDir() {
						return errSkip
					}
					return nil
				}
			}
			if !childNode.isDir() {
				matchedPaths = append(matchedPaths, path.Join(matchRel, childRel))
				matchedFiles = append(matchedFiles, childNode.path)
			}
			return nil
		})
		if matchTreeErr != nil {
			return matchTreeErr
		}
		return errSkip
	})
	if err != nil && err != errSkip {
		return nil, nil, err
	}
	if len(matchedPaths) == 0 {
		return nil, nil, fmt.Errorf("no matching files found")
	}
	return matchedPaths, matchedFiles, nil
}

func validEmbedPattern(pattern string) bool {
	return pattern != "." && fsValidPath(pattern)
}

// validPath reports whether the given path name
// is valid for use in a call to Open.
// Path names passed to open are unrooted, slash-separated
// sequences of path elements, like “x/y/z”.
// Path names must not contain a “.” or “..” or empty element,
// except for the special case that the root directory is named “.”.
//
// Paths are slash-separated on all systems, even Windows.
// Backslashes must not appear in path names.
//
// Copied from io/fs.ValidPath in Go 1.16beta1.
func fsValidPath(name string) bool {
	if name == "." {
		// special case
		return true
	}

	// Iterate over elements in name, checking each.
	for {
		i := 0
		for i < len(name) && name[i] != '/' {
			if name[i] == '\\' {
				return false
			}
			i++
		}
		elem := name[:i]
		if elem == "" || elem == "." || elem == ".." {
			return false
		}
		if i == len(name) {
			return true // reached clean ending
		}
		name = name[i+1:]
	}
}

// isBadEmbedName reports whether name is the base name of a file that
// can't or won't be included in modules and therefore shouldn't be treated
// as existing for embedding.
//
// TODO: This should use the equivalent of golang.org/x/mod/module.CheckFilePath instead of fsValidPath.
// https://cs.opensource.google/go/go/+/master:src/cmd/go/internal/load/pkg.go;l=2200;drc=261fe25c83a94fc3defe064baed3944cd3d16959
func isBadEmbedName(name string) bool {
	if !fsValidPath(name) {
		return true
	}
	switch name {
	// Empty string should be impossible but make it bad.
	case "":
		return true
	// Version control directories won't be present in module.
	case ".bzr", ".hg", ".git", ".svn":
		return true
	}
	return false
}