1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Build toolchain using Go bootstrap version.
6//
7// The general strategy is to copy the source files we need into
8// a new GOPATH workspace, adjust import paths appropriately,
9// invoke the Go bootstrap toolchains go command to build those sources,
10// and then copy the binaries back.
11
12package main
13
14import (
15	"fmt"
16	"os"
17	"path/filepath"
18	"regexp"
19	"strings"
20)
21
22// bootstrapDirs is a list of directories holding code that must be
23// compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
24// All directories in this list are relative to and must be below $GOROOT/src.
25//
26// The list has two kinds of entries: names beginning with cmd/ with
27// no other slashes, which are commands, and other paths, which are packages
28// supporting the commands. Packages in the standard library can be listed
29// if a newer copy needs to be substituted for the Go bootstrap copy when used
30// by the command packages. Paths ending with /... automatically
31// include all packages within subdirectories as well.
32// These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
33var bootstrapDirs = []string{
34	"cmp",
35	"cmd/asm",
36	"cmd/asm/internal/...",
37	"cmd/cgo",
38	"cmd/compile",
39	"cmd/compile/internal/...",
40	"cmd/internal/archive",
41	"cmd/internal/bio",
42	"cmd/internal/codesign",
43	"cmd/internal/dwarf",
44	"cmd/internal/edit",
45	"cmd/internal/gcprog",
46	"cmd/internal/goobj",
47	"cmd/internal/notsha256",
48	"cmd/internal/obj/...",
49	"cmd/internal/objabi",
50	"cmd/internal/pgo",
51	"cmd/internal/pkgpath",
52	"cmd/internal/quoted",
53	"cmd/internal/src",
54	"cmd/internal/sys",
55	"cmd/internal/telemetry",
56	"cmd/internal/telemetry/counter",
57	"cmd/link",
58	"cmd/link/internal/...",
59	"compress/flate",
60	"compress/zlib",
61	"container/heap",
62	"debug/dwarf",
63	"debug/elf",
64	"debug/macho",
65	"debug/pe",
66	"go/build/constraint",
67	"go/constant",
68	"go/version",
69	"internal/abi",
70	"internal/coverage",
71	"cmd/internal/cov/covcmd",
72	"internal/bisect",
73	"internal/buildcfg",
74	"internal/goarch",
75	"internal/godebugs",
76	"internal/goexperiment",
77	"internal/goroot",
78	"internal/gover",
79	"internal/goversion",
80	// internal/lazyregexp is provided by Go 1.17, which permits it to
81	// be imported by other packages in this list, but is not provided
82	// by the Go 1.17 version of gccgo. It's on this list only to
83	// support gccgo, and can be removed if we require gccgo 14 or later.
84	"internal/lazyregexp",
85	"internal/pkgbits",
86	"internal/platform",
87	"internal/profile",
88	"internal/race",
89	"internal/saferio",
90	"internal/syscall/unix",
91	"internal/types/errors",
92	"internal/unsafeheader",
93	"internal/xcoff",
94	"internal/zstd",
95	"math/bits",
96	"sort",
97}
98
99// File prefixes that are ignored by go/build anyway, and cause
100// problems with editor generated temporary files (#18931).
101var ignorePrefixes = []string{
102	".",
103	"_",
104	"#",
105}
106
107// File suffixes that use build tags introduced since Go 1.17.
108// These must not be copied into the bootstrap build directory.
109// Also ignore test files.
110var ignoreSuffixes = []string{
111	"_test.s",
112	"_test.go",
113	// Skip PGO profile. No need to build toolchain1 compiler
114	// with PGO. And as it is not a text file the import path
115	// rewrite will break it.
116	".pgo",
117	// Skip editor backup files.
118	"~",
119}
120
121var tryDirs = []string{
122	"sdk/go1.17",
123	"go1.17",
124}
125
126func bootstrapBuildTools() {
127	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
128	if goroot_bootstrap == "" {
129		home := os.Getenv("HOME")
130		goroot_bootstrap = pathf("%s/go1.4", home)
131		for _, d := range tryDirs {
132			if p := pathf("%s/%s", home, d); isdir(p) {
133				goroot_bootstrap = p
134			}
135		}
136	}
137	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
138
139	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
140	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
141
142	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
143	// We use a subdirectory of $GOROOT/pkg because that's the
144	// space within $GOROOT where we store all generated objects.
145	// We could use a temporary directory outside $GOROOT instead,
146	// but it is easier to debug on failure if the files are in a known location.
147	workspace := pathf("%s/pkg/bootstrap", goroot)
148	xremoveall(workspace)
149	xatexit(func() { xremoveall(workspace) })
150	base := pathf("%s/src/bootstrap", workspace)
151	xmkdirall(base)
152
153	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
154	writefile("module bootstrap\ngo 1.20\n", pathf("%s/%s", base, "go.mod"), 0)
155	for _, dir := range bootstrapDirs {
156		recurse := strings.HasSuffix(dir, "/...")
157		dir = strings.TrimSuffix(dir, "/...")
158		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
159			if err != nil {
160				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
161			}
162
163			name := filepath.Base(path)
164			src := pathf("%s/src/%s", goroot, path)
165			dst := pathf("%s/%s", base, path)
166
167			if info.IsDir() {
168				if !recurse && path != dir || name == "testdata" {
169					return filepath.SkipDir
170				}
171
172				xmkdirall(dst)
173				if path == "cmd/cgo" {
174					// Write to src because we need the file both for bootstrap
175					// and for later in the main build.
176					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
177					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
178				}
179				return nil
180			}
181
182			for _, pre := range ignorePrefixes {
183				if strings.HasPrefix(name, pre) {
184					return nil
185				}
186			}
187			for _, suf := range ignoreSuffixes {
188				if strings.HasSuffix(name, suf) {
189					return nil
190				}
191			}
192
193			text := bootstrapRewriteFile(src)
194			writefile(text, dst, 0)
195			return nil
196		})
197	}
198
199	// Set up environment for invoking Go bootstrap toolchains go command.
200	// GOROOT points at Go bootstrap GOROOT,
201	// GOPATH points at our bootstrap workspace,
202	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
203	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
204	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
205	// Restore GOROOT, GOPATH, and GOBIN when done.
206	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
207	// because setup will take care of those when bootstrapBuildTools returns.
208
209	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
210	os.Setenv("GOROOT", goroot_bootstrap)
211
212	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
213	os.Setenv("GOPATH", workspace)
214
215	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
216	os.Setenv("GOBIN", "")
217
218	os.Setenv("GOOS", "")
219	os.Setenv("GOHOSTOS", "")
220	os.Setenv("GOARCH", "")
221	os.Setenv("GOHOSTARCH", "")
222
223	// Run Go bootstrap to build binaries.
224	// Use the math_big_pure_go build tag to disable the assembly in math/big
225	// which may contain unsupported instructions.
226	// Use the purego build tag to disable other assembly code,
227	// such as in cmd/internal/notsha256.
228	cmd := []string{
229		pathf("%s/bin/go", goroot_bootstrap),
230		"install",
231		"-tags=math_big_pure_go compiler_bootstrap purego",
232	}
233	if vflag > 0 {
234		cmd = append(cmd, "-v")
235	}
236	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
237		cmd = append(cmd, "-toolexec="+tool)
238	}
239	cmd = append(cmd, "bootstrap/cmd/...")
240	run(base, ShowOutput|CheckExit, cmd...)
241
242	// Copy binaries into tool binary directory.
243	for _, name := range bootstrapDirs {
244		if !strings.HasPrefix(name, "cmd/") {
245			continue
246		}
247		name = name[len("cmd/"):]
248		if !strings.Contains(name, "/") {
249			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
250		}
251	}
252
253	if vflag > 0 {
254		xprintf("\n")
255	}
256}
257
258var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
259
260// isUnneededSSARewriteFile reports whether srcFile is a
261// src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
262// architecture that isn't for the given GOARCH.
263//
264// When unneeded is true archCaps is the rewrite base filename without
265// the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
266func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
267	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
268		return "", false
269	}
270	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
271	if fileArch == "" {
272		return "", false
273	}
274	b := fileArch[0]
275	if b == '_' || ('a' <= b && b <= 'z') {
276		return "", false
277	}
278	archCaps = fileArch
279	fileArch = strings.ToLower(fileArch)
280	fileArch = strings.TrimSuffix(fileArch, "splitload")
281	fileArch = strings.TrimSuffix(fileArch, "latelower")
282	if fileArch == goArch {
283		return "", false
284	}
285	if fileArch == strings.TrimSuffix(goArch, "le") {
286		return "", false
287	}
288	return archCaps, true
289}
290
291func bootstrapRewriteFile(srcFile string) string {
292	// During bootstrap, generate dummy rewrite files for
293	// irrelevant architectures. We only need to build a bootstrap
294	// binary that works for the current gohostarch.
295	// This saves 6+ seconds of bootstrap.
296	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
297		return fmt.Sprintf(`%spackage ssa
298
299func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
300func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
301`, generatedHeader, archCaps, archCaps)
302	}
303
304	return bootstrapFixImports(srcFile)
305}
306
307var (
308	importRE      = regexp.MustCompile(`\Aimport\s+(\.|[A-Za-z0-9_]+)?\s*"([^"]+)"\s*(//.*)?\n\z`)
309	importBlockRE = regexp.MustCompile(`\A\s*(?:(\.|[A-Za-z0-9_]+)?\s*"([^"]+)")?\s*(//.*)?\n\z`)
310)
311
312func bootstrapFixImports(srcFile string) string {
313	text := readfile(srcFile)
314	if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
315		text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
316	}
317	lines := strings.SplitAfter(text, "\n")
318	inBlock := false
319	inComment := false
320	for i, line := range lines {
321		if strings.HasSuffix(line, "*/\n") {
322			inComment = false
323		}
324		if strings.HasSuffix(line, "/*\n") {
325			inComment = true
326		}
327		if inComment {
328			continue
329		}
330		if strings.HasPrefix(line, "import (") {
331			inBlock = true
332			continue
333		}
334		if inBlock && strings.HasPrefix(line, ")") {
335			inBlock = false
336			continue
337		}
338
339		var m []string
340		if !inBlock {
341			if !strings.HasPrefix(line, "import ") {
342				continue
343			}
344			m = importRE.FindStringSubmatch(line)
345			if m == nil {
346				fatalf("%s:%d: invalid import declaration: %q", srcFile, i+1, line)
347			}
348		} else {
349			m = importBlockRE.FindStringSubmatch(line)
350			if m == nil {
351				fatalf("%s:%d: invalid import block line", srcFile, i+1)
352			}
353			if m[2] == "" {
354				continue
355			}
356		}
357
358		path := m[2]
359		if strings.HasPrefix(path, "cmd/") {
360			path = "bootstrap/" + path
361		} else {
362			for _, dir := range bootstrapDirs {
363				if path == dir {
364					path = "bootstrap/" + dir
365					break
366				}
367			}
368		}
369
370		// Rewrite use of internal/reflectlite to be plain reflect.
371		if path == "internal/reflectlite" {
372			lines[i] = strings.ReplaceAll(line, `"reflect"`, `reflectlite "reflect"`)
373			continue
374		}
375
376		// Otherwise, reject direct imports of internal packages,
377		// since that implies knowledge of internal details that might
378		// change from one bootstrap toolchain to the next.
379		// There are many internal packages that are listed in
380		// bootstrapDirs and made into bootstrap copies based on the
381		// current repo's source code. Those are fine; this is catching
382		// references to internal packages in the older bootstrap toolchain.
383		if strings.HasPrefix(path, "internal/") {
384			fatalf("%s:%d: bootstrap-copied source file cannot import %s", srcFile, i+1, path)
385		}
386		if path != m[2] {
387			lines[i] = strings.ReplaceAll(line, `"`+m[2]+`"`, `"`+path+`"`)
388		}
389	}
390
391	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
392
393	return strings.Join(lines, "")
394}
395