xref: /aosp_15_r20/external/skia/bazel/deps_parser/deps_parser.go (revision c8dee2aa9b3f27cf6c858bd81872bdeb2c07ed17) 
1// Copyright 2022 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5package main
6
7import (
8	"flag"
9	"fmt"
10	"io"
11	"os"
12	"os/exec"
13	"path/filepath"
14	"regexp"
15	"sort"
16	"strings"
17)
18
19type depConfig struct {
20	bazelNameOverride string // Bazel style uses underscores not dashes, so we fix those if needed.
21	needsBazelFile    bool
22	patchCmds         []string
23	patchCmdsWin      []string
24}
25
26// These are all C++ deps or Rust deps (with a compatible C++ FFI) used by the Bazel build.
27// They are a subset of those listed in DEPS.
28// The key is the name of the repo as specified in DEPS.
29var deps = map[string]depConfig{
30	"abseil-cpp":  {bazelNameOverride: "abseil_cpp"},
31	"brotli":      {},
32	"highway":     {},
33	"spirv-tools": {bazelNameOverride: "spirv_tools"},
34	// This name is important because spirv_tools expects @spirv_headers to exist by that name.
35	"spirv-headers": {bazelNameOverride: "spirv_headers"},
36
37	"dawn":           {needsBazelFile: true},
38	"delaunator-cpp": {bazelNameOverride: "delaunator", needsBazelFile: true},
39	"dng_sdk":        {needsBazelFile: true},
40	"expat":          {needsBazelFile: true},
41	"freetype":       {needsBazelFile: true},
42	"harfbuzz":       {needsBazelFile: true},
43	"icu": {
44		needsBazelFile: true,
45		patchCmds: []string{
46			`"rm source/i18n/BUILD.bazel"`,
47			`"rm source/common/BUILD.bazel"`,
48			`"rm source/stubdata/BUILD.bazel"`,
49		},
50		patchCmdsWin: []string{
51			`"del source/i18n/BUILD.bazel"`,
52			`"del source/common/BUILD.bazel"`,
53			`"del source/stubdata/BUILD.bazel"`,
54		},
55	},
56	"icu4x":                    {needsBazelFile: true},
57	"imgui":                    {needsBazelFile: true},
58	"libavif":                  {needsBazelFile: true},
59	"libgav1":                  {needsBazelFile: true},
60	"libjpeg-turbo":            {bazelNameOverride: "libjpeg_turbo", needsBazelFile: true},
61	"libjxl":                   {needsBazelFile: true},
62	"libpng":                   {needsBazelFile: true},
63	"libwebp":                  {needsBazelFile: true},
64	"libyuv":                   {needsBazelFile: true},
65	"spirv-cross":              {bazelNameOverride: "spirv_cross", needsBazelFile: true},
66	"perfetto":                 {needsBazelFile: true},
67	"piex":                     {needsBazelFile: true},
68	"vello":                    {needsBazelFile: true},
69	"vulkan-headers":           {bazelNameOverride: "vulkan_headers", needsBazelFile: true},
70	"vulkan-tools":             {bazelNameOverride: "vulkan_tools", needsBazelFile: true},
71	"vulkan-utility-libraries": {bazelNameOverride: "vulkan_utility_libraries", needsBazelFile: true},
72	"vulkanmemoryallocator":    {needsBazelFile: true},
73	"wuffs":                    {needsBazelFile: true},
74	// Some other dependency downloads zlib but with their own rules
75	"zlib": {bazelNameOverride: "zlib_skia", needsBazelFile: true},
76}
77
78func main() {
79	var (
80		depsFile      = flag.String("deps_file", "DEPS", "The location of the DEPS file. Usually at the root of the repository")
81		genBzlFile    = flag.String("gen_bzl_file", "bazel/deps.bzl", "The location of the .bzl file that has the generated Bazel repository rules.")
82		workspaceFile = flag.String("workspace_file", "WORKSPACE.bazel", "The location of the WORKSPACE file that should be updated with dep names.")
83		// https://bazel.build/docs/user-manual#running-executables
84		repoDir        = flag.String("repo_dir", os.Getenv("BUILD_WORKSPACE_DIRECTORY"), "The root directory of the repo. Default set by BUILD_WORKSPACE_DIRECTORY env variable.")
85		buildifierPath = flag.String("buildifier", "", "Where to find buildifier. Defaults to Bazel's location")
86	)
87	flag.Parse()
88
89	if *repoDir == "" {
90		fmt.Println(`Must set --repo_dir
91This is done automatically via:
92    bazel run //bazel/deps_parser`)
93		os.Exit(1)
94	}
95
96	buildifier := *buildifierPath
97	if buildifier == "" {
98		// We don't know if this will be buildifier_linux_x64, buildifier_macos_arm64, etc
99		bp, err := filepath.Glob("../buildifier*/file/buildifier")
100		if err != nil || len(bp) != 1 {
101			fmt.Printf("Could not find exactly one buildifier executable %s %v\n", err, bp)
102			os.Exit(1)
103		}
104		buildifier = bp[0]
105	}
106	buildifier, err := filepath.Abs(buildifier)
107	if err != nil {
108		fmt.Printf("Abs path error %s\n", err)
109		os.Exit(1)
110	}
111
112	fmt.Println(os.Environ())
113
114	if *depsFile == "" || *genBzlFile == "" {
115		fmt.Println("Must set --deps_file and --gen_bzl_file")
116		flag.PrintDefaults()
117	}
118
119	if err := os.Chdir(*repoDir); err != nil {
120		fmt.Printf("Could not cd to %s\n", *repoDir)
121		os.Exit(1)
122	}
123
124	b, err := os.ReadFile(*depsFile)
125	if err != nil {
126		fmt.Printf("Could not open %s: %s\n", *depsFile, err)
127		os.Exit(1)
128	}
129	contents := strings.Split(string(b), "\n")
130
131	outputFile, count, err := parseDEPSFile(contents, *workspaceFile)
132	if err != nil {
133		fmt.Printf("Parsing error %s\n", err)
134		os.Exit(1)
135	}
136	if err := exec.Command(buildifier, outputFile).Run(); err != nil {
137		fmt.Printf("Buildifier error %s\n", err)
138		os.Exit(1)
139	}
140	if err := moveWithCopyBackup(outputFile, *genBzlFile); err != nil {
141		fmt.Printf("Could not write comments in workspace file: %s\n", err)
142		os.Exit(1)
143	}
144	fmt.Printf("Wrote %d deps\n", count)
145}
146
147func parseDEPSFile(contents []string, workspaceFile string) (string, int, error) {
148	depsLine := regexp.MustCompile(`externals/(\S+)".+"(https.+)@([a-f0-9]+)"`)
149	outputFile, err := os.CreateTemp("", "genbzl")
150	if err != nil {
151		return "", 0, fmt.Errorf("Could not create output file: %s\n", err)
152	}
153	defer outputFile.Close()
154
155	if _, err := outputFile.WriteString(header); err != nil {
156		return "", 0, fmt.Errorf("Could not write header to output file %s: %s\n", outputFile.Name(), err)
157	}
158
159	var nativeRepos []string
160	var providedRepos []string
161
162	count := 0
163	for _, line := range contents {
164		if match := depsLine.FindStringSubmatch(line); len(match) > 0 {
165			id := match[1]
166			repo := match[2]
167			rev := match[3]
168
169			cfg, ok := deps[id]
170			if !ok {
171				continue
172			}
173			if cfg.bazelNameOverride != "" {
174				id = cfg.bazelNameOverride
175			}
176			if cfg.needsBazelFile {
177				if err := writeNewGitRepositoryRule(outputFile, id, repo, rev, cfg.patchCmds, cfg.patchCmdsWin); err != nil {
178					return "", 0, fmt.Errorf("Could not write to output file %s: %s\n", outputFile.Name(), err)
179				}
180				workspaceLine := fmt.Sprintf("# @%s - //bazel/external/%s:BUILD.bazel", id, id)
181				providedRepos = append(providedRepos, workspaceLine)
182			} else {
183				if err := writeGitRepositoryRule(outputFile, id, repo, rev); err != nil {
184					return "", 0, fmt.Errorf("Could not write to output file %s: %s\n", outputFile.Name(), err)
185				}
186				workspaceLine := fmt.Sprintf("# @%s - %s", id, repo)
187				nativeRepos = append(nativeRepos, workspaceLine)
188			}
189			count++
190		}
191	}
192	if count != len(deps) {
193		return "", 0, fmt.Errorf("Not enough deps written. Maybe the deps dictionary needs a bazelNameOverride or an old dep needs to be removed?")
194	}
195
196	if _, err := outputFile.WriteString(footer); err != nil {
197		return "", 0, fmt.Errorf("Could not write footer to output file %s: %s\n", outputFile.Name(), err)
198	}
199
200	if newWorkspaceFile, err := writeCommentsToWorkspace(workspaceFile, nativeRepos, providedRepos); err != nil {
201		fmt.Printf("Could not parse workspace file %s: %s\n", workspaceFile, err)
202		os.Exit(1)
203	} else {
204		if err := moveWithCopyBackup(newWorkspaceFile, workspaceFile); err != nil {
205			fmt.Printf("Could not write comments in workspace file: %s\n", err)
206			os.Exit(1)
207		}
208	}
209	return outputFile.Name(), count, nil
210}
211
212func writeCommentsToWorkspace(workspaceFile string, nativeRepos, providedRepos []string) (string, error) {
213	b, err := os.ReadFile(workspaceFile)
214	if err != nil {
215		return "", fmt.Errorf("Could not open %s: %s\n", workspaceFile, err)
216	}
217	newWorkspace, err := os.CreateTemp("", "workspace")
218	if err != nil {
219		return "", fmt.Errorf("Could not make tempfile: %s\n", err)
220	}
221	defer newWorkspace.Close()
222
223	workspaceContents := strings.Split(string(b), "\n")
224
225	sort.Strings(nativeRepos)
226	sort.Strings(providedRepos)
227	for _, line := range workspaceContents {
228		if _, err := newWorkspace.WriteString(line + "\n"); err != nil {
229			return "", err
230		}
231		if line == startListString {
232			break
233		}
234	}
235	for _, repoLine := range nativeRepos {
236		if _, err := newWorkspace.WriteString(repoLine + "\n"); err != nil {
237			return "", err
238		}
239	}
240	if _, err := newWorkspace.WriteString("#\n"); err != nil {
241		return "", err
242	}
243	for _, repoLine := range providedRepos {
244		if _, err := newWorkspace.WriteString(repoLine + "\n"); err != nil {
245			return "", err
246		}
247	}
248	if _, err := newWorkspace.WriteString(endListString + "\n"); err != nil {
249		return "", err
250	}
251
252	pastEnd := false
253	// Skip the last line, which is blank. We don't want to end with two empty newlines.
254	for _, line := range workspaceContents[:len(workspaceContents)-1] {
255		if line == endListString {
256			pastEnd = true
257			continue
258		}
259		if !pastEnd {
260			continue
261		}
262		if _, err := newWorkspace.WriteString(line + "\n"); err != nil {
263			return "", err
264		}
265	}
266
267	return newWorkspace.Name(), nil
268}
269
270const (
271	startListString = `#### START GENERATED LIST OF THIRD_PARTY DEPS`
272	endListString   = `#### END GENERATED LIST OF THIRD_PARTY DEPS`
273)
274
275const header = `"""
276This file is auto-generated from //bazel/deps_parser
277DO NOT MODIFY BY HAND.
278Instead, do:
279    bazel run //bazel/deps_parser
280"""
281
282load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
283load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
284load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
285load("//bazel:download_config_files.bzl", "download_config_files")
286load("//bazel:gcs_mirror.bzl", "gcs_mirror_url")
287
288def c_plus_plus_deps(ws = "@skia"):
289    """A list of native Bazel git rules to download third party git repositories
290
291       These are in the order they appear in //DEPS.
292        https://bazel.build/rules/lib/repo/git
293
294    Args:
295      ws: The name of the Skia Bazel workspace. The default, "@", may be when used from within the
296          Skia workspace.
297    """`
298
299// If necessary, we can make a new map for bazel deps
300const footer = `
301def bazel_deps():
302    maybe(
303        http_archive,
304        name = "bazel_skylib",
305        sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d",
306        urls = gcs_mirror_url(
307            sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d",
308            url = "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz",
309        ),
310    )
311
312    maybe(
313        http_archive,
314        name = "bazel_toolchains",
315        sha256 = "e52789d4e89c3e2dc0e3446a9684626a626b6bec3fde787d70bae37c6ebcc47f",
316        strip_prefix = "bazel-toolchains-5.1.1",
317        urls = gcs_mirror_url(
318            sha256 = "e52789d4e89c3e2dc0e3446a9684626a626b6bec3fde787d70bae37c6ebcc47f",
319            url = "https://github.com/bazelbuild/bazel-toolchains/archive/refs/tags/v5.1.1.tar.gz",
320        ),
321    )
322
323def header_based_configs():
324    skia_revision = "d211141c45c9171437fa8e6e07989edb5bffa17a"
325    maybe(
326        download_config_files,
327        name = "expat_config",
328        skia_revision = skia_revision,
329        files = {
330            "BUILD.bazel": "third_party/expat/include/BUILD.bazel",
331            "expat_config/expat_config.h": "third_party/expat/include/expat_config/expat_config.h",
332        },
333    )
334    maybe(
335        download_config_files,
336        name = "freetype_config",
337        skia_revision = skia_revision,
338        files = {
339            "BUILD.bazel": "third_party/freetype2/include/BUILD.bazel",
340            "freetype-android/freetype/config/ftmodule.h": "third_party/freetype2/include/freetype-android/freetype/config/ftmodule.h",
341            "freetype-android/freetype/config/ftoption.h": "third_party/freetype2/include/freetype-android/freetype/config/ftoption.h",
342            "freetype-no-type1/freetype/config/ftmodule.h": "third_party/freetype2/include/freetype-no-type1/freetype/config/ftmodule.h",
343            "freetype-no-type1/freetype/config/ftoption.h": "third_party/freetype2/include/freetype-no-type1/freetype/config/ftoption.h",
344        },
345    )
346    maybe(
347        download_config_files,
348        name = "harfbuzz_config",
349        skia_revision = skia_revision,
350        files = {
351            "BUILD.bazel": "third_party/harfbuzz/BUILD.bazel",
352            "config-override.h": "third_party/harfbuzz/config-override.h",
353        },
354    )
355    maybe(
356        download_config_files,
357        name = "icu_utils",
358        skia_revision = skia_revision,
359        files = {
360            "BUILD.bazel": "third_party/icu/BUILD.bazel",
361            "SkLoadICU.cpp": "third_party/icu/SkLoadICU.cpp",
362            "SkLoadICU.h": "third_party/icu/SkLoadICU.h",
363            "make_data_cpp.py": "third_party/icu/make_data_cpp.py",
364        },
365    )
366`
367
368func writeNewGitRepositoryRule(w io.StringWriter, bazelName, repo, rev string, patchCmds, patchCmdsWin []string) error {
369	if len(patchCmds) == 0 {
370		// TODO(kjlubick) In a newer version of Bazel, new_git_repository can be replaced with just
371		// git_repository
372		_, err := w.WriteString(fmt.Sprintf(`
373    new_git_repository(
374        name = "%s",
375        build_file = ws + "//bazel/external/%s:BUILD.bazel",
376        commit = "%s",
377        remote = "%s",
378    )
379`, bazelName, bazelName, rev, repo))
380		return err
381	}
382	patches := "[" + strings.Join(patchCmds, ",\n") + "]"
383	patches_win := "[" + strings.Join(patchCmdsWin, ",\n") + "]"
384	_, err := w.WriteString(fmt.Sprintf(`
385    new_git_repository(
386        name = "%s",
387        build_file = ws + "//bazel/external/%s:BUILD.bazel",
388        commit = "%s",
389        remote = "%s",
390        patch_cmds = %s,
391		patch_cmds_win = %s,
392    )
393`, bazelName, bazelName, rev, repo, patches, patches_win))
394	return err
395}
396
397func writeGitRepositoryRule(w io.StringWriter, bazelName, repo, rev string) error {
398	_, err := w.WriteString(fmt.Sprintf(`
399    git_repository(
400        name = "%s",
401        commit = "%s",
402        remote = "%s",
403    )
404`, bazelName, rev, repo))
405	return err
406}
407
408func moveWithCopyBackup(src, dst string) error {
409	// Atomically rename temp file to workspace. This should minimize the chance of corruption
410	// or writing a partial file if there is an error or the program is interrupted.
411	if err := os.Rename(src, dst); err != nil {
412		// Errors can happen if the temporary file is on a different partition than the Skia
413		// codebase. In that case, do a manual read/write to copy the data. See
414		// https://github.com/jenkins-x/jx/issues/449 for a similar issue
415		if strings.Contains(err.Error(), "invalid cross-device link") {
416			bytes, err := os.ReadFile(src)
417			if err != nil {
418				return fmt.Errorf("Could not do backup read from %s: %s\n", src, err)
419			}
420			if err := os.WriteFile(dst, bytes, 0644); err != nil {
421				return fmt.Errorf("Could not do backup write of %d bytes to %s: %s\n", len(bytes), dst, err)
422			}
423			// Backup "move" successful
424			return nil
425		}
426		return fmt.Errorf("Could not write %s -> %s: %s\n", src, dst, err)
427	}
428	return nil
429}
430