1// Copyright 2020 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7import ( 8 "context" 9 "encoding/json" 10 "flag" 11 "fmt" 12 "io/ioutil" 13 "os" 14 "path/filepath" 15 "strconv" 16 17 "go.skia.org/infra/go/common" 18 "go.skia.org/infra/go/exec" 19 "go.skia.org/infra/go/httputils" 20 "go.skia.org/infra/go/skerr" 21 "go.skia.org/infra/task_driver/go/lib/os_steps" 22 "go.skia.org/infra/task_driver/go/td" 23) 24 25func main() { 26 var ( 27 // Required properties for this task. 28 builtPath = flag.String("built_path", "", "The directory where the built wasm/js code will be.") 29 gitCommit = flag.String("git_commit", "", "The commit at which we are testing.") 30 goldCtlPath = flag.String("gold_ctl_path", "", "Path to the goldctl binary") 31 goldHashesURL = flag.String("gold_hashes_url", "", "URL from which to download pre-existing hashes") 32 goldKeys = common.NewMultiStringFlag("gold_key", nil, "The keys that will tag this data") 33 nodeBinPath = flag.String("node_bin_path", "", "Path to the node bin directory (should have npm also). This directory *must* be on the PATH when this executable is called, otherwise, the wrong node or npm version may be found (e.g. the one on the system), even if we are explicitly calling npm with the absolute path.") 34 projectID = flag.String("project_id", "", "ID of the Google Cloud project.") 35 resourcePath = flag.String("resource_path", "", "The directory housing the images, fonts, and other assets used by tests.") 36 taskID = flag.String("task_id", "", "task id this data was generated on") 37 taskName = flag.String("task_name", "", "Name of the task.") 38 testHarnessPath = flag.String("test_harness_path", "", "Path to test harness folder (tools/run-wasm-gm-tests)") 39 webGLVersion = flag.Int("webgl_version", 2, "The version of web gl to use. 0 means CPU") 40 workPath = flag.String("work_path", "", "The directory to use to store temporary files (e.g. pngs and JSON)") 41 42 // Provided for tryjobs 43 changelistID = flag.String("changelist_id", "", "The id the Gerrit CL. Omit for primary branch.") 44 tryjobID = flag.String("tryjob_id", "", "The id of the Buildbucket job for tryjobs. Omit for primary branch.") 45 // Because we pass in patchset_order via a placeholder, it can be empty string. As such, we 46 // cannot use flag.Int, because that errors on "" being passed in. 47 patchsetOrder = flag.String("patchset_order", "0", "Represents if this is the nth patchset") 48 49 // Debugging flags. 50 local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)") 51 outputSteps = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") 52 serviceAccountPath = flag.String("service_account_path", "", "Used in local mode for authentication. Non-local mode uses Luci config.") 53 ) 54 55 // Setup. 56 ctx := td.StartRun(projectID, taskID, taskName, outputSteps, local) 57 defer td.EndRun(ctx) 58 59 builtAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *builtPath, "built_path") 60 goldctlAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *goldCtlPath, "gold_ctl_path") 61 nodeBinAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *nodeBinPath, "node_bin_path") 62 resourceAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *resourcePath, "resource_path") 63 testHarnessAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *testHarnessPath, "test_harness_path") 64 workAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *workPath, "work_path") 65 66 goldctlWorkPath := filepath.Join(workAbsPath, "goldctl") 67 if err := os_steps.MkdirAll(ctx, goldctlWorkPath); err != nil { 68 td.Fatal(ctx, err) 69 } 70 testsWorkPath := filepath.Join(workAbsPath, "tests") 71 if err := os_steps.MkdirAll(ctx, testsWorkPath); err != nil { 72 td.Fatal(ctx, err) 73 } 74 if *goldHashesURL == "" { 75 td.Fatalf(ctx, "Must supply --gold_hashes_url") 76 } 77 78 patchset := 0 79 if *patchsetOrder != "" { 80 p, err := strconv.Atoi(*patchsetOrder) 81 if err != nil { 82 td.Fatalf(ctx, "Invalid patchset_order %q", *patchsetOrder) 83 } 84 patchset = p 85 } 86 87 keys := *goldKeys 88 switch *webGLVersion { 89 case 0: 90 keys = append(keys, "cpu_or_gpu:CPU") 91 case 1: 92 keys = append(keys, "cpu_or_gpu:GPU", "extra_config:WebGL1") 93 case 2: 94 keys = append(keys, "cpu_or_gpu:GPU", "extra_config:WebGL2") 95 default: 96 td.Fatalf(ctx, "Invalid value for webgl_version, must be 0, 1, 2 got %d", *webGLVersion) 97 } 98 99 // initialize goldctl 100 if err := setupGoldctl(ctx, *local, *gitCommit, *changelistID, *tryjobID, goldctlAbsPath, goldctlWorkPath, 101 *serviceAccountPath, keys, patchset); err != nil { 102 td.Fatal(ctx, err) 103 } 104 105 if err := downloadKnownHashes(ctx, testsWorkPath, *goldHashesURL); err != nil { 106 td.Fatal(ctx, err) 107 } 108 if err := setupTests(ctx, nodeBinAbsPath, testHarnessAbsPath); err != nil { 109 td.Fatal(ctx, skerr.Wrap(err)) 110 } 111 // Run puppeteer tests. The input is a list of known hashes. The output will be a JSON array and 112 // any new images to be written to disk in the testsWorkPath. See WriteToDisk in DM for how that 113 // is done on the C++ side. 114 if err := runTests(ctx, builtAbsPath, nodeBinAbsPath, resourceAbsPath, testHarnessAbsPath, testsWorkPath, *webGLVersion); err != nil { 115 td.Fatal(ctx, err) 116 } 117 118 // Parse JSON and call goldctl imgtest add them. 119 if err := processTestData(ctx, testsWorkPath, goldctlAbsPath, goldctlWorkPath); err != nil { 120 td.Fatal(ctx, err) 121 } 122 123 // call goldctl finalize to upload stuff. 124 if err := finalizeGoldctl(ctx, goldctlAbsPath, goldctlWorkPath); err != nil { 125 td.Fatal(ctx, err) 126 } 127} 128 129func setupGoldctl(ctx context.Context, local bool, gitCommit, gerritCLID, tryjobID, goldctlPath, workPath, serviceAccountPath string, keys []string, psOrder int) error { 130 ctx = td.StartStep(ctx, td.Props("setup goldctl").Infra()) 131 defer td.EndStep(ctx) 132 133 args := []string{goldctlPath, "auth", "--work-dir", workPath} 134 if !local { 135 args = append(args, "--luci") 136 } else { 137 // When testing locally, it can also be handy to add in --dry-run here. 138 args = append(args, "--service-account", serviceAccountPath) 139 } 140 141 if _, err := exec.RunCwd(ctx, workPath, args...); err != nil { 142 return td.FailStep(ctx, skerr.Wrapf(err, "running %s", args)) 143 } 144 145 args = []string{ 146 goldctlPath, "imgtest", "init", "--work-dir", workPath, "--instance", "skia", "--corpus", "gm", 147 "--commit", gitCommit, "--url", "https://gold.skia.org", "--bucket", "skia-infra-gm", 148 } 149 if gerritCLID != "" { 150 ps := strconv.Itoa(psOrder) 151 args = append(args, "--crs", "gerrit", "--changelist", gerritCLID, "--patchset", ps, 152 "--cis", "buildbucket", "--jobid", tryjobID) 153 } 154 155 for _, key := range keys { 156 args = append(args, "--key", key) 157 } 158 159 if _, err := exec.RunCwd(ctx, workPath, args...); err != nil { 160 return td.FailStep(ctx, skerr.Wrapf(err, "running %s", args)) 161 } 162 return nil 163} 164 165// downloadKnownHashes downloads the known hashes from Gold and stores it as a text file in 166// workPath/hashes.txt 167func downloadKnownHashes(ctx context.Context, workPath, knownHashesURL string) error { 168 ctx = td.StartStep(ctx, td.Props("download known hashes").Infra()) 169 defer td.EndStep(ctx) 170 171 client := httputils.DefaultClientConfig().With2xxOnly().Client() 172 resp, err := client.Get(knownHashesURL) 173 if err != nil { 174 return td.FailStep(ctx, skerr.Wrapf(err, "downloading known hashes")) 175 } 176 defer resp.Body.Close() 177 data, err := ioutil.ReadAll(resp.Body) 178 if err != nil { 179 return td.FailStep(ctx, skerr.Wrapf(err, "reading known hashes")) 180 } 181 return os_steps.WriteFile(ctx, filepath.Join(workPath, "hashes.txt"), data, 0666) 182} 183 184func setupTests(ctx context.Context, nodeBinPath string, testHarnessPath string) error { 185 ctx = td.StartStep(ctx, td.Props("setup npm").Infra()) 186 defer td.EndStep(ctx) 187 188 if _, err := exec.RunCwd(ctx, testHarnessPath, filepath.Join(nodeBinPath, "npm"), "ci"); err != nil { 189 return td.FailStep(ctx, skerr.Wrap(err)) 190 } 191 return nil 192} 193 194func runTests(ctx context.Context, builtPath, nodeBinPath, resourcePath, testHarnessPath, workPath string, webglVersion int) error { 195 ctx = td.StartStep(ctx, td.Props("run GMs and unit tests")) 196 defer td.EndStep(ctx) 197 198 err := td.Do(ctx, td.Props("Run GMs and Unit Tests"), func(ctx context.Context) error { 199 args := []string{filepath.Join(nodeBinPath, "node"), 200 "run-wasm-gm-tests", 201 "--js_file", filepath.Join(builtPath, "wasm_gm_tests.js"), 202 "--wasm_file", filepath.Join(builtPath, "wasm_gm_tests.wasm"), 203 "--known_hashes", filepath.Join(workPath, "hashes.txt"), 204 "--use_gpu", // TODO(kjlubick) use webglVersion and account for CPU 205 "--output", workPath, 206 "--resources", resourcePath, 207 "--timeout", "180", // seconds per batch of 50 tests. 208 } 209 210 _, err := exec.RunCwd(ctx, testHarnessPath, args...) 211 if err != nil { 212 return skerr.Wrap(err) 213 } 214 return nil 215 }) 216 if err != nil { 217 return td.FailStep(ctx, skerr.Wrap(err)) 218 } 219 return nil 220} 221 222type goldResult struct { 223 TestName string `json:"name"` 224 MD5Hash string `json:"digest"` 225} 226 227func processTestData(ctx context.Context, testOutputPath, goldctlPath, goldctlWorkPath string) error { 228 ctx = td.StartStep(ctx, td.Props("process test data").Infra()) 229 defer td.EndStep(ctx) 230 231 // Read in the file, process it as []goldResult 232 var results []goldResult 233 resultFile := filepath.Join(testOutputPath, "gold_results.json") 234 235 err := td.Do(ctx, td.Props("Load results from "+resultFile), func(ctx context.Context) error { 236 b, err := os_steps.ReadFile(ctx, resultFile) 237 if err != nil { 238 return skerr.Wrap(err) 239 } 240 if err := json.Unmarshal(b, &results); err != nil { 241 return skerr.Wrap(err) 242 } 243 return nil 244 }) 245 if err != nil { 246 return td.FailStep(ctx, skerr.Wrap(err)) 247 } 248 249 err = td.Do(ctx, td.Props(fmt.Sprintf("Call goldtl on %d results", len(results))), func(ctx context.Context) error { 250 for _, result := range results { 251 // These args are the same regardless of if we need to upload the png file or not. 252 args := []string{goldctlPath, "imgtest", "add", "--work-dir", goldctlWorkPath, 253 "--test-name", result.TestName, "--png-digest", result.MD5Hash} 254 // check to see if there's an image we need to upload 255 potentialPNGFile := filepath.Join(testOutputPath, result.MD5Hash+".png") 256 _, err := os_steps.Stat(ctx, potentialPNGFile) 257 if os.IsNotExist(err) { 258 // PNG was not produced, we assume it is already uploaded to Gold and just say the digest 259 // we produced. 260 _, err = exec.RunCwd(ctx, goldctlWorkPath, args...) 261 if err != nil { 262 return skerr.Wrapf(err, "reporting result %#v to goldctl", result) 263 } 264 continue 265 } else if err != nil { 266 return skerr.Wrapf(err, "reading %s", potentialPNGFile) 267 } 268 // call goldctl with the png file 269 args = append(args, "--png-file", potentialPNGFile) 270 _, err = exec.RunCwd(ctx, goldctlWorkPath, args...) 271 if err != nil { 272 return skerr.Wrapf(err, "reporting result %#v to goldctl", result) 273 } 274 } 275 return nil 276 }) 277 if err != nil { 278 return td.FailStep(ctx, skerr.Wrap(err)) 279 } 280 return nil 281} 282 283func finalizeGoldctl(ctx context.Context, goldctlPath, workPath string) error { 284 ctx = td.StartStep(ctx, td.Props("finalize goldctl data").Infra()) 285 defer td.EndStep(ctx) 286 287 _, err := exec.RunCwd(ctx, workPath, goldctlPath, "imgtest", "finalize", "--work-dir", workPath) 288 if err != nil { 289 return skerr.Wrapf(err, "Finalizing goldctl") 290 } 291 return nil 292} 293