1// Copyright 2022 The Android Open Source Project 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package local 16 17import ( 18 "bufio" 19 "bytes" 20 "context" 21 "errors" 22 "fmt" 23 "io" 24 "io/ioutil" 25 "os/exec" 26 "strings" 27 "time" 28 29 "tools/treble/build/report/app" 30) 31 32// Performance degrades running multiple CLIs 33const ( 34 MaxNinjaCliWorkers = 4 35 DefaultNinjaTimeout = "100s" 36 DefaultNinjaBuildTimeout = "30m" 37) 38 39// Separate out the executable to allow tests to override the results 40type ninjaExec interface { 41 Command(ctx context.Context, target string) (*bytes.Buffer, error) 42 Input(ctx context.Context, target string) (*bytes.Buffer, error) 43 Query(ctx context.Context, target string) (*bytes.Buffer, error) 44 Path(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) 45 Paths(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) 46 Deps(ctx context.Context) (*bytes.Buffer, error) 47 Build(ctx context.Context, target string) (*bytes.Buffer, error) 48} 49 50// Parse data 51 52// Add all lines to a given array removing any leading whitespace 53func linesToArray(s *bufio.Scanner, arr *[]string) { 54 for s.Scan() { 55 line := strings.TrimSpace(s.Text()) 56 *arr = append(*arr, line) 57 } 58} 59 60// parse -t commands 61func parseCommand(target string, data *bytes.Buffer) (*app.BuildCommand, error) { 62 out := &app.BuildCommand{Target: target, Cmds: []string{}} 63 s := bufio.NewScanner(data) 64 // This tool returns all the commands needed to build a target. 65 // When running against a target like droid the default capacity 66 // will be overrun. Extend the capacity here. 67 const capacity = 1024 * 1024 68 buf := make([]byte, capacity) 69 s.Buffer(buf, capacity) 70 linesToArray(s, &out.Cmds) 71 return out, nil 72} 73 74// parse -t inputs 75func parseInput(target string, data *bytes.Buffer) (*app.BuildInput, error) { 76 out := &app.BuildInput{Target: target, Files: []string{}} 77 s := bufio.NewScanner(data) 78 linesToArray(s, &out.Files) 79 return out, nil 80} 81 82// parse -t query 83func parseQuery(target string, data *bytes.Buffer) (*app.BuildQuery, error) { 84 out := &app.BuildQuery{Target: target, Inputs: []string{}, Outputs: []string{}} 85 const ( 86 unknown = iota 87 inputs 88 outputs 89 ) 90 state := unknown 91 s := bufio.NewScanner(data) 92 for s.Scan() { 93 line := strings.TrimSpace(s.Text()) 94 if strings.HasPrefix(line, "input:") { 95 state = inputs 96 } else if strings.HasPrefix(line, "outputs:") { 97 state = outputs 98 } else { 99 switch state { 100 case inputs: 101 out.Inputs = append(out.Inputs, line) 102 case outputs: 103 out.Outputs = append(out.Outputs, line) 104 } 105 } 106 } 107 return out, nil 108} 109 110// parse -t path 111func parsePath(target string, dependency string, data *bytes.Buffer) (*app.BuildPath, error) { 112 out := &app.BuildPath{Target: target, Dependency: dependency, Paths: []string{}} 113 s := bufio.NewScanner(data) 114 linesToArray(s, &out.Paths) 115 return out, nil 116} 117 118// parse -t paths 119func parsePaths(target string, dependency string, data *bytes.Buffer) ([]*app.BuildPath, error) { 120 out := []*app.BuildPath{} 121 s := bufio.NewScanner(data) 122 for s.Scan() { 123 path := strings.Fields(s.Text()) 124 out = append(out, &app.BuildPath{Target: target, Dependency: dependency, Paths: path}) 125 } 126 return out, nil 127} 128 129// parse build output 130func parseBuild(target string, data *bytes.Buffer, success bool) *app.BuildCmdResult { 131 out := &app.BuildCmdResult{Name: target, Output: []string{}} 132 s := bufio.NewScanner(data) 133 out.Success = success 134 linesToArray(s, &out.Output) 135 return out 136} 137 138// parse deps command 139func parseDeps(data *bytes.Buffer) (*app.BuildDeps, error) { 140 out := &app.BuildDeps{Targets: make(map[string][]string)} 141 s := bufio.NewScanner(data) 142 curTarget := "" 143 var deps []string 144 for s.Scan() { 145 line := strings.TrimSpace(s.Text()) 146 // Check if it's a new target 147 tokens := strings.Split(line, ":") 148 if len(tokens) > 1 { 149 if curTarget != "" { 150 out.Targets[curTarget] = deps 151 } 152 deps = []string{} 153 curTarget = tokens[0] 154 } else if line != "" { 155 deps = append(deps, line) 156 } 157 158 } 159 if curTarget != "" { 160 out.Targets[curTarget] = deps 161 } 162 return out, nil 163} 164 165// 166// Command line interface to ninja binary. 167// 168// This file implements the ninja.Ninja interface by querying 169// the build graph via the ninja binary. The mapping between 170// the interface and the binary are as follows: 171// Command() -t commands 172// Input() -t inputs 173// Query() -t query 174// Path() -t path 175// Paths() -t paths 176// Deps() -t deps 177// 178// 179 180type ninjaCmd struct { 181 cmd string 182 db string 183 184 clientMode bool 185 timeout time.Duration 186 buildTimeout time.Duration 187} 188 189func (n *ninjaCmd) runTool(ctx context.Context, tool string, targets []string) (out *bytes.Buffer, err error) { 190 191 args := []string{"-f", n.db} 192 193 if n.clientMode { 194 args = append(args, []string{ 195 "-t", "client", 196 "-c", tool}...) 197 } else { 198 args = append(args, []string{"-t", tool}...) 199 } 200 args = append(args, targets...) 201 data := []byte{} 202 err, _ = runPipe(ctx, n.timeout, n.cmd, args, func(r io.Reader) { 203 data, _ = ioutil.ReadAll(r) 204 }) 205 return bytes.NewBuffer(data), err 206 207} 208func (n *ninjaCmd) Command(ctx context.Context, target string) (*bytes.Buffer, error) { 209 return n.runTool(ctx, "commands", []string{target}) 210} 211func (n *ninjaCmd) Input(ctx context.Context, target string) (*bytes.Buffer, error) { 212 return n.runTool(ctx, "inputs", []string{target}) 213} 214func (n *ninjaCmd) Query(ctx context.Context, target string) (*bytes.Buffer, error) { 215 return n.runTool(ctx, "query", []string{target}) 216} 217func (n *ninjaCmd) Path(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) { 218 return n.runTool(ctx, "path", []string{target, dependency}) 219} 220func (n *ninjaCmd) Paths(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) { 221 return n.runTool(ctx, "paths", []string{target, dependency}) 222} 223func (n *ninjaCmd) Deps(ctx context.Context) (*bytes.Buffer, error) { 224 return n.runTool(ctx, "deps", []string{}) 225} 226 227func (n *ninjaCmd) Build(ctx context.Context, target string) (*bytes.Buffer, error) { 228 229 args := append([]string{ 230 "-f", n.db, 231 target}) 232 data := []byte{} 233 err, _ := runPipe(ctx, n.buildTimeout, n.cmd, args, func(r io.Reader) { 234 data, _ = ioutil.ReadAll(r) 235 }) 236 237 return bytes.NewBuffer(data), err 238} 239 240// Command line ninja 241type ninjaCli struct { 242 n ninjaExec 243} 244 245// ninja -t commands 246func (cli *ninjaCli) Command(ctx context.Context, target string) (*app.BuildCommand, error) { 247 raw, err := cli.n.Command(ctx, target) 248 if err != nil { 249 return nil, err 250 } 251 return parseCommand(target, raw) 252} 253 254// ninja -t inputs 255func (cli *ninjaCli) Input(ctx context.Context, target string) (*app.BuildInput, error) { 256 raw, err := cli.n.Input(ctx, target) 257 if err != nil { 258 return nil, err 259 } 260 return parseInput(target, raw) 261} 262 263// ninja -t query 264func (cli *ninjaCli) Query(ctx context.Context, target string) (*app.BuildQuery, error) { 265 raw, err := cli.n.Query(ctx, target) 266 if err != nil { 267 return nil, err 268 } 269 return parseQuery(target, raw) 270} 271 272// ninja -t path 273func (cli *ninjaCli) Path(ctx context.Context, target string, dependency string) (*app.BuildPath, error) { 274 raw, err := cli.n.Path(ctx, target, dependency) 275 if err != nil { 276 return nil, err 277 } 278 return parsePath(target, dependency, raw) 279} 280 281// ninja -t paths 282func (cli *ninjaCli) Paths(ctx context.Context, target string, dependency string) ([]*app.BuildPath, error) { 283 raw, err := cli.n.Paths(ctx, target, dependency) 284 if err != nil { 285 return nil, err 286 } 287 return parsePaths(target, dependency, raw) 288} 289 290// ninja -t deps 291func (cli *ninjaCli) Deps(ctx context.Context) (*app.BuildDeps, error) { 292 raw, err := cli.n.Deps(ctx) 293 if err != nil { 294 return nil, err 295 } 296 return parseDeps(raw) 297} 298 299// Build given target 300func (cli *ninjaCli) Build(ctx context.Context, target string) *app.BuildCmdResult { 301 raw, err := cli.n.Build(ctx, target) 302 return parseBuild(target, raw, err == nil) 303 304} 305 306// Wait for server 307func (cli *ninjaCli) WaitForServer(ctx context.Context, maxTries int) error { 308 // Wait for server to response to an empty input request 309 fmt.Printf("Waiting for server.") 310 for i := 0; i < maxTries; i++ { 311 _, err := cli.Input(ctx, "") 312 if err == nil { 313 fmt.Printf("\nConnected\n") 314 return nil 315 } 316 fmt.Printf(".") 317 time.Sleep(time.Second) 318 } 319 fmt.Printf(" failed\n") 320 return errors.New("Failed to connect") 321} 322func NewNinjaCli(cmd string, db string, timeout, buildTimeout time.Duration, client bool) *ninjaCli { 323 cli := &ninjaCli{n: &ninjaCmd{cmd: cmd, db: db, timeout: timeout, buildTimeout: buildTimeout, clientMode: client}} 324 return cli 325} 326 327type ninjaServer struct { 328 cmdName string 329 db string 330 ctx *exec.Cmd 331} 332 333// Run server 334func (srv *ninjaServer) Start(ctx context.Context) error { 335 args := []string{"-f", srv.db, "-t", "server"} 336 srv.ctx = exec.CommandContext(ctx, srv.cmdName, args[0:]...) 337 err := srv.ctx.Start() 338 if err != nil { 339 return err 340 } 341 srv.ctx.Wait() 342 return nil 343} 344func (srv *ninjaServer) Kill() { 345 if srv.ctx != nil { 346 srv.ctx.Process.Kill() 347 } 348} 349func NewNinjaServer(cmd string, db string) *ninjaServer { 350 return &ninjaServer{cmdName: cmd, db: db} 351} 352