1// Copyright 2018 The Bazel Authors. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// Package liteparse does a light parsing of android resources files that can be used at a later 16// stage to generate R.java files. 17package liteparse 18 19import ( 20 "bytes" 21 "context" 22 "flag" 23 "fmt" 24 "io" 25 "io/ioutil" 26 "log" 27 "os" 28 "path" 29 "path/filepath" 30 "strings" 31 "sync" 32 33 "src/common/golang/flags" 34 "src/common/golang/walk" 35 rdpb "src/tools/ak/res/proto/res_data_go_proto" 36 "src/tools/ak/res/res" 37 "src/tools/ak/res/respipe/respipe" 38 "src/tools/ak/res/resxml/resxml" 39 "src/tools/ak/types" 40 "google.golang.org/protobuf/proto" 41) 42 43var ( 44 // Cmd defines the command to run the res parser. 45 Cmd = types.Command{ 46 Init: Init, 47 Run: Run, 48 Desc: desc, 49 Flags: []string{"resourceFiles", "rPbOutput"}, 50 } 51 52 resourceFiles flags.StringList 53 rPbOutput string 54 pkg string 55 56 initOnce sync.Once 57) 58 59const ( 60 numParsers = 25 61) 62 63// Init initializes parse. Flags here need to match flags in AndroidResourceParsingAction. 64func Init() { 65 initOnce.Do(func() { 66 flag.Var(&resourceFiles, "res_files", "Resource files and asset directories to parse.") 67 flag.StringVar(&rPbOutput, "out", "", "Path to the output proto file.") 68 flag.StringVar(&pkg, "pkg", "", "Java package name.") 69 }) 70} 71 72func desc() string { 73 return "Lite parses the resource files to generate an R.pb." 74} 75 76// Run runs the parser. 77func Run() { 78 rscs := ParseAll(context.Background(), resourceFiles, pkg) 79 b, err := proto.Marshal(rscs) 80 if err != nil { 81 log.Fatal(err) 82 } 83 if err = ioutil.WriteFile(rPbOutput, b, 0644); err != nil { 84 log.Fatal(err) 85 } 86} 87 88type resourceFile struct { 89 pathInfo *res.PathInfo 90 contents []byte 91} 92 93// ParseAll parses all the files in resPaths, which can contain both files and directories, 94// and returns pb. 95func ParseAll(ctx context.Context, resPaths []string, packageName string) *rdpb.Resources { 96 resFiles, err := walk.Files(resPaths) 97 if err != nil { 98 log.Fatal(err) 99 } 100 pifs, rscs, err := initializeFileParse(resFiles, packageName) 101 if err != nil { 102 log.Fatal(err) 103 } 104 if len(pifs) == 0 { 105 return rscs 106 } 107 108 piC := make(chan *res.PathInfo, len(pifs)) 109 for _, pi := range pifs { 110 piC <- pi 111 } 112 close(piC) 113 114 ctx, cancel := context.WithCancel(ctx) 115 defer cancel() 116 117 resC, errC := ResParse(ctx, piC) 118 rscs.Resource, err = processResAndErr(resC, errC) 119 if err != nil { 120 cancel() 121 log.Fatal(err) 122 } 123 return rscs 124} 125 126// ResParse consumes a stream of resource paths and converts them into resource protos. These 127// protos will only have the minimal name/type info set. 128func ResParse(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *rdpb.Resource, <-chan error) { 129 parserC := make(chan *res.PathInfo) 130 var parsedResCs []<-chan *rdpb.Resource 131 var parsedErrCs []<-chan error 132 133 for i := 0; i < numParsers; i++ { 134 parsedResC, parsedErrC := xmlParser(ctx, parserC) 135 parsedResCs = append(parsedResCs, parsedResC) 136 parsedErrCs = append(parsedErrCs, parsedErrC) 137 } 138 pathResC := make(chan *rdpb.Resource) 139 pathErrC := make(chan error) 140 go func() { 141 defer close(pathResC) 142 defer close(pathErrC) 143 defer close(parserC) 144 145 for pi := range piC { 146 np, err := needsParse(pi) 147 if err != nil { 148 pathErrC <- err 149 return 150 } else if np { 151 parserC <- pi 152 } 153 if !parsePathInfo(ctx, pi, pathResC, pathErrC) { 154 return 155 } 156 } 157 }() 158 parsedResCs = append(parsedResCs, pathResC) 159 parsedErrCs = append(parsedErrCs, pathErrC) 160 resC := respipe.MergeResStreams(ctx, parsedResCs) 161 errC := respipe.MergeErrStreams(ctx, parsedErrCs) 162 163 return resC, errC 164} 165 166// ParseAllContents parses all resource files with paths and contents and returns pb representing 167// the R class that is generated from the files with the package packageName. 168// paths and contents must have the same length, and a file with paths[i] file path 169// has file contents contents[i]. 170func ParseAllContents(ctx context.Context, paths []string, contents [][]byte, packageName string) (*rdpb.Resources, error) { 171 if len(paths) != len(contents) { 172 return nil, fmt.Errorf("length of paths (%v) and contents (%v) are not equal", len(paths), len(contents)) 173 } 174 pifs, rscs, err := initializeFileParse(paths, packageName) 175 if err != nil { 176 return nil, err 177 } 178 if len(pifs) == 0 { 179 return rscs, nil 180 } 181 182 var rfC []*resourceFile 183 for i, pi := range pifs { 184 rfC = append(rfC, &resourceFile{ 185 pathInfo: pi, 186 contents: contents[i], 187 }) 188 } 189 190 ctx, cancel := context.WithCancel(ctx) 191 defer cancel() 192 193 resC, errC := resParseContents(ctx, rfC) 194 rscs.Resource, err = processResAndErr(resC, errC) 195 if err != nil { 196 return nil, err 197 } 198 return rscs, nil 199} 200 201// resParseContents consumes resource files and converts them into resource protos. 202// These protos will only have the minimal name/type info set. 203// The returned channels will be consumed by processRessAndErr. 204func resParseContents(ctx context.Context, rfC []*resourceFile) (<-chan *rdpb.Resource, <-chan error) { 205 parserC := make(chan *resourceFile) 206 var parsedResCs []<-chan *rdpb.Resource 207 var parsedErrCs []<-chan error 208 209 for i := 0; i < numParsers; i++ { 210 parsedResC, parsedErrC := xmlParserContents(ctx, parserC) 211 parsedResCs = append(parsedResCs, parsedResC) 212 parsedErrCs = append(parsedErrCs, parsedErrC) 213 } 214 pathResC := make(chan *rdpb.Resource) 215 pathErrC := make(chan error) 216 go func() { 217 defer close(pathResC) 218 defer close(pathErrC) 219 defer close(parserC) 220 221 for _, rf := range rfC { 222 if needsParseContents(rf.pathInfo, bytes.NewReader(rf.contents)) { 223 parserC <- rf 224 } 225 if !parsePathInfo(ctx, rf.pathInfo, pathResC, pathErrC) { 226 return 227 } 228 } 229 }() 230 parsedResCs = append(parsedResCs, pathResC) 231 parsedErrCs = append(parsedErrCs, pathErrC) 232 resC := respipe.MergeResStreams(ctx, parsedResCs) 233 errC := respipe.MergeErrStreams(ctx, parsedErrCs) 234 235 return resC, errC 236} 237 238// initializeFileParse returns a slice of all PathInfos of files contained in each file path, 239// which must be a file (not a directory). It also returns Resources with packageName. 240func initializeFileParse(filePaths []string, packageName string) ([]*res.PathInfo, *rdpb.Resources, error) { 241 rscs := &rdpb.Resources{ 242 Pkg: packageName, 243 } 244 245 pifs, err := res.MakePathInfos(filePaths) 246 if err != nil { 247 return nil, nil, err 248 } 249 250 return pifs, rscs, nil 251} 252 253// parsePathInfo attempts to parse the PathInfo and send the provided Resource and error to the 254// provided chan. If the context is canceled, returns false, and otherwise, returns true. 255func parsePathInfo(ctx context.Context, pi *res.PathInfo, pathResC chan<- *rdpb.Resource, pathErrC chan<- error) bool { 256 if rawName, ok := pathAsRes(pi); ok { 257 fqn, err := res.ParseName(rawName, pi.Type) 258 if err != nil { 259 return respipe.SendErr(ctx, pathErrC, respipe.Errorf(ctx, "%s: name parse failed: %v", pi.Path, err)) 260 } 261 r := new(rdpb.Resource) 262 if err := fqn.SetResource(r); err != nil { 263 return respipe.SendErr(ctx, pathErrC, respipe.Errorf(ctx, "%s: name->proto failed: %v", fqn, err)) 264 } 265 return respipe.SendRes(ctx, pathResC, r) 266 } 267 return true 268} 269 270// processResAndErr processes the res and err channels and returns the resources if successful 271// or the first encountered error. 272func processResAndErr(resC <-chan *rdpb.Resource, errC <-chan error) ([]*rdpb.Resource, error) { 273 parseErrChan := make(chan error, 1) 274 go func() { 275 for err := range errC { 276 if err != nil { 277 parseErrChan <- err 278 return 279 } 280 } 281 }() 282 283 doneChan := make(chan struct{}, 1) 284 var res []*rdpb.Resource 285 go func() { 286 for r := range resC { 287 res = append(res, r) 288 } 289 doneChan <- struct{}{} 290 }() 291 292 select { 293 case err := <-parseErrChan: 294 return nil, err 295 case <-doneChan: 296 } 297 298 return res, nil 299} 300 301// xmlParser consumes a stream of paths that need to have their xml contents parsed into resource 302// protos. We only need to get names and types - so the parsing is very quick. 303func xmlParser(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *rdpb.Resource, <-chan error) { 304 resC := make(chan *rdpb.Resource) 305 errC := make(chan error) 306 go func() { 307 defer close(resC) 308 defer close(errC) 309 for p := range piC { 310 if !syncParse(respipe.PrefixErr(ctx, fmt.Sprintf("%s xml-parse: ", p.Path)), p, resC, errC) { 311 // ctx must have been canceled - exit. 312 return 313 } 314 } 315 }() 316 return resC, errC 317} 318 319// xmlParserContents consumes a stream of resource files that need to have their xml contents 320// parsed into resource protos. We only need to get names and types - so the parsing is very quick. 321func xmlParserContents(ctx context.Context, rfC <-chan *resourceFile) (<-chan *rdpb.Resource, <-chan error) { 322 resC := make(chan *rdpb.Resource) 323 errC := make(chan error) 324 go func() { 325 defer close(resC) 326 defer close(errC) 327 for rf := range rfC { 328 if !syncParseContents(respipe.PrefixErr(ctx, fmt.Sprintf("%s xml-parse: ", rf.pathInfo.Path)), rf.pathInfo, bytes.NewReader(rf.contents), resC, errC) { 329 // ctx must have been canceled - exit. 330 return 331 } 332 } 333 }() 334 return resC, errC 335} 336 337func syncParse(ctx context.Context, p *res.PathInfo, resC chan<- *rdpb.Resource, errC chan<- error) bool { 338 f, err := os.Open(p.Path) 339 if err != nil { 340 return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "open failed: %v", err)) 341 } 342 defer f.Close() 343 return syncParseContents(ctx, p, f, resC, errC) 344} 345 346func syncParseContents(ctx context.Context, p *res.PathInfo, fileReader io.Reader, resC chan<- *rdpb.Resource, errC chan<- error) bool { 347 parsedResC, mergedErrC := parseContents(ctx, p, fileReader) 348 for parsedResC != nil || mergedErrC != nil { 349 select { 350 case r, ok := <-parsedResC: 351 if !ok { 352 parsedResC = nil 353 continue 354 } 355 if !respipe.SendRes(ctx, resC, r) { 356 return false 357 } 358 case e, ok := <-mergedErrC: 359 if !ok { 360 mergedErrC = nil 361 continue 362 } 363 if !respipe.SendErr(ctx, errC, e) { 364 return false 365 } 366 } 367 368 } 369 return true 370} 371 372func parseContents(ctx context.Context, filePathInfo *res.PathInfo, fileReader io.Reader) (resC <-chan *rdpb.Resource, errC <-chan error) { 373 xmlC, xmlErrC := resxml.StreamDoc(ctx, fileReader) 374 var parsedErrC <-chan error 375 if filePathInfo.Type == res.ValueType { 376 ctx := respipe.PrefixErr(ctx, "mini-values-parse: ") 377 resC, parsedErrC = valuesParse(ctx, xmlC) 378 } else { 379 ctx := respipe.PrefixErr(ctx, "mini-non-values-parse: ") 380 resC, parsedErrC = nonValuesParse(ctx, xmlC) 381 } 382 errC = respipe.MergeErrStreams(ctx, []<-chan error{parsedErrC, xmlErrC}) 383 return resC, errC 384} 385 386// needsParse determines if a path needs to have a values / nonvalues xml parser run to extract 387// resource information. 388func needsParse(pi *res.PathInfo) (bool, error) { 389 r, err := os.Open(pi.Path) 390 if err != nil { 391 return false, fmt.Errorf("Unable to open file %s: %s", pi.Path, err) 392 } 393 defer r.Close() 394 395 return needsParseContents(pi, r), nil 396} 397 398// needsParseContents determines if a path with the corresponding reader for contents needs to have a 399// values / nonvalues xml parser run to extract resource information. 400func needsParseContents(pi *res.PathInfo, r io.Reader) bool { 401 if pi.Type == res.Raw { 402 return false 403 } 404 if filepath.Ext(pi.Path) == ".xml" { 405 return true 406 } 407 if filepath.Ext(pi.Path) == "" { 408 var header [5]byte 409 _, err := io.ReadFull(r, header[:]) 410 if err != nil && err != io.EOF { 411 log.Fatal("Unable to read file %s: %s", pi.Path, err) 412 } 413 if string(header[:]) == "<?xml" { 414 return true 415 } 416 } 417 return false 418} 419 420// pathAsRes determines if a particular res.PathInfo is also a standalone resource. 421func pathAsRes(pi *res.PathInfo) (string, bool) { 422 if pi.Type.Kind() == res.Value || (pi.Type.Kind() == res.Both && strings.HasPrefix(pi.TypeDir, "values")) { 423 return "", false 424 } 425 p := path.Base(pi.Path) 426 // Only split on last index of dot when the resource is of RAW type. 427 // Some drawable resources (Nine-Patch files) ends with .9.png which should not 428 // be included in the resource name. 429 if dot := strings.LastIndex(p, "."); dot >= 0 && pi.Type == res.Raw { 430 return p[:dot], true 431 } 432 if dot := strings.Index(p, "."); dot >= 0 { 433 return p[:dot], true 434 } 435 return p, true 436} 437