xref: /aosp_15_r20/external/bazelbuild-rules_android/src/tools/ak/liteparse/liteparse.go (revision 9e965d6fece27a77de5377433c2f7e6999b8cc0b)
1// Copyright 2018 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//    http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Package liteparse does a light parsing of android resources files that can be used at a later
16// stage to generate R.java files.
17package liteparse
18
19import (
20	"bytes"
21	"context"
22	"flag"
23	"fmt"
24	"io"
25	"io/ioutil"
26	"log"
27	"os"
28	"path"
29	"path/filepath"
30	"strings"
31	"sync"
32
33	"src/common/golang/flags"
34	"src/common/golang/walk"
35	rdpb "src/tools/ak/res/proto/res_data_go_proto"
36	"src/tools/ak/res/res"
37	"src/tools/ak/res/respipe/respipe"
38	"src/tools/ak/res/resxml/resxml"
39	"src/tools/ak/types"
40	"google.golang.org/protobuf/proto"
41)
42
43var (
44	// Cmd defines the command to run the res parser.
45	Cmd = types.Command{
46		Init:  Init,
47		Run:   Run,
48		Desc:  desc,
49		Flags: []string{"resourceFiles", "rPbOutput"},
50	}
51
52	resourceFiles flags.StringList
53	rPbOutput     string
54	pkg           string
55
56	initOnce sync.Once
57)
58
59const (
60	numParsers = 25
61)
62
63// Init initializes parse. Flags here need to match flags in AndroidResourceParsingAction.
64func Init() {
65	initOnce.Do(func() {
66		flag.Var(&resourceFiles, "res_files", "Resource files and asset directories to parse.")
67		flag.StringVar(&rPbOutput, "out", "", "Path to the output proto file.")
68		flag.StringVar(&pkg, "pkg", "", "Java package name.")
69	})
70}
71
72func desc() string {
73	return "Lite parses the resource files to generate an R.pb."
74}
75
76// Run runs the parser.
77func Run() {
78	rscs := ParseAll(context.Background(), resourceFiles, pkg)
79	b, err := proto.Marshal(rscs)
80	if err != nil {
81		log.Fatal(err)
82	}
83	if err = ioutil.WriteFile(rPbOutput, b, 0644); err != nil {
84		log.Fatal(err)
85	}
86}
87
88type resourceFile struct {
89	pathInfo *res.PathInfo
90	contents []byte
91}
92
93// ParseAll parses all the files in resPaths, which can contain both files and directories,
94// and returns pb.
95func ParseAll(ctx context.Context, resPaths []string, packageName string) *rdpb.Resources {
96	resFiles, err := walk.Files(resPaths)
97	if err != nil {
98		log.Fatal(err)
99	}
100	pifs, rscs, err := initializeFileParse(resFiles, packageName)
101	if err != nil {
102		log.Fatal(err)
103	}
104	if len(pifs) == 0 {
105		return rscs
106	}
107
108	piC := make(chan *res.PathInfo, len(pifs))
109	for _, pi := range pifs {
110		piC <- pi
111	}
112	close(piC)
113
114	ctx, cancel := context.WithCancel(ctx)
115	defer cancel()
116
117	resC, errC := ResParse(ctx, piC)
118	rscs.Resource, err = processResAndErr(resC, errC)
119	if err != nil {
120		cancel()
121		log.Fatal(err)
122	}
123	return rscs
124}
125
126// ResParse consumes a stream of resource paths and converts them into resource protos. These
127// protos will only have the minimal name/type info set.
128func ResParse(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *rdpb.Resource, <-chan error) {
129	parserC := make(chan *res.PathInfo)
130	var parsedResCs []<-chan *rdpb.Resource
131	var parsedErrCs []<-chan error
132
133	for i := 0; i < numParsers; i++ {
134		parsedResC, parsedErrC := xmlParser(ctx, parserC)
135		parsedResCs = append(parsedResCs, parsedResC)
136		parsedErrCs = append(parsedErrCs, parsedErrC)
137	}
138	pathResC := make(chan *rdpb.Resource)
139	pathErrC := make(chan error)
140	go func() {
141		defer close(pathResC)
142		defer close(pathErrC)
143		defer close(parserC)
144
145		for pi := range piC {
146			np, err := needsParse(pi)
147			if err != nil {
148				pathErrC <- err
149				return
150			} else if np {
151				parserC <- pi
152			}
153			if !parsePathInfo(ctx, pi, pathResC, pathErrC) {
154				return
155			}
156		}
157	}()
158	parsedResCs = append(parsedResCs, pathResC)
159	parsedErrCs = append(parsedErrCs, pathErrC)
160	resC := respipe.MergeResStreams(ctx, parsedResCs)
161	errC := respipe.MergeErrStreams(ctx, parsedErrCs)
162
163	return resC, errC
164}
165
166// ParseAllContents parses all resource files with paths and contents and returns pb representing
167// the R class that is generated from the files with the package packageName.
168// paths and contents must have the same length, and a file with paths[i] file path
169// has file contents contents[i].
170func ParseAllContents(ctx context.Context, paths []string, contents [][]byte, packageName string) (*rdpb.Resources, error) {
171	if len(paths) != len(contents) {
172		return nil, fmt.Errorf("length of paths (%v) and contents (%v) are not equal", len(paths), len(contents))
173	}
174	pifs, rscs, err := initializeFileParse(paths, packageName)
175	if err != nil {
176		return nil, err
177	}
178	if len(pifs) == 0 {
179		return rscs, nil
180	}
181
182	var rfC []*resourceFile
183	for i, pi := range pifs {
184		rfC = append(rfC, &resourceFile{
185			pathInfo: pi,
186			contents: contents[i],
187		})
188	}
189
190	ctx, cancel := context.WithCancel(ctx)
191	defer cancel()
192
193	resC, errC := resParseContents(ctx, rfC)
194	rscs.Resource, err = processResAndErr(resC, errC)
195	if err != nil {
196		return nil, err
197	}
198	return rscs, nil
199}
200
201// resParseContents consumes resource files and converts them into resource protos.
202// These protos will only have the minimal name/type info set.
203// The returned channels will be consumed by processRessAndErr.
204func resParseContents(ctx context.Context, rfC []*resourceFile) (<-chan *rdpb.Resource, <-chan error) {
205	parserC := make(chan *resourceFile)
206	var parsedResCs []<-chan *rdpb.Resource
207	var parsedErrCs []<-chan error
208
209	for i := 0; i < numParsers; i++ {
210		parsedResC, parsedErrC := xmlParserContents(ctx, parserC)
211		parsedResCs = append(parsedResCs, parsedResC)
212		parsedErrCs = append(parsedErrCs, parsedErrC)
213	}
214	pathResC := make(chan *rdpb.Resource)
215	pathErrC := make(chan error)
216	go func() {
217		defer close(pathResC)
218		defer close(pathErrC)
219		defer close(parserC)
220
221		for _, rf := range rfC {
222			if needsParseContents(rf.pathInfo, bytes.NewReader(rf.contents)) {
223				parserC <- rf
224			}
225			if !parsePathInfo(ctx, rf.pathInfo, pathResC, pathErrC) {
226				return
227			}
228		}
229	}()
230	parsedResCs = append(parsedResCs, pathResC)
231	parsedErrCs = append(parsedErrCs, pathErrC)
232	resC := respipe.MergeResStreams(ctx, parsedResCs)
233	errC := respipe.MergeErrStreams(ctx, parsedErrCs)
234
235	return resC, errC
236}
237
238// initializeFileParse returns a slice of all PathInfos of files contained in each file path,
239// which must be a file (not a directory). It also returns Resources with packageName.
240func initializeFileParse(filePaths []string, packageName string) ([]*res.PathInfo, *rdpb.Resources, error) {
241	rscs := &rdpb.Resources{
242		Pkg: packageName,
243	}
244
245	pifs, err := res.MakePathInfos(filePaths)
246	if err != nil {
247		return nil, nil, err
248	}
249
250	return pifs, rscs, nil
251}
252
253// parsePathInfo attempts to parse the PathInfo and send the provided Resource and error to the
254// provided chan. If the context is canceled, returns false, and otherwise, returns true.
255func parsePathInfo(ctx context.Context, pi *res.PathInfo, pathResC chan<- *rdpb.Resource, pathErrC chan<- error) bool {
256	if rawName, ok := pathAsRes(pi); ok {
257		fqn, err := res.ParseName(rawName, pi.Type)
258		if err != nil {
259			return respipe.SendErr(ctx, pathErrC, respipe.Errorf(ctx, "%s: name parse failed: %v", pi.Path, err))
260		}
261		r := new(rdpb.Resource)
262		if err := fqn.SetResource(r); err != nil {
263			return respipe.SendErr(ctx, pathErrC, respipe.Errorf(ctx, "%s: name->proto failed: %v", fqn, err))
264		}
265		return respipe.SendRes(ctx, pathResC, r)
266	}
267	return true
268}
269
270// processResAndErr processes the res and err channels and returns the resources if successful
271// or the first encountered error.
272func processResAndErr(resC <-chan *rdpb.Resource, errC <-chan error) ([]*rdpb.Resource, error) {
273	parseErrChan := make(chan error, 1)
274	go func() {
275		for err := range errC {
276			if err != nil {
277				parseErrChan <- err
278				return
279			}
280		}
281	}()
282
283	doneChan := make(chan struct{}, 1)
284	var res []*rdpb.Resource
285	go func() {
286		for r := range resC {
287			res = append(res, r)
288		}
289		doneChan <- struct{}{}
290	}()
291
292	select {
293	case err := <-parseErrChan:
294		return nil, err
295	case <-doneChan:
296	}
297
298	return res, nil
299}
300
301// xmlParser consumes a stream of paths that need to have their xml contents parsed into resource
302// protos. We only need to get names and types - so the parsing is very quick.
303func xmlParser(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *rdpb.Resource, <-chan error) {
304	resC := make(chan *rdpb.Resource)
305	errC := make(chan error)
306	go func() {
307		defer close(resC)
308		defer close(errC)
309		for p := range piC {
310			if !syncParse(respipe.PrefixErr(ctx, fmt.Sprintf("%s xml-parse: ", p.Path)), p, resC, errC) {
311				// ctx must have been canceled - exit.
312				return
313			}
314		}
315	}()
316	return resC, errC
317}
318
319// xmlParserContents consumes a stream of resource files that need to have their xml contents
320// parsed into resource protos. We only need to get names and types - so the parsing is very quick.
321func xmlParserContents(ctx context.Context, rfC <-chan *resourceFile) (<-chan *rdpb.Resource, <-chan error) {
322	resC := make(chan *rdpb.Resource)
323	errC := make(chan error)
324	go func() {
325		defer close(resC)
326		defer close(errC)
327		for rf := range rfC {
328			if !syncParseContents(respipe.PrefixErr(ctx, fmt.Sprintf("%s xml-parse: ", rf.pathInfo.Path)), rf.pathInfo, bytes.NewReader(rf.contents), resC, errC) {
329				// ctx must have been canceled - exit.
330				return
331			}
332		}
333	}()
334	return resC, errC
335}
336
337func syncParse(ctx context.Context, p *res.PathInfo, resC chan<- *rdpb.Resource, errC chan<- error) bool {
338	f, err := os.Open(p.Path)
339	if err != nil {
340		return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "open failed: %v", err))
341	}
342	defer f.Close()
343	return syncParseContents(ctx, p, f, resC, errC)
344}
345
346func syncParseContents(ctx context.Context, p *res.PathInfo, fileReader io.Reader, resC chan<- *rdpb.Resource, errC chan<- error) bool {
347	parsedResC, mergedErrC := parseContents(ctx, p, fileReader)
348	for parsedResC != nil || mergedErrC != nil {
349		select {
350		case r, ok := <-parsedResC:
351			if !ok {
352				parsedResC = nil
353				continue
354			}
355			if !respipe.SendRes(ctx, resC, r) {
356				return false
357			}
358		case e, ok := <-mergedErrC:
359			if !ok {
360				mergedErrC = nil
361				continue
362			}
363			if !respipe.SendErr(ctx, errC, e) {
364				return false
365			}
366		}
367
368	}
369	return true
370}
371
372func parseContents(ctx context.Context, filePathInfo *res.PathInfo, fileReader io.Reader) (resC <-chan *rdpb.Resource, errC <-chan error) {
373	xmlC, xmlErrC := resxml.StreamDoc(ctx, fileReader)
374	var parsedErrC <-chan error
375	if filePathInfo.Type == res.ValueType {
376		ctx := respipe.PrefixErr(ctx, "mini-values-parse: ")
377		resC, parsedErrC = valuesParse(ctx, xmlC)
378	} else {
379		ctx := respipe.PrefixErr(ctx, "mini-non-values-parse: ")
380		resC, parsedErrC = nonValuesParse(ctx, xmlC)
381	}
382	errC = respipe.MergeErrStreams(ctx, []<-chan error{parsedErrC, xmlErrC})
383	return resC, errC
384}
385
386// needsParse determines if a path needs to have a values / nonvalues xml parser run to extract
387// resource information.
388func needsParse(pi *res.PathInfo) (bool, error) {
389	r, err := os.Open(pi.Path)
390	if err != nil {
391		return false, fmt.Errorf("Unable to open file %s: %s", pi.Path, err)
392	}
393	defer r.Close()
394
395	return needsParseContents(pi, r), nil
396}
397
398// needsParseContents determines if a path with the corresponding reader for contents needs to have a
399// values / nonvalues xml parser run to extract resource information.
400func needsParseContents(pi *res.PathInfo, r io.Reader) bool {
401	if pi.Type == res.Raw {
402		return false
403	}
404	if filepath.Ext(pi.Path) == ".xml" {
405		return true
406	}
407	if filepath.Ext(pi.Path) == "" {
408		var header [5]byte
409		_, err := io.ReadFull(r, header[:])
410		if err != nil && err != io.EOF {
411			log.Fatal("Unable to read file %s: %s", pi.Path, err)
412		}
413		if string(header[:]) == "<?xml" {
414			return true
415		}
416	}
417	return false
418}
419
420// pathAsRes determines if a particular res.PathInfo is also a standalone resource.
421func pathAsRes(pi *res.PathInfo) (string, bool) {
422	if pi.Type.Kind() == res.Value || (pi.Type.Kind() == res.Both && strings.HasPrefix(pi.TypeDir, "values")) {
423		return "", false
424	}
425	p := path.Base(pi.Path)
426	// Only split on last index of dot when the resource is of RAW type.
427	// Some drawable resources (Nine-Patch files) ends with .9.png which should not
428	// be included in the resource name.
429	if dot := strings.LastIndex(p, "."); dot >= 0 && pi.Type == res.Raw {
430		return p[:dot], true
431	}
432	if dot := strings.Index(p, "."); dot >= 0 {
433		return p[:dot], true
434	}
435	return p, true
436}
437