xref: /aosp_15_r20/external/cronet/build/check_gn_headers.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b) 
1#!/usr/bin/env python3
2# Copyright 2017 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Find header files missing in GN.
7
8This script gets all the header files from ninja_deps, which is from the true
9dependency generated by the compiler, and report if they don't exist in GN.
10"""
11
12import argparse
13import json
14import os
15import re
16import shutil
17import subprocess
18import sys
19import tempfile
20from multiprocessing import Process, Queue
21
22SRC_DIR = os.path.abspath(
23    os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
24DEPOT_TOOLS_DIR = os.path.join(SRC_DIR, 'third_party', 'depot_tools')
25
26
27def GetHeadersFromNinja(out_dir, skip_obj, q):
28  """Return all the header files from ninja_deps"""
29
30  def NinjaSource():
31    cmd = [
32        os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja'), '-C', out_dir,
33        '-t', 'deps'
34    ]
35    # A negative bufsize means to use the system default, which usually
36    # means fully buffered.
37    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
38    for line in iter(popen.stdout.readline, ''):
39      yield line.rstrip()
40
41    popen.stdout.close()
42    return_code = popen.wait()
43    if return_code:
44      raise subprocess.CalledProcessError(return_code, cmd)
45
46  ans, err = set(), None
47  try:
48    ans = ParseNinjaDepsOutput(NinjaSource(), out_dir, skip_obj)
49  except Exception as e:
50    err = str(e)
51  q.put((ans, err))
52
53
54def ParseNinjaDepsOutput(ninja_out, out_dir, skip_obj):
55  """Parse ninja output and get the header files"""
56  all_headers = {}
57
58  # Ninja always uses "/", even on Windows.
59  prefix = '../../'
60
61  is_valid = False
62  obj_file = ''
63  for line in ninja_out:
64    if line.startswith('    '):
65      if not is_valid:
66        continue
67      if line.endswith('.h') or line.endswith('.hh'):
68        f = line.strip()
69        if f.startswith(prefix):
70          f = f[6:]  # Remove the '../../' prefix
71          # build/ only contains build-specific files like build_config.h
72          # and buildflag.h, and system header files, so they should be
73          # skipped.
74          if f.startswith(out_dir) or f.startswith('out'):
75            continue
76          if not f.startswith('build'):
77            all_headers.setdefault(f, [])
78            if not skip_obj:
79              all_headers[f].append(obj_file)
80    else:
81      is_valid = line.endswith('(VALID)')
82      obj_file = line.split(':')[0]
83
84  return all_headers
85
86
87def GetHeadersFromGN(out_dir, q):
88  """Return all the header files from GN"""
89
90  tmp = None
91  ans, err = set(), None
92  try:
93    # Argument |dir| is needed to make sure it's on the same drive on Windows.
94    # dir='' means dir='.', but doesn't introduce an unneeded prefix.
95    tmp = tempfile.mkdtemp(dir='')
96    shutil.copy2(os.path.join(out_dir, 'args.gn'),
97                 os.path.join(tmp, 'args.gn'))
98    # Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
99    gn_exe = 'gn.bat' if sys.platform == 'win32' else 'gn'
100    subprocess.check_call([
101        os.path.join(DEPOT_TOOLS_DIR, gn_exe), 'gen', tmp, '--ide=json', '-q'])
102    gn_json = json.load(open(os.path.join(tmp, 'project.json')))
103    ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
104  except Exception as e:
105    err = str(e)
106  finally:
107    if tmp:
108      shutil.rmtree(tmp)
109  q.put((ans, err))
110
111
112def ParseGNProjectJSON(gn, out_dir, tmp_out):
113  """Parse GN output and get the header files"""
114  all_headers = set()
115
116  for _target, properties in gn['targets'].items():
117    sources = properties.get('sources', [])
118    public = properties.get('public', [])
119    # Exclude '"public": "*"'.
120    if type(public) is list:
121      sources += public
122    for f in sources:
123      if f.endswith('.h') or f.endswith('.hh'):
124        if f.startswith('//'):
125          f = f[2:]  # Strip the '//' prefix.
126          if f.startswith(tmp_out):
127            f = out_dir + f[len(tmp_out):]
128          all_headers.add(f)
129
130  return all_headers
131
132
133def GetDepsPrefixes(q):
134  """Return all the folders controlled by DEPS file"""
135  prefixes, err = set(), None
136  try:
137    gclient_exe = 'gclient.bat' if sys.platform == 'win32' else 'gclient'
138    gclient_out = subprocess.check_output([
139        os.path.join(DEPOT_TOOLS_DIR, gclient_exe),
140        'recurse', '--no-progress', '-j1',
141        'python', '-c', 'import os;print os.environ["GCLIENT_DEP_PATH"]'],
142        universal_newlines=True)
143    for i in gclient_out.split('\n'):
144      if i.startswith('src/'):
145        i = i[4:]
146        prefixes.add(i)
147  except Exception as e:
148    err = str(e)
149  q.put((prefixes, err))
150
151
152def IsBuildClean(out_dir):
153  cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-n']
154  try:
155    out = subprocess.check_output(cmd)
156    return 'no work to do.' in out
157  except Exception as e:
158    print(e)
159    return False
160
161def ParseWhiteList(whitelist):
162  out = set()
163  for line in whitelist.split('\n'):
164    line = re.sub(r'#.*', '', line).strip()
165    if line:
166      out.add(line)
167  return out
168
169
170def FilterOutDepsedRepo(files, deps):
171  return {f for f in files if not any(f.startswith(d) for d in deps)}
172
173
174def GetNonExistingFiles(lst):
175  out = set()
176  for f in lst:
177    if not os.path.isfile(f):
178      out.add(f)
179  return out
180
181
182def main():
183
184  def DumpJson(data):
185    if args.json:
186      with open(args.json, 'w') as f:
187        json.dump(data, f)
188
189  def PrintError(msg):
190    DumpJson([])
191    parser.error(msg)
192
193  parser = argparse.ArgumentParser(description='''
194      NOTE: Use ninja to build all targets in OUT_DIR before running
195      this script.''')
196  parser.add_argument('--out-dir', metavar='OUT_DIR', default='out/Release',
197                      help='output directory of the build')
198  parser.add_argument('--json',
199                      help='JSON output filename for missing headers')
200  parser.add_argument('--whitelist', help='file containing whitelist')
201  parser.add_argument('--skip-dirty-check', action='store_true',
202                      help='skip checking whether the build is dirty')
203  parser.add_argument('--verbose', action='store_true',
204                      help='print more diagnostic info')
205
206  args, _extras = parser.parse_known_args()
207
208  if not os.path.isdir(args.out_dir):
209    parser.error('OUT_DIR "%s" does not exist.' % args.out_dir)
210
211  if not args.skip_dirty_check and not IsBuildClean(args.out_dir):
212    dirty_msg = 'OUT_DIR looks dirty. You need to build all there.'
213    if args.json:
214      # Assume running on the bots. Silently skip this step.
215      # This is possible because "analyze" step can be wrong due to
216      # underspecified header files. See crbug.com/725877
217      print(dirty_msg)
218      DumpJson([])
219      return 0
220    else:
221      # Assume running interactively.
222      parser.error(dirty_msg)
223
224  d_q = Queue()
225  d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, True, d_q,))
226  d_p.start()
227
228  gn_q = Queue()
229  gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
230  gn_p.start()
231
232  deps_q = Queue()
233  deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
234  deps_p.start()
235
236  d, d_err = d_q.get()
237  gn, gn_err = gn_q.get()
238  missing = set(d.keys()) - gn
239  nonexisting = GetNonExistingFiles(gn)
240
241  deps, deps_err = deps_q.get()
242  missing = FilterOutDepsedRepo(missing, deps)
243  nonexisting = FilterOutDepsedRepo(nonexisting, deps)
244
245  d_p.join()
246  gn_p.join()
247  deps_p.join()
248
249  if d_err:
250    PrintError(d_err)
251  if gn_err:
252    PrintError(gn_err)
253  if deps_err:
254    PrintError(deps_err)
255  if len(GetNonExistingFiles(d)) > 0:
256    print('Non-existing files in ninja deps:', GetNonExistingFiles(d))
257    PrintError('Found non-existing files in ninja deps. You should ' +
258               'build all in OUT_DIR.')
259  if len(d) == 0:
260    PrintError('OUT_DIR looks empty. You should build all there.')
261  if any((('/gen/' in i) for i in nonexisting)):
262    PrintError('OUT_DIR looks wrong. You should build all there.')
263
264  if args.whitelist:
265    whitelist = ParseWhiteList(open(args.whitelist).read())
266    missing -= whitelist
267    nonexisting -= whitelist
268
269  missing = sorted(missing)
270  nonexisting = sorted(nonexisting)
271
272  DumpJson(sorted(missing + nonexisting))
273
274  if len(missing) == 0 and len(nonexisting) == 0:
275    return 0
276
277  if len(missing) > 0:
278    print('\nThe following files should be included in gn files:')
279    for i in missing:
280      print(i)
281
282  if len(nonexisting) > 0:
283    print('\nThe following non-existing files should be removed from gn files:')
284    for i in nonexisting:
285      print(i)
286
287  if args.verbose:
288    # Only get detailed obj dependency here since it is slower.
289    GetHeadersFromNinja(args.out_dir, False, d_q)
290    d, d_err = d_q.get()
291    print('\nDetailed dependency info:')
292    for f in missing:
293      print(f)
294      for cc in d[f]:
295        print('  ', cc)
296
297    print('\nMissing headers sorted by number of affected object files:')
298    count = {k: len(v) for (k, v) in d.items()}
299    for f in sorted(count, key=count.get, reverse=True):
300      if f in missing:
301        print(count[f], f)
302
303  if args.json:
304    # Assume running on the bots. Temporarily return 0 before
305    # https://crbug.com/937847 is fixed.
306    return 0
307  return 1
308
309
310if __name__ == '__main__':
311  sys.exit(main())
312