xref: /aosp_15_r20/external/llvm/utils/lit/lit/util.py (revision 9880d6810fe72a1726cb53787c6711e909410d58) 
1import errno
2import itertools
3import math
4import os
5import platform
6import signal
7import subprocess
8import sys
9import threading
10
11def to_bytes(str):
12    # Encode to UTF-8 to get binary data.
13    return str.encode('utf-8')
14
15def to_string(bytes):
16    if isinstance(bytes, str):
17        return bytes
18    return to_bytes(bytes)
19
20def convert_string(bytes):
21    try:
22        return to_string(bytes.decode('utf-8'))
23    except UnicodeError:
24        return str(bytes)
25
26def detectCPUs():
27    """
28    Detects the number of CPUs on a system. Cribbed from pp.
29    """
30    # Linux, Unix and MacOS:
31    if hasattr(os, "sysconf"):
32        if "SC_NPROCESSORS_ONLN" in os.sysconf_names:
33            # Linux & Unix:
34            ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
35            if isinstance(ncpus, int) and ncpus > 0:
36                return ncpus
37        else: # OSX:
38            return int(capture(['sysctl', '-n', 'hw.ncpu']))
39    # Windows:
40    if "NUMBER_OF_PROCESSORS" in os.environ:
41        ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
42        if ncpus > 0:
43            # With more than 32 processes, process creation often fails with
44            # "Too many open files".  FIXME: Check if there's a better fix.
45            return min(ncpus, 32)
46    return 1 # Default
47
48def mkdir_p(path):
49    """mkdir_p(path) - Make the "path" directory, if it does not exist; this
50    will also make directories for any missing parent directories."""
51    if not path or os.path.exists(path):
52        return
53
54    parent = os.path.dirname(path)
55    if parent != path:
56        mkdir_p(parent)
57
58    try:
59        os.mkdir(path)
60    except OSError:
61        e = sys.exc_info()[1]
62        # Ignore EEXIST, which may occur during a race condition.
63        if e.errno != errno.EEXIST:
64            raise
65
66def capture(args, env=None):
67    """capture(command) - Run the given command (or argv list) in a shell and
68    return the standard output."""
69    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
70                         env=env)
71    out,_ = p.communicate()
72    return convert_string(out)
73
74def which(command, paths = None):
75    """which(command, [paths]) - Look up the given command in the paths string
76    (or the PATH environment variable, if unspecified)."""
77
78    if paths is None:
79        paths = os.environ.get('PATH','')
80
81    # Check for absolute match first.
82    if os.path.isfile(command):
83        return command
84
85    # Would be nice if Python had a lib function for this.
86    if not paths:
87        paths = os.defpath
88
89    # Get suffixes to search.
90    # On Cygwin, 'PATHEXT' may exist but it should not be used.
91    if os.pathsep == ';':
92        pathext = os.environ.get('PATHEXT', '').split(';')
93    else:
94        pathext = ['']
95
96    # Search the paths...
97    for path in paths.split(os.pathsep):
98        for ext in pathext:
99            p = os.path.join(path, command + ext)
100            if os.path.exists(p) and not os.path.isdir(p):
101                return p
102
103    return None
104
105def checkToolsPath(dir, tools):
106    for tool in tools:
107        if not os.path.exists(os.path.join(dir, tool)):
108            return False;
109    return True;
110
111def whichTools(tools, paths):
112    for path in paths.split(os.pathsep):
113        if checkToolsPath(path, tools):
114            return path
115    return None
116
117def printHistogram(items, title = 'Items'):
118    items.sort(key = lambda item: item[1])
119
120    maxValue = max([v for _,v in items])
121
122    # Select first "nice" bar height that produces more than 10 bars.
123    power = int(math.ceil(math.log(maxValue, 10)))
124    for inc in itertools.cycle((5, 2, 2.5, 1)):
125        barH = inc * 10**power
126        N = int(math.ceil(maxValue / barH))
127        if N > 10:
128            break
129        elif inc == 1:
130            power -= 1
131
132    histo = [set() for i in range(N)]
133    for name,v in items:
134        bin = min(int(N * v/maxValue), N-1)
135        histo[bin].add(name)
136
137    barW = 40
138    hr = '-' * (barW + 34)
139    print('\nSlowest %s:' % title)
140    print(hr)
141    for name,value in items[-20:]:
142        print('%.2fs: %s' % (value, name))
143    print('\n%s Times:' % title)
144    print(hr)
145    pDigits = int(math.ceil(math.log(maxValue, 10)))
146    pfDigits = max(0, 3-pDigits)
147    if pfDigits:
148        pDigits += pfDigits + 1
149    cDigits = int(math.ceil(math.log(len(items), 10)))
150    print("[%s] :: [%s] :: [%s]" % ('Range'.center((pDigits+1)*2 + 3),
151                                    'Percentage'.center(barW),
152                                    'Count'.center(cDigits*2 + 1)))
153    print(hr)
154    for i,row in enumerate(histo):
155        pct = float(len(row)) / len(items)
156        w = int(barW * pct)
157        print("[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]" % (
158            pDigits, pfDigits, i*barH, pDigits, pfDigits, (i+1)*barH,
159            '*'*w, ' '*(barW-w), cDigits, len(row), cDigits, len(items)))
160
161class ExecuteCommandTimeoutException(Exception):
162    def __init__(self, msg, out, err, exitCode):
163        assert isinstance(msg, str)
164        assert isinstance(out, str)
165        assert isinstance(err, str)
166        assert isinstance(exitCode, int)
167        self.msg = msg
168        self.out = out
169        self.err = err
170        self.exitCode = exitCode
171
172# Close extra file handles on UNIX (on Windows this cannot be done while
173# also redirecting input).
174kUseCloseFDs = not (platform.system() == 'Windows')
175def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
176    """
177        Execute command ``command`` (list of arguments or string)
178        with
179        * working directory ``cwd`` (str), use None to use the current
180          working directory
181        * environment ``env`` (dict), use None for none
182        * Input to the command ``input`` (str), use string to pass
183          no input.
184        * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
185
186        Returns a tuple (out, err, exitCode) where
187        * ``out`` (str) is the standard output of running the command
188        * ``err`` (str) is the standard error of running the command
189        * ``exitCode`` (int) is the exitCode of running the command
190
191        If the timeout is hit an ``ExecuteCommandTimeoutException``
192        is raised.
193    """
194    p = subprocess.Popen(command, cwd=cwd,
195                         stdin=subprocess.PIPE,
196                         stdout=subprocess.PIPE,
197                         stderr=subprocess.PIPE,
198                         env=env, close_fds=kUseCloseFDs)
199    timerObject = None
200    # FIXME: Because of the way nested function scopes work in Python 2.x we
201    # need to use a reference to a mutable object rather than a plain
202    # bool. In Python 3 we could use the "nonlocal" keyword but we need
203    # to support Python 2 as well.
204    hitTimeOut = [False]
205    try:
206        if timeout > 0:
207            def killProcess():
208                # We may be invoking a shell so we need to kill the
209                # process and all its children.
210                hitTimeOut[0] = True
211                killProcessAndChildren(p.pid)
212
213            timerObject = threading.Timer(timeout, killProcess)
214            timerObject.start()
215
216        out,err = p.communicate(input=input)
217        exitCode = p.wait()
218    finally:
219        if timerObject != None:
220            timerObject.cancel()
221
222    # Ensure the resulting output is always of string type.
223    out = convert_string(out)
224    err = convert_string(err)
225
226    if hitTimeOut[0]:
227        raise ExecuteCommandTimeoutException(
228            msg='Reached timeout of {} seconds'.format(timeout),
229            out=out,
230            err=err,
231            exitCode=exitCode
232            )
233
234    # Detect Ctrl-C in subprocess.
235    if exitCode == -signal.SIGINT:
236        raise KeyboardInterrupt
237
238    return out, err, exitCode
239
240def usePlatformSdkOnDarwin(config, lit_config):
241    # On Darwin, support relocatable SDKs by providing Clang with a
242    # default system root path.
243    if 'darwin' in config.target_triple:
244        try:
245            cmd = subprocess.Popen(['xcrun', '--show-sdk-path'],
246                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
247            out, err = cmd.communicate()
248            out = out.strip()
249            res = cmd.wait()
250        except OSError:
251            res = -1
252        if res == 0 and out:
253            sdk_path = out
254            lit_config.note('using SDKROOT: %r' % sdk_path)
255            config.environment['SDKROOT'] = sdk_path
256
257def killProcessAndChildren(pid):
258    """
259    This function kills a process with ``pid`` and all its
260    running children (recursively). It is currently implemented
261    using the psutil module which provides a simple platform
262    neutral implementation.
263
264    TODO: Reimplement this without using psutil so we can
265          remove our dependency on it.
266    """
267    import psutil
268    try:
269        psutilProc = psutil.Process(pid)
270        # Handle the different psutil API versions
271        try:
272            # psutil >= 2.x
273            children_iterator = psutilProc.children(recursive=True)
274        except AttributeError:
275            # psutil 1.x
276            children_iterator = psutilProc.get_children(recursive=True)
277        for child in children_iterator:
278            try:
279                child.kill()
280            except psutil.NoSuchProcess:
281                pass
282        psutilProc.kill()
283    except psutil.NoSuchProcess:
284        pass
285