xref: /aosp_15_r20/external/autotest/client/bin/result_tools/throttler_lib.py (revision 9c5db1993ded3edbeafc8092d69fe5de2ee02df7) 
1# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Help functions used by different throttlers."""
6
7import os
8import re
9
10try:
11    from autotest_lib.client.bin.result_tools import utils_lib
12except ImportError:
13    import utils_lib
14
15
16# A list of file names that should not be throttled, that is, not modified by
17# deletion, trimming or compression.
18NON_THROTTLEABLE_FILE_NAMES = set([
19        '.autoserv_execute',
20        '.parse.lock',
21        '.parse.log',
22        '.parser_execute',
23        'control',
24        'control.srv',
25        'host_keyvals',
26        'job_report.html',
27        'keyval',
28        'messages',
29        'profiling',
30        'result_summary.html',
31        'sponge_invocation.xml',
32        'status',
33        'status.log',
34
35        # ACTS related files:
36        'test_run_details.txt',
37        'test_run_error.txt',
38        'test_run_info.txt',
39        'test_run_summary.json',
40])
41
42# A list of file name patterns that should not be throttled, that is, not
43# modified by deletion, deduping, trimming or compression.
44NON_THROTTLEABLE_FILE_PATTERNS = [
45        '.*/BUILD_INFO-.*',   # ACTS test result files.
46        '.*/AndroidDevice.*', # ACTS test result files.
47        ]
48
49# Regex of result files sorted based on priority. Files can be throttled first
50# have higher priority.
51RESULT_THROTTLE_PRIORITY = [
52        '(.*/)?sysinfo/var/log/.*',
53        '(.*/)?sysinfo/var/log_diff/.*',
54        '(.*/)?sysinfo/.*',
55        # The last one matches any file.
56        '.*',
57        ]
58
59# Regex of file names for Autotest debug logs. These files should be preserved
60# without throttling if possible.
61AUTOTEST_LOG_PATTERN ='.*\.(DEBUG|ERROR|INFO|WARNING)$'
62
63def _list_files(files, all_files=None):
64    """Get all files in the given directories.
65
66    @param files: A list of ResultInfo objects for files in a directory.
67    @param all_files: A list of ResultInfo objects collected so far.
68    @return: A list of all collected ResultInfo objects.
69    """
70    if all_files is None:
71        all_files = []
72    for info in files:
73        if info.is_dir:
74            _list_files(info.files, all_files)
75        else:
76            all_files.append(info)
77    return all_files
78
79
80def sort_result_files(summary):
81    """Sort result infos based on priority.
82
83    @param summary: A ResultInfo object containing result summary.
84    @return: A tuple of (sorted_files, grouped_files)
85            sorted_files: A list of ResultInfo, sorted based on file size and
86                priority based on RESULT_THROTTLE_PRIORITY.
87            grouped_files: A dictionary of ResultInfo grouped by each item in
88                RESULT_THROTTLE_PRIORITY.
89    """
90    all_files = _list_files(summary.files)
91
92    # Scan all file paths and group them based on the throttle priority.
93    grouped_files = {pattern: [] for pattern in RESULT_THROTTLE_PRIORITY}
94    for info in all_files:
95        for pattern in RESULT_THROTTLE_PRIORITY:
96            if re.match(pattern, info.path):
97                grouped_files[pattern].append(info)
98                break
99
100    sorted_files = []
101    for pattern in RESULT_THROTTLE_PRIORITY:
102        # Sort the files in each group by file size, largest first.
103        infos = grouped_files[pattern]
104        infos.sort(key=lambda info: -info.trimmed_size)
105        sorted_files.extend(infos)
106
107    return sorted_files, grouped_files
108
109
110def get_throttleable_files(file_infos, extra_patterns=[]):
111    """Filter the files can be throttled.
112
113    @param file_infos: A list of ResultInfo objects.
114    @param extra_patterns: Extra patterns of file path that should not be
115            throttled.
116    @yield: ResultInfo objects that can be throttled.
117    """
118    for info in file_infos:
119        # Skip the files being deleted in earlier throttling.
120        if info.trimmed_size == 0:
121            continue
122        if info.name in NON_THROTTLEABLE_FILE_NAMES:
123            continue
124        pattern_matched = False
125        for pattern in extra_patterns + NON_THROTTLEABLE_FILE_PATTERNS:
126            if re.match(pattern, info.path):
127                pattern_matched = True
128                break
129
130        if not pattern_matched:
131            yield info
132
133
134def check_throttle_limit(summary, max_result_size_KB):
135    """Check if throttling is enough already.
136
137    @param summary: A ResultInfo object containing result summary.
138    @param max_result_size_KB: Maximum test result size in KB.
139    @return: True if the result directory has been trimmed to be smaller than
140            max_result_size_KB.
141    """
142    if (summary.trimmed_size <= max_result_size_KB * 1024):
143        utils_lib.LOG('Maximum result size is reached (current result'
144                      'size is %s (limit is %s).' %
145                      (utils_lib.get_size_string(summary.trimmed_size),
146                       utils_lib.get_size_string(max_result_size_KB * 1024)))
147        return True
148    else:
149        return False
150
151
152def try_delete_file_on_disk(path):
153    """Try to delete the give file on disk.
154
155    @param path: Path to the file.
156    @returns: True if the file is deleted, False otherwise.
157    """
158    try:
159        utils_lib.LOG('Deleting file %s.' % path)
160        os.remove(path)
161        return True
162    except OSError as e:
163        utils_lib.LOG('Failed to delete file %s, Error: %s' % (path, e))
164