# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import collections import itertools import json import logging import time from pylib.base import base_test_result def GenerateResultsDict(test_run_results, global_tags=None): """Create a results dict from |test_run_results| suitable for writing to JSON. Args: test_run_results: a list of base_test_result.TestRunResults objects. Returns: A results dict that mirrors the one generated by base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON. """ # Example json output. # { # "global_tags": [], # "all_tests": [ # "test1", # "test2", # ], # "disabled_tests": [], # "per_iteration_data": [ # { # "test1": [ # { # "status": "SUCCESS", # "elapsed_time_ms": 1, # "output_snippet": "", # "output_snippet_base64": "", # "losless_snippet": "", # }, # ... # ], # "test2": [ # { # "status": "FAILURE", # "elapsed_time_ms": 12, # "output_snippet": "", # "output_snippet_base64": "", # "losless_snippet": "", # }, # ... # ], # }, # { # "test1": [ # { # "status": "SUCCESS", # "elapsed_time_ms": 1, # "output_snippet": "", # "output_snippet_base64": "", # "losless_snippet": "", # }, # ], # "test2": [ # { # "status": "FAILURE", # "elapsed_time_ms": 12, # "output_snippet": "", # "output_snippet_base64": "", # "losless_snippet": "", # }, # ], # }, # ... # ], # } all_tests = set() per_iteration_data = [] test_run_links = {} for test_run_result in test_run_results: iteration_data = collections.defaultdict(list) if isinstance(test_run_result, list): results_iterable = itertools.chain(*(t.GetAll() for t in test_run_result)) for tr in test_run_result: test_run_links.update(tr.GetLinks()) else: results_iterable = test_run_result.GetAll() test_run_links.update(test_run_result.GetLinks()) for r in results_iterable: result_dict = { 'status': r.GetType(), 'elapsed_time_ms': r.GetDuration(), 'output_snippet': r.GetLog(), 'losless_snippet': True, 'output_snippet_base64': '', 'links': r.GetLinks(), } iteration_data[r.GetName()].append(result_dict) all_tests = all_tests.union(set(iteration_data.keys())) per_iteration_data.append(iteration_data) return { 'global_tags': global_tags or [], 'all_tests': sorted(list(all_tests)), # TODO(jbudorick): Add support for disabled tests within base_test_result. 'disabled_tests': [], 'per_iteration_data': per_iteration_data, 'links': test_run_links, } def GenerateJsonTestResultFormatDict(test_run_results, interrupted): """Create a results dict from |test_run_results| suitable for writing to JSON. Args: test_run_results: a list of base_test_result.TestRunResults objects. interrupted: True if tests were interrupted, e.g. timeout listing tests Returns: A results dict that mirrors the standard JSON Test Results Format. """ tests = {} counts = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'CRASH': 0, 'TIMEOUT': 0} for test_run_result in test_run_results: if isinstance(test_run_result, list): results_iterable = itertools.chain(*(t.GetAll() for t in test_run_result)) else: results_iterable = test_run_result.GetAll() for r in results_iterable: element = tests for key in r.GetName().split('.'): if key not in element: element[key] = {} element = element[key] element['expected'] = 'PASS' if r.GetType() == base_test_result.ResultType.PASS: result = 'PASS' elif r.GetType() == base_test_result.ResultType.SKIP: result = 'SKIP' elif r.GetType() == base_test_result.ResultType.CRASH: result = 'CRASH' elif r.GetType() == base_test_result.ResultType.TIMEOUT: result = 'TIMEOUT' else: result = 'FAIL' if 'actual' in element: element['actual'] += ' ' + result else: counts[result] += 1 element['actual'] = result if result == 'FAIL': element['is_unexpected'] = True if r.GetDuration() != 0: element['time'] = r.GetDuration() # Fill in required fields. return { 'interrupted': interrupted, 'num_failures_by_type': counts, 'path_delimiter': '.', 'seconds_since_epoch': time.time(), 'tests': tests, 'version': 3, } def GenerateJsonResultsFile(test_run_result, file_path, global_tags=None, **kwargs): """Write |test_run_result| to JSON. This emulates the format of the JSON emitted by base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON. Args: test_run_result: a base_test_result.TestRunResults object. file_path: The path to the JSON file to write. """ with open(file_path, 'w') as json_result_file: json_result_file.write(json.dumps( GenerateResultsDict(test_run_result, global_tags=global_tags), **kwargs)) logging.info('Generated json results file at %s', file_path) def GenerateJsonTestResultFormatFile(test_run_result, interrupted, file_path, **kwargs): """Write |test_run_result| to JSON. This uses the official Chromium Test Results Format. Args: test_run_result: a base_test_result.TestRunResults object. interrupted: True if tests were interrupted, e.g. timeout listing tests file_path: The path to the JSON file to write. """ with open(file_path, 'w') as json_result_file: json_result_file.write( json.dumps( GenerateJsonTestResultFormatDict(test_run_result, interrupted), **kwargs)) logging.info('Generated json results file at %s', file_path) def ParseResultsFromJson(json_results): """Creates a list of BaseTestResult objects from JSON. Args: json_results: A JSON dict in the format created by GenerateJsonResultsFile. """ def string_as_status(s): if s in base_test_result.ResultType.GetTypes(): return s return base_test_result.ResultType.UNKNOWN results_list = [] testsuite_runs = json_results['per_iteration_data'] for testsuite_run in testsuite_runs: for test, test_runs in testsuite_run.items(): results_list.extend( [base_test_result.BaseTestResult(test, string_as_status(tr['status']), duration=tr['elapsed_time_ms'], log=tr.get('output_snippet')) for tr in test_runs]) return results_list