webrtc/android_tools/generate_bp.py

#!/usr/bin/env python3

import json
import os
import sys

PRINT_ORIGINAL_FULL = False

# This flags are augmented with flags added to the json files but not present in .gn or .gni files
IGNORED_FLAGS = [
    '-D_DEBUG',
    '-Werror',
    '-Xclang',
    '-target-feature',
    '+crc',
    '+crypto',
]
IGNORED_DEFINES = [
    'HAVE_ARM64_CRC32C=1'
]
DEFAULT_CFLAGS = [
    '-DHAVE_ARM64_CRC32C=0',
    '-DUSE_AURA=1',
    '-DUSE_GLIB=1',
    '-DUSE_NSS_CERTS=1',
    '-DUSE_UDEV',
    '-DUSE_X11=1',
    '-DWEBRTC_ANDROID_PLATFORM_BUILD=1',
    '-DWEBRTC_APM_DEBUG_DUMP=0',
    '-D_FILE_OFFSET_BITS=64',
    '-D_GNU_SOURCE',
    '-D_LARGEFILE64_SOURCE',
    '-D_LARGEFILE_SOURCE',
    '-Wno-global-constructors',
    '-Wno-implicit-const-int-float-conversion',
    '-Wno-missing-field-initializers',
    '-Wno-unreachable-code-aggressive',
    '-Wno-unreachable-code-break',
]

DEFAULT_CFLAGS_BY_ARCH = {
        'x86': ['-mavx2', '-mfma', '-msse2', '-msse3'],
        'x64': ['-mavx2', '-mfma', '-msse2', '-msse3'],
        'arm': ['-mthumb'],
        'arm64': [],
        'riscv64': [],
        }

FLAGS = ['cflags', 'cflags_c', 'cflags_cc', 'asmflags']
FLAG_NAME_MAP = {
    'cflags': 'cflags',
    'asmflags': 'asflags',
    'cflags_cc': 'cppflags',
    'cflags_c': 'conlyflags',
}

ARCH_NAME_MAP = {n: n for n in DEFAULT_CFLAGS_BY_ARCH.keys()}
ARCH_NAME_MAP['x64'] = 'x86_64'

ARCHS = sorted(ARCH_NAME_MAP.keys())

def FormatList(l):
    return json.dumps(sorted(list(l)))

def IsInclude(name):
    return name.endswith('.h') or name.endswith('.inc')

def FilterIncludes(l):
    return filter(lambda x: not IsInclude(x), l)

def PrintOrigin(target):
    print('/* From target:')
    if PRINT_ORIGINAL_FULL:
        print(json.dumps(target, sort_keys = True, indent = 4))
    else:
        print(target['original_name'])
    print('*/')

def MakeRelatives(l):
    return map(lambda x: x.split('//').pop(), l)

def FormatName(name):
    return 'webrtc_' + name.split('/').pop().replace(':', '__')

def FormatNames(target):
    target['original_name'] = target['name']
    target['name'] = FormatName(target['name'])
    target['deps'] = sorted([FormatName(d) for d in target['deps']])
    return target

def FilterFlags(flags, to_skip = set()):
    skipped_opts = set(IGNORED_FLAGS).union(to_skip)
    return [x for x in flags if not any([x.startswith(y) for y in skipped_opts])]

def PrintHeader():
    print('package {')
    print('    default_applicable_licenses: ["external_webrtc_license"],')
    print('}')
    print('')
    print('// Added automatically by a large-scale-change that took the approach of')
    print('// \'apply every license found to every target\'. While this makes sure we respect')
    print('// every license restriction, it may not be entirely correct.')
    print('//')
    print('// e.g. GPL in an MIT project might only apply to the contrib/ directory.')
    print('//')
    print('// Please consider splitting the single license below into multiple licenses,')
    print('// taking care not to lose any license_kind information, and overriding the')
    print('// default license using the \'licenses: [...]\' property on targets as needed.')
    print('//')
    print('// For unused files, consider creating a \'fileGroup\' with "//visibility:private"')
    print('// to attach the license to, and including a comment whether the files may be')
    print('// used in the current project.')
    print('//')
    print('// large-scale-change included anything that looked like it might be a license')
    print('// text as a license_text. e.g. LICENSE, NOTICE, COPYING etc.')
    print('//')
    print('// Please consider removing redundant or irrelevant files from \'license_text:\'.')
    print('// See: http://go/android-license-faq')
    print('')
    print('///////////////////////////////////////////////////////////////////////////////')
    print('// Do not edit this file directly, it\'s automatically generated by a script. //')
    print('// Modify android_tools/generate_android_bp.py and run that instead.         //')
    print('///////////////////////////////////////////////////////////////////////////////')
    print('')
    print('license {')
    print('    name: "external_webrtc_license",')
    print('    visibility: [":__subpackages__"],')
    print('    license_kinds: [')
    print('        "SPDX-license-identifier-Apache-2.0",')
    print('        "SPDX-license-identifier-BSD",')
    print('        "SPDX-license-identifier-MIT",')
    print('        "SPDX-license-identifier-Zlib",')
    print('        "legacy_notice",')
    print('        "legacy_unencumbered",')
    print('    ],')
    print('    license_text: [')
    print('        "LICENSE",')
    print('        "PATENTS",')
    print('        "license_template.txt",')
    print('    ],')
    print('}')


def GatherDefaultFlags(targets_by_arch):
    # Iterate through all of the targets for each architecture collecting the flags that
    # are the same for all targets in that architecture.  Use a list instead of a set
    # to maintain the flag ordering, which may be significant (e.g. -Wno-shadow has to
    # come after -Wshadow).
    arch_default_flags = {}
    for arch, targets in targets_by_arch.items():
        arch_default_flags[arch] = {}
        for target in targets.values():
            typ = target['type']
            if typ != 'static_library':
                continue
            for flag_type in FLAGS:
                if not flag_type in arch_default_flags:
                    arch_default_flags[arch][flag_type] = target[flag_type]
                else:
                    target_flags = set(target[flag_type])
                    flags = arch_default_flags[arch][flag_type]
                    flags[:]  = [ x for x in flags if x in target_flags ]
        for flag_type, flags in arch_default_flags[arch].items():
            arch_default_flags[arch][flag_type] = FilterFlags(flags)
        # Add in the hardcoded extra default cflags
        arch_default_flags[arch]['cflags'] += DEFAULT_CFLAGS_BY_ARCH.get(arch, [])

    # Iterate through all of the architectures collecting the flags that are the same
    # for all targets in all architectures.
    default_flags = {}
    for arch, flagset in arch_default_flags.items():
        for flag_type, arch_flags in flagset.items():
            if not flag_type in default_flags:
                default_flags[flag_type] = arch_flags.copy()
            else:
                flags = default_flags[flag_type]
                flags[:] = [ x for x in flags if x in arch_flags ]
    # Add in the hardcoded extra default cflags
    default_flags['cflags'] += DEFAULT_CFLAGS

    # Remove the global default flags from the per-architecture default flags
    for arch, flagset in arch_default_flags.items():
        for flag_type in flagset.keys():
            flags = flagset[flag_type]
            flags[:] = [ x for x in flags if x not in default_flags[flag_type] ]

    default_flags['arch'] = arch_default_flags
    return default_flags

def GenerateDefault(targets_by_arch):
    in_default = GatherDefaultFlags(targets_by_arch)
    print('cc_defaults {')
    print('    name: "webrtc_defaults",')
    print('    local_include_dirs: [')
    print('      ".",')
    print('      "webrtc",')
    print('      "third_party/crc32c/src/include",')
    print('    ],')
    for typ in sorted(in_default.keys() - {'arch'}):
        flags = in_default[typ]
        if len(flags) > 0:
            print('    {0}: ['.format(FLAG_NAME_MAP[typ]))
            for flag in flags:
                print('        "{0}",'.format(flag.replace('"', '\\"')))
            print('    ],')
    print('    static_libs: [')
    print('        "libabsl",')
    print('        "libaom",')
    print('        "libevent",')
    print('        "libopus",')
    print('        "libsrtp2",')
    print('        "libvpx",')
    print('        "libyuv",')
    print('        "libpffft",')
    print('        "rnnoise_rnn_vad",')
    print('    ],')
    print('    shared_libs: [')
    print('        "libcrypto",')
    print('        "libprotobuf-cpp-full",')
    print('        "libprotobuf-cpp-lite",')
    print('        "libssl",')
    print('    ],')
    print('    host_supported: true,')
    print('    // vendor needed for libpreprocessing effects.')
    print('    vendor: true,')
    print('    target: {')
    print('        darwin: {')
    print('            enabled: false,')
    print('        },')
    print('    },')
    print('    arch: {')
    for a in ARCHS:
        print('        {0}: {{'.format(ARCH_NAME_MAP[a]))
        for typ in FLAGS:
            flags = in_default['arch'].get(a, {}).get(typ, [])
            if len(flags) > 0:
                print('            {0}: ['.format(FLAG_NAME_MAP[typ]))
                for flag in flags:
                    print('                "{0}",'.format(flag.replace('"', '\\"')))
                print('            ],')
        print('        },')
    print('    },')
    print('    visibility: [')
    print('        "//frameworks/av/media/libeffects/preprocessing:__subpackages__",')
    print('        "//device/google/cuttlefish/host/frontend/webrtc:__subpackages__",')
    print('    ],')
    print('}')

    # The flags in the default entry can be safely removed from the targets
    for arch, targets in targets_by_arch.items():
        for flag_type in FLAGS:
            default_flags = set(in_default[flag_type]) | set(in_default['arch'][arch][flag_type])
            for target in targets.values():
                target[flag_type] = FilterFlags(target.get(flag_type, []), default_flags)
                if len(target[flag_type]) == 0:
                    target.pop(flag_type)

    return in_default


def TransitiveDependencies(name, dep_type, targets):
    target = targets[name]
    field = 'transitive_' + dep_type
    if field in target.keys():
        return target[field]
    target[field] = {'global': set()}
    for a in ARCHS:
        target[field][a] = set()
    if target['type'] == dep_type:
        target[field]['global'].add(name)
    for d in target.get('deps', []):
        if targets[d]['type'] == dep_type:
            target[field]['global'].add(d)
        tDeps = TransitiveDependencies(d, dep_type, targets)
        target[field]['global'] |= tDeps['global']
        for a in ARCHS:
            target[field][a] |= tDeps[a]
    if 'arch' in target:
        for a, x in target['arch'].items():
            for d in x.get('deps', []):
                tDeps = TransitiveDependencies(d, dep_type, targets)
                target[field][a] |= tDeps['global'] | tDeps[a]
            target[field][a] -= target[field]['global']

    return target[field]

def GenerateGroup(target):
    # PrintOrigin(target)
    pass

def GenerateStaticLib(target, targets):
    PrintOrigin(target)
    name = target['name']
    print('cc_library_static {')
    print('    name: "{0}",'.format(name))
    print('    defaults: ["webrtc_defaults"],')
    sources = target.get('sources', [])
    print('    srcs: {0},'.format(FormatList(sources)))
    print('    host_supported: true,')
    if 'asmflags' in target.keys():
        asmflags = target['asmflags']
        if len(asmflags) > 0:
            print('    asflags: {0},'.format(FormatList(asmflags)))
    if 'cflags' in target.keys():
        cflags = target['cflags']
        print('    cflags: {0},'.format(FormatList(cflags)))
    if 'cflags_c' in target.keys():
        cflags_c = target['cflags_c']
        if len(cflags_c) > 0:
            print('    conlyflags: {0},'.format(FormatList(cflags_c)))
    if 'cflags_cc' in target.keys():
        cflags_cc = target['cflags_cc']
        if len(cflags_cc) > 0:
            print('    cppflags: {0},'.format(FormatList(cflags_cc)))
    if 'arch' in target:
        print('   arch: {')
        for arch_name in ARCHS:
            if arch_name not in target['arch'].keys():
                continue
            arch = target['arch'][arch_name]
            print('       ' + ARCH_NAME_MAP[arch_name] + ': {')
            if 'cflags' in arch.keys():
                cflags = arch['cflags']
                print('            cflags: {0},'.format(FormatList(cflags)))
            if 'cflags_c' in arch.keys():
                cflags_c = arch['cflags_c']
                if len(cflags_c) > 0:
                    print('            conlyflags: {0},'.format(FormatList(cflags_c)))
            if 'cflags_cc' in arch.keys():
                cflags_cc = arch['cflags_cc']
                if len(cflags_cc) > 0:
                    print('            cppflags: {0},'.format(FormatList(cflags_cc)))
            if 'sources' in arch:
                  sources = arch['sources']
                  print('            srcs: {0},'.format(FormatList(sources)))
            if 'enabled' in arch:
                print('            enabled: {0},'.format(arch['enabled']))
            print('        },')
        print('   },')
    print('}')
    return name

def DFS(seed, targets):
    visited = set()
    stack = [seed]
    while len(stack) > 0:
        nxt = stack.pop()
        if nxt in visited:
            continue
        visited.add(nxt)
        stack += targets[nxt]['deps']
        if 'arch' not in targets[nxt]:
            continue
        for arch in targets[nxt]['arch']:
            if 'deps' in arch:
                stack += arch['deps']
    return visited

def Preprocess(project):
    targets = {}
    for name, target in project['targets'].items():
        target['name'] = name
        targets[name] = target
        if target['type'] == 'shared_library':
            # Don't bother creating shared libraries
            target['type'] = 'static_library'
        if target['type'] == 'source_set':
            # Convert source_sets to static libraires to avoid recompiling sources multiple times.
            target['type'] = 'static_library'
        if 'defines' in target:
            target['cflags'] = target.get('cflags', []) + ['-D{0}'.format(d) for d in target['defines'] if d not in IGNORED_DEFINES]
            target.pop('defines')
        if 'sources' not in target:
            continue
        sources = list(MakeRelatives(FilterIncludes(target['sources'])))
        if len(sources) > 0:
            target['sources'] = sources
        else:
            target.pop('sources')

    # These dependencies are provided by aosp
    ignored_targets = {
            '//third_party/libaom:libaom',
            '//third_party/libevent:libevent',
            '//third_party/opus:opus',
            '//third_party/libsrtp:libsrtp',
            '//third_party/libvpx:libvpx',
            '//third_party/libyuv:libyuv',
            '//third_party/pffft:pffft',
            '//third_party/rnnoise:rnn_vad',
            '//third_party/boringssl:boringssl',
            '//third_party/android_ndk:cpu_features',
            '//buildtools/third_party/libunwind:libunwind',
            '//buildtools/third_party/libc++:libc++',
        }
    for name, target in targets.items():
        # Skip all "action" targets
        if target['type'] in {'action', 'action_foreach'}:
            ignored_targets.add(name)

    def is_ignored(target):
        if target.startswith('//third_party/abseil-cpp'):
            return True
        return target in ignored_targets

    targets = {name: target for name, target in targets.items() if not is_ignored(name)}

    for target in targets.values():
        # Don't depend on ignored targets
        target['deps'] = [d for d in target['deps'] if not is_ignored(d) ]

    # Ignore empty static libraries
    empty_libs = set()
    for name, target in targets.items():
        if target['type'] == 'static_library' and 'sources' not in target and name != '//:webrtc':
            empty_libs.add(name)
    for empty_lib in empty_libs:
        empty_lib_deps = targets[empty_lib].get('deps', [])
        for target in targets.values():
            target['deps'] = FlattenEmptyLibs(target['deps'], empty_lib, empty_lib_deps)
    for s in empty_libs:
        targets.pop(s)

    # Select libwebrtc, libaudio_processing and its dependencies
    selected = set()
    selected |= DFS('//:webrtc', targets)
    selected |= DFS('//modules/audio_processing:audio_processing', targets)

    return {FormatName(n): FormatNames(targets[n]) for n in selected}

def _FlattenEmptyLibs(deps, empty_lib, empty_lib_deps):
    for x in deps:
        if x == empty_lib:
            yield from empty_lib_deps
        else:
            yield x

def FlattenEmptyLibs(deps, empty_lib, empty_lib_deps):
    return list(_FlattenEmptyLibs(deps, empty_lib, empty_lib_deps))

def NonNoneFrom(l):
    for a in l:
        if a is not None:
            return a
    return None

def MergeListField(target, f, target_by_arch):
    set_by_arch = {}
    for a, t in target_by_arch.items():
        if len(t) == 0:
            # We only care about enabled archs
            continue
        set_by_arch[a] = set(t.get(f, []))

    union = set()
    for _, s in set_by_arch.items():
        union |= s

    common = union
    for a, s in set_by_arch.items():
        common &= s

    not_common = {a: s - common for a,s in set_by_arch.items()}

    if len(common) > 0:
        target[f] = list(common)
    for a, s in not_common.items():
        if len(s) > 0:
            target['arch'][a][f] = sorted(list(s))

def Merge(target_by_arch):
    # The new target shouldn't have the transitive dependencies memoization fields
    # or have the union of those fields from all 4 input targets.
    target = {}
    for f in ['original_name', 'name', 'type']:
        target[f] = NonNoneFrom([t.get(f) for _,t in target_by_arch.items()])

    target['arch'] = {}
    for a, t in target_by_arch.items():
        target['arch'][a] = {}
        if len(t) == 0:
            target['arch'][a]['enabled'] = 'false'

    list_fields = ['sources',
                   'deps',
                   'cflags',
                   'cflags_c',
                   'cflags_cc',
                   'asmflags']
    for lf in list_fields:
        MergeListField(target, lf, target_by_arch)

    # Static libraries should be depended on at the root level and disabled for
    # the corresponding architectures.
    for arch in target['arch'].values():
        if 'deps' not in arch:
            continue
        deps = arch['deps']
        if 'deps' not in target:
            target['deps'] = []
        target['deps'] += deps
        arch.pop('deps')
    if 'deps' in target:
        target['deps'] = sorted(target['deps'])

    # Remove empty sets
    for a in ARCHS:
        if len(target['arch'][a]) == 0:
            target['arch'].pop(a)
    if len(target['arch']) == 0:
        target.pop('arch')

    return target

def DisabledArchs4Target(target):
    ret = set()
    for a in ARCHS:
        if a not in target.get('arch', {}):
            continue
        if target['arch'][a].get('enabled', 'true') == 'false':
            ret.add(a)
    return ret


def HandleDisabledArchs(targets):
    for n, t in targets.items():
        if 'arch' not in t:
            continue
        disabledArchs = DisabledArchs4Target(t)
        if len(disabledArchs) == 0:
            continue
        # Fix targets that depend on this one
        for t in targets.values():
            if DisabledArchs4Target(t) == disabledArchs:
                # With the same disabled archs there is no need to move dependencies
                continue
            if 'deps' in t and n in t['deps']:
                # Remove the dependency from the high level list
                t['deps'] = sorted(set(t['deps']) - {n})
                if 'arch' not in t:
                    t['arch'] = {}
                for a in ARCHS:
                    if a in disabledArchs:
                        continue
                    if a not in t['arch']:
                        t['arch'][a] = {}
                    if 'deps' not in t['arch'][a]:
                        t['arch'][a]['deps'] = []
                    t['arch'][a]['deps'] += [n]

def MergeAll(targets_by_arch):
    names = set()
    for t in targets_by_arch.values():
        names |= t.keys()
    targets = {}
    for name in names:
        targets[name] = Merge({a: t.get(name, {}) for a,t in targets_by_arch.items()})

    HandleDisabledArchs(targets)

    return targets

def GatherAllFlags(obj):
    if type(obj) != type({}):
        # not a dictionary
        return set()
    ret = set()
    for f in FLAGS:
        ret |= set(obj.get(f, []))
    for v in obj.values():
        ret |= GatherAllFlags(v)
    return ret

def FilterFlagsInUse(flags, directory):
    unused = []
    for f in flags:
        nf = f
        if nf.startswith("-D"):
            nf = nf[2:]
            i = nf.find('=')
            if i > 0:
                nf = nf[:i]
        c = os.system(f"find {directory} -name '*.gn*' | xargs grep -q -s -e '{nf}'")
        if c != 0:
            # couldn't find the flag in *.gn or *.gni
            unused.append(f)
    return unused

if len(sys.argv) != 2:
    print('wrong number of arguments', file = sys.stderr)
    exit(1)

dir = sys.argv[1]

targets_by_arch = {}
flags = set()
for arch in ARCHS:
    path = "{0}/project_{1}.json".format(dir, arch)
    json_file = open(path, 'r')
    targets_by_arch[arch] = Preprocess(json.load(json_file))
    flags |= GatherAllFlags(targets_by_arch[arch])

unusedFlags = FilterFlagsInUse(flags, f"{dir}/..")
IGNORED_FLAGS = sorted(set(IGNORED_FLAGS) | set(unusedFlags))

PrintHeader()

GenerateDefault(targets_by_arch)

targets = MergeAll(targets_by_arch)

print('\n\n')

for name, target in sorted(targets.items()):
    typ = target['type']
    if typ == 'static_library':
        GenerateStaticLib(target, targets)
    elif typ == 'group':
        GenerateGroup(target)
    else:
        print('Unknown type: {0} ({1})'.format(typ, target['name']), file = sys.stderr)
        exit(1)
    print('\n\n')

webrtc_libs = TransitiveDependencies(FormatName('//:webrtc'), 'static_library', targets)
print('cc_library_static {')
print('    name: "libwebrtc",')
print('    defaults: ["webrtc_defaults"],')
print('    export_include_dirs: ["."],')
print('    whole_static_libs: {0},'.format(FormatList(sorted(webrtc_libs['global']) + ['libpffft', 'rnnoise_rnn_vad'])))
print('    arch: {')
for a in ARCHS:
    if len(webrtc_libs[a]) > 0:
        print('        {0}: {{'.format(ARCH_NAME_MAP[a]))
        print('            whole_static_libs: {0},'.format(FormatList(sorted(webrtc_libs[a]))))
        print('        },')
print('    },')
print('}')

print('\n\n')

audio_proc_libs = TransitiveDependencies(FormatName('//modules/audio_processing:audio_processing'), 'static_library', targets)
print('cc_library_static {')
print('    name: "webrtc_audio_processing",')
print('    defaults: ["webrtc_defaults"],')
print('    export_include_dirs: [')
print('        ".",')
print('        "modules/include",')
print('        "modules/audio_processing/include",')
print('    ],')
print('    whole_static_libs: {0},'.format(FormatList(sorted(audio_proc_libs['global']) + ['libpffft', 'rnnoise_rnn_vad'])))
print('    arch: {')
for a in ARCHS:
    if len(audio_proc_libs[a]) > 0:
        print('        {0}: {{'.format(ARCH_NAME_MAP[a]))
        print('            whole_static_libs: {0},'.format(FormatList(sorted(audio_proc_libs[a]))))
        print('        },')
print('    },')
print('}')