xref: /aosp_15_r20/tools/asuite/aidegen/project/project_splitter.py (revision c2e18aaa1096c836b086f94603d04f4eb9cf37f5)

#!/usr/bin/env python3
#
# Copyright 2020 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Separate the sources from multiple projects."""

import logging
import os
import shutil

from pathlib import Path

from aidegen import constant
from aidegen.idea import iml
from aidegen.lib import common_util
from aidegen.lib import project_config

_KEY_SOURCE_PATH = 'source_folder_path'
_KEY_TEST_PATH = 'test_folder_path'
_SOURCE_FOLDERS = [_KEY_SOURCE_PATH, _KEY_TEST_PATH]
_KEY_SRCJAR_PATH = 'srcjar_path'
_KEY_R_PATH = 'r_java_path'
_KEY_JAR_PATH = 'jar_path'
_EXCLUDE_ITEM = '\n            <excludeFolder url="file://%s" />'
# Temporarily exclude test-dump and src_stub folders to prevent symbols from
# resolving failure by incorrect reference. These two folders should be removed
# after b/136982078 is resolved.
_EXCLUDE_FOLDERS = ['.idea', '.repo', 'art', 'bionic', 'bootable', 'build',
                    'dalvik', 'developers', 'device', 'hardware', 'kernel',
                    'libnativehelper', 'pdk', 'prebuilts', 'sdk', 'system',
                    'toolchain', 'tools', 'vendor', 'out', 'external',
                    'art/tools/ahat/src/test-dump',
                    'cts/common/device-side/device-info/src_stub',
                    'external/gson/gson/src/main/java'
                    ]
_PERMISSION_DEFINED_PATH = ('frameworks/base/core/res/framework-res/'
                            'android_common/gen/')
_ANDROID = 'android'
_R = 'R'


class ProjectSplitter:
    """Splits the sources from multiple projects.

    It's a specific solution to deal with the source folders in multiple
    project case. Since the IntelliJ does not allow duplicate source folders,
    AIDEGen needs to separate the source folders for each project. The single
    project case has no different with current structure.

    Usage:
    project_splitter = ProjectSplitter(projects)

    # Find the dependencies between the projects.
    project_splitter.get_dependencies()

    # Clear the source folders for each project.
    project_splitter.revise_source_folders()

    Attributes:
        _projects: A list of ProjectInfo.
        _all_srcs: A dictionary contains all sources of multiple projects.
                   e.g.
                   {
                       'module_name': 'test',
                       'path': ['path/to/module'],
                       'srcs': ['src_folder1', 'src_folder2'],
                       'tests': ['test_folder1', 'test_folder2']
                       'jars': ['jar1.jar'],
                       'srcjars': ['1.srcjar', '2.srcjar'],
                       'dependencies': ['framework_srcjars', 'base'],
                       'iml_name': '/abs/path/to/iml.iml'
                   }
        _framework_exist: A boolean, True if framework is one of the projects.
        _framework_iml: A string, the name of the framework's iml.
        _full_repo: A boolean, True if loading with full Android sources.
        _full_repo_iml: A string, the name of the Android folder's iml.
        _permission_r_srcjar: A string, the absolute path of R.srcjar file where
                              the permission relative constants are defined.
        _permission_aapt2: A string, the absolute path of aapt2/R directory
                           where the permission relative constants are defined.
    """
    def __init__(self, projects):
        """ProjectSplitter initialize.

        Args:
            projects: A list of ProjectInfo object.
        """
        self._projects = projects
        self._all_srcs = dict(projects[0].source_path)
        self._framework_iml = None
        self._framework_exist = any(
            {p.project_relative_path == constant.FRAMEWORK_PATH
             for p in self._projects})
        if self._framework_exist:
            self._framework_iml = iml.IMLGenerator.get_unique_iml_name(
                os.path.join(common_util.get_android_root_dir(),
                             constant.FRAMEWORK_PATH))
        self._full_repo = project_config.ProjectConfig.get_instance().full_repo
        if self._full_repo:
            self._full_repo_iml = os.path.basename(
                common_util.get_android_root_dir())
        self._permission_r_srcjar = _get_permission_r_srcjar_rel_path()
        self._permission_aapt2 = _get_permission_aapt2_rel_path()

    def revise_source_folders(self):
        """Resets the source folders of each project.

        There should be no duplicate source root path in IntelliJ. The issue
        doesn't happen in single project case. Once users choose multiple
        projects, there could be several same source paths of different
        projects. In order to prevent that, we should remove the source paths
        in dependencies.iml which are duplicate with the paths in [module].iml
        files.

        Steps to prevent the duplicate source root path in IntelliJ:
        1. Copy all sources from sub-projects to main project.
        2. Delete the source and test folders which are not under the
           sub-projects.
        3. Delete the sub-projects' source and test paths from the main project.
        """
        self._collect_all_srcs()
        self._keep_local_sources()
        self._remove_duplicate_sources()

    def _collect_all_srcs(self):
        """Copies all projects' sources to a dictionary."""
        for project in self._projects[1:]:
            for key, value in project.source_path.items():
                self._all_srcs[key].update(value)

    def _keep_local_sources(self):
        """Removes source folders which are not under the project's path.

        1. Remove the source folders which are not under the project.
        2. Remove the duplicate project's source folders from the _all_srcs.
        """
        for project in self._projects:
            srcs = project.source_path
            relpath = project.project_relative_path
            is_root = not relpath
            for key in _SOURCE_FOLDERS:
                srcs[key] = {s for s in srcs[key]
                             if common_util.is_source_under_relative_path(
                                 s, relpath) or is_root}
                self._all_srcs[key] -= srcs[key]

    def _remove_duplicate_sources(self):
        """Removes the duplicate source folders from each sub-project.

        Priority processing with the longest path length, e.g.
        frameworks/base/packages/SettingsLib must have priority over
        frameworks/base.
        (b/160303006): Remove the parent project's source and test paths under
        the child's project path.
        """
        root = common_util.get_android_root_dir()
        projects = sorted(self._projects, key=lambda k: len(
            k.project_relative_path), reverse=True)
        for child in projects:
            for parent in self._projects:
                is_root = not parent.project_relative_path
                if parent is child:
                    continue
                if (common_util.is_source_under_relative_path(
                        child.project_relative_path,
                        parent.project_relative_path) or is_root):
                    for key in _SOURCE_FOLDERS:
                        parent.source_path[key] -= child.source_path[key]
                        rm_paths = _remove_child_duplicate_sources_from_parent(
                            child, parent.source_path[key], root)
                        parent.source_path[key] -= rm_paths

    def get_dependencies(self):
        """Gets the dependencies between the projects.

        Check if the current project's source folder exists in other projects.
        If so, the current project is a dependency module to the other.
        """
        projects = sorted(self._projects, key=lambda k: len(
            k.project_relative_path))
        for project in projects:
            proj_path = project.project_relative_path
            project.dependencies = [constant.FRAMEWORK_SRCJARS]
            if self._framework_exist and proj_path != constant.FRAMEWORK_PATH:
                project.dependencies.append(self._framework_iml)
            if self._full_repo and proj_path:
                project.dependencies.append(self._full_repo_iml)
            srcs = (project.source_path[_KEY_SOURCE_PATH]
                    | project.source_path[_KEY_TEST_PATH])
            dep_projects = sorted(self._projects, key=lambda k: len(
                k.project_relative_path))
            for dep_proj in dep_projects:
                dep_path = dep_proj.project_relative_path
                is_root = not dep_path
                is_child = common_util.is_source_under_relative_path(dep_path,
                                                                     proj_path)
                is_dep = any({s for s in srcs
                              if common_util.is_source_under_relative_path(
                                  s, dep_path) or is_root})
                if dep_proj is project or is_child or not is_dep:
                    continue
                dep = iml.IMLGenerator.get_unique_iml_name(os.path.join(
                    common_util.get_android_root_dir(), dep_path))
                if dep not in project.dependencies:
                    project.dependencies.append(dep)
            project.dependencies.append(constant.KEY_DEPENDENCIES)

    def gen_framework_srcjars_iml(self):
        """Generates the framework_srcjars.iml.

        Create the iml file with only the srcjars of module framework-all. These
        srcjars will be separated from the modules under frameworks/base.

        Returns:
            A string of the framework_srcjars.iml's absolute path.
        """
        self._remove_permission_definition_srcjar_path()
        mod = dict(self._projects[0].dep_modules[constant.FRAMEWORK_ALL])
        mod[constant.KEY_DEPENDENCIES] = []
        mod[constant.KEY_IML_NAME] = constant.FRAMEWORK_SRCJARS
        if self._framework_exist:
            mod[constant.KEY_DEPENDENCIES].append(self._framework_iml)
        if self._full_repo:
            mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml)
        mod[constant.KEY_DEPENDENCIES].append(constant.KEY_DEPENDENCIES)
        permission_src = self._get_permission_defined_source_path()
        if permission_src:
            mod[constant.KEY_SRCS] = [permission_src]
            srcjar_dict = {constant.KEY_DEP_SRCS: True,
                           constant.KEY_SRCJARS: True,
                           constant.KEY_DEPENDENCIES: True}
        else:
            logging.warning('The permission definition relative paths are '
                            'missing.')
            srcjar_dict = {constant.KEY_SRCJARS: True,
                           constant.KEY_DEPENDENCIES: True}
        framework_srcjars_iml = iml.IMLGenerator(mod)
        framework_srcjars_iml.create(srcjar_dict)
        self._all_srcs[_KEY_SRCJAR_PATH] -= set(mod.get(constant.KEY_SRCJARS,
                                                        []))
        return framework_srcjars_iml.iml_path

    def _get_permission_defined_source_path(self):
        """Gets the source path where permission relative constants are defined.

        For the definition permission constants, the priority is,
        1) If framework-res/android_common/gen/aapt2/R directory exists, return
           it.
        2) If the framework-res/android_common/gen/android/R.srcjar file exists,
           unzip it to 'aidegen_r.srcjar' folder and return the path.

        Returns:
            A string of the path of aapt2/R or android/aidegen_r.srcjar folder,
            else None.
        """
        if os.path.isdir(self._permission_aapt2):
            return self._permission_aapt2
        if os.path.isfile(self._permission_r_srcjar):
            dest = os.path.join(
                os.path.dirname(self._permission_r_srcjar),
                ''.join([constant.UNZIP_SRCJAR_PATH_HEAD,
                         os.path.basename(self._permission_r_srcjar).lower()]))
            if os.path.isdir(dest):
                shutil.rmtree(dest)
            common_util.unzip_file(self._permission_r_srcjar, dest)
            return dest
        return None

    def _gen_dependencies_iml(self):
        """Generates the dependencies.iml."""
        rel_project_soong_paths = self._get_rel_project_soong_paths()
        self._unzip_all_scrjars()
        mod = {
            constant.KEY_SRCS: _get_real_dependencies_jars(
                rel_project_soong_paths, self._all_srcs[_KEY_SOURCE_PATH]),
            constant.KEY_TESTS: _get_real_dependencies_jars(
                rel_project_soong_paths, self._all_srcs[_KEY_TEST_PATH]),
            constant.KEY_JARS: _get_real_dependencies_jars(
                rel_project_soong_paths, self._all_srcs[_KEY_JAR_PATH]),
            constant.KEY_SRCJARS: _get_real_dependencies_jars(
                rel_project_soong_paths,
                self._all_srcs[_KEY_R_PATH] | self._all_srcs[_KEY_SRCJAR_PATH]),
            constant.KEY_DEPENDENCIES: _get_real_dependencies_jars(
                rel_project_soong_paths, [constant.FRAMEWORK_SRCJARS]),
            constant.KEY_PATH: [self._projects[0].project_relative_path],
            constant.KEY_MODULE_NAME: constant.KEY_DEPENDENCIES,
            constant.KEY_IML_NAME: constant.KEY_DEPENDENCIES
        }
        if self._framework_exist:
            mod[constant.KEY_DEPENDENCIES].append(self._framework_iml)
        if self._full_repo:
            mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml)
        dep_iml = iml.IMLGenerator(mod)
        dep_iml.create({constant.KEY_DEP_SRCS: True,
                        constant.KEY_SRCJARS: True,
                        constant.KEY_JARS: True,
                        constant.KEY_DEPENDENCIES: True})

    def _unzip_all_scrjars(self):
        """Unzips all scrjar files to a specific folder 'aidegen_r.srcjar'.

        For some versions of IntelliJ no more supports unzipping srcjar files
        automatically, we have to unzip it to a 'aidegen_r.srcjar' directory.
        The rules of the unzip process are,
        1) If it's a aapt2/R type jar or other directory type sources, add them
           into self._all_srcs[_KEY_SOURCE_PATH].
        2) If it's an R.srcjar file, check if the same path of aapt2/R directory
           exists if so add aapt2/R path into the
           self._all_srcs[_KEY_SOURCE_PATH], otherwise unzip R.srcjar into
           the 'aidegen_r.srcjar' directory and add the unzipped path into
           self._all_srcs[_KEY_SOURCE_PATH].
        """
        sjars = self._all_srcs[_KEY_R_PATH] | self._all_srcs[_KEY_SRCJAR_PATH]
        self._all_srcs[_KEY_R_PATH] = set()
        self._all_srcs[_KEY_SRCJAR_PATH] = set()
        for sjar in sjars:
            if not os.path.exists(sjar):
                continue
            if os.path.isdir(sjar):
                self._all_srcs[_KEY_SOURCE_PATH].add(sjar)
                continue
            sjar_dir = os.path.dirname(sjar)
            sjar_name = os.path.basename(sjar).lower()
            aapt2 = os.path.join(
                os.path.dirname(sjar_dir), constant.NAME_AAPT2, _R)
            if os.path.isdir(aapt2):
                self._all_srcs[_KEY_SOURCE_PATH].add(aapt2)
                continue
            dest = os.path.join(
                sjar_dir, ''.join([constant.UNZIP_SRCJAR_PATH_HEAD, sjar_name]))
            if os.path.isdir(dest):
                shutil.rmtree(dest)
            common_util.unzip_file(sjar, dest)
            self._all_srcs[_KEY_SOURCE_PATH].add(dest)

    def gen_projects_iml(self):
        """Generates the projects' iml file."""
        root_path = common_util.get_android_root_dir()
        excludes = project_config.ProjectConfig.get_instance().exclude_paths
        for project in self._projects:
            relpath = project.project_relative_path
            exclude_folders = []
            if not relpath:
                exclude_folders.extend(get_exclude_content(root_path))
            if excludes:
                exclude_folders.extend(get_exclude_content(root_path, excludes))
            mod_info = {
                constant.KEY_EXCLUDES: ''.join(exclude_folders),
                constant.KEY_SRCS: project.source_path[_KEY_SOURCE_PATH],
                constant.KEY_TESTS: project.source_path[_KEY_TEST_PATH],
                constant.KEY_DEPENDENCIES: project.dependencies,
                constant.KEY_PATH: [relpath],
                constant.KEY_MODULE_NAME: project.module_name,
                constant.KEY_IML_NAME: iml.IMLGenerator.get_unique_iml_name(
                    str(Path(root_path, relpath)))
            }
            dep_iml = iml.IMLGenerator(mod_info)
            dep_iml.create({constant.KEY_SRCS: True,
                            constant.KEY_DEPENDENCIES: True})
            project.iml_path = dep_iml.iml_path
        self._gen_dependencies_iml()

    def _get_rel_project_soong_paths(self):
        """Gets relative projects' paths in 'out/soong/.intermediates' folder.

        Gets relative projects' paths in the 'out/soong/.intermediates'
        directory. For example, if the self.projects = ['frameworks/base'] the
        returned list should be ['out/soong/.intermediates/frameworks/base/'].

        Returns:
            A list of relative projects' paths in out/soong/.intermediates.
        """
        out_soong_dir = os.path.relpath(common_util.get_soong_out_path(),
                                        common_util.get_android_root_dir())
        rel_project_soong_paths = []
        for project in self._projects:
            relpath = project.project_relative_path
            rel_project_soong_paths.append(os.sep.join(
                [out_soong_dir, constant.INTERMEDIATES, relpath]) + os.sep)
        return rel_project_soong_paths

    def _remove_permission_definition_srcjar_path(self):
        """Removes android.Manifest.permission definition srcjar path.

        If framework-res/android_common/gen/aapt2/R directory or
        framework-res/android_common/gen/android/R.srcjar file exists in
        self._all_srcs[_KEY_SRCJAR_PATH], remove them.
        """
        if self._permission_aapt2 in self._all_srcs[_KEY_SRCJAR_PATH]:
            self._all_srcs[_KEY_SRCJAR_PATH].remove(self._permission_aapt2)
        if self._permission_r_srcjar in self._all_srcs[_KEY_SRCJAR_PATH]:
            self._all_srcs[_KEY_SRCJAR_PATH].remove(self._permission_r_srcjar)


def _get_real_dependencies_jars(list_to_check, list_to_be_checked):
    """Gets real dependencies' jar from the input list.

    There are jar files which have the same source codes as the
    self.projects should be removed from dependencies. Otherwise these files
    will cause the duplicated codes in IDE and lead to issues: b/158583214 is an
    example.

    Args:
        list_to_check: A list of relative projects' paths in the folder
                       out/soong/.intermediates to be checked if are contained
                       in the list_to_be_checked list.
        list_to_be_checked: A list of dependencies' paths to be checked.

    Returns:
        A list of dependency jar paths after duplicated ones removed.
    """
    file_exts = [constant.JAR_EXT]
    real_jars = list_to_be_checked.copy()
    for jar in list_to_be_checked:
        ext = os.path.splitext(jar)[-1]
        for check_path in list_to_check:
            if check_path in jar and ext in file_exts:
                real_jars.remove(jar)
                break
    return real_jars


def get_exclude_content(root_path, excludes=None):
    """Get the exclude folder content list.

    It returns the exclude folders content list.
    e.g.
    ['<excludeFolder url="file://a/.idea" />',
    '<excludeFolder url="file://a/.repo" />']

    Args:
        root_path: Android source file path.
        excludes: A list of exclusive directories, the default value is None but
                  will be assigned to _EXCLUDE_FOLDERS.

    Returns:
        String: exclude folder content list.
    """
    exclude_items = []
    if not excludes:
        excludes = _EXCLUDE_FOLDERS
    for folder in excludes:
        folder_path = os.path.join(root_path, folder)
        if os.path.isdir(folder_path):
            exclude_items.append(_EXCLUDE_ITEM % folder_path)
    return exclude_items


def _remove_child_duplicate_sources_from_parent(child, parent_sources, root):
    """Removes the child's duplicate source folders from the parent source list.

    Remove all the child's subdirectories from the parent's source list if there
    is any.

    Args:
        child: A child project of ProjectInfo instance.
        parent_sources: The parent project sources of the ProjectInfo instance.
        root: A string of the Android root.

    Returns:
        A set of the sources to be removed.
    """
    rm_paths = set()
    for path in parent_sources:
        if (common_util.is_source_under_relative_path(
                os.path.relpath(path, root), child.project_relative_path)):
            rm_paths.add(path)
    return rm_paths


def _get_permission_aapt2_rel_path():
    """Gets android.Manifest.permission definition srcjar path."""
    out_soong_dir = os.path.relpath(common_util.get_soong_out_path(),
                                    common_util.get_android_root_dir())
    return os.path.join(out_soong_dir, constant.INTERMEDIATES,
                        _PERMISSION_DEFINED_PATH, constant.NAME_AAPT2, _R)


def _get_permission_r_srcjar_rel_path():
    """Gets android.Manifest.permission definition srcjar path."""
    out_soong_dir = os.path.relpath(common_util.get_soong_out_path(),
                                    common_util.get_android_root_dir())
    return os.path.join(out_soong_dir, constant.INTERMEDIATES,
                        _PERMISSION_DEFINED_PATH, _ANDROID,
                        constant.TARGET_R_SRCJAR)