1#!/usr/bin/python3 -B 2# Copyright 2018 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""Regenerates (just) ICU data source files used to build ICU data files.""" 17 18from __future__ import print_function 19 20import os 21import shutil 22import subprocess 23import sys 24from pathlib import Path 25 26import i18nutil 27import icuutil 28 29 30# Run with no arguments from any directory, with no special setup required. 31# See icu4c/source/data/cldr-icu-readme.txt for the upstream ICU instructions. 32def main(): 33 if subprocess.call(["which", "mvn"]) != 0 or subprocess.call(["which", "ant"]) != 0: 34 print("Can't find the required tools. Run `sudo apt-get install maven ant` to install") 35 exit(1) 36 37 if not os.path.exists(os.path.join(Path.home(), ".m2/settings.xml")): 38 print("Can\'t find `~/.m2/settings.xml`. Please follow the instructions at " 39 "http://cldr.unicode.org/development/maven to create one and the github token.") 40 exit(1) 41 42 cldr_dir = icuutil.cldrDir() 43 print('Found cldr in %s ...' % cldr_dir) 44 icu_dir = icuutil.icuDir() 45 print('Found icu in %s ...' % icu_dir) 46 47 # Ant doesn't have any mechanism for using a build directory separate from the 48 # source directory so this build script creates a temporary directory and then 49 # copies all necessary ICU4J and CLDR source code to here before building it: 50 i18nutil.SwitchToNewTemporaryDirectory() 51 build_dir = os.getcwd() 52 cldr_build_dir = os.path.join(build_dir, 'cldr') 53 icu4c_build_dir = os.path.join(build_dir, 'icu4c') 54 icu4j_build_dir = os.path.join(build_dir, 'icu4j') 55 icu_tools_build_dir = os.path.join(build_dir, 'icu_tools') 56 57 print('Copying CLDR source code ...') 58 shutil.copytree(cldr_dir, cldr_build_dir, symlinks=True) 59 print('Copying ICU4C source code ...') 60 shutil.copytree(os.path.join(icu_dir, 'icu4c'), icu4c_build_dir, symlinks=True) 61 print('Copying ICU4J source code ...') 62 shutil.copytree(os.path.join(icu_dir, 'icu4j'), icu4j_build_dir, symlinks=True) 63 print('Copying ICU tools source code ...') 64 shutil.copytree(os.path.join(icu_dir, 'tools'), icu_tools_build_dir, symlinks=True) 65 66 # Setup environment variables for all subshell 67 os.environ['ANT_OPTS'] = '-Xmx8192m' 68 os.environ['MAVEN_ARGS'] = '--no-transfer-progress' 69 70 # This is the location of the original CLDR source tree (not the temporary 71 # copy of the tools source code) from where the data files are to be read: 72 os.environ['CLDR_DIR'] = cldr_build_dir # os.path.join(os.getcwd(), 'cldr') 73 74 os.environ['ICU4C_ROOT'] = icu4c_build_dir 75 os.environ['ICU4J_ROOT'] = icu4j_build_dir 76 os.environ['TOOLS_ROOT'] = icu_tools_build_dir 77 cldr_tmp_dir = os.path.join(build_dir, 'cldr-staging') 78 os.environ['CLDR_TMP_DIR'] = cldr_tmp_dir 79 cldr_production_tmp_dir = os.path.join(cldr_tmp_dir, 'production') 80 os.environ['CLDR_DATA_DIR'] = cldr_production_tmp_dir 81 82 icu_tools_cldr_dir = os.path.join(icu_tools_build_dir, 'cldr') 83 print('Installing CLDR tools ...') 84 os.chdir(icu_tools_cldr_dir) 85 subprocess.check_call(['ant', 'install-cldr-libs']) 86 87 print('Building ICU data...') 88 icu4c_data_build_dir = os.path.join(icu4c_build_dir, 'source/data') 89 os.chdir(icu4c_data_build_dir) 90 subprocess.check_call(['ant', 'cleanprod']) 91 subprocess.check_call(['ant', 'setup']) 92 subprocess.check_call(['ant', 'proddata']) 93 94 # Finally we "compile" CLDR-data to a "production" form and place it in ICU 95 os.chdir(os.path.join(icu_tools_build_dir, 'cldr', 'cldr-to-icu')) 96 subprocess.check_call([ 97 'ant', 98 '-f', 99 'build-icu-data.xml', 100 '-DcldrDataDir=' + cldr_production_tmp_dir, 101 '-DforceDelete=true', 102 '-DincludePseudoLocales=true' 103 ]) 104 105 os.chdir(icu_tools_cldr_dir) 106 subprocess.check_call([ 107 'ant', 108 'copy-cldr-testdata', 109 ]) 110 111 # Copy the generated data files from the temporary directory into AOSP. 112 icu4c_data_source_dir = os.path.join(icu_dir, 'icu4c/source/data') 113 rmAndCopyTree(icu4c_data_build_dir, icu4c_data_source_dir) 114 115 # Copy test data. It mirrors the copy-cldr-testdata steps in tools/cldr/build.xml. 116 rmAndCopyTree( 117 os.path.join(icu4c_build_dir, 'source/test/testdata/cldr'), 118 os.path.join(icu_dir, 'icu4c/source/test/testdata/cldr')) 119 rmAndCopyTree( 120 os.path.join(icu4j_build_dir, 'main/core/src/test/resources/com/ibm/icu/dev/data/cldr'), 121 os.path.join(icu_dir, 'icu4j/main/core/src/test/resources/com/ibm/icu/dev/data/cldr')) 122 123 # Copy the generated localefallback_data.h and LocaleFallbackData.java 124 shutil.copy( 125 os.path.join(icu4c_build_dir, 'source/common/localefallback_data.h'), 126 os.path.join(icu_dir, 'icu4c/source/common/localefallback_data.h')) 127 shutil.copy( 128 os.path.join(icu4j_build_dir, 129 'main/core/src/main/java/com/ibm/icu/impl/LocaleFallbackData.java'), 130 os.path.join(icu_dir, 'icu4j/main/core/src/main/java/com/ibm/icu/impl/LocaleFallbackData.java')) 131 132 print('Look in %s for new data source files' % icu4c_data_source_dir) 133 sys.exit(0) 134 135 136def rmAndCopyTree(src, dst): 137 if os.path.exists(dst): 138 shutil.rmtree(dst) 139 shutil.copytree(src, dst) 140 141 142if __name__ == '__main__': 143 main() 144