deqp/scripts/make_release.py

# -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

import os
import re
import sys
import copy
import zlib
import time
import shlex
import shutil
import fnmatch
import tarfile
import argparse
import platform
import datetime
import tempfile
import posixpath
import subprocess

from ctsbuild.common import *
from ctsbuild.config import *
from ctsbuild.build import *

pythonExecutable = sys.executable or "python"

def die (msg):
    print(msg)
    sys.exit(-1)

def removeLeadingPath (path, basePath):
    # Both inputs must be normalized already
    assert os.path.normpath(path) == path
    assert os.path.normpath(basePath) == basePath
    return path[len(basePath) + 1:]

def findFile (candidates):
    for file in candidates:
        if os.path.exists(file):
            return file
    return None

def getFileList (basePath):
    allFiles = []
    basePath = os.path.normpath(basePath)
    for root, dirs, files in os.walk(basePath):
        for file in files:
            relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
            allFiles.append(relPath)
    return allFiles

def toDatetime (dateTuple):
    Y, M, D = dateTuple
    return datetime.datetime(Y, M, D)

class PackageBuildInfo:
    def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
        self.releaseConfig = releaseConfig
        self.srcBasePath = srcBasePath
        self.dstBasePath = dstBasePath
        self.tmpBasePath = tmpBasePath

    def getReleaseConfig (self):
        return self.releaseConfig

    def getReleaseVersion (self):
        return self.releaseConfig.getVersion()

    def getReleaseId (self):
        # Release id is crc32(releaseConfig + release)
        return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff

    def getSrcBasePath (self):
        return self.srcBasePath

    def getTmpBasePath (self):
        return self.tmpBasePath

class DstFile (object):
    def __init__ (self, dstFile):
        self.dstFile = dstFile

    def makeDir (self):
        dirName = os.path.dirname(self.dstFile)
        if not os.path.exists(dirName):
            os.makedirs(dirName)

    def make (self, packageBuildInfo):
        assert False # Should not be called

class CopyFile (DstFile):
    def __init__ (self, srcFile, dstFile):
        super(CopyFile, self).__init__(dstFile)
        self.srcFile = srcFile

    def make (self, packageBuildInfo):
        self.makeDir()
        if os.path.exists(self.dstFile):
            die("%s already exists" % self.dstFile)
        shutil.copyfile(self.srcFile, self.dstFile)

class GenReleaseInfoFileTarget (DstFile):
    def __init__ (self, dstFile):
        super(GenReleaseInfoFileTarget, self).__init__(dstFile)

    def make (self, packageBuildInfo):
        self.makeDir()

        scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
        execute([
                pythonExecutable,
                "-B", # no .py[co]
                scriptPath,
                "--name=%s" % packageBuildInfo.getReleaseVersion(),
                "--id=0x%08x" % packageBuildInfo.getReleaseId(),
                "--out=%s" % self.dstFile
            ])

class GenCMake (DstFile):
    def __init__ (self, srcFile, dstFile, replaceVars):
        super(GenCMake, self).__init__(dstFile)
        self.srcFile = srcFile
        self.replaceVars = replaceVars

    def make (self, packageBuildInfo):
        self.makeDir()
        print("    GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath))
        src = readFile(self.srcFile)
        for var, value in self.replaceVars:
            src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
                         'set(%s "%s"' % (var, value), src)
        writeFile(self.dstFile, src)

def createFileTargets (srcBasePath, dstBasePath, files, filters):
    usedFiles = set() # Files that are already included by other filters
    targets = []

    for isMatch, createFileObj in filters:
        # Build list of files that match filter
        matchingFiles = []
        for file in files:
            if not file in usedFiles and isMatch(file):
                matchingFiles.append(file)

        # Build file objects, add to used set
        for file in matchingFiles:
            usedFiles.add(file)
            targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))

    return targets

# Generates multiple file targets based on filters
class FileTargetGroup:
    def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
        self.srcBasePath = srcBasePath
        self.dstBasePath = dstBasePath
        self.filters = filters
        self.getSrcBasePath = srcBasePathFunc

    def make (self, packageBuildInfo):
        fullSrcPath = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
        fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))

        allFiles = getFileList(fullSrcPath)
        targets = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)

        # Make all file targets
        for file in targets:
            file.make(packageBuildInfo)

# Single file target
class SingleFileTarget:
    def __init__ (self, srcFile, dstFile, makeTarget):
        self.srcFile = srcFile
        self.dstFile = dstFile
        self.makeTarget = makeTarget

    def make (self, packageBuildInfo):
        fullSrcPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
        fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))

        target = self.makeTarget(fullSrcPath, fullDstPath)
        target.make(packageBuildInfo)

class BuildTarget:
    def __init__ (self, baseConfig, generator, targets = None):
        self.baseConfig = baseConfig
        self.generator = generator
        self.targets = targets

    def make (self, packageBuildInfo):
        print("    Building %s" % self.baseConfig.getBuildDir())

        # Create config with full build dir path
        config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
                             self.baseConfig.getBuildType(),
                             self.baseConfig.getArgs(),
                             srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))

        assert not os.path.exists(config.getBuildDir())
        build(config, self.generator, self.targets)

class BuildAndroidTarget:
    def __init__ (self, dstFile):
        self.dstFile = dstFile

    def make (self, packageBuildInfo):
        print("    Building Android binary")

        buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")

        assert not os.path.exists(buildRoot)
        os.makedirs(buildRoot)

        # Execute build script
        scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
        execute([
                pythonExecutable,
                "-B", # no .py[co]
                scriptPath,
                "--build-root=%s" % buildRoot,
            ])

        srcFile = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
        dstFile = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))

        CopyFile(srcFile, dstFile).make(packageBuildInfo)

class FetchExternalSourcesTarget:
    def __init__ (self):
        pass

    def make (self, packageBuildInfo):
        scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
        execute([
                pythonExecutable,
                "-B", # no .py[co]
                scriptPath,
            ])

class RemoveSourcesTarget:
    def __init__ (self):
        pass

    def make (self, packageBuildInfo):
        shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)

class Module:
    def __init__ (self, name, targets):
        self.name = name
        self.targets = targets

    def make (self, packageBuildInfo):
        for target in self.targets:
            target.make(packageBuildInfo)

class ReleaseConfig:
    def __init__ (self, name, version, modules, sources = True):
        self.name = name
        self.version = version
        self.modules = modules
        self.sources = sources

    def getName (self):
        return self.name

    def getVersion (self):
        return self.version

    def getModules (self):
        return self.modules

    def packageWithSources (self):
        return self.sources

def matchIncludeExclude (includePatterns, excludePatterns, filename):
    components = os.path.normpath(filename).split(os.sep)
    for pattern in excludePatterns:
        for component in components:
            if fnmatch.fnmatch(component, pattern):
                return False

    for pattern in includePatterns:
        for component in components:
            if fnmatch.fnmatch(component, pattern):
                return True

    return False

def copyFileFilter (includePatterns, excludePatterns=[]):
    return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
            lambda s, d: CopyFile(s, d))

def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
    return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])

def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
    return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)

def makeFileCopy (srcFile, dstFile):
    return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))

def getReleaseFileName (configName, releaseName):
    today = datetime.date.today()
    return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)

def getTempDir ():
    dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
    if not os.path.exists(dirName):
        os.makedirs(dirName)
    return dirName

def makeRelease (releaseConfig):
    releaseName = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
    tmpPath = getTempDir()
    srcBasePath = DEQP_DIR
    dstBasePath = os.path.join(tmpPath, releaseName)
    tmpBasePath = os.path.join(tmpPath, releaseName + "-tmp")
    packageBuildInfo = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
    dstArchiveName = releaseName + ".tar.bz2"

    print("Creating release %s to %s" % (releaseName, tmpPath))

    # Remove old temporary dirs
    for path in [dstBasePath, tmpBasePath]:
        if os.path.exists(path):
            shutil.rmtree(path, ignore_errors=False)

    # Make all modules
    for module in releaseConfig.getModules():
        print("  Processing module %s" % module.name)
        module.make(packageBuildInfo)

    # Remove sources?
    if not releaseConfig.packageWithSources():
        shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)

    # Create archive
    print("Creating %s" % dstArchiveName)
    archive = tarfile.open(dstArchiveName, 'w:bz2')
    archive.add(dstBasePath, arcname=releaseName)
    archive.close()

    # Remove tmp dirs
    for path in [dstBasePath, tmpBasePath]:
        if os.path.exists(path):
            shutil.rmtree(path, ignore_errors=False)

    print("Done!")

# Module declarations

SRC_FILE_PATTERNS = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
TARGET_PATTERNS = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]

BASE = Module("Base", [
    makeFileCopy        ("LICENSE", "src/LICENSE"),
    makeFileCopy        ("CMakeLists.txt", "src/CMakeLists.txt"),
    makeFileCopyGroup    ("targets", "src/targets", TARGET_PATTERNS),
    makeFileCopyGroup    ("execserver", "src/execserver", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("executor", "src/executor", SRC_FILE_PATTERNS),
    makeFileCopy        ("modules/CMakeLists.txt", "src/modules/CMakeLists.txt"),
    makeFileCopyGroup    ("external", "src/external", ["CMakeLists.txt", "*.py"]),

    # Stylesheet for displaying test logs on browser
    makeFileCopyGroup    ("doc/testlog-stylesheet", "doc/testlog-stylesheet", ["*"]),

    # Non-optional parts of framework
    makeFileCopy        ("framework/CMakeLists.txt", "src/framework/CMakeLists.txt"),
    makeFileCopyGroup    ("framework/delibs", "src/framework/delibs", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("framework/common", "src/framework/common", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("framework/qphelper", "src/framework/qphelper", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("framework/platform", "src/framework/platform", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("framework/opengl", "src/framework/opengl", SRC_FILE_PATTERNS, ["simplereference"]),
    makeFileCopyGroup    ("framework/egl", "src/framework/egl", SRC_FILE_PATTERNS),

    # android sources
    makeFileCopyGroup    ("android/package/src", "src/android/package/src", SRC_FILE_PATTERNS),
    makeFileCopy        ("android/package/AndroidManifest.xml", "src/android/package/AndroidManifest.xml"),
    makeFileCopyGroup    ("android/package/res", "src/android/package/res", ["*.png", "*.xml"]),
    makeFileCopyGroup    ("android/scripts", "src/android/scripts", [
        "common.py",
        "build.py",
        "resources.py",
        "install.py",
        "launch.py",
        "debug.py"
        ]),

    # Release info
    GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
])

DOCUMENTATION = Module("Documentation", [
    makeFileCopyGroup    ("doc/pdf", "doc", ["*.pdf"]),
    makeFileCopyGroup    ("doc", "doc", ["porting_layer_changes_*.txt"]),
])

GLSHARED = Module("Shared GL Tests", [
    # Optional framework components
    makeFileCopyGroup    ("framework/randomshaders", "src/framework/randomshaders", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("framework/opengl/simplereference", "src/framework/opengl/simplereference", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("framework/referencerenderer", "src/framework/referencerenderer", SRC_FILE_PATTERNS),

    makeFileCopyGroup    ("modules/glshared", "src/modules/glshared", SRC_FILE_PATTERNS),
])

GLES2 = Module("GLES2", [
    makeFileCopyGroup    ("modules/gles2", "src/modules/gles2", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("data/gles2", "src/data/gles2", ["*.*"]),
    makeFileCopyGroup    ("doc/testspecs/GLES2", "doc/testspecs/GLES2", ["*.txt"])
])

GLES3 = Module("GLES3", [
    makeFileCopyGroup    ("modules/gles3", "src/modules/gles3", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("data/gles3", "src/data/gles3", ["*.*"]),
    makeFileCopyGroup    ("doc/testspecs/GLES3", "doc/testspecs/GLES3", ["*.txt"])
])

GLES31 = Module("GLES31", [
    makeFileCopyGroup    ("modules/gles31", "src/modules/gles31", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("data/gles31", "src/data/gles31", ["*.*"]),
    makeFileCopyGroup    ("doc/testspecs/GLES31", "doc/testspecs/GLES31", ["*.txt"])
])

EGL = Module("EGL", [
    makeFileCopyGroup    ("modules/egl", "src/modules/egl", SRC_FILE_PATTERNS)
])

INTERNAL = Module("Internal", [
    makeFileCopyGroup    ("modules/internal", "src/modules/internal", SRC_FILE_PATTERNS),
    makeFileCopyGroup    ("data/internal", "src/data/internal", ["*.*"]),
])

EXTERNAL_SRCS = Module("External sources", [
    FetchExternalSourcesTarget()
])

ANDROID_BINARIES = Module("Android Binaries", [
    BuildAndroidTarget    ("bin/android/dEQP.apk"),
    makeFileCopyGroup    ("targets/android", "bin/android", ["*.bat", "*.sh"]),
])

COMMON_BUILD_ARGS = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
NULL_X32_CONFIG = BuildConfig('null-x32', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
NULL_X64_CONFIG = BuildConfig('null-x64', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
GLX_X32_CONFIG = BuildConfig('glx-x32', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
GLX_X64_CONFIG = BuildConfig('glx-x64', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)

EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]

LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
    BuildTarget            (NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
    makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES),
    makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES),
])

LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
    BuildTarget            (NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
    makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES),
    makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES),
])

# Special module to remove src dir, for example after binary build
REMOVE_SOURCES = Module("Remove sources from package", [
    RemoveSourcesTarget()
])

# Release configuration

ALL_MODULES = [
    BASE,
    DOCUMENTATION,
    GLSHARED,
    GLES2,
    GLES3,
    GLES31,
    EGL,
    INTERNAL,
    EXTERNAL_SRCS,
]

ALL_BINARIES = [
    LINUX_X64_COMMON_BINARIES,
    ANDROID_BINARIES,
]

RELEASE_CONFIGS = {
    "src": ALL_MODULES,
    "src-bin": ALL_MODULES + ALL_BINARIES,
    "bin": ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
}

def parseArgs ():
    parser = argparse.ArgumentParser(description = "Build release package")
    parser.add_argument("-c",
                        "--config",
                        dest="config",
                        choices=RELEASE_CONFIGS.keys(),
                        required=True,
                        help="Release configuration")
    parser.add_argument("-n",
                        "--name",
                        dest="name",
                        required=True,
                        help="Package-specific name")
    parser.add_argument("-v",
                        "--version",
                        dest="version",
                        required=True,
                        help="Version code")
    return parser.parse_args()

if __name__ == "__main__":
    args = parseArgs()
    config = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
    makeRelease(config)