xref: /aosp_15_r20/external/deqp/external/fetch_sources.py (revision 35238bce31c2a825756842865a792f8cf7f89930) 
1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import os
24import sys
25import shutil
26import tarfile
27import zipfile
28import hashlib
29import argparse
30import subprocess
31import ssl
32import stat
33import platform
34import logging
35
36scriptPath = os.path.join(os.path.dirname(__file__), "..", "scripts")
37sys.path.insert(0, scriptPath)
38
39from ctsbuild.common import *
40
41EXTERNAL_DIR = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
42
43SYSTEM_NAME = platform.system()
44
45def computeChecksum (data):
46    return hashlib.sha256(data).hexdigest()
47
48def onReadonlyRemoveError (func, path, exc_info):
49    os.chmod(path, stat.S_IWRITE)
50    os.unlink(path)
51
52class Source:
53    def __init__(self, baseDir, extractDir):
54        self.baseDir = baseDir
55        self.extractDir = extractDir
56
57    def clean (self):
58        fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
59        # Remove read-only first
60        readonlydir = os.path.join(fullDstPath, ".git")
61        if os.path.exists(readonlydir):
62            logging.debug("Deleting " + readonlydir)
63            shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError)
64        if os.path.exists(fullDstPath):
65            logging.debug("Deleting " + fullDstPath)
66            shutil.rmtree(fullDstPath, ignore_errors=False)
67
68class SourcePackage (Source):
69    def __init__(self, url, checksum, baseDir, extractDir = "src", postExtract=None):
70        Source.__init__(self, baseDir, extractDir)
71        self.url = url
72        self.filename = os.path.basename(self.url)
73        self.checksum = checksum
74        self.archiveDir = "packages"
75        self.postExtract = postExtract
76
77    def clean (self):
78        Source.clean(self)
79        self.removeArchives()
80
81    def update (self, cmdProtocol = None, force = False):
82        if not self.isArchiveUpToDate():
83            self.fetchAndVerifyArchive()
84
85        if self.getExtractedChecksum() != self.checksum:
86            Source.clean(self)
87            self.extract()
88            self.storeExtractedChecksum(self.checksum)
89
90    def removeArchives (self):
91        archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
92        if os.path.exists(archiveDir):
93            logging.debug("Deleting " + archiveDir)
94            shutil.rmtree(archiveDir, ignore_errors=False)
95
96    def isArchiveUpToDate (self):
97        archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
98        if os.path.exists(archiveFile):
99            return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
100        else:
101            return False
102
103    def getExtractedChecksumFilePath (self):
104        return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
105
106    def getExtractedChecksum (self):
107        extractedChecksumFile = self.getExtractedChecksumFilePath()
108
109        if os.path.exists(extractedChecksumFile):
110            return readFile(extractedChecksumFile)
111        else:
112            return None
113
114    def storeExtractedChecksum (self, checksum):
115        checksum_bytes = checksum.encode("utf-8")
116        writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
117
118    def connectToUrl (self, url):
119        result = None
120
121        if sys.version_info < (3, 0):
122            from urllib2 import urlopen
123        else:
124            from urllib.request import urlopen
125
126        if args.insecure:
127            print("Ignoring certificate checks")
128            ssl_context = ssl._create_unverified_context()
129            result = urlopen(url, context=ssl_context)
130        else:
131            result = urlopen(url)
132
133        return result
134
135    def fetchAndVerifyArchive (self):
136        print("Fetching %s" % self.url)
137
138        req = self.connectToUrl(self.url)
139        data = req.read()
140        checksum = computeChecksum(data)
141        dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
142
143        if checksum != self.checksum:
144            raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
145
146        if not os.path.exists(os.path.dirname(dstPath)):
147            os.makedirs(os.path.dirname(dstPath))
148
149        writeBinaryFile(dstPath, data)
150
151    def extract (self):
152        print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
153
154        srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
155        tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
156        dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
157
158        if self.filename.endswith(".zip"):
159            archive = zipfile.ZipFile(srcPath)
160        else:
161            archive = tarfile.open(srcPath)
162
163        if os.path.exists(tmpPath):
164            shutil.rmtree(tmpPath, ignore_errors=False)
165
166        os.mkdir(tmpPath)
167
168        archive.extractall(tmpPath)
169        archive.close()
170
171        extractedEntries = os.listdir(tmpPath)
172        if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
173            raise Exception("%s doesn't contain single top-level directory" % self.filename)
174
175        topLevelPath = os.path.join(tmpPath, extractedEntries[0])
176
177        if not os.path.exists(dstPath):
178            os.mkdir(dstPath)
179
180        for entry in os.listdir(topLevelPath):
181            if os.path.exists(os.path.join(dstPath, entry)):
182                raise Exception("%s exists already" % entry)
183
184            shutil.move(os.path.join(topLevelPath, entry), dstPath)
185
186        shutil.rmtree(tmpPath, ignore_errors=True)
187
188        if self.postExtract != None:
189            self.postExtract(dstPath)
190
191class SourceFile (Source):
192    def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
193        Source.__init__(self, baseDir, extractDir)
194        self.url = url
195        self.filename = filename
196        self.checksum = checksum
197
198    def update (self, cmdProtocol = None, force = False):
199        if not self.isFileUpToDate():
200            Source.clean(self)
201            self.fetchAndVerifyFile()
202
203    def isFileUpToDate (self):
204        file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
205        if os.path.exists(file):
206            data = readFile(file)
207            return computeChecksum(data.encode('utf-8')) == self.checksum
208        else:
209            return False
210
211    def connectToUrl (self, url):
212        result = None
213
214        if sys.version_info < (3, 0):
215            from urllib2 import urlopen
216        else:
217            from urllib.request import urlopen
218
219        if args.insecure:
220            print("Ignoring certificate checks")
221            ssl_context = ssl._create_unverified_context()
222            result = urlopen(url, context=ssl_context)
223        else:
224            result = urlopen(url)
225
226        return result
227
228    def fetchAndVerifyFile (self):
229        print("Fetching %s" % self.url)
230
231        req = self.connectToUrl(self.url)
232        data = req.read()
233        checksum = computeChecksum(data)
234        dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
235
236        if checksum != self.checksum:
237            raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
238
239        if not os.path.exists(os.path.dirname(dstPath)):
240            os.mkdir(os.path.dirname(dstPath))
241
242        writeBinaryFile(dstPath, data)
243
244class GitRepo (Source):
245    def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = [], patch = ""):
246        Source.__init__(self, baseDir, extractDir)
247        self.httpsUrl = httpsUrl
248        self.sshUrl = sshUrl
249        self.revision = revision
250        self.removeTags = removeTags
251        self.patch = patch
252
253    def checkout(self, url, fullDstPath, force):
254        if not os.path.exists(os.path.join(fullDstPath, '.git')):
255            execute(["git", "clone", "--no-checkout", url, fullDstPath])
256
257        pushWorkingDir(fullDstPath)
258        print("Directory: " + fullDstPath)
259        try:
260            for tag in self.removeTags:
261                proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
262                (stdout, stderr) = proc.communicate()
263                if len(stdout) > 0:
264                    execute(["git", "tag", "-d",tag])
265            force_arg = ['--force'] if force else []
266            execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
267            execute(["git", "checkout"] + force_arg + [self.revision])
268
269            if(self.patch != ""):
270                patchFile = os.path.join(EXTERNAL_DIR, self.patch)
271                execute(["git", "reset", "--hard", "HEAD"])
272                execute(["git", "apply", patchFile])
273        finally:
274            popWorkingDir()
275
276    def update (self, cmdProtocol, force = False):
277        fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
278        url         = self.httpsUrl
279        backupUrl   = self.sshUrl
280
281        # If url is none then start with ssh
282        if cmdProtocol == 'ssh' or url == None:
283            url       = self.sshUrl
284            backupUrl = self.httpsUrl
285
286        try:
287            self.checkout(url, fullDstPath, force)
288        except:
289            if backupUrl != None:
290                self.checkout(backupUrl, fullDstPath, force)
291
292def postExtractLibpng (path):
293    shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
294                os.path.join(path, "pnglibconf.h"))
295
296PACKAGES = [
297    SourcePackage(
298        "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz",
299        "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
300        "zlib"),
301    SourcePackage(
302        "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
303        "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
304        "libpng",
305        postExtract = postExtractLibpng),
306    SourceFile(
307        "https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
308        "renderdoc_app.h",
309        "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
310        "renderdoc"),
311    GitRepo(
312        "https://github.com/KhronosGroup/SPIRV-Tools.git",
313        "[email protected]:KhronosGroup/SPIRV-Tools.git",
314        "f9184c6501f7e349e0664d281ac93b1db9c1e5ad",
315        "spirv-tools"),
316    GitRepo(
317        "https://github.com/KhronosGroup/glslang.git",
318        "[email protected]:KhronosGroup/glslang.git",
319        "bada5c87ec6db4441db129d8506742c4a72bd610",
320        "glslang",
321        removeTags = ["main-tot"]),
322    GitRepo(
323        "https://github.com/KhronosGroup/SPIRV-Headers.git",
324        "[email protected]:KhronosGroup/SPIRV-Headers.git",
325        "d3c2a6fa95ad463ca8044d7fc45557db381a6a64",
326        "spirv-headers"),
327    GitRepo(
328        "https://github.com/KhronosGroup/Vulkan-Docs.git",
329        "[email protected]:KhronosGroup/Vulkan-Docs.git",
330        "d99193d3fcc4b2a0dacc0a9d7e4951ea611a3e96",
331        "vulkan-docs"),
332    GitRepo(
333        "https://github.com/google/amber.git",
334        "[email protected]:google/amber.git",
335        "8e90b2d2f532bcd4a80069e3f37a9698209a21bc",
336        "amber"),
337    GitRepo(
338        "https://github.com/open-source-parsers/jsoncpp.git",
339        "[email protected]:open-source-parsers/jsoncpp.git",
340        "9059f5cad030ba11d37818847443a53918c327b1",
341        "jsoncpp"),
342    # NOTE: The samples application is not well suited to external
343    # integration, this fork contains the small fixes needed for use
344    # by the CTS.
345    GitRepo(
346        "https://github.com/Igalia/vk_video_samples.git",
347        "[email protected]:Igalia/vk_video_samples.git",
348        "6821adf11eb4f84a2168264b954c170d03237699",
349        "nvidia-video-samples"),
350]
351
352def parseArgs ():
353    versionsForInsecure = ((2,7,9), (3,4,3))
354    versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
355
356    parser = argparse.ArgumentParser(description = "Fetch external sources")
357    parser.add_argument('--clean', dest='clean', action='store_true', default=False,
358                        help='Remove sources instead of fetching')
359    parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
360                        help="Disable certificate check for external sources."
361                        " Minimum python version required " + versionsForInsecureStr)
362    parser.add_argument('--protocol', dest='protocol', default='https', choices=['ssh', 'https'],
363                        help="Select protocol to checkout git repositories.")
364    parser.add_argument('--force', dest='force', action='store_true', default=False,
365                        help="Pass --force to git fetch and checkout commands")
366    parser.add_argument("-v", "--verbose",
367                        dest="verbose",
368                        action="store_true",
369                        help="Enable verbose logging")
370    args = parser.parse_args()
371
372    if args.insecure:
373        for versionItem in versionsForInsecure:
374            if (sys.version_info.major == versionItem[0]):
375                if sys.version_info < versionItem:
376                    parser.error("For --insecure minimum required python version is " +
377                                versionsForInsecureStr)
378                break;
379
380    return args
381
382def run(*popenargs, **kwargs):
383    process = subprocess.Popen(*popenargs, **kwargs)
384
385    try:
386        stdout, stderr = process.communicate(None)
387    except:
388        process.kill()
389        process.wait()
390        raise
391
392    retcode = process.poll()
393
394    if retcode:
395        raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr)
396
397    return retcode, stdout, stderr
398
399if __name__ == "__main__":
400    args = parseArgs()
401    initializeLogger(args.verbose)
402
403    for pkg in PACKAGES:
404        if args.clean:
405            pkg.clean()
406        else:
407            pkg.update(args.protocol, args.force)
408