1# coding: utf-8
2from __future__ import unicode_literals, division, absolute_import, print_function
3
4import os
5import subprocess
6import sys
7import shutil
8import re
9import json
10import tarfile
11import zipfile
12
13from . import package_root, build_root, other_packages
14from ._pep425 import _pep425tags, _pep425_implementation
15
16if sys.version_info < (3,):
17    str_cls = unicode  # noqa
18else:
19    str_cls = str
20
21
22def run():
23    """
24    Installs required development dependencies. Uses git to checkout other
25    modularcrypto repos for more accurate coverage data.
26    """
27
28    deps_dir = os.path.join(build_root, 'modularcrypto-deps')
29    if os.path.exists(deps_dir):
30        shutil.rmtree(deps_dir, ignore_errors=True)
31    os.mkdir(deps_dir)
32
33    try:
34        print("Staging ci dependencies")
35        _stage_requirements(deps_dir, os.path.join(package_root, 'requires', 'ci'))
36
37        print("Checking out modularcrypto packages for coverage")
38        for other_package in other_packages:
39            pkg_url = 'https://github.com/wbond/%s.git' % other_package
40            pkg_dir = os.path.join(build_root, other_package)
41            if os.path.exists(pkg_dir):
42                print("%s is already present" % other_package)
43                continue
44            print("Cloning %s" % pkg_url)
45            _execute(['git', 'clone', pkg_url], build_root)
46        print()
47
48    except (Exception):
49        if os.path.exists(deps_dir):
50            shutil.rmtree(deps_dir, ignore_errors=True)
51        raise
52
53    return True
54
55
56def _download(url, dest):
57    """
58    Downloads a URL to a directory
59
60    :param url:
61        The URL to download
62
63    :param dest:
64        The path to the directory to save the file in
65
66    :return:
67        The filesystem path to the saved file
68    """
69
70    print('Downloading %s' % url)
71    filename = os.path.basename(url)
72    dest_path = os.path.join(dest, filename)
73
74    if sys.platform == 'win32':
75        powershell_exe = os.path.join('system32\\WindowsPowerShell\\v1.0\\powershell.exe')
76        code = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12;"
77        code += "(New-Object Net.WebClient).DownloadFile('%s', '%s');" % (url, dest_path)
78        _execute([powershell_exe, '-Command', code], dest, 'Unable to connect to')
79
80    else:
81        _execute(
82            ['curl', '-L', '--silent', '--show-error', '-O', url],
83            dest,
84            'Failed to connect to'
85        )
86
87    return dest_path
88
89
90def _tuple_from_ver(version_string):
91    """
92    :param version_string:
93        A unicode dotted version string
94
95    :return:
96        A tuple of integers
97    """
98
99    match = re.search(
100        r'(\d+(?:\.\d+)*)'
101        r'([-._]?(?:alpha|a|beta|b|preview|pre|c|rc)\.?\d*)?'
102        r'(-\d+|(?:[-._]?(?:rev|r|post)\.?\d*))?'
103        r'([-._]?dev\.?\d*)?',
104        version_string
105    )
106    if not match:
107        return tuple()
108
109    nums = tuple(map(int, match.group(1).split('.')))
110
111    pre = match.group(2)
112    if pre:
113        pre = pre.replace('alpha', 'a')
114        pre = pre.replace('beta', 'b')
115        pre = pre.replace('preview', 'rc')
116        pre = pre.replace('pre', 'rc')
117        pre = re.sub(r'(?<!r)c', 'rc', pre)
118        pre = pre.lstrip('._-')
119        pre_dig_match = re.search(r'\d+', pre)
120        if pre_dig_match:
121            pre_dig = int(pre_dig_match.group(0))
122        else:
123            pre_dig = 0
124        pre = pre.rstrip('0123456789')
125
126        pre_num = {
127            'a': -3,
128            'b': -2,
129            'rc': -1,
130        }[pre]
131
132        pre_tup = (pre_num, pre_dig)
133    else:
134        pre_tup = tuple()
135
136    post = match.group(3)
137    if post:
138        post_dig_match = re.search(r'\d+', post)
139        if post_dig_match:
140            post_dig = int(post_dig_match.group(0))
141        else:
142            post_dig = 0
143        post_tup = (1, post_dig)
144    else:
145        post_tup = tuple()
146
147    dev = match.group(4)
148    if dev:
149        dev_dig_match = re.search(r'\d+', dev)
150        if dev_dig_match:
151            dev_dig = int(dev_dig_match.group(0))
152        else:
153            dev_dig = 0
154        dev_tup = (-4, dev_dig)
155    else:
156        dev_tup = tuple()
157
158    normalized = [nums]
159    if pre_tup:
160        normalized.append(pre_tup)
161    if post_tup:
162        normalized.append(post_tup)
163    if dev_tup:
164        normalized.append(dev_tup)
165    # This ensures regular releases happen after dev and prerelease, but
166    # before post releases
167    if not pre_tup and not post_tup and not dev_tup:
168        normalized.append((0, 0))
169
170    return tuple(normalized)
171
172
173def _open_archive(path):
174    """
175    :param path:
176        A unicode string of the filesystem path to the archive
177
178    :return:
179        An archive object
180    """
181
182    if path.endswith('.zip'):
183        return zipfile.ZipFile(path, 'r')
184    return tarfile.open(path, 'r')
185
186
187def _list_archive_members(archive):
188    """
189    :param archive:
190        An archive from _open_archive()
191
192    :return:
193        A list of info objects to be used with _info_name() and _extract_info()
194    """
195
196    if isinstance(archive, zipfile.ZipFile):
197        return archive.infolist()
198    return archive.getmembers()
199
200
201def _archive_single_dir(archive):
202    """
203    Check if all members of the archive are in a single top-level directory
204
205    :param archive:
206        An archive from _open_archive()
207
208    :return:
209        None if not a single top level directory in archive, otherwise a
210        unicode string of the top level directory name
211    """
212
213    common_root = None
214    for info in _list_archive_members(archive):
215        fn = _info_name(info)
216        if fn in set(['.', '/']):
217            continue
218        sep = None
219        if '/' in fn:
220            sep = '/'
221        elif '\\' in fn:
222            sep = '\\'
223        if sep is None:
224            root_dir = fn
225        else:
226            root_dir, _ = fn.split(sep, 1)
227        if common_root is None:
228            common_root = root_dir
229        else:
230            if common_root != root_dir:
231                return None
232    return common_root
233
234
235def _info_name(info):
236    """
237    Returns a normalized file path for an archive info object
238
239    :param info:
240        An info object from _list_archive_members()
241
242    :return:
243        A unicode string with all directory separators normalized to "/"
244    """
245
246    if isinstance(info, zipfile.ZipInfo):
247        return info.filename.replace('\\', '/')
248    return info.name.replace('\\', '/')
249
250
251def _extract_info(archive, info):
252    """
253    Extracts the contents of an archive info object
254
255    ;param archive:
256        An archive from _open_archive()
257
258    :param info:
259        An info object from _list_archive_members()
260
261    :return:
262        None, or a byte string of the file contents
263    """
264
265    if isinstance(archive, zipfile.ZipFile):
266        fn = info.filename
267        is_dir = fn.endswith('/') or fn.endswith('\\')
268        out = archive.read(info)
269        if is_dir and out == b'':
270            return None
271        return out
272
273    info_file = archive.extractfile(info)
274    if info_file:
275        return info_file.read()
276    return None
277
278
279def _extract_package(deps_dir, pkg_path, pkg_dir):
280    """
281    Extract a .whl, .zip, .tar.gz or .tar.bz2 into a package path to
282    use when running CI tasks
283
284    :param deps_dir:
285        A unicode string of the directory the package should be extracted to
286
287    :param pkg_path:
288        A unicode string of the path to the archive
289
290    :param pkg_dir:
291        If running setup.py, change to this dir first - a unicode string
292    """
293
294    if pkg_path.endswith('.exe'):
295        try:
296            zf = None
297            zf = zipfile.ZipFile(pkg_path, 'r')
298            # Exes have a PLATLIB folder containing everything we want
299            for zi in zf.infolist():
300                if not zi.filename.startswith('PLATLIB'):
301                    continue
302                data = _extract_info(zf, zi)
303                if data is not None:
304                    dst_path = os.path.join(deps_dir, zi.filename[8:])
305                    dst_dir = os.path.dirname(dst_path)
306                    if not os.path.exists(dst_dir):
307                        os.makedirs(dst_dir)
308                    with open(dst_path, 'wb') as f:
309                        f.write(data)
310        finally:
311            if zf:
312                zf.close()
313        return
314
315    if pkg_path.endswith('.whl'):
316        try:
317            zf = None
318            zf = zipfile.ZipFile(pkg_path, 'r')
319            # Wheels contain exactly what we need and nothing else
320            zf.extractall(deps_dir)
321        finally:
322            if zf:
323                zf.close()
324        return
325
326    # Source archives may contain a bunch of other things, including mutliple
327    # packages, so we must use setup.py/setuptool to install/extract it
328
329    ar = None
330    staging_dir = os.path.join(deps_dir, '_staging')
331    try:
332        ar = _open_archive(pkg_path)
333
334        common_root = _archive_single_dir(ar)
335
336        members = []
337        for info in _list_archive_members(ar):
338            dst_rel_path = _info_name(info)
339            if common_root is not None:
340                dst_rel_path = dst_rel_path[len(common_root) + 1:]
341            members.append((info, dst_rel_path))
342
343        if not os.path.exists(staging_dir):
344            os.makedirs(staging_dir)
345
346        for info, rel_path in members:
347            info_data = _extract_info(ar, info)
348            # Dirs won't return a file
349            if info_data is not None:
350                dst_path = os.path.join(staging_dir, rel_path)
351                dst_dir = os.path.dirname(dst_path)
352                if not os.path.exists(dst_dir):
353                    os.makedirs(dst_dir)
354                with open(dst_path, 'wb') as f:
355                    f.write(info_data)
356
357        setup_dir = staging_dir
358        if pkg_dir:
359            setup_dir = os.path.join(staging_dir, pkg_dir)
360
361        root = os.path.abspath(os.path.join(deps_dir, '..'))
362        install_lib = os.path.basename(deps_dir)
363
364        _execute(
365            [
366                sys.executable,
367                'setup.py',
368                'install',
369                '--root=%s' % root,
370                '--install-lib=%s' % install_lib,
371                '--no-compile'
372            ],
373            setup_dir
374        )
375
376    finally:
377        if ar:
378            ar.close()
379        if staging_dir:
380            shutil.rmtree(staging_dir)
381
382
383def _sort_pep440_versions(releases, include_prerelease):
384    """
385    :param releases:
386        A list of unicode string PEP 440 version numbers
387
388    :param include_prerelease:
389        A boolean indicating if prerelease versions should be included
390
391    :return:
392        A sorted generator of 2-element tuples:
393         0: A unicode string containing a PEP 440 version number
394         1: A tuple of tuples containing integers - this is the output of
395            _tuple_from_ver() for the PEP 440 version number and is intended
396            for comparing versions
397    """
398
399    parsed_versions = []
400    for v in releases:
401        t = _tuple_from_ver(v)
402        if not include_prerelease and t[1][0] < 0:
403            continue
404        parsed_versions.append((v, t))
405
406    return sorted(parsed_versions, key=lambda v: v[1])
407
408
409def _is_valid_python_version(python_version, requires_python):
410    """
411    Verifies the "python_version" and "requires_python" keys from a PyPi
412    download record are applicable to the current version of Python
413
414    :param python_version:
415        The "python_version" value from a PyPi download JSON structure. This
416        should be one of: "py2", "py3", "py2.py3" or "source".
417
418    :param requires_python:
419        The "requires_python" value from a PyPi download JSON structure. This
420        will be None, or a comma-separated list of conditions that must be
421        true. Ex: ">=3.5", "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
422    """
423
424    if python_version == "py2" and sys.version_info >= (3,):
425        return False
426    if python_version == "py3" and sys.version_info < (3,):
427        return False
428
429    if requires_python is not None:
430
431        def _ver_tuples(ver_str):
432            ver_str = ver_str.strip()
433            if ver_str.endswith('.*'):
434                ver_str = ver_str[:-2]
435            cond_tup = tuple(map(int, ver_str.split('.')))
436            return (sys.version_info[:len(cond_tup)], cond_tup)
437
438        for part in map(str_cls.strip, requires_python.split(',')):
439            if part.startswith('!='):
440                sys_tup, cond_tup = _ver_tuples(part[2:])
441                if sys_tup == cond_tup:
442                    return False
443            elif part.startswith('>='):
444                sys_tup, cond_tup = _ver_tuples(part[2:])
445                if sys_tup < cond_tup:
446                    return False
447            elif part.startswith('>'):
448                sys_tup, cond_tup = _ver_tuples(part[1:])
449                if sys_tup <= cond_tup:
450                    return False
451            elif part.startswith('<='):
452                sys_tup, cond_tup = _ver_tuples(part[2:])
453                if sys_tup > cond_tup:
454                    return False
455            elif part.startswith('<'):
456                sys_tup, cond_tup = _ver_tuples(part[1:])
457                if sys_tup >= cond_tup:
458                    return False
459            elif part.startswith('=='):
460                sys_tup, cond_tup = _ver_tuples(part[2:])
461                if sys_tup != cond_tup:
462                    return False
463
464    return True
465
466
467def _locate_suitable_download(downloads):
468    """
469    :param downloads:
470        A list of dicts containing a key "url", "python_version" and
471        "requires_python"
472
473    :return:
474        A unicode string URL, or None if not a valid release for the current
475        version of Python
476    """
477
478    valid_tags = _pep425tags()
479
480    exe_suffix = None
481    if sys.platform == 'win32' and _pep425_implementation() == 'cp':
482        win_arch = 'win32' if sys.maxsize == 2147483647 else 'win-amd64'
483        version_info = sys.version_info
484        exe_suffix = '.%s-py%d.%d.exe' % (win_arch, version_info[0], version_info[1])
485
486    wheels = {}
487    whl = None
488    tar_bz2 = None
489    tar_gz = None
490    exe = None
491    for download in downloads:
492        if not _is_valid_python_version(download.get('python_version'), download.get('requires_python')):
493            continue
494
495        if exe_suffix and download['url'].endswith(exe_suffix):
496            exe = download['url']
497        if download['url'].endswith('.whl'):
498            parts = os.path.basename(download['url']).split('-')
499            tag_impl = parts[-3]
500            tag_abi = parts[-2]
501            tag_arch = parts[-1].split('.')[0]
502            wheels[(tag_impl, tag_abi, tag_arch)] = download['url']
503        if download['url'].endswith('.tar.bz2'):
504            tar_bz2 = download['url']
505        if download['url'].endswith('.tar.gz'):
506            tar_gz = download['url']
507
508    # Find the most-specific wheel possible
509    for tag in valid_tags:
510        if tag in wheels:
511            whl = wheels[tag]
512            break
513
514    if exe_suffix and exe:
515        url = exe
516    elif whl:
517        url = whl
518    elif tar_bz2:
519        url = tar_bz2
520    elif tar_gz:
521        url = tar_gz
522    else:
523        return None
524
525    return url
526
527
528def _stage_requirements(deps_dir, path):
529    """
530    Installs requirements without using Python to download, since
531    different services are limiting to TLS 1.2, and older version of
532    Python do not support that
533
534    :param deps_dir:
535        A unicode path to a temporary diretory to use for downloads
536
537    :param path:
538        A unicode filesystem path to a requirements file
539    """
540
541    packages = _parse_requires(path)
542    for p in packages:
543        url = None
544        pkg = p['pkg']
545        pkg_sub_dir = None
546        if p['type'] == 'url':
547            anchor = None
548            if '#' in pkg:
549                pkg, anchor = pkg.split('#', 1)
550                if '&' in anchor:
551                    parts = anchor.split('&')
552                else:
553                    parts = [anchor]
554                for part in parts:
555                    param, value = part.split('=')
556                    if param == 'subdirectory':
557                        pkg_sub_dir = value
558
559            if pkg.endswith('.zip') or pkg.endswith('.tar.gz') or pkg.endswith('.tar.bz2') or pkg.endswith('.whl'):
560                url = pkg
561            else:
562                raise Exception('Unable to install package from URL that is not an archive')
563        else:
564            pypi_json_url = 'https://pypi.org/pypi/%s/json' % pkg
565            json_dest = _download(pypi_json_url, deps_dir)
566            with open(json_dest, 'rb') as f:
567                pkg_info = json.loads(f.read().decode('utf-8'))
568            if os.path.exists(json_dest):
569                os.remove(json_dest)
570
571            if p['type'] == '==':
572                if p['ver'] not in pkg_info['releases']:
573                    raise Exception('Unable to find version %s of %s' % (p['ver'], pkg))
574                url = _locate_suitable_download(pkg_info['releases'][p['ver']])
575                if not url:
576                    raise Exception('Unable to find a compatible download of %s == %s' % (pkg, p['ver']))
577            else:
578                p_ver_tup = _tuple_from_ver(p['ver'])
579                for ver_str, ver_tup in reversed(_sort_pep440_versions(pkg_info['releases'], False)):
580                    if p['type'] == '>=' and ver_tup < p_ver_tup:
581                        break
582                    url = _locate_suitable_download(pkg_info['releases'][ver_str])
583                    if url:
584                        break
585                if not url:
586                    if p['type'] == '>=':
587                        raise Exception('Unable to find a compatible download of %s >= %s' % (pkg, p['ver']))
588                    else:
589                        raise Exception('Unable to find a compatible download of %s' % pkg)
590
591        local_path = _download(url, deps_dir)
592
593        _extract_package(deps_dir, local_path, pkg_sub_dir)
594
595        os.remove(local_path)
596
597
598def _parse_requires(path):
599    """
600    Does basic parsing of pip requirements files, to allow for
601    using something other than Python to do actual TLS requests
602
603    :param path:
604        A path to a requirements file
605
606    :return:
607        A list of dict objects containing the keys:
608         - 'type' ('any', 'url', '==', '>=')
609         - 'pkg'
610         - 'ver' (if 'type' == '==' or 'type' == '>=')
611    """
612
613    python_version = '.'.join(map(str_cls, sys.version_info[0:2]))
614    sys_platform = sys.platform
615
616    packages = []
617
618    with open(path, 'rb') as f:
619        contents = f.read().decode('utf-8')
620
621    for line in re.split(r'\r?\n', contents):
622        line = line.strip()
623        if not len(line):
624            continue
625        if re.match(r'^\s*#', line):
626            continue
627        if ';' in line:
628            package, cond = line.split(';', 1)
629            package = package.strip()
630            cond = cond.strip()
631            cond = cond.replace('sys_platform', repr(sys_platform))
632            cond = cond.replace('python_version', repr(python_version))
633            if not eval(cond):
634                continue
635        else:
636            package = line.strip()
637
638        if re.match(r'^\s*-r\s*', package):
639            sub_req_file = re.sub(r'^\s*-r\s*', '', package)
640            sub_req_file = os.path.abspath(os.path.join(os.path.dirname(path), sub_req_file))
641            packages.extend(_parse_requires(sub_req_file))
642            continue
643
644        if re.match(r'https?://', package):
645            packages.append({'type': 'url', 'pkg': package})
646            continue
647
648        if '>=' in package:
649            parts = package.split('>=')
650            package = parts[0].strip()
651            ver = parts[1].strip()
652            packages.append({'type': '>=', 'pkg': package, 'ver': ver})
653            continue
654
655        if '==' in package:
656            parts = package.split('==')
657            package = parts[0].strip()
658            ver = parts[1].strip()
659            packages.append({'type': '==', 'pkg': package, 'ver': ver})
660            continue
661
662        if re.search(r'[^ a-zA-Z0-9\-]', package):
663            raise Exception('Unsupported requirements format version constraint: %s' % package)
664
665        packages.append({'type': 'any', 'pkg': package})
666
667    return packages
668
669
670def _execute(params, cwd, retry=None):
671    """
672    Executes a subprocess
673
674    :param params:
675        A list of the executable and arguments to pass to it
676
677    :param cwd:
678        The working directory to execute the command in
679
680    :param retry:
681        If this string is present in stderr, retry the operation
682
683    :return:
684        A 2-element tuple of (stdout, stderr)
685    """
686
687    proc = subprocess.Popen(
688        params,
689        stdout=subprocess.PIPE,
690        stderr=subprocess.PIPE,
691        cwd=cwd
692    )
693    stdout, stderr = proc.communicate()
694    code = proc.wait()
695    if code != 0:
696        if retry and retry in stderr.decode('utf-8'):
697            return _execute(params, cwd)
698        e = OSError('subprocess exit code for "%s" was %d: %s' % (' '.join(params), code, stderr))
699        e.stdout = stdout
700        e.stderr = stderr
701        raise e
702    return (stdout, stderr)
703