1# Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Help include git hash in tensorflow bazel build. 16 17This creates symlinks from the internal git repository directory so 18that the build system can see changes in the version state. We also 19remember what branch git was on so when the branch changes we can 20detect that the ref file is no longer correct (so we can suggest users 21run ./configure again). 22 23NOTE: this script is only used in opensource. 24 25""" 26import argparse 27from builtins import bytes # pylint: disable=redefined-builtin 28import json 29import os 30import shutil 31import subprocess 32 33 34def parse_branch_ref(filename): 35 """Given a filename of a .git/HEAD file return ref path. 36 37 In particular, if git is in detached head state, this will 38 return None. If git is in attached head, it will return 39 the branch reference. E.g. if on 'master', the HEAD will 40 contain 'ref: refs/heads/master' so 'refs/heads/master' 41 will be returned. 42 43 Example: parse_branch_ref(".git/HEAD") 44 Args: 45 filename: file to treat as a git HEAD file 46 Returns: 47 None if detached head, otherwise ref subpath 48 Raises: 49 RuntimeError: if the HEAD file is unparseable. 50 """ 51 52 data = open(filename).read().strip() 53 items = data.split(" ") 54 if len(items) == 1: 55 return None 56 elif len(items) == 2 and items[0] == "ref:": 57 return items[1].strip() 58 else: 59 raise RuntimeError("Git directory has unparseable HEAD") 60 61 62def configure(src_base_path, gen_path, debug=False): 63 """Configure `src_base_path` to embed git hashes if available.""" 64 65 # TODO(aselle): No files generated or symlinked here are deleted by 66 # the build system. I don't know of a way to do it in bazel. It 67 # should only be a problem if somebody moves a sandbox directory 68 # without running ./configure again. 69 70 git_path = os.path.join(src_base_path, ".git") 71 72 # Remove and recreate the path 73 if os.path.exists(gen_path): 74 if os.path.isdir(gen_path): 75 try: 76 shutil.rmtree(gen_path) 77 except OSError: 78 raise RuntimeError("Cannot delete directory %s due to permission " 79 "error, inspect and remove manually" % gen_path) 80 else: 81 raise RuntimeError("Cannot delete non-directory %s, inspect ", 82 "and remove manually" % gen_path) 83 os.makedirs(gen_path) 84 85 if not os.path.isdir(gen_path): 86 raise RuntimeError("gen_git_source.py: Failed to create dir") 87 88 # file that specifies what the state of the git repo is 89 spec = {} 90 91 # value file names will be mapped to the keys 92 link_map = {"head": None, "branch_ref": None} 93 94 if not os.path.isdir(git_path): 95 # No git directory 96 spec["git"] = False 97 open(os.path.join(gen_path, "head"), "w").write("") 98 open(os.path.join(gen_path, "branch_ref"), "w").write("") 99 else: 100 # Git directory, possibly detached or attached 101 spec["git"] = True 102 spec["path"] = src_base_path 103 git_head_path = os.path.join(git_path, "HEAD") 104 spec["branch"] = parse_branch_ref(git_head_path) 105 link_map["head"] = git_head_path 106 if spec["branch"] is not None: 107 # attached method 108 link_map["branch_ref"] = os.path.join(git_path, * 109 os.path.split(spec["branch"])) 110 # Create symlinks or dummy files 111 for target, src in link_map.items(): 112 if src is None: 113 open(os.path.join(gen_path, target), "w").write("") 114 elif not os.path.exists(src): 115 # Git repo is configured in a way we don't support such as having 116 # packed refs. Even though in a git repo, tf.__git_version__ will not 117 # be accurate. 118 # TODO(mikecase): Support grabbing git info when using packed refs. 119 open(os.path.join(gen_path, target), "w").write("") 120 spec["git"] = False 121 else: 122 try: 123 # In python 3.5, symlink function exists even on Windows. But requires 124 # Windows Admin privileges, otherwise an OSError will be thrown. 125 if hasattr(os, "symlink"): 126 os.symlink(src, os.path.join(gen_path, target)) 127 else: 128 shutil.copy2(src, os.path.join(gen_path, target)) 129 except OSError: 130 shutil.copy2(src, os.path.join(gen_path, target)) 131 132 json.dump(spec, open(os.path.join(gen_path, "spec.json"), "w"), indent=2) 133 if debug: 134 print("gen_git_source.py: list %s" % gen_path) 135 print("gen_git_source.py: %s" + repr(os.listdir(gen_path))) 136 print("gen_git_source.py: spec is %r" % spec) 137 138 139def get_git_version(git_base_path, git_tag_override): 140 """Get the git version from the repository. 141 142 This function runs `git describe ...` in the path given as `git_base_path`. 143 This will return a string of the form: 144 <base-tag>-<number of commits since tag>-<shortened sha hash> 145 146 For example, 'v0.10.0-1585-gbb717a6' means v0.10.0 was the last tag when 147 compiled. 1585 commits are after that commit tag, and we can get back to this 148 version by running `git checkout gbb717a6`. 149 150 Args: 151 git_base_path: where the .git directory is located 152 git_tag_override: Override the value for the git tag. This is useful for 153 releases where we want to build the release before the git tag is 154 created. 155 Returns: 156 A bytestring representing the git version 157 """ 158 unknown_label = b"unknown" 159 try: 160 # Force to bytes so this works on python 2 and python 3 161 val = bytes( 162 subprocess.check_output([ 163 "git", 164 str("--git-dir=%s/.git" % git_base_path), 165 str("--work-tree=%s" % git_base_path), "describe", "--long", 166 "--tags" 167 ]).strip()) 168 version_separator = b"-" 169 if git_tag_override and val: 170 split_val = val.split(version_separator) 171 if len(split_val) < 3: 172 raise Exception( 173 ("Expected git version in format 'TAG-COMMITS AFTER TAG-HASH' " 174 "but got '%s'") % val) 175 # There might be "-" in the tag name. But we can be sure that the final 176 # two "-" are those inserted by the git describe command. 177 abbrev_commit = split_val[-1] 178 val = version_separator.join( 179 [bytes(git_tag_override, "utf-8"), b"0", abbrev_commit]) 180 return val if val else unknown_label 181 except (subprocess.CalledProcessError, OSError): 182 return unknown_label 183 184 185def write_version_info(filename, git_version): 186 """Write a c file that defines the version functions. 187 188 Args: 189 filename: filename to write to. 190 git_version: the result of a git describe. 191 """ 192 if b"\"" in git_version or b"\\" in git_version: 193 git_version = b"git_version_is_invalid" # do not cause build to fail! 194 contents = """ 195/* Generated by gen_git_source.py */ 196 197#ifndef TENSORFLOW_CORE_UTIL_VERSION_INFO_H_ 198#define TENSORFLOW_CORE_UTIL_VERSION_INFO_H_ 199 200#define STRINGIFY(x) #x 201#define TOSTRING(x) STRINGIFY(x) 202 203#define TF_GIT_VERSION "%s" 204#ifdef _MSC_VER 205#define TF_COMPILER_VERSION "MSVC " TOSTRING(_MSC_FULL_VER) 206#else 207#define TF_COMPILER_VERSION __VERSION__ 208#endif 209#ifdef _GLIBCXX_USE_CXX11_ABI 210#define TF_CXX11_ABI_FLAG _GLIBCXX_USE_CXX11_ABI 211#else 212#define TF_CXX11_ABI_FLAG 0 213#endif 214#ifdef TENSORFLOW_MONOLITHIC_BUILD 215#define TF_MONOLITHIC_BUILD 1 216#else 217#define TF_MONOLITHIC_BUILD 0 218#endif 219 220#endif // TENSORFLOW_CORE_UTIL_VERSION_INFO_H_ 221""" % git_version.decode("utf-8") 222 open(filename, "w").write(contents) 223 224 225def generate(arglist, git_tag_override=None): 226 """Generate version_info.cc as given `destination_file`. 227 228 Args: 229 arglist: should be a sequence that contains 230 spec, head_symlink, ref_symlink, destination_file. 231 232 `destination_file` is the filename where version_info.cc will be written 233 234 `spec` is a filename where the file contains a JSON dictionary 235 'git' bool that is true if the source is in a git repo 236 'path' base path of the source code 237 'branch' the name of the ref specification of the current branch/tag 238 239 `head_symlink` is a filename to HEAD that is cross-referenced against 240 what is contained in the json branch designation. 241 242 `ref_symlink` is unused in this script but passed, because the build 243 system uses that file to detect when commits happen. 244 245 git_tag_override: Override the value for the git tag. This is useful for 246 releases where we want to build the release before the git tag is 247 created. 248 249 Raises: 250 RuntimeError: If ./configure needs to be run, RuntimeError will be raised. 251 """ 252 253 # unused ref_symlink arg 254 spec, head_symlink, _, dest_file = arglist 255 data = json.load(open(spec)) 256 git_version = None 257 if not data["git"]: 258 git_version = b"unknown" 259 else: 260 old_branch = data["branch"] 261 new_branch = parse_branch_ref(head_symlink) 262 if new_branch != old_branch: 263 raise RuntimeError( 264 "Run ./configure again, branch was '%s' but is now '%s'" % 265 (old_branch, new_branch)) 266 git_version = get_git_version(data["path"], git_tag_override) 267 write_version_info(dest_file, git_version) 268 269 270def raw_generate(output_file, source_dir, git_tag_override=None): 271 """Simple generator used for cmake/make build systems. 272 273 This does not create any symlinks. It requires the build system 274 to build unconditionally. 275 276 Args: 277 output_file: Output filename for the version info cc 278 source_dir: Base path of the source code 279 git_tag_override: Override the value for the git tag. This is useful for 280 releases where we want to build the release before the git tag is 281 created. 282 """ 283 284 git_version = get_git_version(source_dir, git_tag_override) 285 write_version_info(output_file, git_version) 286 287 288parser = argparse.ArgumentParser(description="""Git hash injection into bazel. 289If used with --configure <path> will search for git directory and put symlinks 290into source so that a bazel genrule can call --generate""") 291 292parser.add_argument( 293 "--debug", 294 type=bool, 295 help="print debugging information about paths", 296 default=False) 297 298parser.add_argument( 299 "--configure", type=str, 300 help="Path to configure as a git repo dependency tracking sentinel") 301 302parser.add_argument( 303 "--gen_root_path", type=str, 304 help="Root path to place generated git files (created by --configure).") 305 306parser.add_argument( 307 "--git_tag_override", type=str, 308 help="Override git tag value in the __git_version__ string. Useful when " 309 "creating release builds before the release tag is created.") 310 311parser.add_argument( 312 "--generate", 313 type=str, 314 help="Generate given spec-file, HEAD-symlink-file, ref-symlink-file", 315 nargs="+") 316 317parser.add_argument( 318 "--raw_generate", 319 type=str, 320 help="Generate version_info.cc (simpler version used for cmake/make)") 321 322parser.add_argument( 323 "--source_dir", 324 type=str, 325 help="Base path of the source code (used for cmake/make)") 326 327args = parser.parse_args() 328 329if args.configure is not None: 330 if args.gen_root_path is None: 331 raise RuntimeError("Must pass --gen_root_path arg when running --configure") 332 configure(args.configure, args.gen_root_path, debug=args.debug) 333elif args.generate is not None: 334 generate(args.generate, args.git_tag_override) 335elif args.raw_generate is not None: 336 source_path = "." 337 if args.source_dir is not None: 338 source_path = args.source_dir 339 raw_generate(args.raw_generate, source_path, args.git_tag_override) 340else: 341 raise RuntimeError("--configure or --generate or --raw_generate " 342 "must be used") 343