1# Copyright 2019 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Module which adds and verifies attributes in Emboss IR. 16 17The main entry point is normalize_and_verify(), which adds attributes and/or 18verifies attributes which may have been manually entered. 19""" 20 21import re 22 23from compiler.front_end import attributes 24from compiler.front_end import type_check 25from compiler.util import attribute_util 26from compiler.util import error 27from compiler.util import ir_data 28from compiler.util import ir_data_utils 29from compiler.util import ir_util 30from compiler.util import traverse_ir 31 32 33# Default value for maximum_bits on an `enum`. 34_DEFAULT_ENUM_MAXIMUM_BITS = 64 35 36# Default value for expected_back_ends -- mostly for legacy 37_DEFAULT_BACK_ENDS = "cpp" 38 39# Attribute type checkers 40_VALID_BYTE_ORDER = attribute_util.string_from_list( 41 {"BigEndian", "LittleEndian", "Null"}) 42_VALID_TEXT_OUTPUT = attribute_util.string_from_list({"Emit", "Skip"}) 43 44 45def _valid_back_ends(attr, module_source_file): 46 if not re.match( 47 r"^(?:\s*[a-z][a-z0-9_]*\s*(?:,\s*[a-z][a-z0-9_]*\s*)*,?)?\s*$", 48 attr.value.string_constant.text): 49 return [[error.error( 50 module_source_file, 51 attr.value.source_location, 52 "Attribute '{name}' must be a comma-delimited list of back end " 53 "specifiers (like \"cpp, proto\")), not \"{value}\".".format( 54 name=attr.name.text, 55 value=attr.value.string_constant.text))]] 56 return [] 57 58 59# Attributes must be the same type no matter where they occur. 60_ATTRIBUTE_TYPES = { 61 attributes.ADDRESSABLE_UNIT_SIZE: attribute_util.INTEGER_CONSTANT, 62 attributes.BYTE_ORDER: _VALID_BYTE_ORDER, 63 attributes.ENUM_MAXIMUM_BITS: attribute_util.INTEGER_CONSTANT, 64 attributes.FIXED_SIZE: attribute_util.INTEGER_CONSTANT, 65 attributes.IS_INTEGER: attribute_util.BOOLEAN_CONSTANT, 66 attributes.IS_SIGNED: attribute_util.BOOLEAN_CONSTANT, 67 attributes.REQUIRES: attribute_util.BOOLEAN, 68 attributes.STATIC_REQUIREMENTS: attribute_util.BOOLEAN, 69 attributes.TEXT_OUTPUT: _VALID_TEXT_OUTPUT, 70 attributes.BACK_ENDS: _valid_back_ends, 71} 72 73_MODULE_ATTRIBUTES = { 74 (attributes.BYTE_ORDER, True), 75 (attributes.BACK_ENDS, False), 76} 77_BITS_ATTRIBUTES = { 78 (attributes.FIXED_SIZE, False), 79 (attributes.REQUIRES, False), 80} 81_STRUCT_ATTRIBUTES = { 82 (attributes.FIXED_SIZE, False), 83 (attributes.BYTE_ORDER, True), 84 (attributes.REQUIRES, False), 85} 86_ENUM_ATTRIBUTES = { 87 (attributes.ENUM_MAXIMUM_BITS, False), 88 (attributes.IS_SIGNED, False), 89} 90_EXTERNAL_ATTRIBUTES = { 91 (attributes.ADDRESSABLE_UNIT_SIZE, False), 92 (attributes.FIXED_SIZE, False), 93 (attributes.IS_INTEGER, False), 94 (attributes.STATIC_REQUIREMENTS, False), 95} 96_STRUCT_PHYSICAL_FIELD_ATTRIBUTES = { 97 (attributes.BYTE_ORDER, False), 98 (attributes.REQUIRES, False), 99 (attributes.TEXT_OUTPUT, False), 100} 101_STRUCT_VIRTUAL_FIELD_ATTRIBUTES = { 102 (attributes.REQUIRES, False), 103 (attributes.TEXT_OUTPUT, False), 104} 105 106 107def _construct_integer_attribute(name, value, source_location): 108 """Constructs an integer Attribute with the given name and value.""" 109 attr_value = ir_data.AttributeValue( 110 expression=ir_data.Expression( 111 constant=ir_data.NumericConstant(value=str(value), 112 source_location=source_location), 113 type=ir_data.ExpressionType( 114 integer=ir_data.IntegerType(modular_value=str(value), 115 modulus="infinity", 116 minimum_value=str(value), 117 maximum_value=str(value))), 118 source_location=source_location), 119 source_location=source_location) 120 return ir_data.Attribute(name=ir_data.Word(text=name, 121 source_location=source_location), 122 value=attr_value, 123 source_location=source_location) 124 125 126def _construct_boolean_attribute(name, value, source_location): 127 """Constructs a boolean Attribute with the given name and value.""" 128 attr_value = ir_data.AttributeValue( 129 expression=ir_data.Expression( 130 boolean_constant=ir_data.BooleanConstant( 131 value=value, source_location=source_location), 132 type=ir_data.ExpressionType(boolean=ir_data.BooleanType(value=value)), 133 source_location=source_location), 134 source_location=source_location) 135 return ir_data.Attribute(name=ir_data.Word(text=name, 136 source_location=source_location), 137 value=attr_value, 138 source_location=source_location) 139 140 141def _construct_string_attribute(name, value, source_location): 142 """Constructs a string Attribute with the given name and value.""" 143 attr_value = ir_data.AttributeValue( 144 string_constant=ir_data.String(text=value, 145 source_location=source_location), 146 source_location=source_location) 147 return ir_data.Attribute(name=ir_data.Word(text=name, 148 source_location=source_location), 149 value=attr_value, 150 source_location=source_location) 151 152 153def _fixed_size_of_struct_or_bits(struct, unit_size): 154 """Returns size of struct in bits or None, if struct is not fixed size.""" 155 size = 0 156 for field in struct.field: 157 if not field.HasField("location"): 158 # Virtual fields do not contribute to the physical size of the struct. 159 continue 160 field_start = ir_util.constant_value(field.location.start) 161 field_size = ir_util.constant_value(field.location.size) 162 if field_start is None or field_size is None: 163 # Technically, start + size could be constant even if start and size are 164 # not; e.g. if start == x and size == 10 - x, but we don't handle that 165 # here. 166 return None 167 # TODO(bolms): knows_own_size 168 # TODO(bolms): compute min/max sizes for variable-sized arrays. 169 field_end = field_start + field_size 170 if field_end >= size: 171 size = field_end 172 return size * unit_size 173 174 175def _verify_size_attributes_on_structure(struct, type_definition, 176 source_file_name, errors): 177 """Verifies size attributes on a struct or bits.""" 178 fixed_size = _fixed_size_of_struct_or_bits(struct, 179 type_definition.addressable_unit) 180 fixed_size_attr = ir_util.get_attribute(type_definition.attribute, 181 attributes.FIXED_SIZE) 182 if not fixed_size_attr: 183 return 184 if fixed_size is None: 185 errors.append([error.error( 186 source_file_name, fixed_size_attr.source_location, 187 "Struct is marked as fixed size, but contains variable-location " 188 "fields.")]) 189 elif ir_util.constant_value(fixed_size_attr.expression) != fixed_size: 190 errors.append([error.error( 191 source_file_name, fixed_size_attr.source_location, 192 "Struct is {} bits, but is marked as {} bits.".format( 193 fixed_size, ir_util.constant_value(fixed_size_attr.expression)))]) 194 195 196# TODO(bolms): remove [fixed_size]; it is superseded by $size_in_{bits,bytes} 197def _add_missing_size_attributes_on_structure(struct, type_definition): 198 """Adds missing size attributes on a struct.""" 199 fixed_size = _fixed_size_of_struct_or_bits(struct, 200 type_definition.addressable_unit) 201 if fixed_size is None: 202 return 203 fixed_size_attr = ir_util.get_attribute(type_definition.attribute, 204 attributes.FIXED_SIZE) 205 if not fixed_size_attr: 206 # TODO(bolms): Use the offset and length of the last field as the 207 # source_location of the fixed_size attribute? 208 type_definition.attribute.extend([ 209 _construct_integer_attribute(attributes.FIXED_SIZE, fixed_size, 210 type_definition.source_location)]) 211 212 213def _field_needs_byte_order(field, type_definition, ir): 214 """Returns true if the given field needs a byte_order attribute.""" 215 if ir_util.field_is_virtual(field): 216 # Virtual fields have no physical type, and thus do not need a byte order. 217 return False 218 field_type = ir_util.find_object( 219 ir_util.get_base_type(field.type).atomic_type.reference.canonical_name, 220 ir) 221 assert field_type is not None 222 assert field_type.addressable_unit != ir_data.AddressableUnit.NONE 223 return field_type.addressable_unit != type_definition.addressable_unit 224 225 226def _field_may_have_null_byte_order(field, type_definition, ir): 227 """Returns true if "Null" is a valid byte order for the given field.""" 228 # If the field is one unit in length, then byte order does not matter. 229 if (ir_util.is_constant(field.location.size) and 230 ir_util.constant_value(field.location.size) == 1): 231 return True 232 unit = type_definition.addressable_unit 233 # Otherwise, if the field's type is either a one-unit-sized type or an array 234 # of a one-unit-sized type, then byte order does not matter. 235 if (ir_util.fixed_size_of_type_in_bits(ir_util.get_base_type(field.type), ir) 236 == unit): 237 return True 238 # In all other cases, byte order does matter. 239 return False 240 241 242def _add_missing_byte_order_attribute_on_field(field, type_definition, ir, 243 defaults): 244 """Adds missing byte_order attributes to fields that need them.""" 245 if _field_needs_byte_order(field, type_definition, ir): 246 byte_order_attr = ir_util.get_attribute(field.attribute, 247 attributes.BYTE_ORDER) 248 if byte_order_attr is None: 249 if attributes.BYTE_ORDER in defaults: 250 field.attribute.extend([defaults[attributes.BYTE_ORDER]]) 251 elif _field_may_have_null_byte_order(field, type_definition, ir): 252 field.attribute.extend( 253 [_construct_string_attribute(attributes.BYTE_ORDER, "Null", 254 field.source_location)]) 255 256 257def _add_missing_back_ends_to_module(module): 258 """Sets the expected_back_ends attribute for a module, if not already set.""" 259 back_ends_attr = ir_util.get_attribute(module.attribute, attributes.BACK_ENDS) 260 if back_ends_attr is None: 261 module.attribute.extend( 262 [_construct_string_attribute(attributes.BACK_ENDS, _DEFAULT_BACK_ENDS, 263 module.source_location)]) 264 265 266def _gather_expected_back_ends(module): 267 """Captures the expected_back_ends attribute for `module`.""" 268 back_ends_attr = ir_util.get_attribute(module.attribute, attributes.BACK_ENDS) 269 back_ends_str = back_ends_attr.string_constant.text 270 return { 271 "expected_back_ends": {x.strip() for x in back_ends_str.split(",")} | {""} 272 } 273 274 275def _add_addressable_unit_to_external(external, type_definition): 276 """Sets the addressable_unit field for an external TypeDefinition.""" 277 # Strictly speaking, addressable_unit isn't an "attribute," but it's close 278 # enough that it makes sense to handle it with attributes. 279 del external # Unused. 280 size = ir_util.get_integer_attribute(type_definition.attribute, 281 attributes.ADDRESSABLE_UNIT_SIZE) 282 if size == 1: 283 type_definition.addressable_unit = ir_data.AddressableUnit.BIT 284 elif size == 8: 285 type_definition.addressable_unit = ir_data.AddressableUnit.BYTE 286 # If the addressable_unit_size is not in (1, 8), it will be caught by 287 # _verify_addressable_unit_attribute_on_external, below. 288 289 290def _add_missing_width_and_sign_attributes_on_enum(enum, type_definition): 291 """Sets the maximum_bits and is_signed attributes for an enum, if needed.""" 292 max_bits_attr = ir_util.get_integer_attribute(type_definition.attribute, 293 attributes.ENUM_MAXIMUM_BITS) 294 if max_bits_attr is None: 295 type_definition.attribute.extend([ 296 _construct_integer_attribute(attributes.ENUM_MAXIMUM_BITS, 297 _DEFAULT_ENUM_MAXIMUM_BITS, 298 type_definition.source_location)]) 299 signed_attr = ir_util.get_boolean_attribute(type_definition.attribute, 300 attributes.IS_SIGNED) 301 if signed_attr is None: 302 for value in enum.value: 303 numeric_value = ir_util.constant_value(value.value) 304 if numeric_value < 0: 305 is_signed = True 306 break 307 else: 308 is_signed = False 309 type_definition.attribute.extend([ 310 _construct_boolean_attribute(attributes.IS_SIGNED, is_signed, 311 type_definition.source_location)]) 312 313 314def _verify_byte_order_attribute_on_field(field, type_definition, 315 source_file_name, ir, errors): 316 """Verifies the byte_order attribute on the given field.""" 317 byte_order_attr = ir_util.get_attribute(field.attribute, 318 attributes.BYTE_ORDER) 319 field_needs_byte_order = _field_needs_byte_order(field, type_definition, ir) 320 if byte_order_attr and not field_needs_byte_order: 321 errors.append([error.error( 322 source_file_name, byte_order_attr.source_location, 323 "Attribute 'byte_order' not allowed on field which is not byte order " 324 "dependent.")]) 325 if not byte_order_attr and field_needs_byte_order: 326 errors.append([error.error( 327 source_file_name, field.source_location, 328 "Attribute 'byte_order' required on field which is byte order " 329 "dependent.")]) 330 if (byte_order_attr and byte_order_attr.string_constant.text == "Null" and 331 not _field_may_have_null_byte_order(field, type_definition, ir)): 332 errors.append([error.error( 333 source_file_name, byte_order_attr.source_location, 334 "Attribute 'byte_order' may only be 'Null' for one-byte fields.")]) 335 336 337def _verify_requires_attribute_on_field(field, source_file_name, ir, errors): 338 """Verifies that [requires] is valid on the given field.""" 339 requires_attr = ir_util.get_attribute(field.attribute, attributes.REQUIRES) 340 if not requires_attr: 341 return 342 if ir_util.field_is_virtual(field): 343 field_expression_type = field.read_transform.type 344 else: 345 if not field.type.HasField("atomic_type"): 346 errors.append([ 347 error.error(source_file_name, requires_attr.source_location, 348 "Attribute 'requires' is only allowed on integer, " 349 "enumeration, or boolean fields, not arrays."), 350 error.note(source_file_name, field.type.source_location, 351 "Field type."), 352 ]) 353 return 354 field_type = ir_util.find_object(field.type.atomic_type.reference, ir) 355 assert field_type, "Field type should be non-None after name resolution." 356 field_expression_type = ( 357 type_check.unbounded_expression_type_for_physical_type(field_type)) 358 if field_expression_type.WhichOneof("type") not in ( 359 "integer", "enumeration", "boolean"): 360 errors.append([error.error( 361 source_file_name, requires_attr.source_location, 362 "Attribute 'requires' is only allowed on integer, enumeration, or " 363 "boolean fields.")]) 364 365 366def _verify_addressable_unit_attribute_on_external(external, type_definition, 367 source_file_name, errors): 368 """Verifies the addressable_unit_size attribute on an external.""" 369 del external # Unused. 370 addressable_unit_size_attr = ir_util.get_integer_attribute( 371 type_definition.attribute, attributes.ADDRESSABLE_UNIT_SIZE) 372 if addressable_unit_size_attr is None: 373 errors.append([error.error( 374 source_file_name, type_definition.source_location, 375 "Expected '{}' attribute for external type.".format( 376 attributes.ADDRESSABLE_UNIT_SIZE))]) 377 elif addressable_unit_size_attr not in (1, 8): 378 errors.append([ 379 error.error(source_file_name, type_definition.source_location, 380 "Only values '1' (bit) and '8' (byte) are allowed for the " 381 "'{}' attribute".format(attributes.ADDRESSABLE_UNIT_SIZE)) 382 ]) 383 384 385def _verify_width_attribute_on_enum(enum, type_definition, source_file_name, 386 errors): 387 """Verifies the maximum_bits attribute for an enum TypeDefinition.""" 388 max_bits_value = ir_util.get_integer_attribute(type_definition.attribute, 389 attributes.ENUM_MAXIMUM_BITS) 390 # The attribute should already have been defaulted, if not originally present. 391 assert max_bits_value is not None, "maximum_bits not set" 392 if max_bits_value > 64 or max_bits_value < 1: 393 max_bits_attr = ir_util.get_attribute(type_definition.attribute, 394 attributes.ENUM_MAXIMUM_BITS) 395 errors.append([ 396 error.error(source_file_name, max_bits_attr.source_location, 397 "'maximum_bits' on an 'enum' must be between 1 and 64.") 398 ]) 399 400 401def _add_missing_attributes_on_ir(ir): 402 """Adds missing attributes in a complete IR.""" 403 traverse_ir.fast_traverse_ir_top_down( 404 ir, [ir_data.Module], _add_missing_back_ends_to_module) 405 traverse_ir.fast_traverse_ir_top_down( 406 ir, [ir_data.External], _add_addressable_unit_to_external) 407 traverse_ir.fast_traverse_ir_top_down( 408 ir, [ir_data.Enum], _add_missing_width_and_sign_attributes_on_enum) 409 traverse_ir.fast_traverse_ir_top_down( 410 ir, [ir_data.Structure], _add_missing_size_attributes_on_structure, 411 incidental_actions={ 412 ir_data.Module: attribute_util.gather_default_attributes, 413 ir_data.TypeDefinition: attribute_util.gather_default_attributes, 414 ir_data.Field: attribute_util.gather_default_attributes, 415 }, 416 parameters={"defaults": {}}) 417 traverse_ir.fast_traverse_ir_top_down( 418 ir, [ir_data.Field], _add_missing_byte_order_attribute_on_field, 419 incidental_actions={ 420 ir_data.Module: attribute_util.gather_default_attributes, 421 ir_data.TypeDefinition: attribute_util.gather_default_attributes, 422 ir_data.Field: attribute_util.gather_default_attributes, 423 }, 424 parameters={"defaults": {}}) 425 return [] 426 427 428def _verify_field_attributes(field, type_definition, source_file_name, ir, 429 errors): 430 _verify_byte_order_attribute_on_field(field, type_definition, 431 source_file_name, ir, errors) 432 _verify_requires_attribute_on_field(field, source_file_name, ir, errors) 433 434 435def _verify_back_end_attributes(attribute, expected_back_ends, source_file_name, 436 ir, errors): 437 back_end_text = ir_data_utils.reader(attribute).back_end.text 438 if back_end_text not in expected_back_ends: 439 expected_back_ends_for_error = expected_back_ends - {""} 440 errors.append([error.error( 441 source_file_name, attribute.back_end.source_location, 442 "Back end specifier '{back_end}' does not match any expected back end " 443 "specifier for this file: '{expected_back_ends}'. Add or update the " 444 "'[expected_back_ends: \"{new_expected_back_ends}\"]' attribute at the " 445 "file level if this back end specifier is intentional.".format( 446 back_end=attribute.back_end.text, 447 expected_back_ends="', '".join( 448 sorted(expected_back_ends_for_error)), 449 new_expected_back_ends=", ".join( 450 sorted(expected_back_ends_for_error | {back_end_text})), 451 ))]) 452 453 454def _verify_attributes_on_ir(ir): 455 """Verifies attributes in a complete IR.""" 456 errors = [] 457 traverse_ir.fast_traverse_ir_top_down( 458 ir, [ir_data.Attribute], _verify_back_end_attributes, 459 incidental_actions={ 460 ir_data.Module: _gather_expected_back_ends, 461 }, 462 parameters={"errors": errors}) 463 traverse_ir.fast_traverse_ir_top_down( 464 ir, [ir_data.Structure], _verify_size_attributes_on_structure, 465 parameters={"errors": errors}) 466 traverse_ir.fast_traverse_ir_top_down( 467 ir, [ir_data.Enum], _verify_width_attribute_on_enum, 468 parameters={"errors": errors}) 469 traverse_ir.fast_traverse_ir_top_down( 470 ir, [ir_data.External], _verify_addressable_unit_attribute_on_external, 471 parameters={"errors": errors}) 472 traverse_ir.fast_traverse_ir_top_down( 473 ir, [ir_data.Field], _verify_field_attributes, 474 parameters={"errors": errors}) 475 return errors 476 477 478def normalize_and_verify(ir): 479 """Performs various normalizations and verifications on ir. 480 481 Checks for duplicate attributes. 482 483 Adds fixed_size_in_bits and addressable_unit_size attributes to types when 484 they are missing, and checks their correctness when they are not missing. 485 486 Arguments: 487 ir: The IR object to normalize. 488 489 Returns: 490 A list of validation errors, or an empty list if no errors were encountered. 491 """ 492 errors = attribute_util.check_attributes_in_ir( 493 ir, 494 types=_ATTRIBUTE_TYPES, 495 module_attributes=_MODULE_ATTRIBUTES, 496 struct_attributes=_STRUCT_ATTRIBUTES, 497 bits_attributes=_BITS_ATTRIBUTES, 498 enum_attributes=_ENUM_ATTRIBUTES, 499 external_attributes=_EXTERNAL_ATTRIBUTES, 500 structure_virtual_field_attributes=_STRUCT_VIRTUAL_FIELD_ATTRIBUTES, 501 structure_physical_field_attributes=_STRUCT_PHYSICAL_FIELD_ATTRIBUTES) 502 if errors: 503 return errors 504 _add_missing_attributes_on_ir(ir) 505 return _verify_attributes_on_ir(ir) 506