1"""Classes for representing properties of STIX Objects and Cyber Observables.""" 2 3import base64 4import binascii 5import copy 6import inspect 7import re 8import uuid 9 10from six import string_types, text_type 11 12import stix2 13 14from .base import _STIXBase 15from .exceptions import ( 16 CustomContentError, DictionaryKeyError, MissingPropertiesError, 17 MutuallyExclusivePropertiesError, STIXError, 18) 19from .parsing import STIX2_OBJ_MAPS, parse, parse_observable 20from .utils import _get_dict, get_class_hierarchy_names, parse_into_datetime 21 22try: 23 from collections.abc import Mapping 24except ImportError: 25 from collections import Mapping 26 27TYPE_REGEX = re.compile(r'^\-?[a-z0-9]+(-[a-z0-9]+)*\-?$') 28TYPE_21_REGEX = re.compile(r'^([a-z][a-z0-9]*)+(-[a-z0-9]+)*\-?$') 29ERROR_INVALID_ID = ( 30 "not a valid STIX identifier, must match <object-type>--<UUID>: {}" 31) 32 33 34def _check_uuid(uuid_str, spec_version): 35 """ 36 Check whether the given UUID string is valid with respect to the given STIX 37 spec version. STIX 2.0 requires UUIDv4; 2.1 only requires the RFC 4122 38 variant. 39 40 :param uuid_str: A UUID as a string 41 :param spec_version: The STIX spec version 42 :return: True if the UUID is valid, False if not 43 :raises ValueError: If uuid_str is malformed 44 """ 45 uuid_obj = uuid.UUID(uuid_str) 46 47 ok = uuid_obj.variant == uuid.RFC_4122 48 if ok and spec_version == "2.0": 49 ok = uuid_obj.version == 4 50 51 return ok 52 53 54def _validate_id(id_, spec_version, required_prefix): 55 """ 56 Check the STIX identifier for correctness, raise an exception if there are 57 errors. 58 59 :param id_: The STIX identifier 60 :param spec_version: The STIX specification version to use 61 :param required_prefix: The required prefix on the identifier, if any. 62 This function doesn't add a "--" suffix to the prefix, so callers must 63 add it if it is important. Pass None to skip the prefix check. 64 :raises ValueError: If there are any errors with the identifier 65 """ 66 if required_prefix: 67 if not id_.startswith(required_prefix): 68 raise ValueError("must start with '{}'.".format(required_prefix)) 69 70 try: 71 if required_prefix: 72 uuid_part = id_[len(required_prefix):] 73 else: 74 idx = id_.index("--") 75 uuid_part = id_[idx+2:] 76 77 result = _check_uuid(uuid_part, spec_version) 78 except ValueError: 79 # replace their ValueError with ours 80 raise ValueError(ERROR_INVALID_ID.format(id_)) 81 82 if not result: 83 raise ValueError(ERROR_INVALID_ID.format(id_)) 84 85 86def _validate_type(type_, spec_version): 87 """ 88 Check the STIX type name for correctness, raise an exception if there are 89 errors. 90 91 :param type_: The STIX type name 92 :param spec_version: The STIX specification version to use 93 :raises ValueError: If there are any errors with the identifier 94 """ 95 if spec_version == "2.0": 96 if not re.match(TYPE_REGEX, type_): 97 raise ValueError( 98 "Invalid type name '%s': must only contain the " 99 "characters a-z (lowercase ASCII), 0-9, and hyphen (-)." % 100 type_, 101 ) 102 else: # 2.1+ 103 if not re.match(TYPE_21_REGEX, type_): 104 raise ValueError( 105 "Invalid type name '%s': must only contain the " 106 "characters a-z (lowercase ASCII), 0-9, and hyphen (-) " 107 "and must begin with an a-z character" % type_, 108 ) 109 110 if len(type_) < 3 or len(type_) > 250: 111 raise ValueError( 112 "Invalid type name '%s': must be between 3 and 250 characters." % type_, 113 ) 114 115 116class Property(object): 117 """Represent a property of STIX data type. 118 119 Subclasses can define the following attributes as keyword arguments to 120 ``__init__()``. 121 122 Args: 123 required (bool): If ``True``, the property must be provided when 124 creating an object with that property. No default value exists for 125 these properties. (Default: ``False``) 126 fixed: This provides a constant default value. Users are free to 127 provide this value explicity when constructing an object (which 128 allows you to copy **all** values from an existing object to a new 129 object), but if the user provides a value other than the ``fixed`` 130 value, it will raise an error. This is semantically equivalent to 131 defining both: 132 133 - a ``clean()`` function that checks if the value matches the fixed 134 value, and 135 - a ``default()`` function that returns the fixed value. 136 137 Subclasses can also define the following functions: 138 139 - ``def clean(self, value) -> any:`` 140 - Return a value that is valid for this property. If ``value`` is not 141 valid for this property, this will attempt to transform it first. If 142 ``value`` is not valid and no such transformation is possible, it 143 should raise an exception. 144 - ``def default(self):`` 145 - provide a default value for this property. 146 - ``default()`` can return the special value ``NOW`` to use the current 147 time. This is useful when several timestamps in the same object 148 need to use the same default value, so calling now() for each 149 property-- likely several microseconds apart-- does not work. 150 151 Subclasses can instead provide a lambda function for ``default`` as a 152 keyword argument. ``clean`` should not be provided as a lambda since 153 lambdas cannot raise their own exceptions. 154 155 When instantiating Properties, ``required`` and ``default`` should not be 156 used together. ``default`` implies that the property is required in the 157 specification so this function will be used to supply a value if none is 158 provided. ``required`` means that the user must provide this; it is 159 required in the specification and we can't or don't want to create a 160 default value. 161 162 """ 163 164 def _default_clean(self, value): 165 if value != self._fixed_value: 166 raise ValueError("must equal '{}'.".format(self._fixed_value)) 167 return value 168 169 def __init__(self, required=False, fixed=None, default=None): 170 self.required = required 171 172 if required and default: 173 raise STIXError( 174 "Cant't use 'required' and 'default' together. 'required'" 175 "really means 'the user must provide this.'", 176 ) 177 178 if fixed: 179 self._fixed_value = fixed 180 self.clean = self._default_clean 181 self.default = lambda: fixed 182 if default: 183 self.default = default 184 185 def clean(self, value): 186 return value 187 188 def __call__(self, value=None): 189 """Used by ListProperty to handle lists that have been defined with 190 either a class or an instance. 191 """ 192 return value 193 194 195class ListProperty(Property): 196 197 def __init__(self, contained, **kwargs): 198 """ 199 ``contained`` should be a Property class or instance, or a _STIXBase 200 subclass. 201 """ 202 self.contained = None 203 204 if inspect.isclass(contained): 205 # Property classes are instantiated; _STIXBase subclasses are left 206 # as-is. 207 if issubclass(contained, Property): 208 self.contained = contained() 209 elif issubclass(contained, _STIXBase): 210 self.contained = contained 211 212 elif isinstance(contained, Property): 213 self.contained = contained 214 215 if not self.contained: 216 raise TypeError( 217 "Invalid list element type: {}".format( 218 str(contained), 219 ), 220 ) 221 222 super(ListProperty, self).__init__(**kwargs) 223 224 def clean(self, value): 225 try: 226 iter(value) 227 except TypeError: 228 raise ValueError("must be an iterable.") 229 230 if isinstance(value, (_STIXBase, string_types)): 231 value = [value] 232 233 if isinstance(self.contained, Property): 234 result = [ 235 self.contained.clean(item) 236 for item in value 237 ] 238 239 else: # self.contained must be a _STIXBase subclass 240 result = [] 241 for item in value: 242 if isinstance(item, self.contained): 243 valid = item 244 245 elif isinstance(item, Mapping): 246 # attempt a mapping-like usage... 247 valid = self.contained(**item) 248 249 else: 250 raise ValueError("Can't create a {} out of {}".format( 251 self.contained._type, str(item), 252 )) 253 254 result.append(valid) 255 256 # STIX spec forbids empty lists 257 if len(result) < 1: 258 raise ValueError("must not be empty.") 259 260 return result 261 262 263class StringProperty(Property): 264 265 def __init__(self, **kwargs): 266 super(StringProperty, self).__init__(**kwargs) 267 268 def clean(self, value): 269 if not isinstance(value, string_types): 270 return text_type(value) 271 return value 272 273 274class TypeProperty(Property): 275 276 def __init__(self, type, spec_version=stix2.DEFAULT_VERSION): 277 _validate_type(type, spec_version) 278 self.spec_version = spec_version 279 super(TypeProperty, self).__init__(fixed=type) 280 281 282class IDProperty(Property): 283 284 def __init__(self, type, spec_version=stix2.DEFAULT_VERSION): 285 self.required_prefix = type + "--" 286 self.spec_version = spec_version 287 super(IDProperty, self).__init__() 288 289 def clean(self, value): 290 _validate_id(value, self.spec_version, self.required_prefix) 291 return value 292 293 def default(self): 294 return self.required_prefix + str(uuid.uuid4()) 295 296 297class IntegerProperty(Property): 298 299 def __init__(self, min=None, max=None, **kwargs): 300 self.min = min 301 self.max = max 302 super(IntegerProperty, self).__init__(**kwargs) 303 304 def clean(self, value): 305 try: 306 value = int(value) 307 except Exception: 308 raise ValueError("must be an integer.") 309 310 if self.min is not None and value < self.min: 311 msg = "minimum value is {}. received {}".format(self.min, value) 312 raise ValueError(msg) 313 314 if self.max is not None and value > self.max: 315 msg = "maximum value is {}. received {}".format(self.max, value) 316 raise ValueError(msg) 317 318 return value 319 320 321class FloatProperty(Property): 322 323 def __init__(self, min=None, max=None, **kwargs): 324 self.min = min 325 self.max = max 326 super(FloatProperty, self).__init__(**kwargs) 327 328 def clean(self, value): 329 try: 330 value = float(value) 331 except Exception: 332 raise ValueError("must be a float.") 333 334 if self.min is not None and value < self.min: 335 msg = "minimum value is {}. received {}".format(self.min, value) 336 raise ValueError(msg) 337 338 if self.max is not None and value > self.max: 339 msg = "maximum value is {}. received {}".format(self.max, value) 340 raise ValueError(msg) 341 342 return value 343 344 345class BooleanProperty(Property): 346 347 def clean(self, value): 348 if isinstance(value, bool): 349 return value 350 351 trues = ['true', 't', '1'] 352 falses = ['false', 'f', '0'] 353 try: 354 if value.lower() in trues: 355 return True 356 if value.lower() in falses: 357 return False 358 except AttributeError: 359 if value == 1: 360 return True 361 if value == 0: 362 return False 363 364 raise ValueError("must be a boolean value.") 365 366 367class TimestampProperty(Property): 368 369 def __init__(self, precision="any", precision_constraint="exact", **kwargs): 370 self.precision = precision 371 self.precision_constraint = precision_constraint 372 373 super(TimestampProperty, self).__init__(**kwargs) 374 375 def clean(self, value): 376 return parse_into_datetime( 377 value, self.precision, self.precision_constraint, 378 ) 379 380 381class DictionaryProperty(Property): 382 383 def __init__(self, spec_version=stix2.DEFAULT_VERSION, **kwargs): 384 self.spec_version = spec_version 385 super(DictionaryProperty, self).__init__(**kwargs) 386 387 def clean(self, value): 388 try: 389 dictified = _get_dict(value) 390 except ValueError: 391 raise ValueError("The dictionary property must contain a dictionary") 392 for k in dictified.keys(): 393 if self.spec_version == '2.0': 394 if len(k) < 3: 395 raise DictionaryKeyError(k, "shorter than 3 characters") 396 elif len(k) > 256: 397 raise DictionaryKeyError(k, "longer than 256 characters") 398 elif self.spec_version == '2.1': 399 if len(k) > 250: 400 raise DictionaryKeyError(k, "longer than 250 characters") 401 if not re.match(r"^[a-zA-Z0-9_-]+$", k): 402 msg = ( 403 "contains characters other than lowercase a-z, " 404 "uppercase A-Z, numerals 0-9, hyphen (-), or " 405 "underscore (_)" 406 ) 407 raise DictionaryKeyError(k, msg) 408 409 if len(dictified) < 1: 410 raise ValueError("must not be empty.") 411 412 return dictified 413 414 415HASHES_REGEX = { 416 "MD5": (r"^[a-fA-F0-9]{32}$", "MD5"), 417 "MD6": (r"^[a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{56}|[a-fA-F0-9]{64}|[a-fA-F0-9]{96}|[a-fA-F0-9]{128}$", "MD6"), 418 "RIPEMD160": (r"^[a-fA-F0-9]{40}$", "RIPEMD-160"), 419 "SHA1": (r"^[a-fA-F0-9]{40}$", "SHA-1"), 420 "SHA224": (r"^[a-fA-F0-9]{56}$", "SHA-224"), 421 "SHA256": (r"^[a-fA-F0-9]{64}$", "SHA-256"), 422 "SHA384": (r"^[a-fA-F0-9]{96}$", "SHA-384"), 423 "SHA512": (r"^[a-fA-F0-9]{128}$", "SHA-512"), 424 "SHA3224": (r"^[a-fA-F0-9]{56}$", "SHA3-224"), 425 "SHA3256": (r"^[a-fA-F0-9]{64}$", "SHA3-256"), 426 "SHA3384": (r"^[a-fA-F0-9]{96}$", "SHA3-384"), 427 "SHA3512": (r"^[a-fA-F0-9]{128}$", "SHA3-512"), 428 "SSDEEP": (r"^[a-zA-Z0-9/+:.]{1,128}$", "SSDEEP"), 429 "WHIRLPOOL": (r"^[a-fA-F0-9]{128}$", "WHIRLPOOL"), 430 "TLSH": (r"^[a-fA-F0-9]{70}$", "TLSH"), 431} 432 433 434class HashesProperty(DictionaryProperty): 435 436 def clean(self, value): 437 clean_dict = super(HashesProperty, self).clean(value) 438 for k, v in copy.deepcopy(clean_dict).items(): 439 key = k.upper().replace('-', '') 440 if key in HASHES_REGEX: 441 vocab_key = HASHES_REGEX[key][1] 442 if vocab_key == "SSDEEP" and self.spec_version == "2.0": 443 vocab_key = vocab_key.lower() 444 if not re.match(HASHES_REGEX[key][0], v): 445 raise ValueError("'{0}' is not a valid {1} hash".format(v, vocab_key)) 446 if k != vocab_key: 447 clean_dict[vocab_key] = clean_dict[k] 448 del clean_dict[k] 449 return clean_dict 450 451 452class BinaryProperty(Property): 453 454 def clean(self, value): 455 try: 456 base64.b64decode(value) 457 except (binascii.Error, TypeError): 458 raise ValueError("must contain a base64 encoded string") 459 return value 460 461 462class HexProperty(Property): 463 464 def clean(self, value): 465 if not re.match(r"^([a-fA-F0-9]{2})+$", value): 466 raise ValueError("must contain an even number of hexadecimal characters") 467 return value 468 469 470class ReferenceProperty(Property): 471 472 def __init__(self, valid_types=None, invalid_types=None, spec_version=stix2.DEFAULT_VERSION, **kwargs): 473 """ 474 references sometimes must be to a specific object type 475 """ 476 self.spec_version = spec_version 477 478 # These checks need to be done prior to the STIX object finishing construction 479 # and thus we can't use base.py's _check_mutually_exclusive_properties() 480 # in the typical location of _check_object_constraints() in sdo.py 481 if valid_types and invalid_types: 482 raise MutuallyExclusivePropertiesError(self.__class__, ['invalid_types', 'valid_types']) 483 elif valid_types is None and invalid_types is None: 484 raise MissingPropertiesError(self.__class__, ['invalid_types', 'valid_types']) 485 486 if valid_types and type(valid_types) is not list: 487 valid_types = [valid_types] 488 elif invalid_types and type(invalid_types) is not list: 489 invalid_types = [invalid_types] 490 491 self.valid_types = valid_types 492 self.invalid_types = invalid_types 493 494 super(ReferenceProperty, self).__init__(**kwargs) 495 496 def clean(self, value): 497 if isinstance(value, _STIXBase): 498 value = value.id 499 value = str(value) 500 501 possible_prefix = value[:value.index('--')] 502 503 if self.valid_types: 504 ref_valid_types = enumerate_types(self.valid_types, 'v' + self.spec_version.replace(".", "")) 505 506 if possible_prefix in ref_valid_types: 507 required_prefix = possible_prefix 508 else: 509 raise ValueError("The type-specifying prefix '%s' for this property is not valid" % (possible_prefix)) 510 elif self.invalid_types: 511 ref_invalid_types = enumerate_types(self.invalid_types, 'v' + self.spec_version.replace(".", "")) 512 513 if possible_prefix not in ref_invalid_types: 514 required_prefix = possible_prefix 515 else: 516 raise ValueError("An invalid type-specifying prefix '%s' was specified for this property" % (possible_prefix)) 517 518 _validate_id(value, self.spec_version, required_prefix) 519 520 return value 521 522 523def enumerate_types(types, spec_version): 524 """ 525 `types` is meant to be a list; it may contain specific object types and/or 526 the any of the words "SCO", "SDO", or "SRO" 527 528 Since "SCO", "SDO", and "SRO" are general types that encompass various specific object types, 529 once each of those words is being processed, that word will be removed from `return_types`, 530 so as not to mistakenly allow objects to be created of types "SCO", "SDO", or "SRO" 531 """ 532 return_types = [] 533 return_types += types 534 535 if "SDO" in types: 536 return_types.remove("SDO") 537 return_types += STIX2_OBJ_MAPS[spec_version]['objects'].keys() 538 if "SCO" in types: 539 return_types.remove("SCO") 540 return_types += STIX2_OBJ_MAPS[spec_version]['observables'].keys() 541 if "SRO" in types: 542 return_types.remove("SRO") 543 return_types += ['relationship', 'sighting'] 544 545 return return_types 546 547 548SELECTOR_REGEX = re.compile(r"^([a-z0-9_-]{3,250}(\.(\[\d+\]|[a-z0-9_-]{1,250}))*|id)$") 549 550 551class SelectorProperty(Property): 552 553 def clean(self, value): 554 if not SELECTOR_REGEX.match(value): 555 raise ValueError("must adhere to selector syntax.") 556 return value 557 558 559class ObjectReferenceProperty(StringProperty): 560 561 def __init__(self, valid_types=None, **kwargs): 562 if valid_types and type(valid_types) is not list: 563 valid_types = [valid_types] 564 self.valid_types = valid_types 565 super(ObjectReferenceProperty, self).__init__(**kwargs) 566 567 568class EmbeddedObjectProperty(Property): 569 570 def __init__(self, type, **kwargs): 571 self.type = type 572 super(EmbeddedObjectProperty, self).__init__(**kwargs) 573 574 def clean(self, value): 575 if type(value) is dict: 576 value = self.type(**value) 577 elif not isinstance(value, self.type): 578 raise ValueError("must be of type {}.".format(self.type.__name__)) 579 return value 580 581 582class EnumProperty(StringProperty): 583 584 def __init__(self, allowed, **kwargs): 585 if type(allowed) is not list: 586 allowed = list(allowed) 587 self.allowed = allowed 588 super(EnumProperty, self).__init__(**kwargs) 589 590 def clean(self, value): 591 cleaned_value = super(EnumProperty, self).clean(value) 592 if cleaned_value not in self.allowed: 593 raise ValueError("value '{}' is not valid for this enumeration.".format(cleaned_value)) 594 595 return cleaned_value 596 597 598class PatternProperty(StringProperty): 599 pass 600 601 602class ObservableProperty(Property): 603 """Property for holding Cyber Observable Objects. 604 """ 605 606 def __init__(self, spec_version=stix2.DEFAULT_VERSION, allow_custom=False, *args, **kwargs): 607 self.allow_custom = allow_custom 608 self.spec_version = spec_version 609 super(ObservableProperty, self).__init__(*args, **kwargs) 610 611 def clean(self, value): 612 try: 613 dictified = _get_dict(value) 614 # get deep copy since we are going modify the dict and might 615 # modify the original dict as _get_dict() does not return new 616 # dict when passed a dict 617 dictified = copy.deepcopy(dictified) 618 except ValueError: 619 raise ValueError("The observable property must contain a dictionary") 620 if dictified == {}: 621 raise ValueError("The observable property must contain a non-empty dictionary") 622 623 valid_refs = dict((k, v['type']) for (k, v) in dictified.items()) 624 625 for key, obj in dictified.items(): 626 parsed_obj = parse_observable( 627 obj, 628 valid_refs, 629 allow_custom=self.allow_custom, 630 version=self.spec_version, 631 ) 632 dictified[key] = parsed_obj 633 634 return dictified 635 636 637class ExtensionsProperty(DictionaryProperty): 638 """Property for representing extensions on Observable objects. 639 """ 640 641 def __init__(self, spec_version=stix2.DEFAULT_VERSION, allow_custom=False, enclosing_type=None, required=False): 642 self.allow_custom = allow_custom 643 self.enclosing_type = enclosing_type 644 super(ExtensionsProperty, self).__init__(spec_version=spec_version, required=required) 645 646 def clean(self, value): 647 try: 648 dictified = _get_dict(value) 649 # get deep copy since we are going modify the dict and might 650 # modify the original dict as _get_dict() does not return new 651 # dict when passed a dict 652 dictified = copy.deepcopy(dictified) 653 except ValueError: 654 raise ValueError("The extensions property must contain a dictionary") 655 656 v = 'v' + self.spec_version.replace('.', '') 657 658 specific_type_map = STIX2_OBJ_MAPS[v]['observable-extensions'].get(self.enclosing_type, {}) 659 for key, subvalue in dictified.items(): 660 if key in specific_type_map: 661 cls = specific_type_map[key] 662 if type(subvalue) is dict: 663 if self.allow_custom: 664 subvalue['allow_custom'] = True 665 dictified[key] = cls(**subvalue) 666 else: 667 dictified[key] = cls(**subvalue) 668 elif type(subvalue) is cls: 669 # If already an instance of an _Extension class, assume it's valid 670 dictified[key] = subvalue 671 else: 672 raise ValueError("Cannot determine extension type.") 673 else: 674 if self.allow_custom: 675 dictified[key] = subvalue 676 else: 677 raise CustomContentError("Can't parse unknown extension type: {}".format(key)) 678 return dictified 679 680 681class STIXObjectProperty(Property): 682 683 def __init__(self, spec_version=stix2.DEFAULT_VERSION, allow_custom=False, *args, **kwargs): 684 self.allow_custom = allow_custom 685 self.spec_version = spec_version 686 super(STIXObjectProperty, self).__init__(*args, **kwargs) 687 688 def clean(self, value): 689 # Any STIX Object (SDO, SRO, or Marking Definition) can be added to 690 # a bundle with no further checks. 691 if any(x in ('_DomainObject', '_RelationshipObject', 'MarkingDefinition') 692 for x in get_class_hierarchy_names(value)): 693 # A simple "is this a spec version 2.1+ object" test. For now, 694 # limit 2.0 bundles to 2.0 objects. It's not possible yet to 695 # have validation co-constraints among properties, e.g. have 696 # validation here depend on the value of another property 697 # (spec_version). So this is a hack, and not technically spec- 698 # compliant. 699 if 'spec_version' in value and self.spec_version == '2.0': 700 raise ValueError( 701 "Spec version 2.0 bundles don't yet support " 702 "containing objects of a different spec " 703 "version.", 704 ) 705 return value 706 try: 707 dictified = _get_dict(value) 708 except ValueError: 709 raise ValueError("This property may only contain a dictionary or object") 710 if dictified == {}: 711 raise ValueError("This property may only contain a non-empty dictionary or object") 712 if 'type' in dictified and dictified['type'] == 'bundle': 713 raise ValueError("This property may not contain a Bundle object") 714 if 'spec_version' in dictified and self.spec_version == '2.0': 715 # See above comment regarding spec_version. 716 raise ValueError( 717 "Spec version 2.0 bundles don't yet support " 718 "containing objects of a different spec version.", 719 ) 720 721 parsed_obj = parse(dictified, allow_custom=self.allow_custom) 722 723 return parsed_obj 724