1#!/usr/bin/env python 2 3""" 4<Program Name> 5 formats.py 6 7<Author> 8 Geremy Condra 9 Vladimir Diaz <vladimir.v.diaz@gmail.com> 10 11<Started> 12 Refactored April 30, 2012. -vladimir.v.diaz 13 14<Copyright> 15 See LICENSE for licensing information. 16 17<Purpose> 18 A central location for all format-related checking of TUF objects. 19 Note: 'formats.py' depends heavily on 'schema.py', so the 'schema.py' 20 module should be read and understood before tackling this module. 21 22 'formats.py' can be broken down into three sections. (1) Schemas and object 23 matching. (2) Classes that represent Role Metadata and help produce 24 correctly formatted files. (3) Functions that help produce or verify TUF 25 objects. 26 27 The first section deals with schemas and object matching based on format. 28 There are two ways of checking the format of objects. The first method 29 raises a 'securesystemslib.exceptions.FormatError' exception if the match 30 fails and the other returns a Boolean result. 31 32 securesystemslib.formats.<SCHEMA>.check_match(object) 33 securesystemslib.formats.<SCHEMA>.matches(object) 34 35 Example: 36 37 rsa_key = {'keytype': 'rsa' 38 'keyid': 34892fc465ac76bc3232fab 39 'keyval': {'public': 'public_key', 40 'private': 'private_key'} 41 42 securesystemslib.formats.RSAKEY_SCHEMA.check_match(rsa_key) 43 securesystemslib.formats.RSAKEY_SCHEMA.matches(rsa_key) 44 45 In this example, if a dict key or dict value is missing or incorrect, 46 the match fails. There are numerous variations of object checking 47 provided by 'formats.py' and 'schema.py'. 48 49 The second section deals with the role metadata classes. There are 50 multiple top-level roles, each with differing metadata formats. 51 Example: 52 53 root_object = securesystemslib.formats.RootFile.from_metadata(root_metadata_file) 54 targets_metadata = securesystemslib.formats.TargetsFile.make_metadata(...) 55 56 The input and output of these classes are checked against their respective 57 schema to ensure correctly formatted metadata. 58 59 The last section contains miscellaneous functions related to the format of 60 TUF objects. 61 Example: 62 63 signable_object = make_signable(unsigned_object) 64""" 65 66# Help with Python 3 compatibility, where the print statement is a function, an 67# implicit relative import is invalid, and the '/' operator performs true 68# division. Example: print 'hello world' raises a 'SyntaxError' exception. 69from __future__ import print_function 70from __future__ import absolute_import 71from __future__ import division 72from __future__ import unicode_literals 73 74import binascii 75import calendar 76import re 77import string 78import datetime 79import time 80import six 81 82import securesystemslib.schema as SCHEMA 83import securesystemslib.exceptions 84 85# Note that in the schema definitions below, the 'SCHEMA.Object' types allow 86# additional keys which are not defined. Thus, any additions to them will be 87# easily backwards compatible with clients that are already deployed. 88 89# A datetime in 'YYYY-MM-DDTHH:MM:SSZ' ISO 8601 format. The "Z" zone designator 90# for the zero UTC offset is always used (i.e., a numerical offset is not 91# supported.) Example: '2015-10-21T13:20:00Z'. Note: This is a simple format 92# check, and an ISO8601 string should be fully verified when it is parsed. 93ISO8601_DATETIME_SCHEMA = SCHEMA.RegularExpression(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z') 94 95# A Unix/POSIX time format. An integer representing the number of seconds 96# since the epoch (January 1, 1970.) Metadata uses this format for the 97# 'expires' field. Set 'hi' to the upper timestamp limit (year 2038), the max 98# value of an int. 99UNIX_TIMESTAMP_SCHEMA = SCHEMA.Integer(lo=0, hi=2147483647) 100 101# A hexadecimal value in '23432df87ab..' format. 102HASH_SCHEMA = SCHEMA.RegularExpression(r'[a-fA-F0-9]+') 103 104# A dict in {'sha256': '23432df87ab..', 'sha512': '34324abc34df..', ...} format. 105HASHDICT_SCHEMA = SCHEMA.DictOf( 106 key_schema = SCHEMA.AnyString(), 107 value_schema = HASH_SCHEMA) 108 109# A hexadecimal value in '23432df87ab..' format. 110HEX_SCHEMA = SCHEMA.RegularExpression(r'[a-fA-F0-9]+') 111 112# A key identifier (e.g., a hexadecimal value identifying an RSA key). 113KEYID_SCHEMA = HASH_SCHEMA 114 115# A list of KEYID_SCHEMA. 116KEYIDS_SCHEMA = SCHEMA.ListOf(KEYID_SCHEMA) 117 118# The signing scheme used by a key to generate a signature (e.g., 119# 'rsassa-pss-sha256' is one of the signing schemes for key type 'rsa'). 120SCHEME_SCHEMA = SCHEMA.AnyString() 121 122# A relative file path (e.g., 'metadata/root/'). 123RELPATH_SCHEMA = SCHEMA.AnyString() 124RELPATHS_SCHEMA = SCHEMA.ListOf(RELPATH_SCHEMA) 125 126# An absolute path. 127PATH_SCHEMA = SCHEMA.AnyString() 128PATHS_SCHEMA = SCHEMA.ListOf(PATH_SCHEMA) 129 130# Uniform Resource Locator identifier (e.g., 'https://www.updateframework.com/'). 131URL_SCHEMA = SCHEMA.AnyString() 132 133# A dictionary holding version information. 134VERSION_SCHEMA = SCHEMA.Object( 135 object_name = 'VERSION_SCHEMA', 136 major = SCHEMA.Integer(lo=0), 137 minor = SCHEMA.Integer(lo=0), 138 fix = SCHEMA.Integer(lo=0)) 139 140# An integer representing the numbered version of a metadata file. 141# Must be 1, or greater. 142METADATAVERSION_SCHEMA = SCHEMA.Integer(lo=0) 143 144# An integer representing length. Must be 0, or greater. 145LENGTH_SCHEMA = SCHEMA.Integer(lo=0) 146 147# An integer representing logger levels, such as logging.CRITICAL (=50). 148# Must be between 0 and 50. 149LOGLEVEL_SCHEMA = SCHEMA.Integer(lo=0, hi=50) 150 151# A string representing a named object. 152NAME_SCHEMA = SCHEMA.AnyString() 153NAMES_SCHEMA = SCHEMA.ListOf(NAME_SCHEMA) 154 155# A byte string representing data. 156DATA_SCHEMA = SCHEMA.AnyBytes() 157 158# A text string. For instance, a string entered by the user. 159TEXT_SCHEMA = SCHEMA.AnyString() 160 161# Supported hash algorithms. 162HASHALGORITHMS_SCHEMA = SCHEMA.ListOf(SCHEMA.OneOf( 163 [SCHEMA.String('md5'), SCHEMA.String('sha1'), 164 SCHEMA.String('sha224'), SCHEMA.String('sha256'), 165 SCHEMA.String('sha384'), SCHEMA.String('sha512')])) 166 167# The contents of an encrypted TUF key. Encrypted TUF keys are saved to files 168# in this format. 169ENCRYPTEDKEY_SCHEMA = SCHEMA.AnyString() 170 171# A value that is either True or False, on or off, etc. 172BOOLEAN_SCHEMA = SCHEMA.Boolean() 173 174# A role's threshold value (i.e., the minimum number 175# of signatures required to sign a metadata file). 176# Must be 1 and greater. 177THRESHOLD_SCHEMA = SCHEMA.Integer(lo=1) 178 179# A string representing a role's name. 180ROLENAME_SCHEMA = SCHEMA.AnyString() 181 182# The minimum number of bits for an RSA key. Must be 2048 bits, or greater 183# (recommended by TUF). Recommended RSA key sizes: 184# http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 185RSAKEYBITS_SCHEMA = SCHEMA.Integer(lo=2048) 186 187# The supported ECDSA signature schemes (ecdsa-sha2-nistp256 is supported by 188# default). 189ECDSA_SCHEME_SCHEMA = SCHEMA.OneOf([SCHEMA.String('ecdsa-sha2-nistp256')]) 190 191# The number of hashed bins, or the number of delegated roles. See 192# delegate_hashed_bins() in 'repository_tool.py' for an example. Note: 193# Tools may require further restrictions on the number of bins, such 194# as requiring them to be a power of 2. 195NUMBINS_SCHEMA = SCHEMA.Integer(lo=1) 196 197# A pyca-cryptography signature. 198PYCACRYPTOSIGNATURE_SCHEMA = SCHEMA.AnyBytes() 199 200# An RSA key in PEM format. 201PEMRSA_SCHEMA = SCHEMA.AnyString() 202 203# An ECDSA key in PEM format. 204PEMECDSA_SCHEMA = SCHEMA.AnyString() 205 206# A string representing a password. 207PASSWORD_SCHEMA = SCHEMA.AnyString() 208 209# A list of passwords. 210PASSWORDS_SCHEMA = SCHEMA.ListOf(PASSWORD_SCHEMA) 211 212# The actual values of a key, as opposed to meta data such as a key type and 213# key identifier ('rsa', 233df889cb). For RSA keys, the key value is a pair of 214# public and private keys in PEM Format stored as strings. 215KEYVAL_SCHEMA = SCHEMA.Object( 216 object_name = 'KEYVAL_SCHEMA', 217 public = SCHEMA.AnyString(), 218 private = SCHEMA.Optional(SCHEMA.AnyString())) 219 220# Public keys CAN have a private portion (for backwards compatibility) which 221# MUST be an empty string 222PUBLIC_KEYVAL_SCHEMA = SCHEMA.Object( 223 object_name = 'KEYVAL_SCHEMA', 224 public = SCHEMA.AnyString(), 225 private = SCHEMA.Optional(SCHEMA.String(""))) 226 227# Supported TUF key types. 228KEYTYPE_SCHEMA = SCHEMA.OneOf( 229 [SCHEMA.String('rsa'), SCHEMA.String('ed25519'), 230 SCHEMA.String('ecdsa-sha2-nistp256')]) 231 232# A generic TUF key. All TUF keys should be saved to metadata files in this 233# format. 234KEY_SCHEMA = SCHEMA.Object( 235 object_name = 'KEY_SCHEMA', 236 keytype = SCHEMA.AnyString(), 237 scheme = SCHEME_SCHEMA, 238 keyval = KEYVAL_SCHEMA, 239 expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA)) 240 241# Like KEY_SCHEMA, but requires keyval's private portion to be unset or empty, 242# and optionally includes the supported keyid hash algorithms used to generate 243# the key's keyid. 244PUBLIC_KEY_SCHEMA = SCHEMA.Object( 245 object_name = 'PUBLIC_KEY_SCHEMA', 246 keytype = SCHEMA.AnyString(), 247 keyid_hash_algorithms = SCHEMA.Optional(HASHALGORITHMS_SCHEMA), 248 keyval = PUBLIC_KEYVAL_SCHEMA, 249 expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA)) 250 251# A TUF key object. This schema simplifies validation of keys that may be one 252# of the supported key types. Supported key types: 'rsa', 'ed25519'. 253ANYKEY_SCHEMA = SCHEMA.Object( 254 object_name = 'ANYKEY_SCHEMA', 255 keytype = KEYTYPE_SCHEMA, 256 scheme = SCHEME_SCHEMA, 257 keyid = KEYID_SCHEMA, 258 keyid_hash_algorithms = SCHEMA.Optional(HASHALGORITHMS_SCHEMA), 259 keyval = KEYVAL_SCHEMA, 260 expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA)) 261 262# A list of TUF key objects. 263ANYKEYLIST_SCHEMA = SCHEMA.ListOf(ANYKEY_SCHEMA) 264 265# RSA signature schemes. 266RSA_SCHEME_SCHEMA = SCHEMA.OneOf([SCHEMA.String('rsassa-pss-sha256')]) 267 268# An RSA TUF key. 269RSAKEY_SCHEMA = SCHEMA.Object( 270 object_name = 'RSAKEY_SCHEMA', 271 keytype = SCHEMA.String('rsa'), 272 scheme = RSA_SCHEME_SCHEMA, 273 keyid = KEYID_SCHEMA, 274 keyid_hash_algorithms = SCHEMA.Optional(HASHALGORITHMS_SCHEMA), 275 keyval = KEYVAL_SCHEMA) 276 277# An ECDSA TUF key. 278ECDSAKEY_SCHEMA = SCHEMA.Object( 279 object_name = 'ECDSAKEY_SCHEMA', 280 keytype = SCHEMA.String('ecdsa-sha2-nistp256'), 281 scheme = ECDSA_SCHEME_SCHEMA, 282 keyid = KEYID_SCHEMA, 283 keyid_hash_algorithms = SCHEMA.Optional(HASHALGORITHMS_SCHEMA), 284 keyval = KEYVAL_SCHEMA) 285 286# An ED25519 raw public key, which must be 32 bytes. 287ED25519PUBLIC_SCHEMA = SCHEMA.LengthBytes(32) 288 289# An ED25519 raw seed key, which must be 32 bytes. 290ED25519SEED_SCHEMA = SCHEMA.LengthBytes(32) 291 292# An ED25519 raw signature, which must be 64 bytes. 293ED25519SIGNATURE_SCHEMA = SCHEMA.LengthBytes(64) 294 295# An ECDSA signature. 296ECDSASIGNATURE_SCHEMA = SCHEMA.AnyBytes() 297 298# Required installation libraries expected by the repository tools and other 299# cryptography modules. 300REQUIRED_LIBRARIES_SCHEMA = SCHEMA.ListOf(SCHEMA.OneOf( 301 [SCHEMA.String('general'), SCHEMA.String('ed25519'), SCHEMA.String('rsa'), 302 SCHEMA.String('ecdsa-sha2-nistp256')])) 303 304# Ed25519 signature schemes. The vanilla Ed25519 signature scheme is currently 305# supported. 306ED25519_SIG_SCHEMA = SCHEMA.OneOf([SCHEMA.String('ed25519')]) 307 308# An ed25519 TUF key. 309ED25519KEY_SCHEMA = SCHEMA.Object( 310 object_name = 'ED25519KEY_SCHEMA', 311 keytype = SCHEMA.String('ed25519'), 312 scheme = ED25519_SIG_SCHEMA, 313 keyid = KEYID_SCHEMA, 314 keyid_hash_algorithms = SCHEMA.Optional(HASHALGORITHMS_SCHEMA), 315 keyval = KEYVAL_SCHEMA) 316 317# Information about target files, like file length and file hash(es). This 318# schema allows the storage of multiple hashes for the same file (e.g., sha256 319# and sha512 may be computed for the same file and stored). 320FILEINFO_SCHEMA = SCHEMA.Object( 321 object_name = 'FILEINFO_SCHEMA', 322 length = LENGTH_SCHEMA, 323 hashes = HASHDICT_SCHEMA, 324 version = SCHEMA.Optional(METADATAVERSION_SCHEMA), 325 custom = SCHEMA.Optional(SCHEMA.Object())) 326 327# Version information specified in "snapshot.json" for each role available on 328# the TUF repository. The 'FILEINFO_SCHEMA' object was previously listed in 329# the snapshot role, but was switched to this object format to reduce the 330# amount of metadata that needs to be downloaded. Listing version numbers in 331# "snapshot.json" also prevents rollback attacks for roles that clients have 332# not downloaded. 333VERSIONINFO_SCHEMA = SCHEMA.Object( 334 object_name = 'VERSIONINFO_SCHEMA', 335 version = METADATAVERSION_SCHEMA) 336 337# A dict holding the version information for a particular metadata role. The 338# dict keys hold the relative file paths, and the dict values the corresponding 339# version numbers. 340VERSIONDICT_SCHEMA = SCHEMA.DictOf( 341 key_schema = RELPATH_SCHEMA, 342 value_schema = VERSIONINFO_SCHEMA) 343 344# A dict holding the information for a particular target / file. The dict keys 345# hold the relative file paths, and the dict values the corresponding file 346# information. 347FILEDICT_SCHEMA = SCHEMA.DictOf( 348 key_schema = RELPATH_SCHEMA, 349 value_schema = FILEINFO_SCHEMA) 350 351# A single signature of an object. Indicates the signature, and the KEYID of 352# the signing key. I debated making the signature schema not contain the key 353# ID and instead have the signatures of a file be a dictionary with the key 354# being the keyid and the value being the signature schema without the keyid. 355# That would be under the argument that a key should only be able to sign a 356# file once. 357SIGNATURE_SCHEMA = SCHEMA.Object( 358 object_name = 'SIGNATURE_SCHEMA', 359 keyid = KEYID_SCHEMA, 360 sig = HEX_SCHEMA) 361 362# List of SIGNATURE_SCHEMA. 363SIGNATURES_SCHEMA = SCHEMA.ListOf(SIGNATURE_SCHEMA) 364 365# A schema holding the result of checking the signatures of a particular 366# 'SIGNABLE_SCHEMA' role. 367# For example, how many of the signatures for the 'Target' role are 368# valid? This SCHEMA holds this information. See 'sig.py' for 369# more information. 370SIGNATURESTATUS_SCHEMA = SCHEMA.Object( 371 object_name = 'SIGNATURESTATUS_SCHEMA', 372 threshold = SCHEMA.Integer(), 373 good_sigs = KEYIDS_SCHEMA, 374 bad_sigs = KEYIDS_SCHEMA, 375 unknown_sigs = KEYIDS_SCHEMA, 376 untrusted_sigs = KEYIDS_SCHEMA) 377 378# A signable object. Holds the signing role and its associated signatures. 379SIGNABLE_SCHEMA = SCHEMA.Object( 380 object_name = 'SIGNABLE_SCHEMA', 381 signed = SCHEMA.Any(), 382 signatures = SCHEMA.ListOf(SIGNATURE_SCHEMA)) 383 384# A dict where the dict keys hold a keyid and the dict values a key object. 385KEYDICT_SCHEMA = SCHEMA.DictOf( 386 key_schema = KEYID_SCHEMA, 387 value_schema = KEY_SCHEMA) 388 389# The format used by the key database to store keys. The dict keys hold a key 390# identifier and the dict values any object. The key database should store 391# key objects in the values (e.g., 'RSAKEY_SCHEMA', 'DSAKEY_SCHEMA'). 392KEYDB_SCHEMA = SCHEMA.DictOf( 393 key_schema = KEYID_SCHEMA, 394 value_schema = SCHEMA.Any()) 395 396# A path hash prefix is a hexadecimal string. 397PATH_HASH_PREFIX_SCHEMA = HEX_SCHEMA 398 399# A list of path hash prefixes. 400PATH_HASH_PREFIXES_SCHEMA = SCHEMA.ListOf(PATH_HASH_PREFIX_SCHEMA) 401 402# Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1, 403# 'paths':[filepaths..]} format. 404ROLE_SCHEMA = SCHEMA.Object( 405 object_name = 'ROLE_SCHEMA', 406 name = SCHEMA.Optional(ROLENAME_SCHEMA), 407 keyids = KEYIDS_SCHEMA, 408 threshold = THRESHOLD_SCHEMA, 409 backtrack = SCHEMA.Optional(BOOLEAN_SCHEMA), 410 paths = SCHEMA.Optional(RELPATHS_SCHEMA), 411 path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA)) 412 413# A dict of roles where the dict keys are role names and the dict values holding 414# the role data/information. 415ROLEDICT_SCHEMA = SCHEMA.DictOf( 416 key_schema = ROLENAME_SCHEMA, 417 value_schema = ROLE_SCHEMA) 418 419# Like ROLEDICT_SCHEMA, except that ROLE_SCHEMA instances are stored in order. 420ROLELIST_SCHEMA = SCHEMA.ListOf(ROLE_SCHEMA) 421 422# The delegated roles of a Targets role (a parent). 423DELEGATIONS_SCHEMA = SCHEMA.Object( 424 keys = KEYDICT_SCHEMA, 425 roles = ROLELIST_SCHEMA) 426 427# The fileinfo format of targets specified in the repository and 428# developer tools. The second element of this list holds custom data about the 429# target, such as file permissions, author(s), last modified, etc. 430CUSTOM_SCHEMA = SCHEMA.Object() 431 432PATH_FILEINFO_SCHEMA = SCHEMA.DictOf( 433 key_schema = RELPATH_SCHEMA, 434 value_schema = CUSTOM_SCHEMA) 435 436# TUF roledb 437ROLEDB_SCHEMA = SCHEMA.Object( 438 object_name = 'ROLEDB_SCHEMA', 439 keyids = SCHEMA.Optional(KEYIDS_SCHEMA), 440 signing_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), 441 previous_keyids = SCHEMA.Optional(KEYIDS_SCHEMA), 442 threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), 443 previous_threshold = SCHEMA.Optional(THRESHOLD_SCHEMA), 444 version = SCHEMA.Optional(METADATAVERSION_SCHEMA), 445 expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA), 446 signatures = SCHEMA.Optional(SIGNATURES_SCHEMA), 447 paths = SCHEMA.Optional(SCHEMA.OneOf([RELPATHS_SCHEMA, PATH_FILEINFO_SCHEMA])), 448 path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA), 449 delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA), 450 partial_loaded = SCHEMA.Optional(BOOLEAN_SCHEMA)) 451 452# Root role: indicates root keys and top-level roles. 453ROOT_SCHEMA = SCHEMA.Object( 454 object_name = 'ROOT_SCHEMA', 455 _type = SCHEMA.String('root'), 456 version = METADATAVERSION_SCHEMA, 457 consistent_snapshot = BOOLEAN_SCHEMA, 458 expires = ISO8601_DATETIME_SCHEMA, 459 keys = KEYDICT_SCHEMA, 460 roles = ROLEDICT_SCHEMA) 461 462# Targets role: Indicates targets and delegates target paths to other roles. 463TARGETS_SCHEMA = SCHEMA.Object( 464 object_name = 'TARGETS_SCHEMA', 465 _type = SCHEMA.String('targets'), 466 version = METADATAVERSION_SCHEMA, 467 expires = ISO8601_DATETIME_SCHEMA, 468 targets = FILEDICT_SCHEMA, 469 delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA)) 470 471# Snapshot role: indicates the latest versions of all metadata (except 472# timestamp). 473SNAPSHOT_SCHEMA = SCHEMA.Object( 474 object_name = 'SNAPSHOT_SCHEMA', 475 _type = SCHEMA.String('snapshot'), 476 version = METADATAVERSION_SCHEMA, 477 expires = ISO8601_DATETIME_SCHEMA, 478 meta = VERSIONDICT_SCHEMA) 479 480# Timestamp role: indicates the latest version of the snapshot file. 481TIMESTAMP_SCHEMA = SCHEMA.Object( 482 object_name = 'TIMESTAMP_SCHEMA', 483 _type = SCHEMA.String('timestamp'), 484 version = METADATAVERSION_SCHEMA, 485 expires = ISO8601_DATETIME_SCHEMA, 486 meta = FILEDICT_SCHEMA) 487 488# project.cfg file: stores information about the project in a json dictionary 489PROJECT_CFG_SCHEMA = SCHEMA.Object( 490 object_name = 'PROJECT_CFG_SCHEMA', 491 project_name = SCHEMA.AnyString(), 492 layout_type = SCHEMA.OneOf([SCHEMA.String('repo-like'), SCHEMA.String('flat')]), 493 targets_location = PATH_SCHEMA, 494 metadata_location = PATH_SCHEMA, 495 prefix = PATH_SCHEMA, 496 public_keys = KEYDICT_SCHEMA, 497 threshold = SCHEMA.Integer(lo = 0, hi = 2) 498 ) 499 500# A schema containing information a repository mirror may require, 501# such as a url, the path of the directory metadata files, etc. 502MIRROR_SCHEMA = SCHEMA.Object( 503 object_name = 'MIRROR_SCHEMA', 504 url_prefix = URL_SCHEMA, 505 metadata_path = RELPATH_SCHEMA, 506 targets_path = RELPATH_SCHEMA, 507 confined_target_dirs = RELPATHS_SCHEMA, 508 custom = SCHEMA.Optional(SCHEMA.Object())) 509 510# A dictionary of mirrors where the dict keys hold the mirror's name and 511# and the dict values the mirror's data (i.e., 'MIRROR_SCHEMA'). 512# The repository class of 'updater.py' accepts dictionaries 513# of this type provided by the TUF client. 514MIRRORDICT_SCHEMA = SCHEMA.DictOf( 515 key_schema = SCHEMA.AnyString(), 516 value_schema = MIRROR_SCHEMA) 517 518# A Mirrorlist: indicates all the live mirrors, and what documents they 519# serve. 520MIRRORLIST_SCHEMA = SCHEMA.Object( 521 object_name = 'MIRRORLIST_SCHEMA', 522 _type = SCHEMA.String('mirrors'), 523 version = METADATAVERSION_SCHEMA, 524 expires = ISO8601_DATETIME_SCHEMA, 525 mirrors = SCHEMA.ListOf(MIRROR_SCHEMA)) 526 527# Any of the role schemas (e.g., TIMESTAMP_SCHEMA, SNAPSHOT_SCHEMA, etc.) 528ANYROLE_SCHEMA = SCHEMA.OneOf([ROOT_SCHEMA, TARGETS_SCHEMA, SNAPSHOT_SCHEMA, 529 TIMESTAMP_SCHEMA, MIRROR_SCHEMA]) 530 531 532 533def datetime_to_unix_timestamp(datetime_object): 534 """ 535 <Purpose> 536 Convert 'datetime_object' (in datetime.datetime()) format) to a Unix/POSIX 537 timestamp. For example, Python's time.time() returns a Unix timestamp, and 538 includes the number of microseconds. 'datetime_object' is converted to UTC. 539 540 >>> datetime_object = datetime.datetime(1985, 10, 26, 1, 22) 541 >>> timestamp = datetime_to_unix_timestamp(datetime_object) 542 >>> timestamp 543 499137720 544 545 <Arguments> 546 datetime_object: 547 The datetime.datetime() object to convert to a Unix timestamp. 548 549 <Exceptions> 550 securesystemslib.exceptions.FormatError, if 'datetime_object' is not a 551 datetime.datetime() object. 552 553 <Side Effects> 554 None. 555 556 <Returns> 557 A unix (posix) timestamp (e.g., 499137660). 558 """ 559 560 # Is 'datetime_object' a datetime.datetime() object? 561 # Raise 'securesystemslib.exceptions.FormatError' if not. 562 if not isinstance(datetime_object, datetime.datetime): 563 message = repr(datetime_object) + ' is not a datetime.datetime() object.' 564 raise securesystemslib.exceptions.FormatError(message) 565 566 unix_timestamp = calendar.timegm(datetime_object.timetuple()) 567 568 return unix_timestamp 569 570 571 572 573def unix_timestamp_to_datetime(unix_timestamp): 574 """ 575 <Purpose> 576 Convert 'unix_timestamp' (i.e., POSIX time, in UNIX_TIMESTAMP_SCHEMA format) 577 to a datetime.datetime() object. 'unix_timestamp' is the number of seconds 578 since the epoch (January 1, 1970.) 579 580 >>> datetime_object = unix_timestamp_to_datetime(1445455680) 581 >>> datetime_object 582 datetime.datetime(2015, 10, 21, 19, 28) 583 584 <Arguments> 585 unix_timestamp: 586 An integer representing the time (e.g., 1445455680). Conformant to 587 'securesystemslib.formats.UNIX_TIMESTAMP_SCHEMA'. 588 589 <Exceptions> 590 securesystemslib.exceptions.FormatError, if 'unix_timestamp' is improperly 591 formatted. 592 593 <Side Effects> 594 None. 595 596 <Returns> 597 A datetime.datetime() object corresponding to 'unix_timestamp'. 598 """ 599 600 # Is 'unix_timestamp' properly formatted? 601 # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. 602 securesystemslib.formats.UNIX_TIMESTAMP_SCHEMA.check_match(unix_timestamp) 603 604 # Convert 'unix_timestamp' to a 'time.struct_time', in UTC. The Daylight 605 # Savings Time (DST) flag is set to zero. datetime.fromtimestamp() is not 606 # used because it returns a local datetime. 607 struct_time = time.gmtime(unix_timestamp) 608 609 # Extract the (year, month, day, hour, minutes, seconds) arguments for the 610 # datetime object to be returned. 611 datetime_object = datetime.datetime(*struct_time[:6]) 612 613 return datetime_object 614 615 616 617 618def format_base64(data): 619 """ 620 <Purpose> 621 Return the base64 encoding of 'data' with whitespace and '=' signs omitted. 622 623 <Arguments> 624 data: 625 Binary or buffer of data to convert. 626 627 <Exceptions> 628 securesystemslib.exceptions.FormatError, if the base64 encoding fails or the 629 argument is invalid. 630 631 <Side Effects> 632 None. 633 634 <Returns> 635 A base64-encoded string. 636 """ 637 638 try: 639 return binascii.b2a_base64(data).decode('utf-8').rstrip('=\n ') 640 641 except (TypeError, binascii.Error) as e: 642 raise securesystemslib.exceptions.FormatError('Invalid base64' 643 ' encoding: ' + str(e)) 644 645 646 647 648def parse_base64(base64_string): 649 """ 650 <Purpose> 651 Parse a base64 encoding with whitespace and '=' signs omitted. 652 653 <Arguments> 654 base64_string: 655 A string holding a base64 value. 656 657 <Exceptions> 658 securesystemslib.exceptions.FormatError, if 'base64_string' cannot be parsed 659 due to an invalid base64 encoding. 660 661 <Side Effects> 662 None. 663 664 <Returns> 665 A byte string representing the parsed based64 encoding of 666 'base64_string'. 667 """ 668 669 if not isinstance(base64_string, six.string_types): 670 message = 'Invalid argument: '+repr(base64_string) 671 raise securesystemslib.exceptions.FormatError(message) 672 673 extra = len(base64_string) % 4 674 if extra: 675 padding = '=' * (4 - extra) 676 base64_string = base64_string + padding 677 678 try: 679 return binascii.a2b_base64(base64_string.encode('utf-8')) 680 681 except (TypeError, binascii.Error) as e: 682 raise securesystemslib.exceptions.FormatError('Invalid base64' 683 ' encoding: ' + str(e)) 684 685 686 687 688def _canonical_string_encoder(string): 689 """ 690 <Purpose> 691 Encode 'string' to canonical string format. 692 693 <Arguments> 694 string: 695 The string to encode. 696 697 <Exceptions> 698 None. 699 700 <Side Effects> 701 None. 702 703 <Returns> 704 A string with the canonical-encoded 'string' embedded. 705 """ 706 707 string = '"%s"' % re.sub(r'(["\\])', r'\\\1', string) 708 709 return string 710 711 712def _encode_canonical(object, output_function): 713 # Helper for encode_canonical. Older versions of json.encoder don't 714 # even let us replace the separators. 715 716 if isinstance(object, six.string_types): 717 output_function(_canonical_string_encoder(object)) 718 elif object is True: 719 output_function("true") 720 elif object is False: 721 output_function("false") 722 elif object is None: 723 output_function("null") 724 elif isinstance(object, six.integer_types): 725 output_function(str(object)) 726 elif isinstance(object, (tuple, list)): 727 output_function("[") 728 if len(object): 729 for item in object[:-1]: 730 _encode_canonical(item, output_function) 731 output_function(",") 732 _encode_canonical(object[-1], output_function) 733 output_function("]") 734 elif isinstance(object, dict): 735 output_function("{") 736 if len(object): 737 items = sorted(six.iteritems(object)) 738 for key, value in items[:-1]: 739 output_function(_canonical_string_encoder(key)) 740 output_function(":") 741 _encode_canonical(value, output_function) 742 output_function(",") 743 key, value = items[-1] 744 output_function(_canonical_string_encoder(key)) 745 output_function(":") 746 _encode_canonical(value, output_function) 747 output_function("}") 748 else: 749 raise securesystemslib.exceptions.FormatError('I cannot encode '+repr(object)) 750 751 752def encode_canonical(object, output_function=None): 753 """ 754 <Purpose> 755 Encode 'object' in canonical JSON form, as specified at 756 http://wiki.laptop.org/go/Canonical_JSON . It's a restricted 757 dialect of JSON in which keys are always lexically sorted, 758 there is no whitespace, floats aren't allowed, and only quote 759 and backslash get escaped. The result is encoded in UTF-8, 760 and the resulting bits are passed to output_function (if provided), 761 or joined into a string and returned. 762 763 Note: This function should be called prior to computing the hash or 764 signature of a JSON object in TUF. For example, generating a signature 765 of a signing role object such as 'ROOT_SCHEMA' is required to ensure 766 repeatable hashes are generated across different json module versions 767 and platforms. Code elsewhere is free to dump JSON objects in any format 768 they wish (e.g., utilizing indentation and single quotes around object 769 keys). These objects are only required to be in "canonical JSON" format 770 when their hashes or signatures are needed. 771 772 >>> encode_canonical("") 773 '""' 774 >>> encode_canonical([1, 2, 3]) 775 '[1,2,3]' 776 >>> encode_canonical([]) 777 '[]' 778 >>> encode_canonical({"A": [99]}) 779 '{"A":[99]}' 780 >>> encode_canonical({"x" : 3, "y" : 2}) 781 '{"x":3,"y":2}' 782 783 <Arguments> 784 object: 785 The object to be encoded. 786 787 output_function: 788 The result will be passed as arguments to 'output_function' 789 (e.g., output_function('result')). 790 791 <Exceptions> 792 securesystemslib.exceptions.FormatError, if 'object' cannot be encoded or 793 'output_function' is not callable. 794 795 <Side Effects> 796 The results are fed to 'output_function()' if 'output_function' is set. 797 798 <Returns> 799 A string representing the 'object' encoded in canonical JSON form. 800 """ 801 802 result = None 803 # If 'output_function' is unset, treat it as 804 # appending to a list. 805 if output_function is None: 806 result = [] 807 output_function = result.append 808 809 try: 810 _encode_canonical(object, output_function) 811 812 except (TypeError, securesystemslib.exceptions.FormatError) as e: 813 message = 'Could not encode ' + repr(object) + ': ' + str(e) 814 raise securesystemslib.exceptions.FormatError(message) 815 816 # Return the encoded 'object' as a string. 817 # Note: Implies 'output_function' is None, 818 # otherwise results are sent to 'output_function'. 819 if result is not None: 820 return ''.join(result) 821