1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 5# 6 7""" 8Move config options from headers to defconfig files. 9 10Since Kconfig was introduced to U-Boot, we have worked on moving 11config options from headers to Kconfig (defconfig). 12 13This tool intends to help this tremendous work. 14 15Installing 16---------- 17 18You may need to install 'python3-asteval' for the 'asteval' module. 19 20Usage 21----- 22 23First, you must edit the Kconfig to add the menu entries for the configs 24you are moving. 25 26And then run this tool giving CONFIG names you want to move. 27For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 28simply type as follows: 29 30 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 31 32The tool walks through all the defconfig files and move the given CONFIGs. 33 34The log is also displayed on the terminal. 35 36The log is printed for each defconfig as follows: 37 38<defconfig_name> 39 <action1> 40 <action2> 41 <action3> 42 ... 43 44<defconfig_name> is the name of the defconfig. 45 46<action*> shows what the tool did for that defconfig. 47It looks like one of the following: 48 49 - Move 'CONFIG_... ' 50 This config option was moved to the defconfig 51 52 - CONFIG_... is not defined in Kconfig. Do nothing. 53 The entry for this CONFIG was not found in Kconfig. The option is not 54 defined in the config header, either. So, this case can be just skipped. 55 56 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 57 This option is defined in the config header, but its entry was not found 58 in Kconfig. 59 There are two common cases: 60 - You forgot to create an entry for the CONFIG before running 61 this tool, or made a typo in a CONFIG passed to this tool. 62 - The entry was hidden due to unmet 'depends on'. 63 The tool does not know if the result is reasonable, so please check it 64 manually. 65 66 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 67 The define in the config header matched the one in Kconfig. 68 We do not need to touch it. 69 70 - Compiler is missing. Do nothing. 71 The compiler specified for this architecture was not found 72 in your PATH environment. 73 (If -e option is passed, the tool exits immediately.) 74 75 - Failed to process. 76 An error occurred during processing this defconfig. Skipped. 77 (If -e option is passed, the tool exits immediately on error.) 78 79Finally, you will be asked, Clean up headers? [y/n]: 80 81If you say 'y' here, the unnecessary config defines are removed 82from the config headers (include/configs/*.h). 83It just uses the regex method, so you should not rely on it. 84Just in case, please do 'git diff' to see what happened. 85 86 87How does it work? 88----------------- 89 90This tool runs configuration and builds include/autoconf.mk for every 91defconfig. The config options defined in Kconfig appear in the .config 92file (unless they are hidden because of unmet dependency.) 93On the other hand, the config options defined by board headers are seen 94in include/autoconf.mk. The tool looks for the specified options in both 95of them to decide the appropriate action for the options. If the given 96config option is found in the .config, but its value does not match the 97one from the board header, the config option in the .config is replaced 98with the define in the board header. Then, the .config is synced by 99"make savedefconfig" and the defconfig is updated with it. 100 101For faster processing, this tool handles multi-threading. It creates 102separate build directories where the out-of-tree build is run. The 103temporary build directories are automatically created and deleted as 104needed. The number of threads are chosen based on the number of the CPU 105cores of your system although you can change it via -j (--jobs) option. 106 107 108Toolchains 109---------- 110 111Appropriate toolchain are necessary to generate include/autoconf.mk 112for all the architectures supported by U-Boot. Most of them are available 113at the kernel.org site, some are not provided by kernel.org. This tool uses 114the same tools as buildman, so see that tool for setup (e.g. --fetch-arch). 115 116 117Tips and trips 118-------------- 119 120To sync only X86 defconfigs: 121 122 ./tools/moveconfig.py -s -d <(grep -l X86 configs/*) 123 124or: 125 126 grep -l X86 configs/* | ./tools/moveconfig.py -s -d - 127 128To process CONFIG_CMD_FPGAD only for a subset of configs based on path match: 129 130 ls configs/{hrcon*,iocon*,strider*} | \ 131 ./tools/moveconfig.py -Cy CONFIG_CMD_FPGAD -d - 132 133 134Finding implied CONFIGs 135----------------------- 136 137Some CONFIG options can be implied by others and this can help to reduce 138the size of the defconfig files. For example, CONFIG_X86 implies 139CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 140all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 141each of the x86 defconfig files. 142 143This tool can help find such configs. To use it, first build a database: 144 145 ./tools/moveconfig.py -b 146 147Then try to query it: 148 149 ./tools/moveconfig.py -i CONFIG_CMD_IRQ 150 CONFIG_CMD_IRQ found in 311/2384 defconfigs 151 44 : CONFIG_SYS_FSL_ERRATUM_IFC_A002769 152 41 : CONFIG_SYS_FSL_ERRATUM_A007075 153 31 : CONFIG_SYS_FSL_DDR_VER_44 154 28 : CONFIG_ARCH_P1010 155 28 : CONFIG_SYS_FSL_ERRATUM_P1010_A003549 156 28 : CONFIG_SYS_FSL_ERRATUM_SEC_A003571 157 28 : CONFIG_SYS_FSL_ERRATUM_IFC_A003399 158 25 : CONFIG_SYS_FSL_ERRATUM_A008044 159 22 : CONFIG_ARCH_P1020 160 21 : CONFIG_SYS_FSL_DDR_VER_46 161 20 : CONFIG_MAX_PIRQ_LINKS 162 20 : CONFIG_HPET_ADDRESS 163 20 : CONFIG_X86 164 20 : CONFIG_PCIE_ECAM_SIZE 165 20 : CONFIG_IRQ_SLOT_COUNT 166 20 : CONFIG_I8259_PIC 167 20 : CONFIG_CPU_ADDR_BITS 168 20 : CONFIG_RAMBASE 169 20 : CONFIG_SYS_FSL_ERRATUM_A005871 170 20 : CONFIG_PCIE_ECAM_BASE 171 20 : CONFIG_X86_TSC_TIMER 172 20 : CONFIG_I8254_TIMER 173 20 : CONFIG_CMD_GETTIME 174 19 : CONFIG_SYS_FSL_ERRATUM_A005812 175 18 : CONFIG_X86_RUN_32BIT 176 17 : CONFIG_CMD_CHIP_CONFIG 177 ... 178 179This shows a list of config options which might imply CONFIG_CMD_EEPROM along 180with how many defconfigs they cover. From this you can see that CONFIG_X86 181implies CONFIG_CMD_EEPROM. Therefore, instead of adding CONFIG_CMD_EEPROM to 182the defconfig of every x86 board, you could add a single imply line to the 183Kconfig file: 184 185 config X86 186 bool "x86 architecture" 187 ... 188 imply CMD_EEPROM 189 190That will cover 20 defconfigs. Many of the options listed are not suitable as 191they are not related. E.g. it would be odd for CONFIG_CMD_GETTIME to imply 192CMD_EEPROM. 193 194Using this search you can reduce the size of moveconfig patches. 195 196You can automatically add 'imply' statements in the Kconfig with the -a 197option: 198 199 ./tools/moveconfig.py -s -i CONFIG_SCSI \ 200 -a CONFIG_ARCH_LS1021A,CONFIG_ARCH_LS1043A 201 202This will add 'imply SCSI' to the two CONFIG options mentioned, assuming that 203the database indicates that they do actually imply CONFIG_SCSI and do not 204already have an 'imply SCSI'. 205 206The output shows where the imply is added: 207 208 18 : CONFIG_ARCH_LS1021A arch/arm/cpu/armv7/ls102xa/Kconfig:1 209 13 : CONFIG_ARCH_LS1043A arch/arm/cpu/armv8/fsl-layerscape/Kconfig:11 210 12 : CONFIG_ARCH_LS1046A arch/arm/cpu/armv8/fsl-layerscape/Kconfig:31 211 212The first number is the number of boards which can avoid having a special 213CONFIG_SCSI option in their defconfig file if this 'imply' is added. 214The location at the right is the Kconfig file and line number where the config 215appears. For example, adding 'imply CONFIG_SCSI' to the 'config ARCH_LS1021A' 216in arch/arm/cpu/armv7/ls102xa/Kconfig at line 1 will help 18 boards to reduce 217the size of their defconfig files. 218 219If you want to add an 'imply' to every imply config in the list, you can use 220 221 ./tools/moveconfig.py -s -i CONFIG_SCSI -a all 222 223To control which ones are displayed, use -I <list> where list is a list of 224options (use '-I help' to see possible options and their meaning). 225 226To skip showing you options that already have an 'imply' attached, use -A. 227 228When you have finished adding 'imply' options you can regenerate the 229defconfig files for affected boards with something like: 230 231 git show --stat | ./tools/moveconfig.py -s -d - 232 233This will regenerate only those defconfigs changed in the current commit. 234If you start with (say) 100 defconfigs being changed in the commit, and add 235a few 'imply' options as above, then regenerate, hopefully you can reduce the 236number of defconfigs changed in the commit. 237 238 239Available options 240----------------- 241 242 -c, --color 243 Surround each portion of the log with escape sequences to display it 244 in color on the terminal. 245 246 -C, --commit 247 Create a git commit with the changes when the operation is complete. A 248 standard commit message is used which may need to be edited. 249 250 -d, --defconfigs 251 Specify a file containing a list of defconfigs to move. The defconfig 252 files can be given with shell-style wildcards. Use '-' to read from stdin. 253 254 -n, --dry-run 255 Perform a trial run that does not make any changes. It is useful to 256 see what is going to happen before one actually runs it. 257 258 -e, --exit-on-error 259 Exit immediately if Make exits with a non-zero status while processing 260 a defconfig file. 261 262 -s, --force-sync 263 Do "make savedefconfig" forcibly for all the defconfig files. 264 If not specified, "make savedefconfig" only occurs for cases 265 where at least one CONFIG was moved. 266 267 -S, --spl 268 Look for moved config options in spl/include/autoconf.mk instead of 269 include/autoconf.mk. This is useful for moving options for SPL build 270 because SPL related options (mostly prefixed with CONFIG_SPL_) are 271 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 272 273 -H, --headers-only 274 Only cleanup the headers; skip the defconfig processing 275 276 -j, --jobs 277 Specify the number of threads to run simultaneously. If not specified, 278 the number of threads is the same as the number of CPU cores. 279 280 -r, --git-ref 281 Specify the git ref to clone for building the autoconf.mk. If unspecified 282 use the CWD. This is useful for when changes to the Kconfig affect the 283 default values and you want to capture the state of the defconfig from 284 before that change was in effect. If in doubt, specify a ref pre-Kconfig 285 changes (use HEAD if Kconfig changes are not committed). Worst case it will 286 take a bit longer to run, but will always do the right thing. 287 288 -v, --verbose 289 Show any build errors as boards are built 290 291 -y, --yes 292 Instead of prompting, automatically go ahead with all operations. This 293 includes cleaning up headers, CONFIG_SYS_EXTRA_OPTIONS, the config whitelist 294 and the README. 295 296To see the complete list of supported options, run 297 298 $ tools/moveconfig.py -h 299 300""" 301 302import asteval 303import collections 304import copy 305import difflib 306import filecmp 307import fnmatch 308import glob 309import multiprocessing 310import optparse 311import os 312import queue 313import re 314import shutil 315import subprocess 316import sys 317import tempfile 318import threading 319import time 320 321from buildman import bsettings 322from buildman import kconfiglib 323from buildman import toolchain 324 325SHOW_GNU_MAKE = 'scripts/show-gnu-make' 326SLEEP_TIME=0.03 327 328STATE_IDLE = 0 329STATE_DEFCONFIG = 1 330STATE_AUTOCONF = 2 331STATE_SAVEDEFCONFIG = 3 332 333ACTION_MOVE = 0 334ACTION_NO_ENTRY = 1 335ACTION_NO_ENTRY_WARN = 2 336ACTION_NO_CHANGE = 3 337 338COLOR_BLACK = '0;30' 339COLOR_RED = '0;31' 340COLOR_GREEN = '0;32' 341COLOR_BROWN = '0;33' 342COLOR_BLUE = '0;34' 343COLOR_PURPLE = '0;35' 344COLOR_CYAN = '0;36' 345COLOR_LIGHT_GRAY = '0;37' 346COLOR_DARK_GRAY = '1;30' 347COLOR_LIGHT_RED = '1;31' 348COLOR_LIGHT_GREEN = '1;32' 349COLOR_YELLOW = '1;33' 350COLOR_LIGHT_BLUE = '1;34' 351COLOR_LIGHT_PURPLE = '1;35' 352COLOR_LIGHT_CYAN = '1;36' 353COLOR_WHITE = '1;37' 354 355AUTO_CONF_PATH = 'include/config/auto.conf' 356CONFIG_DATABASE = 'moveconfig.db' 357 358CONFIG_LEN = len('CONFIG_') 359 360SIZES = { 361 "SZ_1": 0x00000001, "SZ_2": 0x00000002, 362 "SZ_4": 0x00000004, "SZ_8": 0x00000008, 363 "SZ_16": 0x00000010, "SZ_32": 0x00000020, 364 "SZ_64": 0x00000040, "SZ_128": 0x00000080, 365 "SZ_256": 0x00000100, "SZ_512": 0x00000200, 366 "SZ_1K": 0x00000400, "SZ_2K": 0x00000800, 367 "SZ_4K": 0x00001000, "SZ_8K": 0x00002000, 368 "SZ_16K": 0x00004000, "SZ_32K": 0x00008000, 369 "SZ_64K": 0x00010000, "SZ_128K": 0x00020000, 370 "SZ_256K": 0x00040000, "SZ_512K": 0x00080000, 371 "SZ_1M": 0x00100000, "SZ_2M": 0x00200000, 372 "SZ_4M": 0x00400000, "SZ_8M": 0x00800000, 373 "SZ_16M": 0x01000000, "SZ_32M": 0x02000000, 374 "SZ_64M": 0x04000000, "SZ_128M": 0x08000000, 375 "SZ_256M": 0x10000000, "SZ_512M": 0x20000000, 376 "SZ_1G": 0x40000000, "SZ_2G": 0x80000000, 377 "SZ_4G": 0x100000000 378} 379 380### helper functions ### 381def get_devnull(): 382 """Get the file object of '/dev/null' device.""" 383 try: 384 devnull = subprocess.DEVNULL # py3k 385 except AttributeError: 386 devnull = open(os.devnull, 'wb') 387 return devnull 388 389def check_top_directory(): 390 """Exit if we are not at the top of source directory.""" 391 for f in ('README', 'Licenses'): 392 if not os.path.exists(f): 393 sys.exit('Please run at the top of source directory.') 394 395def check_clean_directory(): 396 """Exit if the source tree is not clean.""" 397 for f in ('.config', 'include/config'): 398 if os.path.exists(f): 399 sys.exit("source tree is not clean, please run 'make mrproper'") 400 401def get_make_cmd(): 402 """Get the command name of GNU Make. 403 404 U-Boot needs GNU Make for building, but the command name is not 405 necessarily "make". (for example, "gmake" on FreeBSD). 406 Returns the most appropriate command name on your system. 407 """ 408 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 409 ret = process.communicate() 410 if process.returncode: 411 sys.exit('GNU Make not found') 412 return ret[0].rstrip() 413 414def get_matched_defconfig(line): 415 """Get the defconfig files that match a pattern 416 417 Args: 418 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 419 'k2*_defconfig'. If no directory is provided, 'configs/' is 420 prepended 421 422 Returns: 423 a list of matching defconfig files 424 """ 425 dirname = os.path.dirname(line) 426 if dirname: 427 pattern = line 428 else: 429 pattern = os.path.join('configs', line) 430 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 431 432def get_matched_defconfigs(defconfigs_file): 433 """Get all the defconfig files that match the patterns in a file. 434 435 Args: 436 defconfigs_file: File containing a list of defconfigs to process, or 437 '-' to read the list from stdin 438 439 Returns: 440 A list of paths to defconfig files, with no duplicates 441 """ 442 defconfigs = [] 443 if defconfigs_file == '-': 444 fd = sys.stdin 445 defconfigs_file = 'stdin' 446 else: 447 fd = open(defconfigs_file) 448 for i, line in enumerate(fd): 449 line = line.strip() 450 if not line: 451 continue # skip blank lines silently 452 if ' ' in line: 453 line = line.split(' ')[0] # handle 'git log' input 454 matched = get_matched_defconfig(line) 455 if not matched: 456 print("warning: %s:%d: no defconfig matched '%s'" % \ 457 (defconfigs_file, i + 1, line), file=sys.stderr) 458 459 defconfigs += matched 460 461 # use set() to drop multiple matching 462 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 463 464def get_all_defconfigs(): 465 """Get all the defconfig files under the configs/ directory.""" 466 defconfigs = [] 467 for (dirpath, dirnames, filenames) in os.walk('configs'): 468 dirpath = dirpath[len('configs') + 1:] 469 for filename in fnmatch.filter(filenames, '*_defconfig'): 470 defconfigs.append(os.path.join(dirpath, filename)) 471 472 return defconfigs 473 474def color_text(color_enabled, color, string): 475 """Return colored string.""" 476 if color_enabled: 477 # LF should not be surrounded by the escape sequence. 478 # Otherwise, additional whitespace or line-feed might be printed. 479 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 480 for s in string.split('\n') ]) 481 else: 482 return string 483 484def show_diff(a, b, file_path, color_enabled): 485 """Show unidified diff. 486 487 Arguments: 488 a: A list of lines (before) 489 b: A list of lines (after) 490 file_path: Path to the file 491 color_enabled: Display the diff in color 492 """ 493 494 diff = difflib.unified_diff(a, b, 495 fromfile=os.path.join('a', file_path), 496 tofile=os.path.join('b', file_path)) 497 498 for line in diff: 499 if line[0] == '-' and line[1] != '-': 500 print(color_text(color_enabled, COLOR_RED, line), end=' ') 501 elif line[0] == '+' and line[1] != '+': 502 print(color_text(color_enabled, COLOR_GREEN, line), end=' ') 503 else: 504 print(line, end=' ') 505 506def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 507 extend_post): 508 """Extend matched lines if desired patterns are found before/after already 509 matched lines. 510 511 Arguments: 512 lines: A list of lines handled. 513 matched: A list of line numbers that have been already matched. 514 (will be updated by this function) 515 pre_patterns: A list of regular expression that should be matched as 516 preamble. 517 post_patterns: A list of regular expression that should be matched as 518 postamble. 519 extend_pre: Add the line number of matched preamble to the matched list. 520 extend_post: Add the line number of matched postamble to the matched list. 521 """ 522 extended_matched = [] 523 524 j = matched[0] 525 526 for i in matched: 527 if i == 0 or i < j: 528 continue 529 j = i 530 while j in matched: 531 j += 1 532 if j >= len(lines): 533 break 534 535 for p in pre_patterns: 536 if p.search(lines[i - 1]): 537 break 538 else: 539 # not matched 540 continue 541 542 for p in post_patterns: 543 if p.search(lines[j]): 544 break 545 else: 546 # not matched 547 continue 548 549 if extend_pre: 550 extended_matched.append(i - 1) 551 if extend_post: 552 extended_matched.append(j) 553 554 matched += extended_matched 555 matched.sort() 556 557def confirm(options, prompt): 558 if not options.yes: 559 while True: 560 choice = input('{} [y/n]: '.format(prompt)) 561 choice = choice.lower() 562 print(choice) 563 if choice == 'y' or choice == 'n': 564 break 565 566 if choice == 'n': 567 return False 568 569 return True 570 571def cleanup_empty_blocks(header_path, options): 572 """Clean up empty conditional blocks 573 574 Arguments: 575 header_path: path to the cleaned file. 576 options: option flags. 577 """ 578 pattern = re.compile(r'^\s*#\s*if.*$\n^\s*#\s*endif.*$\n*', flags=re.M) 579 with open(header_path) as f: 580 try: 581 data = f.read() 582 except UnicodeDecodeError as e: 583 print("Failed on file %s': %s" % (header_path, e)) 584 return 585 586 new_data = pattern.sub('\n', data) 587 588 show_diff(data.splitlines(True), new_data.splitlines(True), header_path, 589 options.color) 590 591 if options.dry_run: 592 return 593 594 with open(header_path, 'w') as f: 595 f.write(new_data) 596 597def cleanup_one_header(header_path, patterns, options): 598 """Clean regex-matched lines away from a file. 599 600 Arguments: 601 header_path: path to the cleaned file. 602 patterns: list of regex patterns. Any lines matching to these 603 patterns are deleted. 604 options: option flags. 605 """ 606 with open(header_path) as f: 607 try: 608 lines = f.readlines() 609 except UnicodeDecodeError as e: 610 print("Failed on file %s': %s" % (header_path, e)) 611 return 612 613 matched = [] 614 for i, line in enumerate(lines): 615 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 616 matched.append(i) 617 continue 618 for pattern in patterns: 619 if pattern.search(line): 620 matched.append(i) 621 break 622 623 if not matched: 624 return 625 626 # remove empty #ifdef ... #endif, successive blank lines 627 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 628 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 629 pattern_endif = re.compile(r'#\s*endif\W') # #endif 630 pattern_blank = re.compile(r'^\s*$') # empty line 631 632 while True: 633 old_matched = copy.copy(matched) 634 extend_matched_lines(lines, matched, [pattern_if], 635 [pattern_endif], True, True) 636 extend_matched_lines(lines, matched, [pattern_elif], 637 [pattern_elif, pattern_endif], True, False) 638 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 639 [pattern_blank], False, True) 640 extend_matched_lines(lines, matched, [pattern_blank], 641 [pattern_elif, pattern_endif], True, False) 642 extend_matched_lines(lines, matched, [pattern_blank], 643 [pattern_blank], True, False) 644 if matched == old_matched: 645 break 646 647 tolines = copy.copy(lines) 648 649 for i in reversed(matched): 650 tolines.pop(i) 651 652 show_diff(lines, tolines, header_path, options.color) 653 654 if options.dry_run: 655 return 656 657 with open(header_path, 'w') as f: 658 for line in tolines: 659 f.write(line) 660 661def cleanup_headers(configs, options): 662 """Delete config defines from board headers. 663 664 Arguments: 665 configs: A list of CONFIGs to remove. 666 options: option flags. 667 """ 668 if not confirm(options, 'Clean up headers?'): 669 return 670 671 patterns = [] 672 for config in configs: 673 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 674 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 675 676 for dir in 'include', 'arch', 'board': 677 for (dirpath, dirnames, filenames) in os.walk(dir): 678 if dirpath == os.path.join('include', 'generated'): 679 continue 680 for filename in filenames: 681 if not filename.endswith(('~', '.dts', '.dtsi', '.bin', 682 '.elf','.aml','.dat')): 683 header_path = os.path.join(dirpath, filename) 684 # This file contains UTF-16 data and no CONFIG symbols 685 if header_path == 'include/video_font_data.h': 686 continue 687 cleanup_one_header(header_path, patterns, options) 688 cleanup_empty_blocks(header_path, options) 689 690def cleanup_one_extra_option(defconfig_path, configs, options): 691 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 692 693 Arguments: 694 defconfig_path: path to the cleaned defconfig file. 695 configs: A list of CONFIGs to remove. 696 options: option flags. 697 """ 698 699 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 700 end = '"\n' 701 702 with open(defconfig_path) as f: 703 lines = f.readlines() 704 705 for i, line in enumerate(lines): 706 if line.startswith(start) and line.endswith(end): 707 break 708 else: 709 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 710 return 711 712 old_tokens = line[len(start):-len(end)].split(',') 713 new_tokens = [] 714 715 for token in old_tokens: 716 pos = token.find('=') 717 if not (token[:pos] if pos >= 0 else token) in configs: 718 new_tokens.append(token) 719 720 if new_tokens == old_tokens: 721 return 722 723 tolines = copy.copy(lines) 724 725 if new_tokens: 726 tolines[i] = start + ','.join(new_tokens) + end 727 else: 728 tolines.pop(i) 729 730 show_diff(lines, tolines, defconfig_path, options.color) 731 732 if options.dry_run: 733 return 734 735 with open(defconfig_path, 'w') as f: 736 for line in tolines: 737 f.write(line) 738 739def cleanup_extra_options(configs, options): 740 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 741 742 Arguments: 743 configs: A list of CONFIGs to remove. 744 options: option flags. 745 """ 746 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 747 return 748 749 configs = [ config[len('CONFIG_'):] for config in configs ] 750 751 defconfigs = get_all_defconfigs() 752 753 for defconfig in defconfigs: 754 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 755 options) 756 757def cleanup_whitelist(configs, options): 758 """Delete config whitelist entries 759 760 Arguments: 761 configs: A list of CONFIGs to remove. 762 options: option flags. 763 """ 764 if not confirm(options, 'Clean up whitelist entries?'): 765 return 766 767 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 768 lines = f.readlines() 769 770 lines = [x for x in lines if x.strip() not in configs] 771 772 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 773 f.write(''.join(lines)) 774 775def find_matching(patterns, line): 776 for pat in patterns: 777 if pat.search(line): 778 return True 779 return False 780 781def cleanup_readme(configs, options): 782 """Delete config description in README 783 784 Arguments: 785 configs: A list of CONFIGs to remove. 786 options: option flags. 787 """ 788 if not confirm(options, 'Clean up README?'): 789 return 790 791 patterns = [] 792 for config in configs: 793 patterns.append(re.compile(r'^\s+%s' % config)) 794 795 with open('README') as f: 796 lines = f.readlines() 797 798 found = False 799 newlines = [] 800 for line in lines: 801 if not found: 802 found = find_matching(patterns, line) 803 if found: 804 continue 805 806 if found and re.search(r'^\s+CONFIG', line): 807 found = False 808 809 if not found: 810 newlines.append(line) 811 812 with open('README', 'w') as f: 813 f.write(''.join(newlines)) 814 815def try_expand(line): 816 """If value looks like an expression, try expanding it 817 Otherwise just return the existing value 818 """ 819 if line.find('=') == -1: 820 return line 821 822 try: 823 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True ) 824 cfg, val = re.split("=", line) 825 val= val.strip('\"') 826 if re.search("[*+-/]|<<|SZ_+|\(([^\)]+)\)", val): 827 newval = hex(aeval(val)) 828 print("\tExpanded expression %s to %s" % (val, newval)) 829 return cfg+'='+newval 830 except: 831 print("\tFailed to expand expression in %s" % line) 832 833 return line 834 835 836### classes ### 837class Progress: 838 839 """Progress Indicator""" 840 841 def __init__(self, total): 842 """Create a new progress indicator. 843 844 Arguments: 845 total: A number of defconfig files to process. 846 """ 847 self.current = 0 848 self.total = total 849 850 def inc(self): 851 """Increment the number of processed defconfig files.""" 852 853 self.current += 1 854 855 def show(self): 856 """Display the progress.""" 857 print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ') 858 sys.stdout.flush() 859 860 861class KconfigScanner: 862 """Kconfig scanner.""" 863 864 def __init__(self): 865 """Scan all the Kconfig files and create a Config object.""" 866 # Define environment variables referenced from Kconfig 867 os.environ['srctree'] = os.getcwd() 868 os.environ['UBOOTVERSION'] = 'dummy' 869 os.environ['KCONFIG_OBJDIR'] = '' 870 self.conf = kconfiglib.Kconfig() 871 872 873class KconfigParser: 874 875 """A parser of .config and include/autoconf.mk.""" 876 877 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 878 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 879 880 def __init__(self, configs, options, build_dir): 881 """Create a new parser. 882 883 Arguments: 884 configs: A list of CONFIGs to move. 885 options: option flags. 886 build_dir: Build directory. 887 """ 888 self.configs = configs 889 self.options = options 890 self.dotconfig = os.path.join(build_dir, '.config') 891 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 892 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 893 'autoconf.mk') 894 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) 895 self.defconfig = os.path.join(build_dir, 'defconfig') 896 897 def get_arch(self): 898 """Parse .config file and return the architecture. 899 900 Returns: 901 Architecture name (e.g. 'arm'). 902 """ 903 arch = '' 904 cpu = '' 905 for line in open(self.dotconfig): 906 m = self.re_arch.match(line) 907 if m: 908 arch = m.group(1) 909 continue 910 m = self.re_cpu.match(line) 911 if m: 912 cpu = m.group(1) 913 914 if not arch: 915 return None 916 917 # fix-up for aarch64 918 if arch == 'arm' and cpu == 'armv8': 919 arch = 'aarch64' 920 921 return arch 922 923 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 924 """Parse .config, defconfig, include/autoconf.mk for one config. 925 926 This function looks for the config options in the lines from 927 defconfig, .config, and include/autoconf.mk in order to decide 928 which action should be taken for this defconfig. 929 930 Arguments: 931 config: CONFIG name to parse. 932 dotconfig_lines: lines from the .config file. 933 autoconf_lines: lines from the include/autoconf.mk file. 934 935 Returns: 936 A tupple of the action for this defconfig and the line 937 matched for the config. 938 """ 939 not_set = '# %s is not set' % config 940 941 for line in autoconf_lines: 942 line = line.rstrip() 943 if line.startswith(config + '='): 944 new_val = line 945 break 946 else: 947 new_val = not_set 948 949 new_val = try_expand(new_val) 950 951 for line in dotconfig_lines: 952 line = line.rstrip() 953 if line.startswith(config + '=') or line == not_set: 954 old_val = line 955 break 956 else: 957 if new_val == not_set: 958 return (ACTION_NO_ENTRY, config) 959 else: 960 return (ACTION_NO_ENTRY_WARN, config) 961 962 # If this CONFIG is neither bool nor trisate 963 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 964 # tools/scripts/define2mk.sed changes '1' to 'y'. 965 # This is a problem if the CONFIG is int type. 966 # Check the type in Kconfig and handle it correctly. 967 if new_val[-2:] == '=y': 968 new_val = new_val[:-1] + '1' 969 970 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 971 new_val) 972 973 def update_dotconfig(self): 974 """Parse files for the config options and update the .config. 975 976 This function parses the generated .config and include/autoconf.mk 977 searching the target options. 978 Move the config option(s) to the .config as needed. 979 980 Arguments: 981 defconfig: defconfig name. 982 983 Returns: 984 Return a tuple of (updated flag, log string). 985 The "updated flag" is True if the .config was updated, False 986 otherwise. The "log string" shows what happend to the .config. 987 """ 988 989 results = [] 990 updated = False 991 suspicious = False 992 rm_files = [self.config_autoconf, self.autoconf] 993 994 if self.options.spl: 995 if os.path.exists(self.spl_autoconf): 996 autoconf_path = self.spl_autoconf 997 rm_files.append(self.spl_autoconf) 998 else: 999 for f in rm_files: 1000 os.remove(f) 1001 return (updated, suspicious, 1002 color_text(self.options.color, COLOR_BROWN, 1003 "SPL is not enabled. Skipped.") + '\n') 1004 else: 1005 autoconf_path = self.autoconf 1006 1007 with open(self.dotconfig) as f: 1008 dotconfig_lines = f.readlines() 1009 1010 with open(autoconf_path) as f: 1011 autoconf_lines = f.readlines() 1012 1013 for config in self.configs: 1014 result = self.parse_one_config(config, dotconfig_lines, 1015 autoconf_lines) 1016 results.append(result) 1017 1018 log = '' 1019 1020 for (action, value) in results: 1021 if action == ACTION_MOVE: 1022 actlog = "Move '%s'" % value 1023 log_color = COLOR_LIGHT_GREEN 1024 elif action == ACTION_NO_ENTRY: 1025 actlog = "%s is not defined in Kconfig. Do nothing." % value 1026 log_color = COLOR_LIGHT_BLUE 1027 elif action == ACTION_NO_ENTRY_WARN: 1028 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 1029 log_color = COLOR_YELLOW 1030 suspicious = True 1031 elif action == ACTION_NO_CHANGE: 1032 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 1033 % value 1034 log_color = COLOR_LIGHT_PURPLE 1035 elif action == ACTION_SPL_NOT_EXIST: 1036 actlog = "SPL is not enabled for this defconfig. Skip." 1037 log_color = COLOR_PURPLE 1038 else: 1039 sys.exit("Internal Error. This should not happen.") 1040 1041 log += color_text(self.options.color, log_color, actlog) + '\n' 1042 1043 with open(self.dotconfig, 'a') as f: 1044 for (action, value) in results: 1045 if action == ACTION_MOVE: 1046 f.write(value + '\n') 1047 updated = True 1048 1049 self.results = results 1050 for f in rm_files: 1051 os.remove(f) 1052 1053 return (updated, suspicious, log) 1054 1055 def check_defconfig(self): 1056 """Check the defconfig after savedefconfig 1057 1058 Returns: 1059 Return additional log if moved CONFIGs were removed again by 1060 'make savedefconfig'. 1061 """ 1062 1063 log = '' 1064 1065 with open(self.defconfig) as f: 1066 defconfig_lines = f.readlines() 1067 1068 for (action, value) in self.results: 1069 if action != ACTION_MOVE: 1070 continue 1071 if not value + '\n' in defconfig_lines: 1072 log += color_text(self.options.color, COLOR_YELLOW, 1073 "'%s' was removed by savedefconfig.\n" % 1074 value) 1075 1076 return log 1077 1078 1079class DatabaseThread(threading.Thread): 1080 """This thread processes results from Slot threads. 1081 1082 It collects the data in the master config directary. There is only one 1083 result thread, and this helps to serialise the build output. 1084 """ 1085 def __init__(self, config_db, db_queue): 1086 """Set up a new result thread 1087 1088 Args: 1089 builder: Builder which will be sent each result 1090 """ 1091 threading.Thread.__init__(self) 1092 self.config_db = config_db 1093 self.db_queue= db_queue 1094 1095 def run(self): 1096 """Called to start up the result thread. 1097 1098 We collect the next result job and pass it on to the build. 1099 """ 1100 while True: 1101 defconfig, configs = self.db_queue.get() 1102 self.config_db[defconfig] = configs 1103 self.db_queue.task_done() 1104 1105 1106class Slot: 1107 1108 """A slot to store a subprocess. 1109 1110 Each instance of this class handles one subprocess. 1111 This class is useful to control multiple threads 1112 for faster processing. 1113 """ 1114 1115 def __init__(self, toolchains, configs, options, progress, devnull, 1116 make_cmd, reference_src_dir, db_queue): 1117 """Create a new process slot. 1118 1119 Arguments: 1120 toolchains: Toolchains object containing toolchains. 1121 configs: A list of CONFIGs to move. 1122 options: option flags. 1123 progress: A progress indicator. 1124 devnull: A file object of '/dev/null'. 1125 make_cmd: command name of GNU Make. 1126 reference_src_dir: Determine the true starting config state from this 1127 source tree. 1128 db_queue: output queue to write config info for the database 1129 """ 1130 self.toolchains = toolchains 1131 self.options = options 1132 self.progress = progress 1133 self.build_dir = tempfile.mkdtemp() 1134 self.devnull = devnull 1135 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 1136 self.reference_src_dir = reference_src_dir 1137 self.db_queue = db_queue 1138 self.parser = KconfigParser(configs, options, self.build_dir) 1139 self.state = STATE_IDLE 1140 self.failed_boards = set() 1141 self.suspicious_boards = set() 1142 1143 def __del__(self): 1144 """Delete the working directory 1145 1146 This function makes sure the temporary directory is cleaned away 1147 even if Python suddenly dies due to error. It should be done in here 1148 because it is guaranteed the destructor is always invoked when the 1149 instance of the class gets unreferenced. 1150 1151 If the subprocess is still running, wait until it finishes. 1152 """ 1153 if self.state != STATE_IDLE: 1154 while self.ps.poll() == None: 1155 pass 1156 shutil.rmtree(self.build_dir) 1157 1158 def add(self, defconfig): 1159 """Assign a new subprocess for defconfig and add it to the slot. 1160 1161 If the slot is vacant, create a new subprocess for processing the 1162 given defconfig and add it to the slot. Just returns False if 1163 the slot is occupied (i.e. the current subprocess is still running). 1164 1165 Arguments: 1166 defconfig: defconfig name. 1167 1168 Returns: 1169 Return True on success or False on failure 1170 """ 1171 if self.state != STATE_IDLE: 1172 return False 1173 1174 self.defconfig = defconfig 1175 self.log = '' 1176 self.current_src_dir = self.reference_src_dir 1177 self.do_defconfig() 1178 return True 1179 1180 def poll(self): 1181 """Check the status of the subprocess and handle it as needed. 1182 1183 Returns True if the slot is vacant (i.e. in idle state). 1184 If the configuration is successfully finished, assign a new 1185 subprocess to build include/autoconf.mk. 1186 If include/autoconf.mk is generated, invoke the parser to 1187 parse the .config and the include/autoconf.mk, moving 1188 config options to the .config as needed. 1189 If the .config was updated, run "make savedefconfig" to sync 1190 it, update the original defconfig, and then set the slot back 1191 to the idle state. 1192 1193 Returns: 1194 Return True if the subprocess is terminated, False otherwise 1195 """ 1196 if self.state == STATE_IDLE: 1197 return True 1198 1199 if self.ps.poll() == None: 1200 return False 1201 1202 if self.ps.poll() != 0: 1203 self.handle_error() 1204 elif self.state == STATE_DEFCONFIG: 1205 if self.reference_src_dir and not self.current_src_dir: 1206 self.do_savedefconfig() 1207 else: 1208 self.do_autoconf() 1209 elif self.state == STATE_AUTOCONF: 1210 if self.current_src_dir: 1211 self.current_src_dir = None 1212 self.do_defconfig() 1213 elif self.options.build_db: 1214 self.do_build_db() 1215 else: 1216 self.do_savedefconfig() 1217 elif self.state == STATE_SAVEDEFCONFIG: 1218 self.update_defconfig() 1219 else: 1220 sys.exit("Internal Error. This should not happen.") 1221 1222 return True if self.state == STATE_IDLE else False 1223 1224 def handle_error(self): 1225 """Handle error cases.""" 1226 1227 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1228 "Failed to process.\n") 1229 if self.options.verbose: 1230 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1231 self.ps.stderr.read().decode()) 1232 self.finish(False) 1233 1234 def do_defconfig(self): 1235 """Run 'make <board>_defconfig' to create the .config file.""" 1236 1237 cmd = list(self.make_cmd) 1238 cmd.append(self.defconfig) 1239 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1240 stderr=subprocess.PIPE, 1241 cwd=self.current_src_dir) 1242 self.state = STATE_DEFCONFIG 1243 1244 def do_autoconf(self): 1245 """Run 'make AUTO_CONF_PATH'.""" 1246 1247 arch = self.parser.get_arch() 1248 try: 1249 toolchain = self.toolchains.Select(arch) 1250 except ValueError: 1251 self.log += color_text(self.options.color, COLOR_YELLOW, 1252 "Tool chain for '%s' is missing. Do nothing.\n" % arch) 1253 self.finish(False) 1254 return 1255 env = toolchain.MakeEnvironment(False) 1256 1257 cmd = list(self.make_cmd) 1258 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1259 cmd.append(AUTO_CONF_PATH) 1260 self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env, 1261 stderr=subprocess.PIPE, 1262 cwd=self.current_src_dir) 1263 self.state = STATE_AUTOCONF 1264 1265 def do_build_db(self): 1266 """Add the board to the database""" 1267 configs = {} 1268 with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: 1269 for line in fd.readlines(): 1270 if line.startswith('CONFIG'): 1271 config, value = line.split('=', 1) 1272 configs[config] = value.rstrip() 1273 self.db_queue.put([self.defconfig, configs]) 1274 self.finish(True) 1275 1276 def do_savedefconfig(self): 1277 """Update the .config and run 'make savedefconfig'.""" 1278 1279 (updated, suspicious, log) = self.parser.update_dotconfig() 1280 if suspicious: 1281 self.suspicious_boards.add(self.defconfig) 1282 self.log += log 1283 1284 if not self.options.force_sync and not updated: 1285 self.finish(True) 1286 return 1287 if updated: 1288 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1289 "Syncing by savedefconfig...\n") 1290 else: 1291 self.log += "Syncing by savedefconfig (forced by option)...\n" 1292 1293 cmd = list(self.make_cmd) 1294 cmd.append('savedefconfig') 1295 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1296 stderr=subprocess.PIPE) 1297 self.state = STATE_SAVEDEFCONFIG 1298 1299 def update_defconfig(self): 1300 """Update the input defconfig and go back to the idle state.""" 1301 1302 log = self.parser.check_defconfig() 1303 if log: 1304 self.suspicious_boards.add(self.defconfig) 1305 self.log += log 1306 orig_defconfig = os.path.join('configs', self.defconfig) 1307 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1308 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1309 1310 if updated: 1311 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1312 "defconfig was updated.\n") 1313 1314 if not self.options.dry_run and updated: 1315 shutil.move(new_defconfig, orig_defconfig) 1316 self.finish(True) 1317 1318 def finish(self, success): 1319 """Display log along with progress and go to the idle state. 1320 1321 Arguments: 1322 success: Should be True when the defconfig was processed 1323 successfully, or False when it fails. 1324 """ 1325 # output at least 30 characters to hide the "* defconfigs out of *". 1326 log = self.defconfig.ljust(30) + '\n' 1327 1328 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1329 # Some threads are running in parallel. 1330 # Print log atomically to not mix up logs from different threads. 1331 print(log, file=(sys.stdout if success else sys.stderr)) 1332 1333 if not success: 1334 if self.options.exit_on_error: 1335 sys.exit("Exit on error.") 1336 # If --exit-on-error flag is not set, skip this board and continue. 1337 # Record the failed board. 1338 self.failed_boards.add(self.defconfig) 1339 1340 self.progress.inc() 1341 self.progress.show() 1342 self.state = STATE_IDLE 1343 1344 def get_failed_boards(self): 1345 """Returns a set of failed boards (defconfigs) in this slot. 1346 """ 1347 return self.failed_boards 1348 1349 def get_suspicious_boards(self): 1350 """Returns a set of boards (defconfigs) with possible misconversion. 1351 """ 1352 return self.suspicious_boards - self.failed_boards 1353 1354class Slots: 1355 1356 """Controller of the array of subprocess slots.""" 1357 1358 def __init__(self, toolchains, configs, options, progress, 1359 reference_src_dir, db_queue): 1360 """Create a new slots controller. 1361 1362 Arguments: 1363 toolchains: Toolchains object containing toolchains. 1364 configs: A list of CONFIGs to move. 1365 options: option flags. 1366 progress: A progress indicator. 1367 reference_src_dir: Determine the true starting config state from this 1368 source tree. 1369 db_queue: output queue to write config info for the database 1370 """ 1371 self.options = options 1372 self.slots = [] 1373 devnull = get_devnull() 1374 make_cmd = get_make_cmd() 1375 for i in range(options.jobs): 1376 self.slots.append(Slot(toolchains, configs, options, progress, 1377 devnull, make_cmd, reference_src_dir, 1378 db_queue)) 1379 1380 def add(self, defconfig): 1381 """Add a new subprocess if a vacant slot is found. 1382 1383 Arguments: 1384 defconfig: defconfig name to be put into. 1385 1386 Returns: 1387 Return True on success or False on failure 1388 """ 1389 for slot in self.slots: 1390 if slot.add(defconfig): 1391 return True 1392 return False 1393 1394 def available(self): 1395 """Check if there is a vacant slot. 1396 1397 Returns: 1398 Return True if at lease one vacant slot is found, False otherwise. 1399 """ 1400 for slot in self.slots: 1401 if slot.poll(): 1402 return True 1403 return False 1404 1405 def empty(self): 1406 """Check if all slots are vacant. 1407 1408 Returns: 1409 Return True if all the slots are vacant, False otherwise. 1410 """ 1411 ret = True 1412 for slot in self.slots: 1413 if not slot.poll(): 1414 ret = False 1415 return ret 1416 1417 def show_failed_boards(self): 1418 """Display all of the failed boards (defconfigs).""" 1419 boards = set() 1420 output_file = 'moveconfig.failed' 1421 1422 for slot in self.slots: 1423 boards |= slot.get_failed_boards() 1424 1425 if boards: 1426 boards = '\n'.join(boards) + '\n' 1427 msg = "The following boards were not processed due to error:\n" 1428 msg += boards 1429 msg += "(the list has been saved in %s)\n" % output_file 1430 print(color_text(self.options.color, COLOR_LIGHT_RED, 1431 msg), file=sys.stderr) 1432 1433 with open(output_file, 'w') as f: 1434 f.write(boards) 1435 1436 def show_suspicious_boards(self): 1437 """Display all boards (defconfigs) with possible misconversion.""" 1438 boards = set() 1439 output_file = 'moveconfig.suspicious' 1440 1441 for slot in self.slots: 1442 boards |= slot.get_suspicious_boards() 1443 1444 if boards: 1445 boards = '\n'.join(boards) + '\n' 1446 msg = "The following boards might have been converted incorrectly.\n" 1447 msg += "It is highly recommended to check them manually:\n" 1448 msg += boards 1449 msg += "(the list has been saved in %s)\n" % output_file 1450 print(color_text(self.options.color, COLOR_YELLOW, 1451 msg), file=sys.stderr) 1452 1453 with open(output_file, 'w') as f: 1454 f.write(boards) 1455 1456class ReferenceSource: 1457 1458 """Reference source against which original configs should be parsed.""" 1459 1460 def __init__(self, commit): 1461 """Create a reference source directory based on a specified commit. 1462 1463 Arguments: 1464 commit: commit to git-clone 1465 """ 1466 self.src_dir = tempfile.mkdtemp() 1467 print("Cloning git repo to a separate work directory...") 1468 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1469 cwd=self.src_dir) 1470 print("Checkout '%s' to build the original autoconf.mk." % \ 1471 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip()) 1472 subprocess.check_output(['git', 'checkout', commit], 1473 stderr=subprocess.STDOUT, cwd=self.src_dir) 1474 1475 def __del__(self): 1476 """Delete the reference source directory 1477 1478 This function makes sure the temporary directory is cleaned away 1479 even if Python suddenly dies due to error. It should be done in here 1480 because it is guaranteed the destructor is always invoked when the 1481 instance of the class gets unreferenced. 1482 """ 1483 shutil.rmtree(self.src_dir) 1484 1485 def get_dir(self): 1486 """Return the absolute path to the reference source directory.""" 1487 1488 return self.src_dir 1489 1490def move_config(toolchains, configs, options, db_queue): 1491 """Move config options to defconfig files. 1492 1493 Arguments: 1494 configs: A list of CONFIGs to move. 1495 options: option flags 1496 """ 1497 if len(configs) == 0: 1498 if options.force_sync: 1499 print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ') 1500 elif options.build_db: 1501 print('Building %s database' % CONFIG_DATABASE) 1502 else: 1503 print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ') 1504 else: 1505 print('Move ' + ', '.join(configs), end=' ') 1506 print('(jobs: %d)\n' % options.jobs) 1507 1508 if options.git_ref: 1509 reference_src = ReferenceSource(options.git_ref) 1510 reference_src_dir = reference_src.get_dir() 1511 else: 1512 reference_src_dir = None 1513 1514 if options.defconfigs: 1515 defconfigs = get_matched_defconfigs(options.defconfigs) 1516 else: 1517 defconfigs = get_all_defconfigs() 1518 1519 progress = Progress(len(defconfigs)) 1520 slots = Slots(toolchains, configs, options, progress, reference_src_dir, 1521 db_queue) 1522 1523 # Main loop to process defconfig files: 1524 # Add a new subprocess into a vacant slot. 1525 # Sleep if there is no available slot. 1526 for defconfig in defconfigs: 1527 while not slots.add(defconfig): 1528 while not slots.available(): 1529 # No available slot: sleep for a while 1530 time.sleep(SLEEP_TIME) 1531 1532 # wait until all the subprocesses finish 1533 while not slots.empty(): 1534 time.sleep(SLEEP_TIME) 1535 1536 print('') 1537 slots.show_failed_boards() 1538 slots.show_suspicious_boards() 1539 1540def find_kconfig_rules(kconf, config, imply_config): 1541 """Check whether a config has a 'select' or 'imply' keyword 1542 1543 Args: 1544 kconf: Kconfiglib.Kconfig object 1545 config: Name of config to check (without CONFIG_ prefix) 1546 imply_config: Implying config (without CONFIG_ prefix) which may or 1547 may not have an 'imply' for 'config') 1548 1549 Returns: 1550 Symbol object for 'config' if found, else None 1551 """ 1552 sym = kconf.syms.get(imply_config) 1553 if sym: 1554 for sel in sym.get_selected_symbols() | sym.get_implied_symbols(): 1555 if sel.get_name() == config: 1556 return sym 1557 return None 1558 1559def check_imply_rule(kconf, config, imply_config): 1560 """Check if we can add an 'imply' option 1561 1562 This finds imply_config in the Kconfig and looks to see if it is possible 1563 to add an 'imply' for 'config' to that part of the Kconfig. 1564 1565 Args: 1566 kconf: Kconfiglib.Kconfig object 1567 config: Name of config to check (without CONFIG_ prefix) 1568 imply_config: Implying config (without CONFIG_ prefix) which may or 1569 may not have an 'imply' for 'config') 1570 1571 Returns: 1572 tuple: 1573 filename of Kconfig file containing imply_config, or None if none 1574 line number within the Kconfig file, or 0 if none 1575 message indicating the result 1576 """ 1577 sym = kconf.syms.get(imply_config) 1578 if not sym: 1579 return 'cannot find sym' 1580 locs = sym.get_def_locations() 1581 if len(locs) != 1: 1582 return '%d locations' % len(locs) 1583 fname, linenum = locs[0] 1584 cwd = os.getcwd() 1585 if cwd and fname.startswith(cwd): 1586 fname = fname[len(cwd) + 1:] 1587 file_line = ' at %s:%d' % (fname, linenum) 1588 with open(fname) as fd: 1589 data = fd.read().splitlines() 1590 if data[linenum - 1] != 'config %s' % imply_config: 1591 return None, 0, 'bad sym format %s%s' % (data[linenum], file_line) 1592 return fname, linenum, 'adding%s' % file_line 1593 1594def add_imply_rule(config, fname, linenum): 1595 """Add a new 'imply' option to a Kconfig 1596 1597 Args: 1598 config: config option to add an imply for (without CONFIG_ prefix) 1599 fname: Kconfig filename to update 1600 linenum: Line number to place the 'imply' before 1601 1602 Returns: 1603 Message indicating the result 1604 """ 1605 file_line = ' at %s:%d' % (fname, linenum) 1606 data = open(fname).read().splitlines() 1607 linenum -= 1 1608 1609 for offset, line in enumerate(data[linenum:]): 1610 if line.strip().startswith('help') or not line: 1611 data.insert(linenum + offset, '\timply %s' % config) 1612 with open(fname, 'w') as fd: 1613 fd.write('\n'.join(data) + '\n') 1614 return 'added%s' % file_line 1615 1616 return 'could not insert%s' 1617 1618(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = ( 1619 1, 2, 4, 8) 1620 1621IMPLY_FLAGS = { 1622 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'], 1623 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'], 1624 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'], 1625 'non-arch-board': [ 1626 IMPLY_NON_ARCH_BOARD, 1627 'Allow Kconfig options outside arch/ and /board/ to imply'], 1628}; 1629 1630def do_imply_config(config_list, add_imply, imply_flags, skip_added, 1631 check_kconfig=True, find_superset=False): 1632 """Find CONFIG options which imply those in the list 1633 1634 Some CONFIG options can be implied by others and this can help to reduce 1635 the size of the defconfig files. For example, CONFIG_X86 implies 1636 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 1637 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 1638 each of the x86 defconfig files. 1639 1640 This function uses the moveconfig database to find such options. It 1641 displays a list of things that could possibly imply those in the list. 1642 The algorithm ignores any that start with CONFIG_TARGET since these 1643 typically refer to only a few defconfigs (often one). It also does not 1644 display a config with less than 5 defconfigs. 1645 1646 The algorithm works using sets. For each target config in config_list: 1647 - Get the set 'defconfigs' which use that target config 1648 - For each config (from a list of all configs): 1649 - Get the set 'imply_defconfig' of defconfigs which use that config 1650 - 1651 - If imply_defconfigs contains anything not in defconfigs then 1652 this config does not imply the target config 1653 1654 Params: 1655 config_list: List of CONFIG options to check (each a string) 1656 add_imply: Automatically add an 'imply' for each config. 1657 imply_flags: Flags which control which implying configs are allowed 1658 (IMPLY_...) 1659 skip_added: Don't show options which already have an imply added. 1660 check_kconfig: Check if implied symbols already have an 'imply' or 1661 'select' for the target config, and show this information if so. 1662 find_superset: True to look for configs which are a superset of those 1663 already found. So for example if CONFIG_EXYNOS5 implies an option, 1664 but CONFIG_EXYNOS covers a larger set of defconfigs and also 1665 implies that option, this will drop the former in favour of the 1666 latter. In practice this option has not proved very used. 1667 1668 Note the terminoloy: 1669 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') 1670 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') 1671 """ 1672 kconf = KconfigScanner().conf if check_kconfig else None 1673 if add_imply and add_imply != 'all': 1674 add_imply = add_imply.split() 1675 1676 # key is defconfig name, value is dict of (CONFIG_xxx, value) 1677 config_db = {} 1678 1679 # Holds a dict containing the set of defconfigs that contain each config 1680 # key is config, value is set of defconfigs using that config 1681 defconfig_db = collections.defaultdict(set) 1682 1683 # Set of all config options we have seen 1684 all_configs = set() 1685 1686 # Set of all defconfigs we have seen 1687 all_defconfigs = set() 1688 1689 # Read in the database 1690 configs = {} 1691 with open(CONFIG_DATABASE) as fd: 1692 for line in fd.readlines(): 1693 line = line.rstrip() 1694 if not line: # Separator between defconfigs 1695 config_db[defconfig] = configs 1696 all_defconfigs.add(defconfig) 1697 configs = {} 1698 elif line[0] == ' ': # CONFIG line 1699 config, value = line.strip().split('=', 1) 1700 configs[config] = value 1701 defconfig_db[config].add(defconfig) 1702 all_configs.add(config) 1703 else: # New defconfig 1704 defconfig = line 1705 1706 # Work through each target config option in tern, independently 1707 for config in config_list: 1708 defconfigs = defconfig_db.get(config) 1709 if not defconfigs: 1710 print('%s not found in any defconfig' % config) 1711 continue 1712 1713 # Get the set of defconfigs without this one (since a config cannot 1714 # imply itself) 1715 non_defconfigs = all_defconfigs - defconfigs 1716 num_defconfigs = len(defconfigs) 1717 print('%s found in %d/%d defconfigs' % (config, num_defconfigs, 1718 len(all_configs))) 1719 1720 # This will hold the results: key=config, value=defconfigs containing it 1721 imply_configs = {} 1722 rest_configs = all_configs - set([config]) 1723 1724 # Look at every possible config, except the target one 1725 for imply_config in rest_configs: 1726 if 'ERRATUM' in imply_config: 1727 continue 1728 if not (imply_flags & IMPLY_CMD): 1729 if 'CONFIG_CMD' in imply_config: 1730 continue 1731 if not (imply_flags & IMPLY_TARGET): 1732 if 'CONFIG_TARGET' in imply_config: 1733 continue 1734 1735 # Find set of defconfigs that have this config 1736 imply_defconfig = defconfig_db[imply_config] 1737 1738 # Get the intersection of this with defconfigs containing the 1739 # target config 1740 common_defconfigs = imply_defconfig & defconfigs 1741 1742 # Get the set of defconfigs containing this config which DO NOT 1743 # also contain the taret config. If this set is non-empty it means 1744 # that this config affects other defconfigs as well as (possibly) 1745 # the ones affected by the target config. This means it implies 1746 # things we don't want to imply. 1747 not_common_defconfigs = imply_defconfig & non_defconfigs 1748 if not_common_defconfigs: 1749 continue 1750 1751 # If there are common defconfigs, imply_config may be useful 1752 if common_defconfigs: 1753 skip = False 1754 if find_superset: 1755 for prev in list(imply_configs.keys()): 1756 prev_count = len(imply_configs[prev]) 1757 count = len(common_defconfigs) 1758 if (prev_count > count and 1759 (imply_configs[prev] & common_defconfigs == 1760 common_defconfigs)): 1761 # skip imply_config because prev is a superset 1762 skip = True 1763 break 1764 elif count > prev_count: 1765 # delete prev because imply_config is a superset 1766 del imply_configs[prev] 1767 if not skip: 1768 imply_configs[imply_config] = common_defconfigs 1769 1770 # Now we have a dict imply_configs of configs which imply each config 1771 # The value of each dict item is the set of defconfigs containing that 1772 # config. Rank them so that we print the configs that imply the largest 1773 # number of defconfigs first. 1774 ranked_iconfigs = sorted(imply_configs, 1775 key=lambda k: len(imply_configs[k]), reverse=True) 1776 kconfig_info = '' 1777 cwd = os.getcwd() 1778 add_list = collections.defaultdict(list) 1779 for iconfig in ranked_iconfigs: 1780 num_common = len(imply_configs[iconfig]) 1781 1782 # Don't bother if there are less than 5 defconfigs affected. 1783 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5): 1784 continue 1785 missing = defconfigs - imply_configs[iconfig] 1786 missing_str = ', '.join(missing) if missing else 'all' 1787 missing_str = '' 1788 show = True 1789 if kconf: 1790 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:], 1791 iconfig[CONFIG_LEN:]) 1792 kconfig_info = '' 1793 if sym: 1794 locs = sym.get_def_locations() 1795 if len(locs) == 1: 1796 fname, linenum = locs[0] 1797 if cwd and fname.startswith(cwd): 1798 fname = fname[len(cwd) + 1:] 1799 kconfig_info = '%s:%d' % (fname, linenum) 1800 if skip_added: 1801 show = False 1802 else: 1803 sym = kconf.syms.get(iconfig[CONFIG_LEN:]) 1804 fname = '' 1805 if sym: 1806 locs = sym.get_def_locations() 1807 if len(locs) == 1: 1808 fname, linenum = locs[0] 1809 if cwd and fname.startswith(cwd): 1810 fname = fname[len(cwd) + 1:] 1811 in_arch_board = not sym or (fname.startswith('arch') or 1812 fname.startswith('board')) 1813 if (not in_arch_board and 1814 not (imply_flags & IMPLY_NON_ARCH_BOARD)): 1815 continue 1816 1817 if add_imply and (add_imply == 'all' or 1818 iconfig in add_imply): 1819 fname, linenum, kconfig_info = (check_imply_rule(kconf, 1820 config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) 1821 if fname: 1822 add_list[fname].append(linenum) 1823 1824 if show and kconfig_info != 'skip': 1825 print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30), 1826 kconfig_info, missing_str)) 1827 1828 # Having collected a list of things to add, now we add them. We process 1829 # each file from the largest line number to the smallest so that 1830 # earlier additions do not affect our line numbers. E.g. if we added an 1831 # imply at line 20 it would change the position of each line after 1832 # that. 1833 for fname, linenums in add_list.items(): 1834 for linenum in sorted(linenums, reverse=True): 1835 add_imply_rule(config[CONFIG_LEN:], fname, linenum) 1836 1837 1838def main(): 1839 try: 1840 cpu_count = multiprocessing.cpu_count() 1841 except NotImplementedError: 1842 cpu_count = 1 1843 1844 parser = optparse.OptionParser() 1845 # Add options here 1846 parser.add_option('-a', '--add-imply', type='string', default='', 1847 help='comma-separated list of CONFIG options to add ' 1848 "an 'imply' statement to for the CONFIG in -i") 1849 parser.add_option('-A', '--skip-added', action='store_true', default=False, 1850 help="don't show options which are already marked as " 1851 'implying others') 1852 parser.add_option('-b', '--build-db', action='store_true', default=False, 1853 help='build a CONFIG database') 1854 parser.add_option('-c', '--color', action='store_true', default=False, 1855 help='display the log in color') 1856 parser.add_option('-C', '--commit', action='store_true', default=False, 1857 help='Create a git commit for the operation') 1858 parser.add_option('-d', '--defconfigs', type='string', 1859 help='a file containing a list of defconfigs to move, ' 1860 "one per line (for example 'snow_defconfig') " 1861 "or '-' to read from stdin") 1862 parser.add_option('-i', '--imply', action='store_true', default=False, 1863 help='find options which imply others') 1864 parser.add_option('-I', '--imply-flags', type='string', default='', 1865 help="control the -i option ('help' for help") 1866 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1867 help='perform a trial run (show log with no changes)') 1868 parser.add_option('-e', '--exit-on-error', action='store_true', 1869 default=False, 1870 help='exit immediately on any error') 1871 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1872 help='force sync by savedefconfig') 1873 parser.add_option('-S', '--spl', action='store_true', default=False, 1874 help='parse config options defined for SPL build') 1875 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1876 action='store_true', default=False, 1877 help='only cleanup the headers') 1878 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1879 help='the number of jobs to run simultaneously') 1880 parser.add_option('-r', '--git-ref', type='string', 1881 help='the git ref to clone for building the autoconf.mk') 1882 parser.add_option('-y', '--yes', action='store_true', default=False, 1883 help="respond 'yes' to any prompts") 1884 parser.add_option('-v', '--verbose', action='store_true', default=False, 1885 help='show any build errors as boards are built') 1886 parser.usage += ' CONFIG ...' 1887 1888 (options, configs) = parser.parse_args() 1889 1890 if len(configs) == 0 and not any((options.force_sync, options.build_db, 1891 options.imply)): 1892 parser.print_usage() 1893 sys.exit(1) 1894 1895 # prefix the option name with CONFIG_ if missing 1896 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1897 for config in configs ] 1898 1899 check_top_directory() 1900 1901 if options.imply: 1902 imply_flags = 0 1903 if options.imply_flags == 'all': 1904 imply_flags = -1 1905 1906 elif options.imply_flags: 1907 for flag in options.imply_flags.split(','): 1908 bad = flag not in IMPLY_FLAGS 1909 if bad: 1910 print("Invalid flag '%s'" % flag) 1911 if flag == 'help' or bad: 1912 print("Imply flags: (separate with ',')") 1913 for name, info in IMPLY_FLAGS.items(): 1914 print(' %-15s: %s' % (name, info[1])) 1915 parser.print_usage() 1916 sys.exit(1) 1917 imply_flags |= IMPLY_FLAGS[flag][0] 1918 1919 do_imply_config(configs, options.add_imply, imply_flags, 1920 options.skip_added) 1921 return 1922 1923 config_db = {} 1924 db_queue = queue.Queue() 1925 t = DatabaseThread(config_db, db_queue) 1926 t.setDaemon(True) 1927 t.start() 1928 1929 if not options.cleanup_headers_only: 1930 check_clean_directory() 1931 bsettings.Setup('') 1932 toolchains = toolchain.Toolchains() 1933 toolchains.GetSettings() 1934 toolchains.Scan(verbose=False) 1935 move_config(toolchains, configs, options, db_queue) 1936 db_queue.join() 1937 1938 if configs: 1939 cleanup_headers(configs, options) 1940 cleanup_extra_options(configs, options) 1941 cleanup_whitelist(configs, options) 1942 cleanup_readme(configs, options) 1943 1944 if options.commit: 1945 subprocess.call(['git', 'add', '-u']) 1946 if configs: 1947 msg = 'Convert %s %sto Kconfig' % (configs[0], 1948 'et al ' if len(configs) > 1 else '') 1949 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1950 '\n '.join(configs)) 1951 else: 1952 msg = 'configs: Resync with savedefconfig' 1953 msg += '\n\nRsync all defconfig files using moveconfig.py' 1954 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1955 1956 if options.build_db: 1957 with open(CONFIG_DATABASE, 'w') as fd: 1958 for defconfig, configs in config_db.items(): 1959 fd.write('%s\n' % defconfig) 1960 for config in sorted(configs.keys()): 1961 fd.write(' %s=%s\n' % (config, configs[config])) 1962 fd.write('\n') 1963 1964if __name__ == '__main__': 1965 main() 1966