1"""Access and/or modify INI files 2 3* Compatiable with ConfigParser 4* Preserves order of sections & options 5* Preserves comments/blank lines/etc 6* More conveninet access to data 7 8Example: 9 10 >>> from six import StringIO 11 >>> sio = StringIO('''# configure foo-application 12 ... [foo] 13 ... bar1 = qualia 14 ... bar2 = 1977 15 ... [foo-ext] 16 ... special = 1''') 17 18 >>> cfg = INIConfig(sio) 19 >>> print(cfg.foo.bar1) 20 qualia 21 >>> print(cfg['foo-ext'].special) 22 1 23 >>> cfg.foo.newopt = 'hi!' 24 >>> cfg.baz.enabled = 0 25 26 >>> print(cfg) 27 # configure foo-application 28 [foo] 29 bar1 = qualia 30 bar2 = 1977 31 newopt = hi! 32 [foo-ext] 33 special = 1 34 <BLANKLINE> 35 [baz] 36 enabled = 0 37 38""" 39 40# An ini parser that supports ordered sections/options 41# Also supports updates, while preserving structure 42# Backward-compatiable with ConfigParser 43 44import re 45from .configparser import DEFAULTSECT, ParsingError, MissingSectionHeaderError 46 47import six 48 49from . import config 50 51 52class LineType(object): 53 line = None 54 55 def __init__(self, line=None): 56 if line is not None: 57 self.line = line.strip('\n') 58 59 # Return the original line for unmodified objects 60 # Otherwise construct using the current attribute values 61 def __str__(self): 62 if self.line is not None: 63 return self.line 64 else: 65 return self.to_string() 66 67 # If an attribute is modified after initialization 68 # set line to None since it is no longer accurate. 69 def __setattr__(self, name, value): 70 if hasattr(self,name): 71 self.__dict__['line'] = None 72 self.__dict__[name] = value 73 74 def to_string(self): 75 raise Exception('This method must be overridden in derived classes') 76 77 78class SectionLine(LineType): 79 regex = re.compile(r'^\[' 80 r'(?P<name>[^]]+)' 81 r'\]\s*' 82 r'((?P<csep>;|#)(?P<comment>.*))?$') 83 84 def __init__(self, name, comment=None, comment_separator=None, 85 comment_offset=-1, line=None): 86 super(SectionLine, self).__init__(line) 87 self.name = name 88 self.comment = comment 89 self.comment_separator = comment_separator 90 self.comment_offset = comment_offset 91 92 def to_string(self): 93 out = '[' + self.name + ']' 94 if self.comment is not None: 95 # try to preserve indentation of comments 96 out = (out+' ').ljust(self.comment_offset) 97 out = out + self.comment_separator + self.comment 98 return out 99 100 def parse(cls, line): 101 m = cls.regex.match(line.rstrip()) 102 if m is None: 103 return None 104 return cls(m.group('name'), m.group('comment'), 105 m.group('csep'), m.start('csep'), 106 line) 107 parse = classmethod(parse) 108 109 110class OptionLine(LineType): 111 def __init__(self, name, value, separator=' = ', comment=None, 112 comment_separator=None, comment_offset=-1, line=None): 113 super(OptionLine, self).__init__(line) 114 self.name = name 115 self.value = value 116 self.separator = separator 117 self.comment = comment 118 self.comment_separator = comment_separator 119 self.comment_offset = comment_offset 120 121 def to_string(self): 122 out = '%s%s%s' % (self.name, self.separator, self.value) 123 if self.comment is not None: 124 # try to preserve indentation of comments 125 out = (out+' ').ljust(self.comment_offset) 126 out = out + self.comment_separator + self.comment 127 return out 128 129 regex = re.compile(r'^(?P<name>[^:=\s[][^:=]*)' 130 r'(?P<sep>[:=]\s*)' 131 r'(?P<value>.*)$') 132 133 def parse(cls, line): 134 m = cls.regex.match(line.rstrip()) 135 if m is None: 136 return None 137 138 name = m.group('name').rstrip() 139 value = m.group('value') 140 sep = m.group('name')[len(name):] + m.group('sep') 141 142 # comments are not detected in the regex because 143 # ensuring total compatibility with ConfigParser 144 # requires that: 145 # option = value ;comment // value=='value' 146 # option = value;1 ;comment // value=='value;1 ;comment' 147 # 148 # Doing this in a regex would be complicated. I 149 # think this is a bug. The whole issue of how to 150 # include ';' in the value needs to be addressed. 151 # Also, '#' doesn't mark comments in options... 152 153 coff = value.find(';') 154 if coff != -1 and value[coff-1].isspace(): 155 comment = value[coff+1:] 156 csep = value[coff] 157 value = value[:coff].rstrip() 158 coff = m.start('value') + coff 159 else: 160 comment = None 161 csep = None 162 coff = -1 163 164 return cls(name, value, sep, comment, csep, coff, line) 165 parse = classmethod(parse) 166 167 168def change_comment_syntax(comment_chars='%;#', allow_rem=False): 169 comment_chars = re.sub(r'([\]\-\^])', r'\\\1', comment_chars) 170 regex = r'^(?P<csep>[%s]' % comment_chars 171 if allow_rem: 172 regex += '|[rR][eE][mM]' 173 regex += r')(?P<comment>.*)$' 174 CommentLine.regex = re.compile(regex) 175 176 177class CommentLine(LineType): 178 regex = re.compile(r'^(?P<csep>[;#]|[rR][eE][mM])' 179 r'(?P<comment>.*)$') 180 181 def __init__(self, comment='', separator='#', line=None): 182 super(CommentLine, self).__init__(line) 183 self.comment = comment 184 self.separator = separator 185 186 def to_string(self): 187 return self.separator + self.comment 188 189 def parse(cls, line): 190 m = cls.regex.match(line.rstrip()) 191 if m is None: 192 return None 193 return cls(m.group('comment'), m.group('csep'), line) 194 195 parse = classmethod(parse) 196 197 198class EmptyLine(LineType): 199 # could make this a singleton 200 def to_string(self): 201 return '' 202 203 value = property(lambda self: '') 204 205 def parse(cls, line): 206 if line.strip(): 207 return None 208 return cls(line) 209 210 parse = classmethod(parse) 211 212 213class ContinuationLine(LineType): 214 regex = re.compile(r'^\s+(?P<value>.*)$') 215 216 def __init__(self, value, value_offset=None, line=None): 217 super(ContinuationLine, self).__init__(line) 218 self.value = value 219 if value_offset is None: 220 value_offset = 8 221 self.value_offset = value_offset 222 223 def to_string(self): 224 return ' '*self.value_offset + self.value 225 226 def parse(cls, line): 227 m = cls.regex.match(line.rstrip()) 228 if m is None: 229 return None 230 return cls(m.group('value'), m.start('value'), line) 231 232 parse = classmethod(parse) 233 234 235class LineContainer(object): 236 def __init__(self, d=None): 237 self.contents = [] 238 self.orgvalue = None 239 if d: 240 if isinstance(d, list): self.extend(d) 241 else: self.add(d) 242 243 def add(self, x): 244 self.contents.append(x) 245 246 def extend(self, x): 247 for i in x: self.add(i) 248 249 def get_name(self): 250 return self.contents[0].name 251 252 def set_name(self, data): 253 self.contents[0].name = data 254 255 def get_value(self): 256 if self.orgvalue is not None: 257 return self.orgvalue 258 elif len(self.contents) == 1: 259 return self.contents[0].value 260 else: 261 return '\n'.join([('%s' % x.value) for x in self.contents 262 if not isinstance(x, CommentLine)]) 263 264 def set_value(self, data): 265 self.orgvalue = data 266 lines = ('%s' % data).split('\n') 267 268 # If there is an existing ContinuationLine, use its offset 269 value_offset = None 270 for v in self.contents: 271 if isinstance(v, ContinuationLine): 272 value_offset = v.value_offset 273 break 274 275 # Rebuild contents list, preserving initial OptionLine 276 self.contents = self.contents[0:1] 277 self.contents[0].value = lines[0] 278 del lines[0] 279 for line in lines: 280 if line.strip(): 281 self.add(ContinuationLine(line, value_offset)) 282 else: 283 self.add(EmptyLine()) 284 285 name = property(get_name, set_name) 286 287 value = property(get_value, set_value) 288 289 def __str__(self): 290 s = [x.__str__() for x in self.contents] 291 return '\n'.join(s) 292 293 def finditer(self, key): 294 for x in self.contents[::-1]: 295 if hasattr(x, 'name') and x.name==key: 296 yield x 297 298 def find(self, key): 299 for x in self.finditer(key): 300 return x 301 raise KeyError(key) 302 303 304def _make_xform_property(myattrname, srcattrname=None): 305 private_attrname = myattrname + 'value' 306 private_srcname = myattrname + 'source' 307 if srcattrname is None: 308 srcattrname = myattrname 309 310 def getfn(self): 311 srcobj = getattr(self, private_srcname) 312 if srcobj is not None: 313 return getattr(srcobj, srcattrname) 314 else: 315 return getattr(self, private_attrname) 316 317 def setfn(self, value): 318 srcobj = getattr(self, private_srcname) 319 if srcobj is not None: 320 setattr(srcobj, srcattrname, value) 321 else: 322 setattr(self, private_attrname, value) 323 324 return property(getfn, setfn) 325 326 327class INISection(config.ConfigNamespace): 328 _lines = None 329 _options = None 330 _defaults = None 331 _optionxformvalue = None 332 _optionxformsource = None 333 _compat_skip_empty_lines = set() 334 335 def __init__(self, lineobj, defaults=None, optionxformvalue=None, optionxformsource=None): 336 self._lines = [lineobj] 337 self._defaults = defaults 338 self._optionxformvalue = optionxformvalue 339 self._optionxformsource = optionxformsource 340 self._options = {} 341 342 _optionxform = _make_xform_property('_optionxform') 343 344 def _compat_get(self, key): 345 # identical to __getitem__ except that _compat_XXX 346 # is checked for backward-compatible handling 347 if key == '__name__': 348 return self._lines[-1].name 349 if self._optionxform: key = self._optionxform(key) 350 try: 351 value = self._options[key].value 352 del_empty = key in self._compat_skip_empty_lines 353 except KeyError: 354 if self._defaults and key in self._defaults._options: 355 value = self._defaults._options[key].value 356 del_empty = key in self._defaults._compat_skip_empty_lines 357 else: 358 raise 359 if del_empty: 360 value = re.sub('\n+', '\n', value) 361 return value 362 363 def _getitem(self, key): 364 if key == '__name__': 365 return self._lines[-1].name 366 if self._optionxform: key = self._optionxform(key) 367 try: 368 return self._options[key].value 369 except KeyError: 370 if self._defaults and key in self._defaults._options: 371 return self._defaults._options[key].value 372 else: 373 raise 374 375 def __setitem__(self, key, value): 376 if self._optionxform: xkey = self._optionxform(key) 377 else: xkey = key 378 if xkey in self._compat_skip_empty_lines: 379 self._compat_skip_empty_lines.remove(xkey) 380 if xkey not in self._options: 381 # create a dummy object - value may have multiple lines 382 obj = LineContainer(OptionLine(key, '')) 383 self._lines[-1].add(obj) 384 self._options[xkey] = obj 385 # the set_value() function in LineContainer 386 # automatically handles multi-line values 387 self._options[xkey].value = value 388 389 def __delitem__(self, key): 390 if self._optionxform: key = self._optionxform(key) 391 if key in self._compat_skip_empty_lines: 392 self._compat_skip_empty_lines.remove(key) 393 for l in self._lines: 394 remaining = [] 395 for o in l.contents: 396 if isinstance(o, LineContainer): 397 n = o.name 398 if self._optionxform: n = self._optionxform(n) 399 if key != n: remaining.append(o) 400 else: 401 remaining.append(o) 402 l.contents = remaining 403 del self._options[key] 404 405 def __iter__(self): 406 d = set() 407 for l in self._lines: 408 for x in l.contents: 409 if isinstance(x, LineContainer): 410 if self._optionxform: 411 ans = self._optionxform(x.name) 412 else: 413 ans = x.name 414 if ans not in d: 415 yield ans 416 d.add(ans) 417 if self._defaults: 418 for x in self._defaults: 419 if x not in d: 420 yield x 421 d.add(x) 422 423 def _new_namespace(self, name): 424 raise Exception('No sub-sections allowed', name) 425 426 427def make_comment(line): 428 return CommentLine(line.rstrip('\n')) 429 430 431def readline_iterator(f): 432 """iterate over a file by only using the file object's readline method""" 433 434 have_newline = False 435 while True: 436 line = f.readline() 437 438 if not line: 439 if have_newline: 440 yield "" 441 return 442 443 if line.endswith('\n'): 444 have_newline = True 445 else: 446 have_newline = False 447 448 yield line 449 450 451def lower(x): 452 return x.lower() 453 454 455class INIConfig(config.ConfigNamespace): 456 _data = None 457 _sections = None 458 _defaults = None 459 _optionxformvalue = None 460 _optionxformsource = None 461 _sectionxformvalue = None 462 _sectionxformsource = None 463 _parse_exc = None 464 _bom = False 465 466 def __init__(self, fp=None, defaults=None, parse_exc=True, 467 optionxformvalue=lower, optionxformsource=None, 468 sectionxformvalue=None, sectionxformsource=None): 469 self._data = LineContainer() 470 self._parse_exc = parse_exc 471 self._optionxformvalue = optionxformvalue 472 self._optionxformsource = optionxformsource 473 self._sectionxformvalue = sectionxformvalue 474 self._sectionxformsource = sectionxformsource 475 self._sections = {} 476 if defaults is None: defaults = {} 477 self._defaults = INISection(LineContainer(), optionxformsource=self) 478 for name, value in defaults.items(): 479 self._defaults[name] = value 480 if fp is not None: 481 self._readfp(fp) 482 483 _optionxform = _make_xform_property('_optionxform', 'optionxform') 484 _sectionxform = _make_xform_property('_sectionxform', 'optionxform') 485 486 def _getitem(self, key): 487 if key == DEFAULTSECT: 488 return self._defaults 489 if self._sectionxform: key = self._sectionxform(key) 490 return self._sections[key] 491 492 def __setitem__(self, key, value): 493 raise Exception('Values must be inside sections', key, value) 494 495 def __delitem__(self, key): 496 if self._sectionxform: key = self._sectionxform(key) 497 for line in self._sections[key]._lines: 498 self._data.contents.remove(line) 499 del self._sections[key] 500 501 def __iter__(self): 502 d = set() 503 d.add(DEFAULTSECT) 504 for x in self._data.contents: 505 if isinstance(x, LineContainer): 506 if x.name not in d: 507 yield x.name 508 d.add(x.name) 509 510 def _new_namespace(self, name): 511 if self._data.contents: 512 self._data.add(EmptyLine()) 513 obj = LineContainer(SectionLine(name)) 514 self._data.add(obj) 515 if self._sectionxform: name = self._sectionxform(name) 516 if name in self._sections: 517 ns = self._sections[name] 518 ns._lines.append(obj) 519 else: 520 ns = INISection(obj, defaults=self._defaults, 521 optionxformsource=self) 522 self._sections[name] = ns 523 return ns 524 525 def __str__(self): 526 if self._bom: 527 fmt = u'\ufeff%s' 528 else: 529 fmt = '%s' 530 return fmt % self._data.__str__() 531 532 __unicode__ = __str__ 533 534 _line_types = [EmptyLine, CommentLine, 535 SectionLine, OptionLine, 536 ContinuationLine] 537 538 def _parse(self, line): 539 for linetype in self._line_types: 540 lineobj = linetype.parse(line) 541 if lineobj: 542 return lineobj 543 else: 544 # can't parse line 545 return None 546 547 def _readfp(self, fp): 548 cur_section = None 549 cur_option = None 550 cur_section_name = None 551 cur_option_name = None 552 pending_lines = [] 553 pending_empty_lines = False 554 try: 555 fname = fp.name 556 except AttributeError: 557 fname = '<???>' 558 line_count = 0 559 exc = None 560 line = None 561 562 for line in readline_iterator(fp): 563 # Check for BOM on first line 564 if line_count == 0 and isinstance(line, six.text_type): 565 if line[0] == u'\ufeff': 566 line = line[1:] 567 self._bom = True 568 569 line_obj = self._parse(line) 570 line_count += 1 571 572 if not cur_section and not isinstance(line_obj, (CommentLine, EmptyLine, SectionLine)): 573 if self._parse_exc: 574 raise MissingSectionHeaderError(fname, line_count, line) 575 else: 576 line_obj = make_comment(line) 577 578 if line_obj is None: 579 if self._parse_exc: 580 if exc is None: 581 exc = ParsingError(fname) 582 exc.append(line_count, line) 583 line_obj = make_comment(line) 584 585 if isinstance(line_obj, ContinuationLine): 586 if cur_option: 587 if pending_lines: 588 cur_option.extend(pending_lines) 589 pending_lines = [] 590 if pending_empty_lines: 591 optobj._compat_skip_empty_lines.add(cur_option_name) 592 pending_empty_lines = False 593 cur_option.add(line_obj) 594 else: 595 # illegal continuation line - convert to comment 596 if self._parse_exc: 597 if exc is None: 598 exc = ParsingError(fname) 599 exc.append(line_count, line) 600 line_obj = make_comment(line) 601 602 if isinstance(line_obj, OptionLine): 603 if pending_lines: 604 cur_section.extend(pending_lines) 605 pending_lines = [] 606 pending_empty_lines = False 607 cur_option = LineContainer(line_obj) 608 cur_section.add(cur_option) 609 if self._optionxform: 610 cur_option_name = self._optionxform(cur_option.name) 611 else: 612 cur_option_name = cur_option.name 613 if cur_section_name == DEFAULTSECT: 614 optobj = self._defaults 615 else: 616 optobj = self._sections[cur_section_name] 617 optobj._options[cur_option_name] = cur_option 618 619 if isinstance(line_obj, SectionLine): 620 self._data.extend(pending_lines) 621 pending_lines = [] 622 pending_empty_lines = False 623 cur_section = LineContainer(line_obj) 624 self._data.add(cur_section) 625 cur_option = None 626 cur_option_name = None 627 if cur_section.name == DEFAULTSECT: 628 self._defaults._lines.append(cur_section) 629 cur_section_name = DEFAULTSECT 630 else: 631 if self._sectionxform: 632 cur_section_name = self._sectionxform(cur_section.name) 633 else: 634 cur_section_name = cur_section.name 635 if cur_section_name not in self._sections: 636 self._sections[cur_section_name] = \ 637 INISection(cur_section, defaults=self._defaults, 638 optionxformsource=self) 639 else: 640 self._sections[cur_section_name]._lines.append(cur_section) 641 642 if isinstance(line_obj, (CommentLine, EmptyLine)): 643 pending_lines.append(line_obj) 644 if isinstance(line_obj, EmptyLine): 645 pending_empty_lines = True 646 647 self._data.extend(pending_lines) 648 if line and line[-1] == '\n': 649 self._data.add(EmptyLine()) 650 651 if exc: 652 raise exc 653