1#!/usr/bin/env python3 2 3import sys 4import apsw 5import shlex 6import os 7import csv 8import re 9import textwrap 10import time 11import codecs 12import base64 13 14if sys.platform == "win32": 15 _win_colour = False 16 try: 17 import colorama 18 colorama.init() 19 del colorama 20 _win_colour = True 21 except: # there are several failure reasons, ignore them all 22 pass 23 24 25class Shell(object): 26 """Implements a SQLite shell 27 28 :param stdin: Where to read input from (default sys.stdin) 29 :param stdout: Where to send output (default sys.stdout) 30 :param stderr: Where to send errors (default sys.stderr) 31 :param encoding: Default encoding for files opened/created by the 32 Shell. If you want stdin/out/err to use a particular encoding 33 then you need to provide them `already configured <http://docs.python.org/library/codecs.html#codecs.open>`__ that way. 34 :param args: This should be program arguments only (ie if 35 passing in sys.argv do not include sys.argv[0] which is the 36 program name. You can also pass in None and then call 37 :meth:`process_args` if you want to catch any errors 38 in handling the arguments yourself. 39 :param db: A existing :class:`Connection` you wish to use 40 41 The commands and behaviour are modelled after the `interactive 42 shell <https://sqlite.org/sqlite.html>`__ that is part of 43 SQLite. 44 45 You can inherit from this class to embed in your own code and user 46 interface. Internally everything is handled as unicode. 47 Conversions only happen at the point of input or output which you 48 can override in your own code. 49 50 Errors and diagnostics are only ever sent to error output 51 (self.stderr) and never to the regular output (self.stdout). This 52 means using shell output is always easy and consistent. 53 54 Shell commands begin with a dot (eg .help). They are implemented 55 as a method named after the command (eg command_help). The method 56 is passed one parameter which is the list of arguments to the 57 command. 58 59 Output modes are implemented by functions named after the mode (eg 60 output_column). 61 62 When you request help the help information is automatically 63 generated from the docstrings for the command and output 64 functions. 65 66 You should not use a Shell object concurrently from multiple 67 threads. It is one huge set of state information which would 68 become inconsistent if used simultaneously, and then give baffling 69 errors. It is safe to call methods one at a time from different 70 threads. ie it doesn't care what thread calls methods as long as 71 you don't call more than one concurrently. 72 """ 73 class Error(Exception): 74 """Class raised on errors. The expectation is that the error 75 will be displayed by the shell as text so there are no 76 specific subclasses as the distinctions between different 77 types of errors doesn't matter.""" 78 pass 79 80 def __init__(self, stdin=None, stdout=None, stderr=None, encoding="utf8", args=None, db=None): 81 """Create instance, set defaults and do argument processing.""" 82 super(Shell, self).__init__() 83 # The parameter doc has to be in main class doc as sphinx 84 # ignores any described here 85 self.exceptions = False 86 self.history_file = "~/.sqlite_history" 87 self._db = None 88 self.dbfilename = None 89 if db: 90 self.db = db, db.filename 91 else: 92 self.db = None, None 93 self.prompt = "sqlite> " 94 self.moreprompt = " ..> " 95 self.separator = "|" 96 self.bail = False 97 self.echo = False 98 self.timer = False 99 self.header = False 100 self.nullvalue = "" 101 self.output = self.output_list 102 self._output_table = self._fmt_sql_identifier("table") 103 self.widths = [] 104 # do we truncate output in list mode? (explain doesn't, regular does) 105 self.truncate = True 106 # a stack of previous outputs. turning on explain saves previous, off restores 107 self._output_stack = [] 108 109 # other stuff 110 self.set_encoding(encoding) 111 if stdin is None: stdin = sys.stdin 112 if stdout is None: stdout = sys.stdout 113 if stderr is None: stderr = sys.stderr 114 self.stdin = stdin 115 self.stdout = stdout 116 self._original_stdout = stdout 117 self.stderr = stderr 118 # we don't become interactive until the command line args are 119 # successfully parsed and acted upon 120 self.interactive = None 121 # current colouring object 122 self.command_colour() # set to default 123 self._using_readline = False 124 self._input_stack = [] 125 self.input_line_number = 0 126 self.push_input() 127 self.push_output() 128 self._input_descriptions = [] 129 130 if args: 131 try: 132 self.process_args(args) 133 except: 134 if len(self._input_descriptions): 135 self._input_descriptions.append("Processing command line arguments") 136 self.handle_exception() 137 raise 138 139 if self.interactive is None: 140 self.interactive = getattr(self.stdin, "isatty", False) and self.stdin.isatty() and getattr( 141 self.stdout, "isatty", False) and self.stdout.isatty() 142 143 def _ensure_db(self): 144 "The database isn't opened until first use. This function ensures it is now open." 145 if not self._db: 146 if not self.dbfilename: 147 self.dbfilename = ":memory:" 148 self._db = apsw.Connection(self.dbfilename, 149 flags=apsw.SQLITE_OPEN_URI | apsw.SQLITE_OPEN_READWRITE 150 | apsw.SQLITE_OPEN_CREATE) 151 return self._db 152 153 def _set_db(self, newv): 154 "Sets the open database (or None) and filename" 155 (db, dbfilename) = newv 156 if self._db: 157 self._db.close(True) 158 self._db = None 159 self._db = db 160 self.dbfilename = dbfilename 161 162 db = property(_ensure_db, _set_db, None, "The current :class:`Connection`") 163 164 def process_args(self, args): 165 """Process command line options specified in args. It is safe to 166 call this multiple times. We try to be compatible with SQLite shell 167 argument parsing. 168 169 :param args: A list of string options. Do not include the 170 program as args[0] 171 172 :returns: A tuple of (databasefilename, initfiles, 173 sqlncommands). This is provided for informational purposes 174 only - they have already been acted upon. An example use 175 is that the SQLite shell does not enter the main interactive 176 loop if any sql/commands were provided. 177 178 The first non-option is the database file name. Each 179 remaining non-option is treated as a complete input (ie it 180 isn't joined with others looking for a trailing semi-colon). 181 182 The SQLite shell uses single dash in front of options. We 183 allow both single and double dashes. When an unrecognized 184 argument is encountered then 185 :meth:`process_unknown_args` is called. 186 """ 187 # we don't use optparse as we need to use single dashes for 188 # options - all hand parsed 189 if not args: 190 return None, [], [] 191 192 # are options still valid? 193 options = True 194 # have we seen the database name? 195 havedbname = False 196 # List of init files to read 197 inits = [] 198 # List of sql/dot commands 199 sqls = [] 200 201 while args: 202 if not options or not args[0].startswith("-"): 203 options = False 204 if not havedbname: 205 # grab new database 206 self.db = None, args[0] 207 havedbname = True 208 else: 209 sqls.append(args[0]) 210 args = args[1:] 211 continue 212 213 # remove initial single or double dash 214 args[0] = args[0][1:] 215 if args[0].startswith("-"): 216 args[0] = args[0][1:] 217 218 if args[0] == "init": 219 if len(args) < 2: 220 raise self.Error("You need to specify a filename after -init") 221 inits.append(args[1]) 222 args = args[2:] 223 continue 224 225 if args[0] == "header" or args[0] == "noheader": 226 self.header = args[0] == "header" 227 args = args[1:] 228 continue 229 230 if args[0] in ("echo", "bail", "interactive"): 231 setattr(self, args[0], True) 232 args = args[1:] 233 continue 234 235 if args[0] == "batch": 236 self.interactive = False 237 args = args[1:] 238 continue 239 240 if args[0] in ("separator", "nullvalue", "encoding"): 241 if len(args) < 2: 242 raise self.Error("You need to specify a value after -" + args[0]) 243 getattr(self, "command_" + args[0])([args[1]]) 244 args = args[2:] 245 continue 246 247 if args[0] == "version": 248 self.write(self.stdout, apsw.sqlitelibversion() + "\n") 249 # A pretty gnarly thing to do 250 sys.exit(0) 251 252 if args[0] == "help": 253 self.write(self.stderr, self.usage()) 254 sys.exit(0) 255 256 if args[0] in ("no-colour", "no-color", "nocolour", "nocolor"): 257 self.colour_scheme = "off" 258 self._out_colour() 259 args = args[1:] 260 continue 261 262 # only remaining known args are output modes 263 if getattr(self, "output_" + args[0], None): 264 self.command_mode(args[:1]) 265 args = args[1:] 266 continue 267 268 newargs = self.process_unknown_args(args) 269 if newargs is None: 270 raise self.Error("Unrecognized argument '" + args[0] + "'") 271 args = newargs 272 273 for f in inits: 274 self.command_read([f]) 275 276 for s in sqls: 277 self.process_complete_line(s) 278 279 return self.dbfilename, inits, sqls 280 281 def process_unknown_args(self, args): 282 """This is called when :meth:`process_args` encounters an 283 argument it doesn't understand. Override this method if you 284 want to be able to understand additional command line arguments. 285 286 :param args: A list of the remaining arguments. The initial one will 287 have had the leading dashes removed (eg if it was --foo on the command 288 line then args[0] will be "foo" 289 :returns: None if you don't recognize the argument either. Otherwise 290 return the list of remaining arguments after you have processed 291 yours. 292 """ 293 return None 294 295 def usage(self): 296 "Returns the usage message. Make sure it is newline terminated" 297 298 msg = """ 299Usage: program [OPTIONS] FILENAME [SQL|CMD] [SQL|CMD]... 300FILENAME is the name of a SQLite database. A new database is 301created if the file does not exist. 302OPTIONS include: 303 -init filename read/process named file 304 -echo print commands before execution 305 -[no]header turn headers on or off 306 -bail stop after hitting an error 307 -interactive force interactive I/O 308 -batch force batch I/O 309 -column set output mode to 'column' 310 -csv set output mode to 'csv' 311 -html set output mode to 'html' 312 -line set output mode to 'line' 313 -list set output mode to 'list' 314 -python set output mode to 'python' 315 -separator 'x' set output field separator (|) 316 -nullvalue 'text' set text string for NULL values 317 -version show SQLite version 318 -encoding 'name' the encoding to use for files 319 opened via .import, .read & .output 320 -nocolour disables colour output to screen 321""" 322 return msg.lstrip() 323 324 ### 325 ### Value formatting routines. They take a value and return a 326 ### text formatting of them. Mostly used by the various output's 327 ### but also by random other pieces of code. 328 ### 329 330 _binary_type = eval(("buffer", "bytes")[sys.version_info >= (3, 0)]) 331 _basestring = eval(("basestring", "str")[sys.version_info >= (3, 0)]) 332 333 # bytes that are ok in C strings - no need for quoting 334 _printable = [ 335 ord(x) for x in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~!@#$%^&*()`_-+={}[]:;,.<>/?|" 336 ] 337 338 def _fmt_c_string(self, v): 339 "Format as a C string including surrounding double quotes" 340 if isinstance(v, self._basestring): 341 op = ['"'] 342 for c in v: 343 if c == "\\": 344 op.append("\\\\") 345 elif c == "\r": 346 op.append("\\r") 347 elif c == "\n": 348 op.append("\\n") 349 elif c == "\t": 350 op.append("\\t") 351 elif ord(c) not in self._printable: 352 op.append("\\" + c) 353 else: 354 op.append(c) 355 op.append('"') 356 return "".join(op) 357 elif v is None: 358 return '"' + self.nullvalue + '"' 359 elif isinstance(v, self._binary_type): 360 if sys.version_info < (3, 0): 361 o = lambda x: ord(x) 362 fromc = lambda x: x 363 else: 364 o = lambda x: x 365 fromc = lambda x: chr(x) 366 res = ['"'] 367 for c in v: 368 if o(c) in self._printable: 369 res.append(fromc(c)) 370 else: 371 res.append("\\x%02X" % (o(c), )) 372 res.append('"') 373 return "".join(res) 374 else: 375 # number of some kind 376 return '"%s"' % (v, ) 377 378 def _fmt_html_col(self, v): 379 "Format as HTML (mainly escaping &/</>" 380 return self._fmt_text_col(v).\ 381 replace("&", "&"). \ 382 replace(">", ">"). \ 383 replace("<", "<"). \ 384 replace("'", "'"). \ 385 replace('"', """) 386 387 def _fmt_json_value(self, v): 388 "Format a value." 389 if isinstance(v, self._basestring): 390 # we assume utf8 so only some characters need to be escaed 391 op = ['"'] 392 for c in v: 393 if c == "\\": 394 op.append("\\\\") 395 elif c == "\r": 396 op.append("\\r") 397 elif c == "\n": 398 op.append("\\n") 399 elif c == "\t": 400 op.append("\\t") 401 elif c == "/": # yes you have to escape forward slash for some reason 402 op.append("\\/") 403 elif c == '"': 404 op.append("\\" + c) 405 elif c == "\\b": 406 op.append("\\b") 407 elif c == "\\f": 408 op.append("\\f") 409 else: 410 # It isn't clear when \u sequences *must* be used. 411 # Assuming not needed due to utf8 output which 412 # corresponds to what rfc4627 implies. 413 op.append(c) 414 op.append('"') 415 return "".join(op) 416 elif v is None: 417 return 'null' 418 elif isinstance(v, self._binary_type): 419 if sys.version_info < (3, 0): 420 o = base64.encodestring(v) 421 else: 422 o = base64.encodebytes(v).decode("ascii") 423 if o[-1] == "\n": 424 o = o[:-1] 425 return '"' + o + '"' 426 else: 427 # number of some kind 428 return '%s' % (v, ) 429 430 def _fmt_python(self, v): 431 "Format as python literal" 432 if v is None: 433 return "None" 434 elif isinstance(v, self._basestring): 435 return repr(v) 436 elif isinstance(v, self._binary_type): 437 if sys.version_info < (3, 0): 438 res = ["buffer(\""] 439 for i in v: 440 if ord(i) in self._printable: 441 res.append(i) 442 else: 443 res.append("\\x%02X" % (ord(i), )) 444 res.append("\")") 445 return "".join(res) 446 else: 447 res = ['b"'] 448 for i in v: 449 if i in self._printable: 450 res.append(chr(i)) 451 else: 452 res.append("\\x%02X" % (i, )) 453 res.append('"') 454 return "".join(res) 455 else: 456 return "%s" % (v, ) 457 458 def _fmt_sql_identifier(self, v): 459 "Return the identifier quoted in SQL syntax if needed (eg table and column names)" 460 if not len(v): # yes sqlite does allow zero length identifiers 461 return '""' 462 nonalnum = re.sub("[A-Za-z_0-9]+", "", v) 463 if len(nonalnum) == 0: 464 if v.upper() not in self._sqlite_reserved: 465 # Ok providing it doesn't start with a digit 466 if v[0] not in "0123456789": 467 return v 468 # double quote it unless there are any double quotes in it 469 if '"' in nonalnum: 470 return "[%s]" % (v, ) 471 return '"%s"' % (v, ) 472 473 def _fmt_text_col(self, v): 474 "Regular text formatting" 475 if v is None: 476 return self.nullvalue 477 elif isinstance(v, self._basestring): 478 return v 479 elif isinstance(v, self._binary_type): 480 # sqlite gives back raw bytes! 481 return "<Binary data>" 482 else: 483 return "%s" % (v, ) 484 485 ### 486 ### The various output routines. They are always called with the 487 ### header irrespective of the setting allowing for some per query 488 ### setup. (see output_column for example). The doc strings are 489 ### used to generate help. 490 ### 491 492 def output_column(self, header, line): 493 """ 494 Items left aligned in space padded columns. They are 495 truncated if they do not fit. If the width hasn't been 496 specified for a column then 10 is used unless the column name 497 (header) is longer in which case that width is used. Use the 498 .width command to change column sizes. 499 """ 500 # as an optimization we calculate self._actualwidths which is 501 # reset for each query 502 if header: 503 504 def gw(n): 505 if n < len(self.widths) and self.widths[n] != 0: 506 return self.widths[n] 507 # if width is not present or 0 then autosize 508 text = self._fmt_text_col(line[n]) 509 return max(len(text), 10) 510 511 widths = [gw(i) for i in range(len(line))] 512 513 if self.truncate: 514 self._actualwidths = ["%" + ("-%d.%ds", "%d.%ds")[w < 0] % (abs(w), abs(w)) for w in widths] 515 else: 516 self._actualwidths = ["%" + ("-%ds", "%ds")[w < 0] % (abs(w), ) for w in widths] 517 518 if self.header: 519 # output the headers 520 c = self.colour 521 cols = [ 522 c.header + (self._actualwidths[i] % (self._fmt_text_col(line[i]), )) + c.header_ 523 for i in range(len(line)) 524 ] 525 # sqlite shell uses two spaces between columns 526 self.write(self.stdout, " ".join(cols) + "\n") 527 if c is self._colours["off"]: 528 self.output_column(False, ["-" * abs(widths[i]) for i in range(len(widths))]) 529 return 530 cols = [ 531 self.colour.colour_value(line[i], self._actualwidths[i] % (self._fmt_text_col(line[i]), )) 532 for i in range(len(line)) 533 ] 534 # sqlite shell uses two spaces between columns 535 self.write(self.stdout, " ".join(cols) + "\n") 536 537 output_columns = output_column 538 539 def output_csv(self, header, line): 540 """ 541 Items in csv format (comma separated). Use tabs mode for tab 542 separated. You can use the .separator command to use a 543 different one after switching mode. A separator of comma uses 544 double quotes for quoting while other separators do not do any 545 quoting. The Python csv library used for this only supports 546 single character separators. 547 """ 548 549 # we use self._csv for the work, setup when header is 550 # supplied. _csv is a tuple of a StringIO and the csv.writer 551 # instance. 552 553 # Sigh 554 if sys.version_info < (3, 0): 555 fixdata = lambda x: x.encode("utf8") 556 else: 557 fixdata = lambda x: x 558 559 if header: 560 if sys.version_info < (3, 0): 561 import StringIO as io 562 else: 563 import io 564 s = io.StringIO() 565 kwargs = {} 566 if self.separator == ",": 567 kwargs["dialect"] = "excel" 568 elif self.separator == "\t": 569 kwargs["dialect"] = "excel-tab" 570 else: 571 kwargs["quoting"] = csv.QUOTE_NONE 572 kwargs["delimiter"] = fixdata(self.separator) 573 kwargs["doublequote"] = False 574 # csv module is bug ridden junk - I already say no 575 # quoting so it still looks for the quotechar and then 576 # gets upset that it can't be quoted. Which bit of no 577 # quoting was ambiguous? 578 kwargs["quotechar"] = "\x00" 579 580 writer = csv.writer(s, **kwargs) 581 self._csv = (s, writer) 582 if self.header: 583 self.output_csv(None, line) 584 return 585 586 if header is None: 587 c = self.colour 588 line = [c.header + fixdata(self._fmt_text_col(l)) + c.header_ for l in line] 589 else: 590 fmt = lambda x: self.colour.colour_value(x, fixdata(self._fmt_text_col(x))) 591 line = [fmt(l) for l in line] 592 self._csv[1].writerow(line) 593 t = self._csv[0].getvalue() 594 if sys.version_info < (3, 0): 595 t = t.decode("utf8") 596 # csv lib always does DOS eol 597 assert (t.endswith("\r\n")) 598 t = t[:-2] 599 # should not be other eol irregularities 600 assert (not t.endswith("\r") and not t.endswith("\n")) 601 self.write(self.stdout, t + "\n") 602 self._csv[0].truncate(0) 603 self._csv[0].seek(0) 604 605 def output_html(self, header, line): 606 "HTML table style" 607 if header: 608 if not self.header: 609 return 610 fmt = lambda x: self.colour.header + self._fmt_html_col(x) + self.colour.header_ 611 else: 612 fmt = lambda x: self.colour.colour_value(x, self._fmt_html_col(x)) 613 line = [fmt(l) for l in line] 614 out = ["<TR>"] 615 for l in line: 616 out.append(("<TD>", "<TH>")[header]) 617 out.append(l) 618 out.append(("</TD>\n", "</TH>\n")[header]) 619 out.append("</TR>\n") 620 self.write(self.stdout, "".join(out)) 621 622 def output_insert(self, header, line): 623 """ 624 Lines as SQL insert statements. The table name is "table" 625 unless you specified a different one as the second parameter 626 to the .mode command. 627 """ 628 if header: 629 return 630 fmt = lambda x: self.colour.colour_value(x, apsw.format_sql_value(x)) 631 out = "INSERT INTO " + self._output_table + " VALUES(" + ",".join([fmt(l) for l in line]) + ");\n" 632 self.write(self.stdout, out) 633 634 def output_json(self, header, line): 635 """ 636 Each line as a JSON object with a trailing comma. Blobs are 637 output as base64 encoded strings. You should be using UTF8 638 output encoding. 639 """ 640 if header: 641 self._output_json_cols = line 642 return 643 fmt = lambda x: self.colour.colour_value(x, self._fmt_json_value(x)) 644 out = ["%s: %s" % (self._fmt_json_value(k), fmt(line[i])) for i, k in enumerate(self._output_json_cols)] 645 self.write(self.stdout, "{ " + ", ".join(out) + "},\n") 646 647 def output_line(self, header, line): 648 """ 649 One value per line in the form 'column = value' with a blank 650 line between rows. 651 """ 652 if header: 653 w = 5 654 for l in line: 655 if len(l) > w: 656 w = len(l) 657 self._line_info = (w, line) 658 return 659 fmt = lambda x: self.colour.colour_value(x, self._fmt_text_col(x)) 660 w = self._line_info[0] 661 for i in range(len(line)): 662 self.write(self.stdout, "%*s = %s\n" % (w, self._line_info[1][i], fmt(line[i]))) 663 self.write(self.stdout, "\n") 664 665 output_lines = output_line 666 667 def output_list(self, header, line): 668 "All items on one line with separator" 669 if header: 670 if not self.header: 671 return 672 c = self.colour 673 fmt = lambda x: c.header + x + c.header_ 674 else: 675 fmt = lambda x: self.colour.colour_value(x, self._fmt_text_col(x)) 676 self.write(self.stdout, self.separator.join([fmt(x) for x in line]) + "\n") 677 678 def output_python(self, header, line): 679 "Tuples in Python source form for each row" 680 if header: 681 if not self.header: 682 return 683 c = self.colour 684 fmt = lambda x: c.header + self._fmt_python(x) + c.header_ 685 else: 686 fmt = lambda x: self.colour.colour_value(x, self._fmt_python(x)) 687 self.write(self.stdout, '(' + ", ".join([fmt(l) for l in line]) + "),\n") 688 689 def output_tcl(self, header, line): 690 "Outputs TCL/C style strings using current separator" 691 # In theory you could paste the output into your source ... 692 if header: 693 if not self.header: 694 return 695 c = self.colour 696 fmt = lambda x: c.header + self._fmt_c_string(x) + c.header_ 697 else: 698 fmt = lambda x: self.colour.colour_value(x, self._fmt_c_string(x)) 699 self.write(self.stdout, self.separator.join([fmt(l) for l in line]) + "\n") 700 701 def _output_summary(self, summary): 702 # internal routine to output a summary line or two 703 self.write(self.stdout, self.colour.summary + summary + self.colour.summary_) 704 705 ### 706 ### Various routines 707 ### 708 709 def cmdloop(self, intro=None): 710 """Runs the main interactive command loop. 711 712 :param intro: Initial text banner to display instead of the 713 default. Make sure you newline terminate it. 714 """ 715 if intro is None: 716 intro = """ 717SQLite version %s (APSW %s) 718Enter ".help" for instructions 719Enter SQL statements terminated with a ";" 720""" % (apsw.sqlitelibversion(), apsw.apswversion()) 721 intro = intro.lstrip() 722 if self.interactive and intro: 723 if sys.version_info < (3, 0): 724 intro = unicode(intro) 725 c = self.colour 726 self.write(self.stdout, c.intro + intro + c.intro_) 727 728 using_readline = False 729 try: 730 if self.interactive and self.stdin is sys.stdin: 731 import readline 732 old_completer = readline.get_completer() 733 readline.set_completer(self.complete) 734 readline.parse_and_bind("tab: complete") 735 using_readline = True 736 try: 737 readline.read_history_file(os.path.expanduser(self.history_file)) 738 except: 739 # We only expect IOError here but if the history 740 # file does not exist and this code has been 741 # compiled into the module it is possible to get 742 # an IOError that doesn't match the IOError from 743 # Python parse time resulting in an IOError 744 # exception being raised. Consequently we just 745 # catch all exceptions. 746 pass 747 except ImportError: 748 pass 749 750 try: 751 while True: 752 self._input_descriptions = [] 753 if using_readline: 754 # we drop completion cache because it contains 755 # table and column names which could have changed 756 # with last executed SQL 757 self._completion_cache = None 758 self._using_readline = True 759 try: 760 command = self.getcompleteline() 761 if command is None: # EOF 762 return 763 self.process_complete_line(command) 764 except: 765 self._append_input_description() 766 try: 767 self.handle_exception() 768 except UnicodeDecodeError: 769 self.handle_exception() 770 finally: 771 if using_readline: 772 readline.set_completer(old_completer) 773 readline.set_history_length(256) 774 readline.write_history_file(os.path.expanduser(self.history_file)) 775 776 def handle_exception(self): 777 """Handles the current exception, printing a message to stderr as appropriate. 778 It will reraise the exception if necessary (eg if bail is true)""" 779 eclass, eval, etb = sys.exc_info() # py2&3 compatible way of doing this 780 if isinstance(eval, SystemExit): 781 eval._handle_exception_saw_this = True 782 raise 783 784 self._out_colour() 785 self.write(self.stderr, self.colour.error) 786 787 if isinstance(eval, KeyboardInterrupt): 788 self.handle_interrupt() 789 text = "Interrupted" 790 else: 791 text = str(eval) 792 793 if not text.endswith("\n"): 794 text = text + "\n" 795 796 if len(self._input_descriptions): 797 for i in range(len(self._input_descriptions)): 798 if i == 0: 799 pref = "At " 800 else: 801 pref = " " * i + "From " 802 self.write(self.stderr, pref + self._input_descriptions[i] + "\n") 803 804 self.write(self.stderr, text) 805 if self.exceptions: 806 stack = [] 807 while etb: 808 stack.append(etb.tb_frame) 809 etb = etb.tb_next 810 811 for frame in stack: 812 self.write( 813 self.stderr, 814 "\nFrame %s in %s at line %d\n" % (frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno)) 815 vars = list(frame.f_locals.items()) 816 vars.sort() 817 for k, v in vars: 818 try: 819 v = repr(v)[:80] 820 except: 821 v = "<Unable to convert to string>" 822 self.write(self.stderr, "%10s = %s\n" % (k, v)) 823 self.write(self.stderr, "\n%s: %s\n" % (eclass, repr(eval))) 824 825 self.write(self.stderr, self.colour.error_) 826 827 eval._handle_exception_saw_this = True 828 if self.bail: 829 raise 830 831 def process_sql(self, sql, bindings=None, internal=False, summary=None): 832 """Processes SQL text consisting of one or more statements 833 834 :param sql: SQL to execute 835 836 :param bindings: bindings for the *sql* 837 838 :param internal: If True then this is an internal execution 839 (eg the .tables or .database command). When executing 840 internal sql timings are not shown nor is the SQL echoed. 841 842 :param summary: If not None then should be a tuple of two 843 items. If the ``sql`` returns any data then the first item 844 is printed before the first row, and the second item is 845 printed after the last row. An example usage is the .find 846 command which shows table names. 847 """ 848 cur = self.db.cursor() 849 # we need to know when each new statement is executed 850 state = {'newsql': True, 'timing': None} 851 852 def et(cur, sql, bindings): 853 state['newsql'] = True 854 # if time reporting, do so now 855 if not internal and self.timer: 856 if state['timing']: 857 self.display_timing(state['timing'], self.get_resource_usage()) 858 # print statement if echo is on 859 if not internal and self.echo: 860 # ? should we strip leading and trailing whitespace? backslash quote stuff? 861 if bindings: 862 self.write(self.stderr, "%s [%s]\n" % (sql, bindings)) 863 else: 864 self.write(self.stderr, sql + "\n") 865 # save resource from beginning of command (ie don't include echo time above) 866 if not internal and self.timer: 867 state['timing'] = self.get_resource_usage() 868 return True 869 870 cur.setexectrace(et) 871 # processing loop 872 try: 873 for row in cur.execute(sql, bindings): 874 if state['newsql']: 875 # summary line? 876 if summary: 877 self._output_summary(summary[0]) 878 # output a header always 879 cols = [h for h, d in cur.getdescription()] 880 self.output(True, cols) 881 state['newsql'] = False 882 self.output(False, row) 883 if not state['newsql'] and summary: 884 self._output_summary(summary[1]) 885 except: 886 # If echo is on and the sql to execute is a syntax error 887 # then the exec tracer won't have seen it so it won't be 888 # printed and the user will be wondering exactly what sql 889 # had the error. We look in the traceback and deduce if 890 # the error was happening in a prepare or not. Also we 891 # need to ignore the case where SQLITE_SCHEMA happened and 892 # a reprepare is being done since the exec tracer will 893 # have been called in that situation. 894 if not internal and self.echo: 895 tb = sys.exc_info()[2] 896 last = None 897 while tb: 898 last = tb.tb_frame 899 tb = tb.tb_next 900 901 if last and last.f_code.co_name=="sqlite3_prepare" \ 902 and last.f_code.co_filename.endswith("statementcache.c") \ 903 and "sql" in last.f_locals: 904 self.write(self.stderr, last.f_locals["sql"] + "\n") 905 raise 906 907 if not internal and self.timer: 908 self.display_timing(state['timing'], self.get_resource_usage()) 909 910 def process_command(self, cmd): 911 """Processes a dot command. It is split into parts using the 912 `shlex.split 913 <http://docs.python.org/library/shlex.html#shlex.split>`__ 914 function which is roughly the same method used by Unix/POSIX 915 shells. 916 """ 917 if self.echo: 918 self.write(self.stderr, cmd + "\n") 919 # broken with unicode on Python 2!!! 920 if sys.version_info < (3, 0): 921 cmd = cmd.encode("utf8") 922 cmd = [c.decode("utf8") for c in shlex.split(cmd)] 923 else: 924 cmd = shlex.split(cmd) 925 assert cmd[0][0] == "." 926 cmd[0] = cmd[0][1:] 927 fn = getattr(self, "command_" + cmd[0], None) 928 if not fn: 929 raise self.Error("Unknown command \"%s\". Enter \".help\" for help" % (cmd[0], )) 930 res = fn(cmd[1:]) 931 932 ### 933 ### Commands start here 934 ### 935 936 def _boolean_command(self, name, cmd): 937 "Parse and verify boolean parameter" 938 if len(cmd) != 1 or cmd[0].lower() not in ("on", "off"): 939 raise self.Error(name + " expected ON or OFF") 940 return cmd[0].lower() == "on" 941 942 # Note that doc text is used for generating help output. 943 944 def command_backup(self, cmd): 945 """backup ?DB? FILE: Backup DB (default "main") to FILE 946 947 Copies the contents of the current database to FILE 948 overwriting whatever was in FILE. If you have attached databases 949 then you can specify their name instead of the default of "main". 950 951 The backup is done at the page level - SQLite copies the pages 952 as is. There is no round trip through SQL code. 953 """ 954 dbname = "main" 955 if len(cmd) == 1: 956 fname = cmd[0] 957 elif len(cmd) == 2: 958 dbname = cmd[0] 959 fname = cmd[1] 960 else: 961 raise self.Error("Backup takes one or two parameters") 962 out = apsw.Connection(fname) 963 b = out.backup("main", self.db, dbname) 964 try: 965 while not b.done: 966 b.step() 967 finally: 968 b.finish() 969 out.close() 970 971 def command_bail(self, cmd): 972 """bail ON|OFF: Stop after hitting an error (default OFF) 973 974 If an error is encountered while processing commands or SQL 975 then exit. (Note this is different than SQLite shell which 976 only exits for errors in SQL.) 977 """ 978 self.bail = self._boolean_command("bail", cmd) 979 980 def command_colour(self, cmd=[]): 981 """colour SCHEME: Selects a colour scheme 982 983 Residents of both countries that have not adopted the metric 984 system may also spell this command without a 'u'. If using a 985 colour terminal in interactive mode then output is 986 automatically coloured to make it more readable. Use 'off' to 987 turn off colour, and no name or 'default' for the default. 988 """ 989 if len(cmd) > 1: 990 raise self.Error("Too many colour schemes") 991 c = cmd and cmd[0] or "default" 992 if c not in self._colours: 993 raise self.Error("No such colour scheme: " + c) 994 self.colour_scheme = c 995 self._out_colour() 996 997 command_color = command_colour 998 999 def command_databases(self, cmd): 1000 """databases: Lists names and files of attached databases 1001 1002 """ 1003 if len(cmd): 1004 raise self.Error("databases command doesn't take any parameters") 1005 self.push_output() 1006 self.header = True 1007 self.output = self.output_column 1008 self.truncate = False 1009 self.widths = [3, 15, 58] 1010 try: 1011 self.process_sql("pragma database_list", internal=True) 1012 finally: 1013 self.pop_output() 1014 1015 def command_dump(self, cmd): 1016 """dump ?TABLE? [TABLE...]: Dumps all or specified tables in SQL text format 1017 1018 The table name is treated as like pattern so you can use % as 1019 a wildcard. You can use dump to make a text based backup of 1020 the database. It is also useful for comparing differences or 1021 making the data available to other databases. Indices and 1022 triggers for the table(s) are also dumped. Finally views 1023 matching the table pattern name are dumped (it isn't possible 1024 to work out which views access which table and views can 1025 access multiple tables anyway). 1026 1027 Note that if you are dumping virtual tables such as used by 1028 the FTS3 module then they may use other tables to store 1029 information. For example if you create a FTS3 table named 1030 *recipes* then it also creates *recipes_content*, 1031 *recipes_segdir* etc. Consequently to dump this example 1032 correctly use:: 1033 1034 .dump recipes recipes_% 1035 1036 If the database is empty or no tables/views match then there 1037 is no output. 1038 """ 1039 # Simple tables are easy to dump. More complicated is dealing 1040 # with virtual tables, foreign keys etc. 1041 1042 # Lock the database while doing the dump so nothing changes 1043 # under our feet 1044 self.process_sql("BEGIN IMMEDIATE", internal=True) 1045 1046 # Used in comment() - see issue 142 1047 outputstrtype = str 1048 if sys.version_info < (3, 0): 1049 outputstrtype = unicode 1050 1051 # Python 2.3 can end up with nonsense like "en_us" so we fall 1052 # back to ascii in that case 1053 outputstrencoding = getattr(self.stdout, "encoding", "ascii") 1054 try: 1055 codecs.lookup(outputstrencoding) 1056 except: 1057 outputstrencoding = "ascii" 1058 1059 def unicodify(s): 1060 if not isinstance(s, outputstrtype): 1061 # See issue 142 - it may not be in an expected encoding 1062 return s.decode(outputstrencoding, "replace") 1063 return s 1064 1065 try: 1066 # first pass -see if virtual tables or foreign keys are in 1067 # use. If they are we emit pragmas to deal with them, but 1068 # prefer not to emit them 1069 v = {"virtuals": False, "foreigns": False} 1070 1071 def check(name, sql): 1072 if name.lower().startswith("sqlite_"): 1073 return False 1074 sql = sql.lower() 1075 if re.match(r"^\s*create\s+virtual\s+.*", sql): 1076 v["virtuals"] = True 1077 # pragma table_info doesn't tell us if foreign keys 1078 # are involved so we guess if any the various strings are 1079 # in the sql somewhere 1080 if re.match(r".*\b(foreign\s*key|references)\b.*", sql): 1081 v["foreigns"] = True 1082 return True 1083 1084 if len(cmd) == 0: 1085 cmd = ["%"] 1086 1087 tables = [] 1088 for pattern in cmd: 1089 for name, sql in self.db.cursor().execute( 1090 "SELECT name,sql FROM sqlite_master " 1091 "WHERE sql NOT NULL AND type IN ('table','view') " 1092 "AND tbl_name LIKE ?1", (pattern, )): 1093 if check(name, sql) and name not in tables: 1094 tables.append(name) 1095 1096 if not tables: 1097 return 1098 1099 # will we need to analyze anything later? 1100 analyze_needed = [] 1101 for stat in self.db.cursor().execute( 1102 "select name from sqlite_master where sql not null and type='table' and tbl_name like 'sqlite_stat%'" 1103 ): 1104 for name in tables: 1105 if len(self.db.cursor().execute( 1106 "select * from " + self._fmt_sql_identifier(stat[0]) + " WHERE tbl=?", 1107 (name, )).fetchall()): 1108 if name not in analyze_needed: 1109 analyze_needed.append(name) 1110 analyze_needed.sort() 1111 1112 def blank(): 1113 self.write(self.stdout, "\n") 1114 1115 def comment(s): 1116 s = unicodify(s) 1117 self.write(self.stdout, textwrap.fill(s, 78, initial_indent="-- ", subsequent_indent="-- ") + "\n") 1118 1119 pats = ", ".join([(x, "(All)")[x == "%"] for x in cmd]) 1120 comment("SQLite dump (by APSW %s)" % (apsw.apswversion(), )) 1121 comment("SQLite version " + apsw.sqlitelibversion()) 1122 comment("Date: " + unicodify(time.strftime("%c"))) 1123 comment("Tables like: " + pats) 1124 comment("Database: " + self.db.filename) 1125 try: 1126 import getpass 1127 import socket 1128 comment("User: %s @ %s" % (unicodify(getpass.getuser()), unicodify(socket.gethostname()))) 1129 except ImportError: 1130 pass 1131 blank() 1132 1133 comment("The values of various per-database settings") 1134 self.write(self.stdout, 1135 "PRAGMA page_size=" + str(self.db.cursor().execute("pragma page_size").fetchall()[0][0]) + ";\n") 1136 comment("PRAGMA encoding='" + self.db.cursor().execute("pragma encoding").fetchall()[0][0] + "';\n") 1137 vac = {0: "NONE", 1: "FULL", 2: "INCREMENTAL"} 1138 vacvalue = self.db.cursor().execute("pragma auto_vacuum").fetchall()[0][0] 1139 comment("PRAGMA auto_vacuum=" + vac.get(vacvalue, str(vacvalue)) + ";\n") 1140 comment("PRAGMA max_page_count=" + str(self.db.cursor().execute("pragma max_page_count").fetchall()[0][0]) + 1141 ";\n") 1142 blank() 1143 1144 # different python versions have different requirements 1145 # about specifying cmp to sort routine so we use this 1146 # portable workaround with a decorated list instead 1147 dectables = [(x.lower(), x) for x in tables] 1148 dectables.sort() 1149 tables = [y for x, y in dectables] 1150 1151 virtuals = v["virtuals"] 1152 foreigns = v["foreigns"] 1153 1154 if virtuals: 1155 comment("This pragma is needed to restore virtual tables") 1156 self.write(self.stdout, "PRAGMA writable_schema=ON;\n") 1157 if foreigns: 1158 comment("This pragma turns off checking of foreign keys " 1159 "as tables would be inconsistent while restoring. It was introduced " 1160 "in SQLite 3.6.19.") 1161 self.write(self.stdout, "PRAGMA foreign_keys=OFF;\n") 1162 1163 if virtuals or foreigns: 1164 blank() 1165 1166 self.write(self.stdout, "BEGIN TRANSACTION;\n") 1167 blank() 1168 1169 def sqldef(s): 1170 # return formatted sql watching out for embedded 1171 # comments at the end forcing trailing ; onto next 1172 # line https://sqlite.org/src/info/c04a8b8a4f 1173 if "--" in s.split("\n")[-1]: 1174 nl = "\n" 1175 else: 1176 nl = "" 1177 return s + nl + ";\n" 1178 1179 # do the table dumping loops 1180 oldtable = self._output_table 1181 try: 1182 self.push_output() 1183 self.output = self.output_insert 1184 # Dump the table 1185 for table in tables: 1186 for sql in self.db.cursor().execute("SELECT sql FROM sqlite_master WHERE name=?1 AND type='table'", 1187 (table, )): 1188 comment("Table " + table) 1189 # Special treatment for virtual tables - they 1190 # get called back on drops and creates and 1191 # could thwart us so we have to manipulate 1192 # sqlite_master directly 1193 if sql[0].lower().split()[:3] == ["create", "virtual", "table"]: 1194 self.write( 1195 self.stdout, "DELETE FROM sqlite_master WHERE name=" + apsw.format_sql_value(table) + 1196 " AND type='table';\n") 1197 self.write( 1198 self.stdout, 1199 "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql) VALUES('table',%s,%s,0,%s);\n" 1200 % (apsw.format_sql_value(table), apsw.format_sql_value(table), 1201 apsw.format_sql_value(sql[0]))) 1202 else: 1203 self.write(self.stdout, "DROP TABLE IF EXISTS " + self._fmt_sql_identifier(table) + ";\n") 1204 self.write(self.stdout, sqldef(sql[0])) 1205 self._output_table = self._fmt_sql_identifier(table) 1206 self.process_sql("select * from " + self._fmt_sql_identifier(table), internal=True) 1207 # Now any indices or triggers 1208 first = True 1209 for name, sql in self.db.cursor().execute( 1210 "SELECT name,sql FROM sqlite_master " 1211 "WHERE sql NOT NULL AND type IN ('index', 'trigger') " 1212 "AND tbl_name=?1 AND name NOT LIKE 'sqlite_%' " 1213 "ORDER BY lower(name)", (table, )): 1214 if first: 1215 comment("Triggers and indices on " + table) 1216 first = False 1217 self.write(self.stdout, sqldef(sql)) 1218 blank() 1219 # Views done last. They have to be done in the same order as they are in sqlite_master 1220 # as they could refer to each other 1221 first = True 1222 for name, sql in self.db.cursor().execute("SELECT name,sql FROM sqlite_master " 1223 "WHERE sql NOT NULL AND type='view' " 1224 "AND name IN ( " + 1225 ",".join([apsw.format_sql_value(i) 1226 for i in tables]) + ") ORDER BY _ROWID_"): 1227 if first: 1228 comment("Views") 1229 first = False 1230 self.write(self.stdout, "DROP VIEW IF EXISTS %s;\n" % (self._fmt_sql_identifier(name), )) 1231 self.write(self.stdout, sqldef(sql)) 1232 if not first: 1233 blank() 1234 1235 # sqlite sequence 1236 # does it exist 1237 if len(self.db.cursor().execute("select * from sqlite_master where name='sqlite_sequence'").fetchall()): 1238 first = True 1239 for t in tables: 1240 v = self.db.cursor().execute("select seq from main.sqlite_sequence where name=?1", 1241 (t, )).fetchall() 1242 if len(v): 1243 assert len(v) == 1 1244 if first: 1245 comment("For primary key autoincrements the next id " 1246 "to use is stored in sqlite_sequence") 1247 first = False 1248 self.write( 1249 self.stdout, 1250 'DELETE FROM main.sqlite_sequence WHERE name=%s;\n' % (apsw.format_sql_value(t), )) 1251 self.write( 1252 self.stdout, 'INSERT INTO main.sqlite_sequence VALUES (%s, %s);\n' % 1253 (apsw.format_sql_value(t), v[0][0])) 1254 if not first: 1255 blank() 1256 finally: 1257 self.pop_output() 1258 self._output_table = oldtable 1259 1260 # analyze 1261 if analyze_needed: 1262 comment("You had used the analyze command on these tables before. Rerun for this new data.") 1263 for n in analyze_needed: 1264 self.write(self.stdout, "ANALYZE " + self._fmt_sql_identifier(n) + ";\n") 1265 blank() 1266 1267 # user version pragma 1268 uv = self.db.cursor().execute("pragma user_version").fetchall()[0][0] 1269 if uv: 1270 comment( 1271 "Your database may need this. It is sometimes used to keep track of the schema version (eg Firefox does this)." 1272 ) 1273 self.write(self.stdout, "pragma user_version=%d;" % (uv, )) 1274 blank() 1275 1276 # Save it all 1277 self.write(self.stdout, "COMMIT TRANSACTION;\n") 1278 1279 # cleanup pragmas 1280 if foreigns: 1281 blank() 1282 comment("Restoring foreign key checking back on. Note that SQLite 3.6.19 is off by default") 1283 self.write(self.stdout, "PRAGMA foreign_keys=ON;\n") 1284 if virtuals: 1285 blank() 1286 comment("Restoring writable schema back to default") 1287 self.write(self.stdout, "PRAGMA writable_schema=OFF;\n") 1288 # schema reread 1289 blank() 1290 comment("We need to force SQLite to reread the schema because otherwise it doesn't know that " 1291 "the virtual tables we inserted directly into sqlite_master exist. See " 1292 "last comments of https://sqlite.org/cvstrac/tktview?tn=3425") 1293 self.write(self.stdout, "BEGIN;\nCREATE TABLE no_such_table(x,y,z);\nROLLBACK;\n") 1294 1295 finally: 1296 self.process_sql("END", internal=True) 1297 1298 def command_echo(self, cmd): 1299 """echo ON|OFF: If ON then each SQL statement or command is printed before execution (default OFF) 1300 1301 The SQL statement or command is sent to error output so that 1302 it is not intermingled with regular output. 1303 """ 1304 self.echo = self._boolean_command("echo", cmd) 1305 1306 def set_encoding(self, enc): 1307 """Saves *enc* as the default encoding, after verifying that 1308 it is valid. You can also include :error to specify error 1309 handling - eg 'cp437:replace' 1310 1311 Raises an exception on invalid encoding or error 1312 """ 1313 enc = enc.split(":", 1) 1314 if len(enc) > 1: 1315 enc, errors = enc 1316 else: 1317 enc = enc[0] 1318 errors = None 1319 try: 1320 codecs.lookup(enc) 1321 except LookupError: 1322 raise self.Error("No known encoding '%s'" % (enc, )) 1323 try: 1324 if errors is not None: 1325 codecs.lookup_error(errors) 1326 except LookupError: 1327 raise self.Error("No known codec error handler '%s'" % (errors, )) 1328 self.encoding = enc, errors 1329 1330 def command_encoding(self, cmd): 1331 """encoding ENCODING: Set the encoding used for new files opened via .output and imports 1332 1333 SQLite and APSW work internally using Unicode and characters. 1334 Files however are a sequence of bytes. An encoding describes 1335 how to convert between bytes and characters. The default 1336 encoding is utf8 and that is generally the best value to use 1337 when other programs give you a choice. 1338 1339 You can also specify an error handler. For example 1340 'cp437:replace' will use code page 437 and any Unicode 1341 codepoints not present in cp437 will be replaced (typically 1342 with something like a question mark). Other error handlers 1343 include 'ignore', 'strict' (default) and 'xmlcharrefreplace'. 1344 1345 For the default input/output/error streams on startup the 1346 shell defers to Python's detection of encoding. For example 1347 on Windows it asks what code page is in use and on Unix it 1348 looks at the LC_CTYPE environment variable. You can set the 1349 PYTHONIOENCODING environment variable to override this 1350 detection. 1351 1352 This command affects files opened after setting the encoding 1353 as well as imports. 1354 1355 See the online APSW documentation for more details. 1356 """ 1357 if len(cmd) != 1: 1358 raise self.Error("Encoding takes one argument") 1359 self.set_encoding(cmd[0]) 1360 1361 def command_exceptions(self, cmd): 1362 """exceptions ON|OFF: If ON then detailed tracebacks are shown on exceptions (default OFF) 1363 1364 Normally when an exception occurs the error string only is 1365 displayed. However it is sometimes useful to get a full 1366 traceback. An example would be when you are developing 1367 virtual tables and using the shell to exercise them. In 1368 addition to displaying each stack frame, the local variables 1369 within each frame are also displayed. 1370 """ 1371 self.exceptions = self._boolean_command("exceptions", cmd) 1372 1373 def command_exit(self, cmd): 1374 """exit:Exit this program""" 1375 if len(cmd): 1376 raise self.Error("Exit doesn't take any parameters") 1377 sys.exit(0) 1378 1379 def command_quit(self, cmd): 1380 """quit:Exit this program""" 1381 if len(cmd): 1382 raise self.Error("Quit doesn't take any parameters") 1383 sys.exit(0) 1384 1385 def command_explain(self, cmd): 1386 """explain ON|OFF: Set output mode suitable for explain (default OFF) 1387 1388 Explain shows the underlying SQLite virtual machine code for a 1389 statement. You need to prefix the SQL with explain. For example: 1390 1391 explain select * from table; 1392 1393 This output mode formats the explain output nicely. If you do 1394 '.explain OFF' then the output mode and settings in place when 1395 you did '.explain ON' are restored. 1396 """ 1397 if len(cmd) == 0 or self._boolean_command("explain", cmd): 1398 self.push_output() 1399 self.header = True 1400 self.widths = [4, 13, 4, 4, 4, 13, 2, 13] 1401 self.truncate = False 1402 self.output = self.output_column 1403 else: 1404 self.pop_output() 1405 1406 def command_find(self, cmd): 1407 """find what ?TABLE?: Searches all columns of all tables for a value 1408 1409 The find command helps you locate data across your database 1410 for example to find a string or any references to an id. 1411 1412 You can specify a like pattern to limit the search to a subset 1413 of tables (eg specifying 'CUSTOMER%' for all tables beginning 1414 with CUSTOMER). 1415 1416 The what value will be treated as a string and/or integer if 1417 possible. If what contains % or _ then it is also treated as 1418 a like pattern. 1419 1420 This command will take a long time to execute needing to read 1421 all of the relevant tables. 1422 """ 1423 if len(cmd) < 1 or len(cmd) > 2: 1424 raise self.Error("At least one argument required and at most two accepted") 1425 tablefilter = "%" 1426 if len(cmd) == 2: 1427 tablefilter = cmd[1] 1428 querytemplate = [] 1429 queryparams = [] 1430 1431 def qp(): # binding for current queryparams 1432 return "?" + str(len(queryparams)) 1433 1434 s = cmd[0] 1435 if '%' in s or '_' in s: 1436 queryparams.append(s) 1437 querytemplate.append("%s LIKE " + qp()) 1438 queryparams.append(s) 1439 querytemplate.append("%s = " + qp()) 1440 try: 1441 i = int(s) 1442 queryparams.append(i) 1443 querytemplate.append("%s = " + qp()) 1444 except ValueError: 1445 pass 1446 querytemplate = " OR ".join(querytemplate) 1447 for (table, ) in self.db.cursor().execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ?1", 1448 (tablefilter, )): 1449 t = self._fmt_sql_identifier(table) 1450 query = "SELECT * from %s WHERE " % (t, ) 1451 colq = [] 1452 for _, column, _, _, _, _ in self.db.cursor().execute("pragma table_info(%s)" % (t, )): 1453 colq.append(querytemplate % ((self._fmt_sql_identifier(column), ) * len(queryparams))) 1454 query = query + " OR ".join(colq) 1455 self.process_sql(query, queryparams, internal=True, summary=("Table " + table + "\n", "\n")) 1456 1457 def command_header(self, cmd): 1458 """header(s) ON|OFF: Display the column names in output (default OFF) 1459 1460 """ 1461 self.header = self._boolean_command("header", cmd) 1462 1463 command_headers = command_header 1464 1465 _help_info = None 1466 1467 def command_help(self, cmd): 1468 """help ?COMMAND?: Shows list of commands and their usage. If COMMAND is specified then shows detail about that COMMAND. ('.help all' will show detailed help about all commands.) 1469 """ 1470 if not self._help_info: 1471 # buildup help database 1472 self._help_info = {} 1473 for c in dir(self): 1474 if not c.startswith("command_"): 1475 continue 1476 # help is 3 parts 1477 # - the syntax string (eg backup ?dbname? filename) 1478 # - the one liner description (eg saves database to filename) 1479 # - the multi-liner detailed description 1480 # We grab this from the doc string for the function in the form 1481 # syntax: one liner\nmulti\nliner 1482 d = getattr(self, c).__doc__ 1483 assert d, c + " command must have documentation" 1484 c = c[len("command_"):] 1485 if c in ("headers", "color"): continue 1486 while d[0] == "\n": 1487 d = d[1:] 1488 parts = d.split("\n", 1) 1489 firstline = parts[0].strip().split(":", 1) 1490 assert len(firstline) == 2, c + " command must have usage: description doc" 1491 if len(parts) == 1 or len(parts[1].strip()) == 0: # work around textwrap bug 1492 multi = "" 1493 else: 1494 multi = textwrap.dedent(parts[1]) 1495 if c == "mode": 1496 if not self._output_modes: 1497 self._cache_output_modes() 1498 firstline[1] = firstline[1] + " " + " ".join(self._output_modes) 1499 multi = multi + "\n\n" + "\n\n".join(self._output_modes_detail) 1500 if c == "colour": 1501 colours = list(self._colours.keys()) 1502 colours.sort() 1503 firstline[1] = firstline[1] + " from " + ", ".join(colours) 1504 if len(multi.strip()) == 0: # All whitespace 1505 multi = None 1506 else: 1507 multi = multi.strip("\n") 1508 # we need to keep \n\n as a newline but turn all others into spaces 1509 multi = multi.replace("\n\n", "\x00") 1510 multi = multi.replace("\n", " ") 1511 multi = multi.replace("\x00", "\n\n") 1512 multi = multi.split("\n\n") 1513 self._help_info[c] = ('.' + firstline[0].strip(), firstline[1].strip(), multi) 1514 1515 self.write(self.stderr, "\n") 1516 1517 tw = self._terminal_width() 1518 if tw < 32: 1519 tw = 32 1520 if len(cmd) == 0: 1521 commands = list(self._help_info.keys()) 1522 commands.sort() 1523 w = 0 1524 for command in commands: 1525 if len(self._help_info[command][0]) > w: 1526 w = len(self._help_info[command][0]) 1527 out = [] 1528 for command in commands: 1529 hi = self._help_info[command] 1530 # usage string 1531 out.append(hi[0]) 1532 # space padding (including 2 for between columns) 1533 out.append(" " * (2 + w - len(hi[0]))) 1534 # usage message wrapped if need be 1535 out.append(("\n" + " " * (2 + w)).join(textwrap.wrap(hi[1], tw - w - 2))) 1536 # newline 1537 out.append("\n") 1538 self.write(self.stderr, "".join(out)) 1539 else: 1540 if cmd[0] == "all": 1541 cmd = list(self._help_info.keys()) 1542 cmd.sort() 1543 w = 0 1544 for command in self._help_info: 1545 if len(self._help_info[command][0]) > w: 1546 w = len(self._help_info[command][0]) 1547 1548 for command in cmd: 1549 if command == "headers": command = "header" 1550 if command not in self._help_info: 1551 raise self.Error("No such command \"%s\"" % (command, )) 1552 out = [] 1553 hi = self._help_info[command] 1554 # usage string 1555 out.append(hi[0]) 1556 # space padding (2) 1557 out.append(" " * (2 + w - len(hi[0]))) 1558 # usage message wrapped if need be 1559 out.append(("\n" + " " * (2 + w)).join(textwrap.wrap(hi[1], tw - w - 2)) + "\n") 1560 if hi[2]: 1561 # newlines 1562 out.append("\n") 1563 # detailed message 1564 for i, para in enumerate(hi[2]): 1565 out.append(textwrap.fill(para, tw) + "\n") 1566 if i < len(hi[2]) - 1: 1567 out.append("\n") 1568 # if not first one then print separator header 1569 if command != cmd[0]: 1570 self.write(self.stderr, "\n" + "=" * tw + "\n") 1571 self.write(self.stderr, "".join(out)) 1572 self.write(self.stderr, "\n") 1573 1574 def command_import(self, cmd): 1575 """import FILE TABLE: Imports separated data from FILE into TABLE 1576 1577 Reads data from the file into the named table using the 1578 current separator and encoding. For example if the separator 1579 is currently a comma then the file should be CSV (comma 1580 separated values). 1581 1582 All values read in are supplied to SQLite as strings. If you 1583 want SQLite to treat them as other types then declare your 1584 columns appropriately. For example declaring a column 'REAL' 1585 will result in the values being stored as floating point if 1586 they can be safely converted. See this page for more details: 1587 1588 https://sqlite.org/datatype3.html 1589 1590 Another alternative is to create a tempory table, insert the 1591 values into that and then use casting. 1592 1593 CREATE TEMPORARY TABLE import(a,b,c); 1594 1595 .import filename import 1596 1597 CREATE TABLE final AS SELECT cast(a as BLOB), cast(b as INTEGER), cast(c as CHAR) from import; 1598 1599 DROP TABLE import; 1600 1601 You can also get more sophisticated using the SQL CASE 1602 operator. For example this will turn zero length strings into 1603 null: 1604 1605 SELECT CASE col WHEN '' THEN null ELSE col END FROM ... 1606 """ 1607 if len(cmd) != 2: 1608 raise self.Error("import takes two parameters") 1609 1610 try: 1611 final = None 1612 # start transaction so database can't be changed 1613 # underneath us 1614 self.db.cursor().execute("BEGIN IMMEDIATE") 1615 final = "ROLLBACK" 1616 1617 # how many columns? 1618 ncols = len(self.db.cursor().execute("pragma table_info(" + self._fmt_sql_identifier(cmd[1]) + 1619 ")").fetchall()) 1620 if ncols < 1: 1621 raise self.Error("No such table '%s'" % (cmd[1], )) 1622 1623 cur = self.db.cursor() 1624 sql = "insert into %s values(%s)" % (self._fmt_sql_identifier(cmd[1]), ",".join("?" * ncols)) 1625 1626 kwargs = {} 1627 if self.separator == ",": 1628 kwargs["dialect"] = "excel" 1629 elif self.separator == "\t": 1630 kwargs["dialect"] = "excel-tab" 1631 else: 1632 kwargs["quoting"] = csv.QUOTE_NONE 1633 kwargs["delimiter"] = self.separator 1634 kwargs["doublequote"] = False 1635 kwargs["quotechar"] = "\x00" 1636 row = 1 1637 for line in self._csvin_wrapper(cmd[0], kwargs): 1638 if len(line) != ncols: 1639 raise self.Error("row %d has %d columns but should have %d" % (row, len(line), ncols)) 1640 try: 1641 cur.execute(sql, line) 1642 except: 1643 self.write(self.stderr, "Error inserting row %d" % (row, )) 1644 raise 1645 row += 1 1646 self.db.cursor().execute("COMMIT") 1647 1648 except: 1649 if final: 1650 self.db.cursor().execute(final) 1651 raise 1652 1653 def _csvin_wrapper(self, filename, dialect): 1654 # Returns a csv reader that works around python bugs and uses 1655 # dialect dict to configure reader 1656 1657 # Very easy for python 3 1658 if sys.version_info >= (3, 0): 1659 thefile = codecs.open(filename, "r", self.encoding[0]) 1660 for line in csv.reader(thefile, **dialect.copy()): 1661 yield line 1662 thefile.close() 1663 return 1664 1665 ### 1666 ### csv module is not good at unicode so we have to 1667 ### indirect unless utf8 is in use 1668 ### 1669 if self.encoding[0].lower() == "utf8": # no need for tempfile 1670 thefile = open(filename, "rb") 1671 else: 1672 import tempfile 1673 thefile = tempfile.TemporaryFile(prefix="apsw_import") 1674 thefile.write(codecs.open(filename, "r", self.encoding[0]).read().encode("utf8")) 1675 # move back to beginning 1676 thefile.seek(0, 0) 1677 1678 # Ensure all values are utf8 not unicode 1679 for k, v in dialect.items(): 1680 if isinstance(v, unicode): 1681 dialect[k] = v.encode("utf8") 1682 for line in csv.reader(thefile, **dialect): 1683 # back to unicode again 1684 yield [x.decode("utf8") for x in line] 1685 thefile.close() 1686 1687 def command_autoimport(self, cmd): 1688 """autoimport FILENAME ?TABLE?: Imports filename creating a table and automatically working out separators and data types (alternative to .import command) 1689 1690 The import command requires that you precisely pre-setup the 1691 table and schema, and set the data separators (eg commas or 1692 tabs). In many cases this information can be automatically 1693 deduced from the file contents which is what this command 1694 does. There must be at least two columns and two rows. 1695 1696 If the table is not specified then the basename of the file 1697 will be used. 1698 1699 Additionally the type of the contents of each column is also 1700 deduced - for example if it is a number or date. Empty values 1701 are turned into nulls. Dates are normalized into YYYY-MM-DD 1702 format and DateTime are normalized into ISO8601 format to 1703 allow easy sorting and searching. 4 digit years must be used 1704 to detect dates. US (swapped day and month) versus rest of 1705 the world is also detected providing there is at least one 1706 value that resolves the ambiguity. 1707 1708 Care is taken to ensure that columns looking like numbers are 1709 only treated as numbers if they do not have unnecessary 1710 leading zeroes or plus signs. This is to avoid treating phone 1711 numbers and similar number like strings as integers. 1712 1713 This command can take quite some time on large files as they 1714 are effectively imported twice. The first time is to 1715 determine the format and the types for each column while the 1716 second pass actually imports the data. 1717 """ 1718 if len(cmd) < 1 or len(cmd) > 2: 1719 raise self.Error("Expected one or two parameters") 1720 if not os.path.exists(cmd[0]): 1721 raise self.Error("File \"%s\" does not exist" % (cmd[0], )) 1722 if len(cmd) == 2: 1723 tablename = cmd[1] 1724 else: 1725 tablename = None 1726 try: 1727 final = None 1728 c = self.db.cursor() 1729 c.execute("BEGIN IMMEDIATE") 1730 final = "ROLLBACK" 1731 1732 if not tablename: 1733 tablename = os.path.splitext(os.path.basename(cmd[0]))[0] 1734 1735 if c.execute("pragma table_info(%s)" % (self._fmt_sql_identifier(tablename), )).fetchall(): 1736 raise self.Error("Table \"%s\" already exists" % (tablename, )) 1737 1738 # The types we support deducing 1739 def DateUS(v): # US formatted date with wrong ordering of day and month 1740 return DateWorld(v, switchdm=True) 1741 1742 def DateWorld(v, switchdm=False): # Sensibly formatted date as used anywhere else in the world 1743 y, m, d = self._getdate(v) 1744 if switchdm: m, d = d, m 1745 if m < 1 or m > 12 or d < 1 or d > 31: 1746 raise ValueError 1747 return "%d-%02d-%02d" % (y, m, d) 1748 1749 def DateTimeUS(v): # US date and time 1750 return DateTimeWorld(v, switchdm=True) 1751 1752 def DateTimeWorld(v, switchdm=False): # Sensible date and time 1753 y, m, d, h, M, s = self._getdatetime(v) 1754 if switchdm: m, d = d, m 1755 if m < 1 or m > 12 or d < 1 or d > 31 or h < 0 or h > 23 or M < 0 or M > 59 or s < 0 or s > 65: 1756 raise ValueError 1757 return "%d-%02d-%02dT%02d:%02d:%02d" % (y, m, d, h, M, s) 1758 1759 def Number(v): # we really don't want phone numbers etc to match 1760 # Python's float & int constructors allow whitespace which we don't 1761 if re.search(r"\s", v): 1762 raise ValueError 1763 if v == "0": return 0 1764 if v[0] == "+": # idd prefix 1765 raise ValueError 1766 if re.match("^[0-9]+$", v): 1767 if v[0] == "0": raise ValueError # also a phone number 1768 return int(v) 1769 if v[0] == "0" and not v.startswith("0."): # deceptive not a number 1770 raise ValueError 1771 return float(v) 1772 1773 # Work out the file format 1774 formats = [{"dialect": "excel"}, {"dialect": "excel-tab"}] 1775 seps = ["|", ";", ":"] 1776 if self.separator not in seps: 1777 seps.append(self.separator) 1778 for sep in seps: 1779 formats.append({"quoting": csv.QUOTE_NONE, "delimiter": sep, "doublequote": False, "quotechar": "\x00"}) 1780 possibles = [] 1781 errors = [] 1782 encodingissue = False 1783 # format is copy() on every use. This appears bizarre and 1784 # unnecessary. However Python 2.3 and 2.4 somehow manage 1785 # to empty it if not copied. 1786 for format in formats: 1787 ncols = -1 1788 lines = 0 1789 try: 1790 for line in self._csvin_wrapper(cmd[0], format.copy()): 1791 if lines == 0: 1792 lines = 1 1793 ncols = len(line) 1794 # data type guess setup 1795 datas = [] 1796 for i in range(ncols): 1797 datas.append([DateUS, DateWorld, DateTimeUS, DateTimeWorld, Number]) 1798 allblanks = [True] * ncols 1799 continue 1800 if len(line) != ncols: 1801 raise ValueError("Expected %d columns - got %d" % (ncols, len(line))) 1802 lines += 1 1803 for i in range(ncols): 1804 if not line[i]: 1805 continue 1806 allblanks[i] = False 1807 if not datas[i]: 1808 continue 1809 # remove datas that give ValueError 1810 d = [] 1811 for dd in datas[i]: 1812 try: 1813 dd(line[i]) 1814 d.append(dd) 1815 except ValueError: 1816 pass 1817 datas[i] = d 1818 if ncols > 1 and lines > 1: 1819 # if a particular column was allblank then clear datas for it 1820 for i in range(ncols): 1821 if allblanks[i]: 1822 datas[i] = [] 1823 possibles.append((format.copy(), ncols, lines, datas)) 1824 except UnicodeDecodeError: 1825 encodingissue = True 1826 except: 1827 s = str(sys.exc_info()[1]) 1828 if s not in errors: 1829 errors.append(s) 1830 1831 if len(possibles) == 0: 1832 if encodingissue: 1833 raise self.Error( 1834 "The file is probably not in the current encoding \"%s\" and didn't match a known file format" % 1835 (self.encoding[0], )) 1836 v = "File doesn't appear to match a known type." 1837 if len(errors): 1838 v += " Errors reported:\n" + "\n".join([" " + e for e in errors]) 1839 raise self.Error(v) 1840 if len(possibles) > 1: 1841 raise self.Error("File matches more than one type!") 1842 format, ncols, lines, datas = possibles[0] 1843 fmt = format.get("dialect", None) 1844 if fmt is None: 1845 fmt = "(delimited by \"%s\")" % (format["delimiter"], ) 1846 self.write(self.stdout, "Detected Format %s Columns %d Rows %d\n" % (fmt, ncols, lines)) 1847 # Header row 1848 reader = self._csvin_wrapper(cmd[0], format) 1849 for header in reader: 1850 break 1851 # Check schema 1852 identity = lambda x: x 1853 for i in range(ncols): 1854 if len(datas[i]) > 1: 1855 raise self.Error("Column #%d \"%s\" has ambiguous data format - %s" % 1856 (i + 1, header[i], ", ".join([d.__name__ for d in datas[i]]))) 1857 if datas[i]: 1858 datas[i] = datas[i][0] 1859 else: 1860 datas[i] = identity 1861 # Make the table 1862 sql = "CREATE TABLE %s(%s)" % (self._fmt_sql_identifier(tablename), ", ".join( 1863 [self._fmt_sql_identifier(h) for h in header])) 1864 c.execute(sql) 1865 # prep work for each row 1866 sql = "INSERT INTO %s VALUES(%s)" % (self._fmt_sql_identifier(tablename), ",".join(["?"] * ncols)) 1867 for line in reader: 1868 vals = [] 1869 for i in range(ncols): 1870 l = line[i] 1871 if not l: 1872 vals.append(None) 1873 else: 1874 vals.append(datas[i](l)) 1875 c.execute(sql, vals) 1876 1877 c.execute("COMMIT") 1878 self.write(self.stdout, "Auto-import into table \"%s\" complete\n" % (tablename, )) 1879 except: 1880 if final: 1881 self.db.cursor().execute(final) 1882 raise 1883 1884 def _getdate(self, v): 1885 # Returns a tuple of 3 items y,m,d from string v 1886 m = re.match(r"^([0-9]+)[^0-9]([0-9]+)[^0-9]([0-9]+)$", v) 1887 if not m: 1888 raise ValueError 1889 y, m, d = int(m.group(1)), int(m.group(2)), int(m.group(3)) 1890 if d > 1000: # swap order 1891 y, m, d = d, m, y 1892 if y < 1000 or y > 9999: 1893 raise ValueError 1894 return y, m, d 1895 1896 def _getdatetime(self, v): 1897 # must be at least HH:MM 1898 m = re.match(r"^([0-9]+)[^0-9]([0-9]+)[^0-9]([0-9]+)[^0-9]+([0-9]+)[^0-9]([0-9]+)([^0-9]([0-9]+))?$", v) 1899 if not m: 1900 raise ValueError 1901 items = list(m.group(1, 2, 3, 4, 5, 7)) 1902 for i in range(len(items)): 1903 if items[i] is None: 1904 items[i] = 0 1905 items = [int(i) for i in items] 1906 if items[2] > 1000: 1907 items = [items[2], items[1], items[0]] + items[3:] 1908 if items[0] < 1000 or items[0] > 9999: 1909 raise ValueError 1910 return items 1911 1912 def command_indices(self, cmd): 1913 """indices TABLE: Lists all indices on table TABLE 1914 1915 """ 1916 if len(cmd) != 1: 1917 raise self.Error("indices takes one table name") 1918 self.push_output() 1919 self.header = False 1920 self.output = self.output_list 1921 try: 1922 self.process_sql( 1923 "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name LIKE ?1 " 1924 "UNION ALL SELECT name FROM sqlite_temp_master WHERE type='index' AND tbl_name LIKE " 1925 "?1 ORDER by name", 1926 cmd, 1927 internal=True) 1928 finally: 1929 self.pop_output() 1930 1931 def command_load(self, cmd): 1932 """load FILE ?ENTRY?: Loads a SQLite extension library 1933 1934 Note: Extension loading may not be enabled in the SQLite 1935 library version you are using. 1936 1937 Extensions are an easy way to add new functions and 1938 functionality. For a useful extension look at the bottom of 1939 https://sqlite.org/contrib 1940 1941 By default sqlite3_extension_init is called in the library but 1942 you can specify an alternate entry point. 1943 1944 If you get an error about the extension not being found you 1945 may need to explicitly specify the directory. For example if 1946 it is in the current directory then use: 1947 1948 .load ./extension.so 1949 """ 1950 if len(cmd) < 1 or len(cmd) > 2: 1951 raise self.Error("load takes one or two parameters") 1952 try: 1953 self.db.enableloadextension(True) 1954 except: 1955 raise self.Error("Extension loading is not supported") 1956 1957 self.db.loadextension(*cmd) 1958 1959 _output_modes = None 1960 1961 def command_mode(self, cmd): 1962 """mode MODE ?TABLE?: Sets output mode to one of""" 1963 if len(cmd) in (1, 2): 1964 w = cmd[0] 1965 if w == "tabs": 1966 w = "list" 1967 m = getattr(self, "output_" + w, None) 1968 if w != "insert": 1969 if len(cmd) == 2: 1970 raise self.Error("Output mode %s doesn't take parameters" % (cmd[0])) 1971 if m: 1972 self.output = m 1973 # set some defaults 1974 self.truncate = True 1975 if cmd[0] == "csv": 1976 self.separator = "," 1977 elif cmd[0] == "tabs": 1978 self.separator = "\t" 1979 else: 1980 pass 1981 #self.separator=self._output_stack[0]["separator"] 1982 if w == "insert": 1983 if len(cmd) == 2: 1984 self._output_table = cmd[1] 1985 else: 1986 self._output_table = "table" 1987 self._output_table = self._fmt_sql_identifier(self._output_table) 1988 return 1989 if not self._output_modes: 1990 self._cache_output_modes() 1991 raise self.Error("Expected a valid output mode: " + ", ".join(self._output_modes)) 1992 1993 # needed so command completion and help can use it 1994 def _cache_output_modes(self): 1995 modes = [m[len("output_"):] for m in dir(self) if m.startswith("output_")] 1996 modes.append("tabs") 1997 modes.sort() 1998 self._output_modes = modes 1999 2000 detail = [] 2001 2002 for m in modes: 2003 if m == 'tabs': continue 2004 d = getattr(self, "output_" + m).__doc__ 2005 assert d, "output mode " + m + " needs doc" 2006 d = d.replace("\n", " ").strip() 2007 while " " in d: 2008 d = d.replace(" ", " ") 2009 detail.append(m + ": " + d) 2010 self._output_modes_detail = detail 2011 2012 def command_nullvalue(self, cmd): 2013 """nullvalue STRING: Print STRING in place of null values 2014 2015 This affects textual output modes like column and list and 2016 sets how SQL null values are shown. The default is a zero 2017 length string. Insert mode and dumps are not affected by this 2018 setting. You can use double quotes to supply a zero length 2019 string. For example: 2020 2021 .nullvalue "" # the default 2022 .nullvalue <NULL> # rather obvious 2023 .nullvalue " \\t " # A tab surrounded by spaces 2024 """ 2025 if len(cmd) != 1: 2026 raise self.Error("nullvalue takes exactly one parameter") 2027 self.nullvalue = self.fixup_backslashes(cmd[0]) 2028 2029 def command_open(self, cmd): 2030 """open ?OPTIONS? ?FILE?: Closes existing database and opens a different one 2031 2032 Options are: --new which deletes the file if it already exists 2033 2034 If FILE is omitted then a memory database is opened 2035 """ 2036 new = False 2037 dbname = None 2038 c = cmd 2039 while c: 2040 p = c.pop(0) 2041 if p.startswith("--"): 2042 if p == "--new": 2043 new = True 2044 continue 2045 raise self.Error("Unknown open param: " + p) 2046 if dbname: 2047 raise self.Error("Too many arguments: " + p) 2048 dbname = p 2049 2050 if new and dbname: 2051 # close it first in case re-opening existing. windows doesn't 2052 # allow deleting open files, tag alongs cause problems etc 2053 # hence retry and sleeps 2054 self.db = (None, None) 2055 for suffix in "", "-journal", "-wal", "-shm": 2056 fn = dbname + suffix 2057 for retry in range(1, 5): 2058 try: 2059 os.remove(fn) 2060 break 2061 except OSError: 2062 if not os.path.isfile(fn): 2063 break 2064 # under windows tag alongs can delay being able to 2065 # delete after we have closed the file 2066 import gc 2067 gc.collect(2) 2068 time.sleep(.05 * retry) 2069 else: 2070 os.rename(fn, fn + "-DELETEME") 2071 2072 self.db = (None, dbname) 2073 2074 def command_output(self, cmd): 2075 """output FILENAME: Send output to FILENAME (or stdout) 2076 2077 If the FILENAME is stdout then output is sent to standard 2078 output from when the shell was started. The file is opened 2079 using the current encoding (change with .encoding command). 2080 """ 2081 # Flush everything 2082 self.stdout.flush() 2083 self.stderr.flush() 2084 if hasattr(self.stdin, "flush"): 2085 try: 2086 self.stdin.flush() 2087 except IOError: # see issue 117 2088 pass 2089 2090 # we will also close stdout but only do so once we have a 2091 # replacement so that stdout is always valid 2092 2093 if len(cmd) != 1: 2094 raise self.Error("You must specify a filename") 2095 2096 try: 2097 fname = cmd[0] 2098 if fname == "stdout": 2099 old = None 2100 if self.stdout != self._original_stdout: 2101 old = self.stdout 2102 self.stdout = self._original_stdout 2103 if old is not None: # done here in case close raises exception 2104 old.close() 2105 return 2106 2107 newf = codecs.open(fname, "w", self.encoding[0], self.encoding[1]) 2108 old = None 2109 if self.stdout != self._original_stdout: 2110 old = self.stdout 2111 self.stdout = newf 2112 if old is not None: 2113 old.close() 2114 finally: 2115 self._out_colour() 2116 2117 def command_print(self, cmd): 2118 """print STRING: print the literal STRING 2119 2120 If more than one argument is supplied then they are printed 2121 space separated. You can use backslash escapes such as \\n 2122 and \\t. 2123 """ 2124 self.write(self.stdout, " ".join([self.fixup_backslashes(i) for i in cmd]) + "\n") 2125 2126 def command_prompt(self, cmd): 2127 """prompt MAIN ?CONTINUE?: Changes the prompts for first line and continuation lines 2128 2129 The default is to print 'sqlite> ' for the main prompt where 2130 you can enter a dot command or a SQL statement. If the SQL 2131 statement is complete (eg not ; terminated) then you are 2132 prompted for more using the continuation prompt which defaults 2133 to ' ..> '. Example: 2134 2135 .prompt "Yes, Master> " "More, Master> " 2136 2137 You can use backslash escapes such as \\n and \\t. 2138 """ 2139 if len(cmd) < 1 or len(cmd) > 2: 2140 raise self.Error("prompt takes one or two arguments") 2141 self.prompt = self.fixup_backslashes(cmd[0]) 2142 if len(cmd) == 2: 2143 self.moreprompt = self.fixup_backslashes(cmd[1]) 2144 2145 def command_read(self, cmd): 2146 """read FILENAME: Processes SQL and commands in FILENAME (or Python if FILENAME ends with .py) 2147 2148 Treats the specified file as input (a mixture or SQL and/or 2149 dot commands). If the filename ends in .py then it is treated 2150 as Python code instead. 2151 2152 For Python code the symbol 'shell' refers to the instance of 2153 the shell and 'apsw' is the apsw module. 2154 """ 2155 if len(cmd) != 1: 2156 raise self.Error("read takes a single filename") 2157 if cmd[0].lower().endswith(".py"): 2158 g = {} 2159 g.update({'apsw': apsw, 'shell': self}) 2160 if sys.version_info < (3, 0): 2161 execfile(cmd[0], g, g) 2162 else: 2163 # compile step is needed to associate name with code 2164 f=open(cmd[0]) 2165 try: 2166 exec(compile(f.read(), cmd[0], 'exec'), g, g) 2167 finally: 2168 f.close() 2169 else: 2170 f = codecs.open(cmd[0], "r", self.encoding[0]) 2171 try: 2172 try: 2173 self.push_input() 2174 self.stdin = f 2175 self.interactive = False 2176 self.input_line_number = 0 2177 while True: 2178 line = self.getcompleteline() 2179 if line is None: 2180 break 2181 self.process_complete_line(line) 2182 except: 2183 eval = sys.exc_info()[1] 2184 if not isinstance(eval, SystemExit): 2185 self._append_input_description() 2186 raise 2187 2188 finally: 2189 self.pop_input() 2190 f.close() 2191 2192 def command_restore(self, cmd): 2193 """restore ?DB? FILE: Restore database from FILE into DB (default "main") 2194 2195 Copies the contents of FILE to the current database (default "main"). 2196 The backup is done at the page level - SQLite copies the pages as 2197 is. There is no round trip through SQL code. 2198 """ 2199 dbname = "main" 2200 if len(cmd) == 1: 2201 fname = cmd[0] 2202 elif len(cmd) == 2: 2203 dbname = cmd[0] 2204 fname = cmd[1] 2205 else: 2206 raise self.Error("Restore takes one or two parameters") 2207 input = apsw.Connection(fname) 2208 b = self.db.backup(dbname, input, "main") 2209 try: 2210 while not b.done: 2211 b.step() 2212 finally: 2213 b.finish() 2214 input.close() 2215 2216 def command_schema(self, cmd): 2217 """schema ?TABLE? [TABLE...]: Shows SQL for table 2218 2219 If you give one or more tables then their schema is listed 2220 (including indices). If you don't specify any then all 2221 schemas are listed. TABLE is a like pattern so you can % for 2222 wildcards. 2223 """ 2224 self.push_output() 2225 self.output = self.output_list 2226 self.header = False 2227 try: 2228 if len(cmd) == 0: 2229 cmd = ['%'] 2230 for n in cmd: 2231 self.process_sql( 2232 "SELECT sql||';' FROM " 2233 "(SELECT sql sql, type type, tbl_name tbl_name, name name " 2234 "FROM sqlite_master UNION ALL " 2235 "SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " 2236 "WHERE tbl_name LIKE ?1 AND type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " 2237 "ORDER BY substr(type,2,1), name", (n, ), 2238 internal=True) 2239 finally: 2240 self.pop_output() 2241 2242 def command_separator(self, cmd): 2243 """separator STRING: Change separator for output mode and .import 2244 2245 You can use quotes and backslashes. For example to set the 2246 separator to space tab space you can use: 2247 2248 .separator " \\t " 2249 2250 The setting is automatically changed when you switch to csv or 2251 tabs output mode. You should also set it before doing an 2252 import (ie , for CSV and \\t for TSV). 2253 """ 2254 if len(cmd) != 1: 2255 raise self.Error("separator takes exactly one parameter") 2256 self.separator = self.fixup_backslashes(cmd[0]) 2257 2258 _shows = ("echo", "explain", "headers", "mode", "nullvalue", "output", "separator", "width", "exceptions", 2259 "encoding") 2260 2261 def command_show(self, cmd): 2262 """show: Show the current values for various settings.""" 2263 if len(cmd) > 1: 2264 raise self.Error("show takes at most one parameter") 2265 if len(cmd): 2266 what = cmd[0] 2267 if what not in self._shows: 2268 raise self.Error("Unknown show: '%s'" % (what, )) 2269 else: 2270 what = None 2271 2272 outs = [] 2273 for i in self._shows: 2274 k = i 2275 if what and i != what: 2276 continue 2277 # boolean settings 2278 if i in ("echo", "headers", "exceptions"): 2279 if i == "headers": i = "header" 2280 v = "off" 2281 if getattr(self, i): 2282 v = "on" 2283 elif i == "explain": 2284 # we cheat by looking at truncate setting! 2285 v = "on" 2286 if self.truncate: 2287 v = "off" 2288 elif i in ("nullvalue", "separator"): 2289 v = self._fmt_c_string(getattr(self, i)) 2290 elif i == "mode": 2291 if not self._output_modes: 2292 self._cache_output_modes() 2293 for v in self._output_modes: 2294 if self.output == getattr(self, "output_" + v): 2295 break 2296 else: 2297 assert False, "Bug: didn't find output mode" 2298 elif i == "output": 2299 if self.stdout is self._original_stdout: 2300 v = "stdout" 2301 else: 2302 v = getattr(self.stdout, "name", "<unknown stdout>") 2303 elif i == "width": 2304 v = " ".join(["%d" % (i, ) for i in self.widths]) 2305 elif i == "encoding": 2306 v = self.encoding[0] 2307 if self.encoding[1]: 2308 v += " (Errors " + self.encoding[1] + ")" 2309 else: 2310 assert False, "Bug: unknown show handling" 2311 outs.append((k, v)) 2312 2313 # find width of k column 2314 l = 0 2315 for k, v in outs: 2316 if len(k) > l: 2317 l = len(k) 2318 2319 for k, v in outs: 2320 self.write(self.stderr, "%*.*s: %s\n" % (l, l, k, v)) 2321 2322 def command_tables(self, cmd): 2323 """tables ?PATTERN?: Lists names of tables matching LIKE pattern 2324 2325 This also returns views. 2326 """ 2327 self.push_output() 2328 self.output = self.output_list 2329 self.header = False 2330 try: 2331 if len(cmd) == 0: 2332 cmd = ['%'] 2333 2334 # The SQLite shell code filters out sqlite_ prefixes if 2335 # you specified an argument else leaves them in. It also 2336 # has a hand coded output mode that does space separation 2337 # plus wrapping at 80 columns. 2338 for n in cmd: 2339 self.process_sql( 2340 "SELECT name FROM sqlite_master " 2341 "WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' " 2342 "AND name like ?1 " 2343 "UNION ALL " 2344 "SELECT name FROM sqlite_temp_master " 2345 "WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' " 2346 "ORDER BY 1", (n, ), 2347 internal=True) 2348 finally: 2349 self.pop_output() 2350 2351 def command_timeout(self, cmd): 2352 """timeout MS: Try opening locked tables for MS milliseconds 2353 2354 If a database is locked by another process SQLite will keep 2355 retrying. This sets how many thousandths of a second it will 2356 keep trying for. If you supply zero or a negative number then 2357 all busy handlers are disabled. 2358 """ 2359 if len(cmd) != 1: 2360 raise self.Error("timeout takes a number") 2361 try: 2362 t = int(cmd[0]) 2363 except: 2364 raise self.Error("%s is not a number" % (cmd[0], )) 2365 self.db.setbusytimeout(t) 2366 2367 def command_timer(self, cmd): 2368 """timer ON|OFF: Control printing of time and resource usage after each query 2369 2370 The values displayed are in seconds when shown as floating 2371 point or an absolute count. Only items that have changed 2372 since starting the query are shown. On non-Windows platforms 2373 considerably more information can be shown. 2374 """ 2375 if self._boolean_command("timer", cmd): 2376 try: 2377 self.get_resource_usage() 2378 except: 2379 raise self.Error("Timing not supported by this Python version/platform") 2380 self.timer = True 2381 else: 2382 self.timer = False 2383 2384 def command_width(self, cmd): 2385 """width NUM NUM ...: Set the column widths for "column" mode 2386 2387 In "column" output mode, each column is a fixed width with values truncated to 2388 fit. Specify new widths using this command. Use a negative number 2389 to right justify and zero for default column width. 2390 """ 2391 if len(cmd) == 0: 2392 raise self.Error("You need to specify some widths!") 2393 w = [] 2394 for i in cmd: 2395 try: 2396 w.append(int(i)) 2397 except: 2398 raise self.Error("'%s' is not a valid number" % (i, )) 2399 self.widths = w 2400 2401 def _terminal_width(self): 2402 """Works out the terminal width which is used for word wrapping 2403 some output (eg .help)""" 2404 try: 2405 if sys.platform == "win32": 2406 import ctypes, struct 2407 h = ctypes.windll.kernel32.GetStdHandle(-12) # -12 is stderr 2408 buf = ctypes.create_string_buffer(22) 2409 if ctypes.windll.kernel32.GetConsoleScreenBufferInfo(h, buf): 2410 _, _, _, _, _, left, top, right, bottom, _, _ = struct.unpack("hhhhHhhhhhh", buf.raw) 2411 return right - left 2412 raise Exception() 2413 else: 2414 # posix 2415 import struct, fcntl, termios 2416 s = struct.pack('HHHH', 0, 0, 0, 0) 2417 x = fcntl.ioctl(2, termios.TIOCGWINSZ, s) 2418 return struct.unpack('HHHH', x)[1] 2419 except: 2420 try: 2421 v = int(os.getenv("COLUMNS")) 2422 if v < 10: 2423 return 80 2424 return v 2425 except: 2426 return 80 2427 2428 def push_output(self): 2429 """Saves the current output settings onto a stack. See 2430 :meth:`pop_output` for more details as to why you would use 2431 this.""" 2432 o = {} 2433 for k in "separator", "header", "nullvalue", "output", "widths", "truncate": 2434 o[k] = getattr(self, k) 2435 self._output_stack.append(o) 2436 2437 def pop_output(self): 2438 """Restores most recently pushed output. There are many 2439 output parameters such as nullvalue, mode 2440 (list/tcl/html/insert etc), column widths, header etc. If you 2441 temporarily need to change some settings then 2442 :meth:`push_output`, change the settings and then pop the old 2443 ones back. 2444 2445 A simple example is implementing a command like .dump. Push 2446 the current output, change the mode to insert so we get SQL 2447 inserts printed and then pop to go back to what was there 2448 before. 2449 2450 """ 2451 # first item should always be present 2452 assert len(self._output_stack) 2453 if len(self._output_stack) == 1: 2454 o = self._output_stack[0] 2455 else: 2456 o = self._output_stack.pop() 2457 for k, v in o.items(): 2458 setattr(self, k, v) 2459 2460 def _append_input_description(self): 2461 """When displaying an error in :meth:`handle_exception` we 2462 want to give context such as when the commands being executed 2463 came from a .read command (which in turn could execute another 2464 .read). 2465 """ 2466 if self.interactive: 2467 return 2468 res = [] 2469 res.append("Line %d" % (self.input_line_number, )) 2470 res.append(": " + getattr(self.stdin, "name", "<stdin>")) 2471 self._input_descriptions.append(" ".join(res)) 2472 2473 def fixup_backslashes(self, s): 2474 """Implements the various backlash sequences in s such as 2475 turning backslash t into a tab. 2476 2477 This function is needed because shlex does not do it for us. 2478 """ 2479 if "\\" not in s: return s 2480 # See the resolve_backslashes function in SQLite shell source 2481 res = [] 2482 i = 0 2483 while i < len(s): 2484 if s[i] != "\\": 2485 res.append(s[i]) 2486 i += 1 2487 continue 2488 i += 1 2489 if i >= len(s): 2490 raise self.Error("Backslash with nothing following") 2491 c = s[i] 2492 res.append({"\\": "\\", "r": "\r", "n": "\n", "t": "\t"}.get(c, None)) 2493 i += 1 # advance again 2494 if res[-1] is None: 2495 raise self.Error("Unknown backslash sequence \\" + c) 2496 return "".join(res) 2497 2498 if sys.version_info < (3, 0): 2499 2500 def write(self, dest, text): 2501 """Writes text to dest. dest will typically be one of self.stdout or self.stderr.""" 2502 # ensure text is unicode to catch codeset issues here 2503 if type(text) != unicode: 2504 text = unicode(text) 2505 try: 2506 dest.write(text) 2507 except UnicodeEncodeError: 2508 ev = sys.exc_info()[1] 2509 # See issue108 and try to work around it 2510 if ev.args[0]=="ascii" and dest.encoding and ev.args[0]!=dest.encoding and hasattr(dest, "fileno") and \ 2511 isinstance(dest.fileno(), int) and dest.fileno()>=0: 2512 args = [ 2513 dest.encoding, 2514 ] 2515 if dest.errors: 2516 args.append(dest.errors) 2517 dest.write(text.encode(*args)) 2518 else: 2519 raise 2520 2521 _raw_input = raw_input 2522 else: 2523 2524 def write(self, dest, text): 2525 "Writes text to dest. dest will typically be one of self.stdout or self.stderr." 2526 dest.write(text) 2527 2528 _raw_input = input 2529 2530 def getline(self, prompt=""): 2531 """Returns a single line of input (may be incomplete SQL) from self.stdin. 2532 2533 If EOF is reached then return None. Do not include trailing 2534 newline in return. 2535 """ 2536 self.stdout.flush() 2537 self.stderr.flush() 2538 try: 2539 if self.interactive: 2540 if self.stdin is sys.stdin: 2541 c = self.colour.prompt, self.colour.prompt_ 2542 if self._using_readline and sys.platform != "win32": 2543 # these are needed so that readline knows they are non-printing characters 2544 c = "\x01" + c[0] + "\x02", "\x01" + c[1] + "\x02", 2545 line = self._raw_input(c[0] + prompt + c[1]) + "\n" # raw_input excludes newline 2546 else: 2547 self.write(self.stdout, prompt) 2548 line = self.stdin.readline() # includes newline unless last line of file doesn't have one 2549 else: 2550 line = self.stdin.readline() # includes newline unless last line of file doesn't have one 2551 self.input_line_number += 1 2552 if sys.version_info < (3, 0): 2553 if type(line) != unicode: 2554 enc = getattr(self.stdin, "encoding", self.encoding[0]) 2555 if not enc: enc = self.encoding[0] 2556 line = line.decode(enc) 2557 except EOFError: 2558 return None 2559 if len(line) == 0: # always a \n on the end normally so this is EOF 2560 return None 2561 if line[-1] == "\n": 2562 line = line[:-1] 2563 return line 2564 2565 def getcompleteline(self): 2566 """Returns a complete input. 2567 2568 For dot commands it will be one line. For SQL statements it 2569 will be as many as is necessary to have a 2570 :meth:`~apsw.complete` statement (ie semicolon terminated). 2571 Returns None on end of file.""" 2572 try: 2573 self._completion_first = True 2574 command = self.getline(self.prompt) 2575 if command is None: 2576 return None 2577 if len(command.strip()) == 0: 2578 return "" 2579 if command[0] == "?": command = ".help " + command[1:] 2580 # incomplete SQL? 2581 while command[0] != "." and not apsw.complete(command): 2582 self._completion_first = False 2583 line = self.getline(self.moreprompt) 2584 if line is None: # unexpected eof 2585 raise self.Error("Incomplete SQL (line %d of %s): %s\n" % 2586 (self.input_line_number, getattr(self.stdin, "name", "<stdin>"), command)) 2587 if line in ("go", "/"): 2588 break 2589 command = command + "\n" + line 2590 return command 2591 except KeyboardInterrupt: 2592 self.handle_interrupt() 2593 return "" 2594 2595 def handle_interrupt(self): 2596 """Deal with keyboard interrupt (typically Control-C). It 2597 will :meth:`~Connection.interrupt` the database and print"^C" if interactive.""" 2598 self.db.interrupt() 2599 if not self.bail and self.interactive: 2600 self.write(self.stderr, "^C\n") 2601 return 2602 raise 2603 2604 def process_complete_line(self, command): 2605 """Given some text will call the appropriate method to process 2606 it (eg :meth:`process_sql` or :meth:`process_command`)""" 2607 try: 2608 if len(command.strip()) == 0: 2609 return 2610 if command[0] == ".": 2611 self.process_command(command) 2612 else: 2613 self.process_sql(command) 2614 except KeyboardInterrupt: 2615 self.handle_interrupt() 2616 2617 def push_input(self): 2618 """Saves the current input parameters to a stack. See :meth:`pop_input`.""" 2619 d = {} 2620 for i in "interactive", "stdin", "input_line_number": 2621 d[i] = getattr(self, i) 2622 self._input_stack.append(d) 2623 2624 def pop_input(self): 2625 """Restore most recently pushed input parameters (interactive, 2626 self.stdin, linenumber etc). Use this if implementing a 2627 command like read. Push the current input, read the file and 2628 then pop the input to go back to before. 2629 """ 2630 assert (len(self._input_stack)) > 1 2631 d = self._input_stack.pop() 2632 for k, v in d.items(): 2633 setattr(self, k, v) 2634 2635 def complete(self, token, state): 2636 """Return a possible completion for readline 2637 2638 This function is called with state starting at zero to get the 2639 first completion, then one/two/three etc until you return None. The best 2640 implementation is to generate the list when state==0, save it, 2641 and provide members on each increase. 2642 2643 The default implementation extracts the current full input 2644 from readline and then calls :meth:`complete_command` or 2645 :meth:`complete_sql` as appropriate saving the results for 2646 subsequent calls. 2647 """ 2648 if state == 0: 2649 import readline 2650 # the whole line 2651 line = readline.get_line_buffer() 2652 # beginning and end(+1) of the token in line 2653 beg = readline.get_begidx() 2654 end = readline.get_endidx() 2655 # Are we matching a command? 2656 try: 2657 if self._completion_first and line.startswith("."): 2658 self.completions = self.complete_command(line, token, beg, end) 2659 else: 2660 self.completions = self.complete_sql(line, token, beg, end) 2661 except: 2662 # Readline swallows any exceptions we raise. We 2663 # shouldn't be raising any so this is to catch that 2664 import traceback 2665 traceback.print_exc() 2666 raise 2667 2668 if state > len(self.completions): 2669 return None 2670 return self.completions[state] 2671 2672 # Taken from https://sqlite.org/lang_keywords.html 2673 _sqlite_keywords = """ABORT ACTION ADD AFTER ALL ALTER ANALYZE AND AS ASC ATTACH 2674 AUTOINCREMENT BEFORE BEGIN BETWEEN BY CASCADE CASE CAST CHECK 2675 COLLATE COLUMN COMMIT CONFLICT CONSTRAINT CREATE CROSS 2676 CURRENT_DATE CURRENT_TIME CURRENT_TIMESTAMP DATABASE DEFAULT 2677 DEFERRABLE DEFERRED DELETE DESC DETACH DISTINCT DROP EACH ELSE END 2678 ESCAPE EXCEPT EXCLUSIVE EXISTS EXPLAIN FAIL FOR FOREIGN FROM FULL 2679 GLOB GROUP HAVING IF IGNORE IMMEDIATE IN INDEX INDEXED INITIALLY 2680 INNER INSERT INSTEAD INTERSECT INTO IS ISNULL JOIN KEY LEFT LIKE 2681 LIMIT MATCH NATURAL NO NOT NOTNULL NULL OF OFFSET ON OR ORDER 2682 OUTER PLAN PRAGMA PRIMARY QUERY RAISE RECURSIVE REFERENCES REGEXP 2683 REINDEX RELEASE RENAME REPLACE RESTRICT RIGHT ROLLBACK ROW 2684 SAVEPOINT SELECT SET TABLE TEMP TEMPORARY THEN TO TRANSACTION 2685 TRIGGER UNION UNIQUE UPDATE USING VACUUM VALUES VIEW VIRTUAL WHEN 2686 WHERE WITH WITHOUT""".split() 2687 2688 _sqlite_keywords.extend(getattr(apsw, "keywords", [])) 2689 2690 # reserved words need to be quoted. Only a subset of the above are reserved 2691 # but what the heck 2692 _sqlite_reserved = _sqlite_keywords 2693 # add a space after each of them except functions which get parentheses 2694 _sqlite_keywords = [x + (" ", "(")[x in ("VALUES", "CAST")] for x in _sqlite_keywords] 2695 2696 _sqlite_special_names = """_ROWID_ OID ROWID SQLITE_MASTER 2697 SQLITE_SEQUENCE""".split() 2698 2699 _sqlite_functions = """abs( changes() char( coalesce( glob( ifnull( hex( instr( 2700 last_insert_rowid() length( like( likelihood( likely( 2701 load_extension( lower( ltrim( max( min( nullif( printf( 2702 quote( random() randomblob( replace( round( rtrim( soundex( 2703 sqlite_compileoption_get( sqlite_compileoption_used( 2704 sqlite_source_id() sqlite_version() substr( total_changes() 2705 trim( typeof( unlikely( unicode( upper( zeroblob( date( 2706 time( datetime( julianday( strftime( avg( count( 2707 group_concat( sum( total(""".split() 2708 2709 _pragmas_bool = ("yes", "true", "on", "no", "false", "off") 2710 _pragmas = { 2711 "application_id": None, 2712 "auto_vacuum=": ("NONE", "FULL", "INCREMENTAL"), 2713 "automatic_index=": _pragmas_bool, 2714 "busy_timeout=": None, 2715 "cache_size=": None, 2716 "case_sensitive_like=": _pragmas_bool, 2717 "cache_spill=": _pragmas_bool, 2718 "cell_size_check=": _pragmas_bool, 2719 "checkpoint_fullfsync=": _pragmas_bool, 2720 "collation_list": None, 2721 "compile_options": None, 2722 "data_version": None, 2723 "database_list": None, 2724 "defer_foreign_keys=": _pragmas_bool, 2725 "encoding=": None, 2726 # ('"UTF-8"', '"UTF-16"', '"UTF-16le"', '"UTF16-16be"'), 2727 # too hard to get " to be part of token just in this special case 2728 "foreign_key_check": None, 2729 "foreign_key_list(": None, 2730 "foreign_keys": _pragmas_bool, 2731 "freelist_count": None, 2732 "fullfsync=": _pragmas_bool, 2733 "ignore_check_constraints": _pragmas_bool, 2734 "incremental_vacuum(": None, 2735 "index_info(": None, 2736 "index_list(": None, 2737 "index_xinfo(": None, 2738 "integrity_check": None, 2739 "journal_mode=": ("DELETE", "TRUNCATE", "PERSIST", "MEMORY", "OFF", "WAL"), 2740 "journal_size_limit=": None, 2741 "legacy_file_format=": _pragmas_bool, 2742 "locking_mode=": ("NORMAL", "EXCLUSIVE"), 2743 "max_page_count=": None, 2744 "mmap_size=": None, 2745 "optimize(": None, 2746 "page_count;": None, 2747 "page_size=": None, 2748 "query_only=": _pragmas_bool, 2749 "quick_check": None, 2750 "read_uncommitted=": _pragmas_bool, 2751 "recursive_triggers=": _pragmas_bool, 2752 "reverse_unordered_selects=": _pragmas_bool, 2753 "schema_version": None, 2754 "secure_delete=": _pragmas_bool, 2755 "shrink_memory": None, 2756 "soft_heap_limit=": None, 2757 "synchronous=": ("OFF", "NORMAL", "FULL"), 2758 "table_info(": None, 2759 "temp_store=": ("DEFAULT", "FILE", "MEMORY"), 2760 "threads=": None, 2761 "user_version=": None, 2762 "wal_autocheckpoint=": None, 2763 "wal_checkpoint": None, 2764 "writable_schema": _pragmas_bool, 2765 } 2766 2767 def _get_prev_tokens(self, line, end): 2768 "Returns the tokens prior to pos end in the line" 2769 return re.findall(r'"?\w+"?', line[:end]) 2770 2771 def complete_sql(self, line, token, beg, end): 2772 """Provide some completions for SQL 2773 2774 :param line: The current complete input line 2775 :param token: The word readline is looking for matches 2776 :param beg: Integer offset of token in line 2777 :param end: Integer end of token in line 2778 :return: A list of completions, or an empty list if none 2779 """ 2780 if self._completion_cache is None: 2781 cur = self.db.cursor() 2782 collations = [row[1] for row in cur.execute("pragma collation_list")] 2783 databases = [row[1] for row in cur.execute("pragma database_list")] 2784 other = [] 2785 for db in databases: 2786 if db == "temp": 2787 master = "sqlite_temp_master" 2788 else: 2789 master = "[%s].sqlite_master" % (db, ) 2790 for row in cur.execute("select * from " + master).fetchall(): 2791 for col in (1, 2): 2792 if row[col] not in other and not row[col].startswith("sqlite_"): 2793 other.append(row[col]) 2794 if row[0] == "table": 2795 try: 2796 for table in cur.execute("pragma [%s].table_info([%s])" % ( 2797 db, 2798 row[1], 2799 )).fetchall(): 2800 if table[1] not in other: 2801 other.append(table[1]) 2802 for item in table[2].split(): 2803 if item not in other: 2804 other.append(item) 2805 except apsw.SQLError: 2806 # See https://github.com/rogerbinns/apsw/issues/86 2807 pass 2808 functions = {} 2809 for row in cur.execute("pragma function_list"): 2810 name = row[0] 2811 narg = row[4] 2812 functions[name] = max(narg, functions.get(name, -1)) 2813 2814 def fmtfunc(name, nargs): 2815 if nargs == 0: 2816 return name + "()" 2817 return name + "(" 2818 2819 func_list = [fmtfunc(name, narg) for name, narg in functions.items()] 2820 2821 self._completion_cache = [ 2822 self._sqlite_keywords, func_list, self._sqlite_special_names, collations, databases, other 2823 ] 2824 for i in range(len(self._completion_cache)): 2825 self._completion_cache[i].sort() 2826 2827 # be somewhat sensible about pragmas 2828 if "pragma " in line.lower(): 2829 t = self._get_prev_tokens(line.lower(), end) 2830 2831 # pragma foo = bar 2832 if len(t) > 2 and t[-3] == "pragma": 2833 # t[-2] should be a valid one 2834 for p in self._pragmas: 2835 if p.replace("=", "") == t[-2]: 2836 vals = self._pragmas[p] 2837 if not vals: 2838 return [] 2839 return [x + ";" for x in vals if x.startswith(token)] 2840 # at equals? 2841 if len(t) > 1 and t[-2] == "pragma" and line[:end].replace(" ", "").endswith("="): 2842 for p in self._pragmas: 2843 if p.replace("=", "") == t[-1]: 2844 vals = self._pragmas[p] 2845 if not vals: 2846 return [] 2847 return vals 2848 # pragma foo 2849 if len(t) > 1 and t[-2] == "pragma": 2850 res = [x for x in self._pragmas.keys() if x.startswith(token)] 2851 res.sort() 2852 return res 2853 2854 # pragma 2855 if len(t) and t[-1] == "pragma": 2856 res = list(self._pragmas.keys()) 2857 res.sort() 2858 return res 2859 2860 # This is currently not context sensitive (eg it doesn't look 2861 # to see if last token was 'FROM' and hence next should only 2862 # be table names. That is a SMOP like pragmas above 2863 res = [] 2864 ut = token.upper() 2865 for corpus in self._completion_cache: 2866 for word in corpus: 2867 if word.upper().startswith(ut): 2868 # potential match - now match case 2869 if word.startswith(token): # exact 2870 if word not in res: 2871 res.append(word) 2872 elif word.lower().startswith(token): # lower 2873 if word.lower() not in res: 2874 res.append(word.lower()) 2875 elif word.upper().startswith(token): # upper 2876 if word.upper() not in res: 2877 res.append(word.upper()) 2878 else: 2879 # match letter by letter otherwise readline mangles what was typed in 2880 w = token + word[len(token):] 2881 if w not in res: 2882 res.append(w) 2883 return res 2884 2885 _builtin_commands = None 2886 2887 def complete_command(self, line, token, beg, end): 2888 """Provide some completions for dot commands 2889 2890 :param line: The current complete input line 2891 :param token: The word readline is looking for matches 2892 :param beg: Integer offset of token in line 2893 :param end: Integer end of token in line 2894 :return: A list of completions, or an empty list if none 2895 """ 2896 if not self._builtin_commands: 2897 self._builtin_commands = [ 2898 "." + x[len("command_"):] for x in dir(self) if x.startswith("command_") and x != "command_headers" 2899 ] 2900 if beg == 0: 2901 # some commands don't need a space because they take no 2902 # params but who cares? 2903 return [x + " " for x in self._builtin_commands if x.startswith(token)] 2904 return None 2905 2906 def get_resource_usage(self): 2907 """Return a dict of various numbers (ints or floats). The 2908 .timer command shows the difference between before and after 2909 results of what this returns by calling :meth:`display_timing`""" 2910 if sys.platform == "win32": 2911 import ctypes, time, platform 2912 ctypes.windll.kernel32.GetProcessTimes.argtypes = [ 2913 platform.architecture()[0] == '64bit' and ctypes.c_int64 or ctypes.c_int32, ctypes.c_void_p, 2914 ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p 2915 ] 2916 2917 # All 4 out params have to be present. FILETIME is really 2918 # just a 64 bit quantity in 100 nanosecond granularity 2919 dummy = ctypes.c_ulonglong() 2920 utime = ctypes.c_ulonglong() 2921 stime = ctypes.c_ulonglong() 2922 rc = ctypes.windll.kernel32.GetProcessTimes( 2923 ctypes.windll.kernel32.GetCurrentProcess(), 2924 ctypes.byref(dummy), # creation time 2925 ctypes.byref(dummy), # exit time 2926 ctypes.byref(stime), 2927 ctypes.byref(utime)) 2928 if rc: 2929 return { 2930 'Wall clock': time.time(), 2931 'User time': float(utime.value) / 10000000, 2932 'System time': float(stime.value) / 10000000 2933 } 2934 return {} 2935 else: 2936 import resource, time 2937 r = resource.getrusage(resource.RUSAGE_SELF) 2938 res = {'Wall clock': time.time()} 2939 for i, desc in ( 2940 ("utime", "User time"), 2941 ("stime", "System time"), 2942 ("maxrss", "Max rss"), 2943 ("idrss", "Memory"), 2944 ("isrss", "Stack"), 2945 ("ixrss", "Shared Memory"), 2946 ("minflt", "PF (no I/O)"), 2947 ("majflt", "PF (I/O)"), 2948 ("inblock", "Blocks in"), 2949 ("oublock", "Blocks out"), 2950 ("nsignals", "Signals"), 2951 ("nvcsw", "Voluntary context switches"), 2952 ("nivcsw", "Involunary context switches"), 2953 ("msgrcv", "Messages received"), 2954 ("msgsnd", "Messages sent"), 2955 ("nswap", "Swaps"), 2956 ): 2957 f = "ru_" + i 2958 if hasattr(r, f): 2959 res[desc] = getattr(r, f) 2960 return res 2961 2962 def display_timing(self, b4, after): 2963 """Writes the difference between b4 and after to self.stderr. 2964 The data is dictionaries returned from 2965 :meth:`get_resource_usage`.""" 2966 v = list(b4.keys()) 2967 for i in after: 2968 if i not in v: 2969 v.append(i) 2970 v.sort() 2971 for k in v: 2972 if k in b4 and k in after: 2973 one = b4[k] 2974 two = after[k] 2975 val = two - one 2976 if val: 2977 if type(val) == float: 2978 self.write(self.stderr, "+ %s: %.4f\n" % (k, val)) 2979 else: 2980 self.write(self.stderr, "+ %s: %d\n" % (k, val)) 2981 2982 ### Colour support 2983 2984 def _out_colour(self): 2985 # Sets up color for output. Input being interactive doesn't 2986 # matter. This method needs to be called on all changes to 2987 # output. 2988 if getattr(self.stdout, "isatty", False) and self.stdout.isatty(): 2989 self.colour = self._colours[self.colour_scheme] 2990 else: 2991 self.colour = self._colours["off"] 2992 2993 # This class returns an empty string for all undefined attributes 2994 # so that it doesn't matter if a colour scheme leaves something 2995 # out. 2996 class _colourscheme: 2997 def __init__(self, **kwargs): 2998 for k, v in kwargs.items(): 2999 setattr(self, k, v) 3000 3001 def __nonzero__(self): 3002 return True 3003 3004 def __str__(self): 3005 return "_colourscheme(" + str(self.__dict__) + ")" 3006 3007 def __getattr__(self, k): 3008 return "" 3009 3010 def colour_value(self, val, formatted): 3011 c = self.colour 3012 if val is None: 3013 return self.vnull + formatted + self.vnull_ 3014 if isinstance(val, Shell._basestring): 3015 return self.vstring + formatted + self.vstring_ 3016 if isinstance(val, Shell._binary_type): 3017 return self.vblob + formatted + self.vblob_ 3018 # must be a number - we don't distinguish between float/int 3019 return self.vnumber + formatted + self.vnumber_ 3020 3021 # The colour definitions - the convention is the name to turn 3022 # something on and the name with an underscore suffix to turn it 3023 # off 3024 d = _colourscheme(**dict([(v, "\x1b[" + str(n) + "m") for n, v in { 3025 0: "reset", 3026 1: "bold", 3027 4: "underline", 3028 22: "bold_", 3029 24: "underline_", 3030 7: "inverse", 3031 27: "inverse_", 3032 30: "fg_black", 3033 31: "fg_red", 3034 32: "fg_green", 3035 33: "fg_yellow", 3036 34: "fg_blue", 3037 35: "fg_magenta", 3038 36: "fg_cyan", 3039 37: "fg_white", 3040 39: "fg_", 3041 40: "bg_black", 3042 41: "bg_red", 3043 42: "bg_green", 3044 43: "bg_yellow", 3045 44: "bg_blue", 3046 45: "bg_magenta", 3047 46: "bg_cyan", 3048 47: "bg_white", 3049 49: "bg_" 3050 }.items()])) 3051 3052 _colours = {"off": _colourscheme(colour_value=lambda x, y: y)} 3053 3054 _colours["default"] = _colourscheme(prompt=d.bold, 3055 prompt_=d.bold_, 3056 error=d.fg_red + d.bold, 3057 error_=d.bold_ + d.fg_, 3058 intro=d.fg_blue + d.bold, 3059 intro_=d.bold_ + d.fg_, 3060 summary=d.fg_blue + d.bold, 3061 summary_=d.bold_ + d.fg_, 3062 header=sys.platform == "win32" and d.inverse or d.underline, 3063 header_=sys.platform == "win32" and d.inverse_ or d.underline_, 3064 vnull=d.fg_red, 3065 vnull_=d.fg_, 3066 vstring=d.fg_yellow, 3067 vstring_=d.fg_, 3068 vblob=d.fg_blue, 3069 vblob_=d.fg_, 3070 vnumber=d.fg_magenta, 3071 vnumber_=d.fg_) 3072 if sys.platform == "win32": 3073 if not _win_colour: 3074 for k in _colours: 3075 _colours[k] = _colours["off"] 3076 # unpollute namespace 3077 del d 3078 del _colourscheme 3079 try: 3080 del n 3081 del x 3082 del v 3083 except: 3084 pass 3085 3086 3087def main(): 3088 # Docstring must start on second line so dedenting works correctly 3089 """ 3090 Call this to run the interactive shell. It automatically passes 3091 in sys.argv[1:] and exits Python when done. 3092 3093 """ 3094 try: 3095 s = Shell() 3096 _, _, cmds = s.process_args(sys.argv[1:]) 3097 if len(cmds) == 0: 3098 s.cmdloop() 3099 except: 3100 v = sys.exc_info()[1] 3101 if getattr(v, "_handle_exception_saw_this", False): 3102 pass 3103 else: 3104 # Where did this exception come from? 3105 import traceback 3106 traceback.print_exc() 3107 sys.exit(1) 3108 3109 3110if __name__ == '__main__': 3111 main() 3112