1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 * You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5"use strict";
6
7this.EXPORTED_SYMBOLS = ["ESEDBReader"]; /* exported ESEDBReader */
8
9const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
10
11Cu.import("resource://gre/modules/ctypes.jsm");
12Cu.import("resource://gre/modules/XPCOMUtils.jsm");
13Cu.import("resource://gre/modules/Services.jsm");
14XPCOMUtils.defineLazyGetter(this, "log", () => {
15  let ConsoleAPI = Cu.import("resource://gre/modules/Console.jsm", {}).ConsoleAPI;
16  let consoleOptions = {
17    maxLogLevelPref: "browser.esedbreader.loglevel",
18    prefix: "ESEDBReader",
19  };
20  return new ConsoleAPI(consoleOptions);
21});
22
23// We have a globally unique identifier for ESE instances. A new one
24// is used for each different database opened.
25let gESEInstanceCounter = 0;
26
27// We limit the length of strings that we read from databases.
28const MAX_STR_LENGTH = 64 * 1024;
29
30// Kernel-related types:
31const KERNEL = {};
32KERNEL.FILETIME = new ctypes.StructType("FILETIME", [
33  {dwLowDateTime: ctypes.uint32_t},
34  {dwHighDateTime: ctypes.uint32_t}
35]);
36KERNEL.SYSTEMTIME = new ctypes.StructType("SYSTEMTIME", [
37  {wYear: ctypes.uint16_t},
38  {wMonth: ctypes.uint16_t},
39  {wDayOfWeek: ctypes.uint16_t},
40  {wDay: ctypes.uint16_t},
41  {wHour: ctypes.uint16_t},
42  {wMinute: ctypes.uint16_t},
43  {wSecond: ctypes.uint16_t},
44  {wMilliseconds: ctypes.uint16_t}
45]);
46
47// DB column types, cribbed from the ESE header
48var COLUMN_TYPES = {
49  JET_coltypBit:           1, /* True, False, or NULL */
50  JET_coltypUnsignedByte:  2, /* 1-byte integer, unsigned */
51  JET_coltypShort:         3, /* 2-byte integer, signed */
52  JET_coltypLong:          4, /* 4-byte integer, signed */
53  JET_coltypCurrency:      5, /* 8 byte integer, signed */
54  JET_coltypIEEESingle:    6, /* 4-byte IEEE single precision */
55  JET_coltypIEEEDouble:    7, /* 8-byte IEEE double precision */
56  JET_coltypDateTime:      8, /* Integral date, fractional time */
57  JET_coltypBinary:        9, /* Binary data, < 255 bytes */
58  JET_coltypText:         10, /* ANSI text, case insensitive, < 255 bytes */
59  JET_coltypLongBinary:   11, /* Binary data, long value */
60  JET_coltypLongText:     12, /* ANSI text, long value */
61
62  JET_coltypUnsignedLong: 14, /* 4-byte unsigned integer */
63  JET_coltypLongLong:     15, /* 8-byte signed integer */
64  JET_coltypGUID:         16, /* 16-byte globally unique identifier */
65};
66
67// Not very efficient, but only used for error messages
68function getColTypeName(numericValue) {
69  return Object.keys(COLUMN_TYPES).find(t => COLUMN_TYPES[t] == numericValue) || "unknown";
70}
71
72// All type constants and method wrappers go on this object:
73const ESE = {};
74ESE.JET_ERR = ctypes.long;
75ESE.JET_PCWSTR = ctypes.char16_t.ptr;
76// The ESE header calls this JET_API_PTR, but because it isn't ever used as a
77// pointer and because OS.File code implies that the name you give a type
78// matters, I opted for a different name.
79// Note that this is defined differently on 32 vs. 64-bit in the header.
80ESE.JET_API_ITEM = ctypes.voidptr_t.size == 4 ? ctypes.unsigned_long : ctypes.uint64_t;
81ESE.JET_INSTANCE = ESE.JET_API_ITEM;
82ESE.JET_SESID = ESE.JET_API_ITEM;
83ESE.JET_TABLEID = ESE.JET_API_ITEM;
84ESE.JET_COLUMNID = ctypes.unsigned_long;
85ESE.JET_GRBIT = ctypes.unsigned_long;
86ESE.JET_COLTYP = ctypes.unsigned_long;
87ESE.JET_DBID = ctypes.unsigned_long;
88
89ESE.JET_COLUMNDEF = new ctypes.StructType("JET_COLUMNDEF", [
90  {"cbStruct": ctypes.unsigned_long},
91  {"columnid": ESE.JET_COLUMNID	},
92  {"coltyp": ESE.JET_COLTYP },
93  {"wCountry": ctypes.unsigned_short }, // sepcifies the country/region for the column definition
94  {"langid": ctypes.unsigned_short },
95  {"cp": ctypes.unsigned_short },
96  {"wCollate": ctypes.unsigned_short }, /* Must be 0 */
97  {"cbMax": ctypes.unsigned_long },
98  {"grbit": ESE.JET_GRBIT }
99]);
100
101// Track open databases
102let gOpenDBs = new Map();
103
104// Track open libraries
105let gLibs = {};
106this.ESE = ESE; // Required for tests.
107this.KERNEL = KERNEL; // ditto
108this.gLibs = gLibs; // ditto
109
110function convertESEError(errorCode) {
111  switch (errorCode) {
112    case -1213 /* JET_errPageSizeMismatch */:
113    case -1002 /* JET_errInvalidName*/:
114    case -1507 /* JET_errColumnNotFound */:
115      // The DB format has changed and we haven't updated this migration code:
116      return "The database format has changed, error code: " + errorCode;
117    case -1207 /* JET_errDatabaseLocked */:
118    case -1302 /* JET_errTableLocked */:
119      return "The database or table is locked, error code: " + errorCode;
120    case -1809 /* JET_errPermissionDenied*/:
121    case -1907 /* JET_errAccessDenied */:
122      return "Access or permission denied, error code: " + errorCode;
123    case -1044 /* JET_errInvalidFilename */:
124      return "Invalid file name";
125    case -1811 /* JET_errFileNotFound */:
126      return "File not found";
127    case -550 /* JET_errDatabaseDirtyShutdown */:
128      return "Database in dirty shutdown state (without the requisite logs?)";
129    case -514 /* JET_errBadLogVersion */:
130      return "Database log version does not match the version of ESE in use.";
131    default:
132      return "Unknown error: " + errorCode;
133  }
134}
135
136function handleESEError(method, methodName, shouldThrow = true, errorLog = true) {
137  return function () {
138    let rv;
139    try {
140      rv = method.apply(null, arguments);
141    } catch (ex) {
142      log.error("Error calling into ctypes method", methodName, ex);
143      throw ex;
144    }
145    let resultCode = parseInt(rv.toString(10), 10);
146    if (resultCode < 0) {
147      if (errorLog) {
148        log.error("Got error " + resultCode + " calling " + methodName);
149      }
150      if (shouldThrow) {
151        throw new Error(convertESEError(rv));
152      }
153    } else if (resultCode > 0 && errorLog) {
154      log.warn("Got warning " + resultCode + " calling " + methodName);
155    }
156    return resultCode;
157  };
158}
159
160
161function declareESEFunction(methodName, ...args) {
162  let declaration = ["Jet" + methodName, ctypes.winapi_abi, ESE.JET_ERR].concat(args);
163  let ctypeMethod = gLibs.ese.declare.apply(gLibs.ese, declaration);
164  ESE[methodName] = handleESEError(ctypeMethod, methodName);
165  ESE["FailSafe" + methodName] = handleESEError(ctypeMethod, methodName, false);
166  ESE["Manual" + methodName] = handleESEError(ctypeMethod, methodName, false, false);
167}
168
169function declareESEFunctions() {
170  declareESEFunction("GetDatabaseFileInfoW", ESE.JET_PCWSTR, ctypes.voidptr_t,
171                     ctypes.unsigned_long, ctypes.unsigned_long);
172
173  declareESEFunction("GetSystemParameterW", ESE.JET_INSTANCE, ESE.JET_SESID,
174                     ctypes.unsigned_long, ESE.JET_API_ITEM.ptr,
175                     ESE.JET_PCWSTR, ctypes.unsigned_long);
176  declareESEFunction("SetSystemParameterW", ESE.JET_INSTANCE.ptr,
177                     ESE.JET_SESID, ctypes.unsigned_long, ESE.JET_API_ITEM,
178                     ESE.JET_PCWSTR);
179  declareESEFunction("CreateInstanceW", ESE.JET_INSTANCE.ptr, ESE.JET_PCWSTR);
180  declareESEFunction("Init", ESE.JET_INSTANCE.ptr);
181
182  declareESEFunction("BeginSessionW", ESE.JET_INSTANCE, ESE.JET_SESID.ptr,
183                     ESE.JET_PCWSTR, ESE.JET_PCWSTR);
184  declareESEFunction("AttachDatabaseW", ESE.JET_SESID, ESE.JET_PCWSTR,
185                     ESE.JET_GRBIT);
186  declareESEFunction("DetachDatabaseW", ESE.JET_SESID, ESE.JET_PCWSTR);
187  declareESEFunction("OpenDatabaseW", ESE.JET_SESID, ESE.JET_PCWSTR,
188                     ESE.JET_PCWSTR, ESE.JET_DBID.ptr, ESE.JET_GRBIT);
189  declareESEFunction("OpenTableW", ESE.JET_SESID, ESE.JET_DBID, ESE.JET_PCWSTR,
190                     ctypes.voidptr_t, ctypes.unsigned_long, ESE.JET_GRBIT,
191                     ESE.JET_TABLEID.ptr);
192
193  declareESEFunction("GetColumnInfoW", ESE.JET_SESID, ESE.JET_DBID,
194                     ESE.JET_PCWSTR, ESE.JET_PCWSTR, ctypes.voidptr_t,
195                     ctypes.unsigned_long, ctypes.unsigned_long);
196
197  declareESEFunction("Move", ESE.JET_SESID, ESE.JET_TABLEID, ctypes.long,
198                     ESE.JET_GRBIT);
199
200  declareESEFunction("RetrieveColumn", ESE.JET_SESID, ESE.JET_TABLEID,
201                     ESE.JET_COLUMNID, ctypes.voidptr_t, ctypes.unsigned_long,
202                     ctypes.unsigned_long.ptr, ESE.JET_GRBIT, ctypes.voidptr_t);
203
204  declareESEFunction("CloseTable", ESE.JET_SESID, ESE.JET_TABLEID);
205  declareESEFunction("CloseDatabase", ESE.JET_SESID, ESE.JET_DBID,
206                     ESE.JET_GRBIT);
207
208  declareESEFunction("EndSession", ESE.JET_SESID, ESE.JET_GRBIT);
209
210  declareESEFunction("Term", ESE.JET_INSTANCE);
211}
212
213function unloadLibraries() {
214  log.debug("Unloading");
215  if (gOpenDBs.size) {
216    log.error("Shouldn't unload libraries before DBs are closed!");
217    for (let db of gOpenDBs.values()) {
218      db._close();
219    }
220  }
221  for (let k of Object.keys(ESE)) {
222    delete ESE[k];
223  }
224  gLibs.ese.close();
225  gLibs.kernel.close();
226  delete gLibs.ese;
227  delete gLibs.kernel;
228}
229
230function loadLibraries() {
231  Services.obs.addObserver(unloadLibraries, "xpcom-shutdown", false);
232  gLibs.ese = ctypes.open("esent.dll");
233  gLibs.kernel = ctypes.open("kernel32.dll");
234  KERNEL.FileTimeToSystemTime = gLibs.kernel.declare("FileTimeToSystemTime",
235    ctypes.default_abi, ctypes.int, KERNEL.FILETIME.ptr, KERNEL.SYSTEMTIME.ptr);
236
237  declareESEFunctions();
238}
239
240function ESEDB(rootPath, dbPath, logPath) {
241  log.info("Created db");
242  this.rootPath = rootPath;
243  this.dbPath = dbPath;
244  this.logPath = logPath;
245  this._references = 0;
246  this._init();
247}
248
249ESEDB.prototype = {
250  rootPath: null,
251  dbPath: null,
252  logPath: null,
253  _opened: false,
254  _attached: false,
255  _sessionCreated: false,
256  _instanceCreated: false,
257  _dbId: null,
258  _sessionId: null,
259  _instanceId: null,
260
261  _init() {
262    if (!gLibs.ese) {
263      loadLibraries();
264    }
265    this.incrementReferenceCounter();
266    this._internalOpen();
267  },
268
269  _internalOpen() {
270    try {
271      let dbinfo = new ctypes.unsigned_long();
272      ESE.GetDatabaseFileInfoW(this.dbPath, dbinfo.address(),
273                               ctypes.unsigned_long.size, 17);
274
275      let pageSize = ctypes.UInt64.lo(dbinfo.value);
276      ESE.SetSystemParameterW(null, 0, 64 /* JET_paramDatabasePageSize*/,
277                              pageSize, null);
278
279      this._instanceId = new ESE.JET_INSTANCE();
280      ESE.CreateInstanceW(this._instanceId.address(),
281                          "firefox-dbreader-" + (gESEInstanceCounter++));
282      this._instanceCreated = true;
283
284      ESE.SetSystemParameterW(this._instanceId.address(), 0,
285                              0 /* JET_paramSystemPath*/, 0, this.rootPath);
286      ESE.SetSystemParameterW(this._instanceId.address(), 0,
287                              1 /* JET_paramTempPath */, 0, this.rootPath);
288      ESE.SetSystemParameterW(this._instanceId.address(), 0,
289                              2 /* JET_paramLogFilePath*/, 0, this.logPath);
290
291      // Shouldn't try to call JetTerm if the following call fails.
292      this._instanceCreated = false;
293      ESE.Init(this._instanceId.address());
294      this._instanceCreated = true;
295      this._sessionId = new ESE.JET_SESID();
296      ESE.BeginSessionW(this._instanceId, this._sessionId.address(), null,
297                        null);
298      this._sessionCreated = true;
299
300      const JET_bitDbReadOnly = 1;
301      ESE.AttachDatabaseW(this._sessionId, this.dbPath, JET_bitDbReadOnly);
302      this._attached = true;
303      this._dbId = new ESE.JET_DBID();
304      ESE.OpenDatabaseW(this._sessionId, this.dbPath, null,
305                        this._dbId.address(), JET_bitDbReadOnly);
306      this._opened = true;
307    } catch (ex) {
308      try {
309        this._close();
310      } catch (innerException) {
311        Cu.reportError(innerException);
312      }
313      // Make sure caller knows we failed.
314      throw ex;
315    }
316    gOpenDBs.set(this.dbPath, this);
317  },
318
319  checkForColumn(tableName, columnName) {
320    if (!this._opened) {
321      throw new Error("The database was closed!");
322    }
323
324    let columnInfo;
325    try {
326      columnInfo = this._getColumnInfo(tableName, [{name: columnName}]);
327    } catch (ex) {
328      return null;
329    }
330    return columnInfo[0];
331  },
332
333  tableExists(tableName) {
334    if (!this._opened) {
335      throw new Error("The database was closed!");
336    }
337
338    let tableId = new ESE.JET_TABLEID();
339    let rv = ESE.ManualOpenTableW(this._sessionId, this._dbId, tableName, null,
340                                  0, 4 /* JET_bitTableReadOnly */,
341                                  tableId.address());
342    if (rv == -1305 /* JET_errObjectNotFound */) {
343      return false;
344    }
345    if (rv < 0) {
346      log.error("Got error " + rv + " calling OpenTableW");
347      throw new Error(convertESEError(rv));
348    }
349
350    if (rv > 0) {
351      log.error("Got warning " + rv + " calling OpenTableW");
352    }
353    ESE.FailSafeCloseTable(this._sessionId, tableId);
354    return true;
355  },
356
357  tableItems: function*(tableName, columns) {
358    if (!this._opened) {
359      throw new Error("The database was closed!");
360    }
361
362    let tableOpened = false;
363    let tableId;
364    try {
365      tableId = this._openTable(tableName);
366      tableOpened = true;
367
368      let columnInfo = this._getColumnInfo(tableName, columns);
369
370      let rv = ESE.ManualMove(this._sessionId, tableId,
371                              -2147483648 /* JET_MoveFirst */, 0);
372      if (rv == -1603 /* JET_errNoCurrentRecord */) {
373        // There are no rows in the table.
374        this._closeTable(tableId);
375        return;
376      }
377      if (rv != 0) {
378        throw new Error(convertESEError(rv));
379      }
380
381      do {
382        let rowContents = {};
383        for (let column of columnInfo) {
384          let [buffer, bufferSize] = this._getBufferForColumn(column);
385          // We handle errors manually so we accurately deal with NULL values.
386          let err = ESE.ManualRetrieveColumn(this._sessionId, tableId,
387                                             column.id, buffer.address(),
388                                             bufferSize, null, 0, null);
389          rowContents[column.name] = this._convertResult(column, buffer, err);
390        }
391        yield rowContents;
392      } while (ESE.ManualMove(this._sessionId, tableId, 1 /* JET_MoveNext */, 0) === 0);
393    } catch (ex) {
394      if (tableOpened) {
395        this._closeTable(tableId);
396      }
397      throw ex;
398    }
399    this._closeTable(tableId);
400  },
401
402  _openTable(tableName) {
403    let tableId = new ESE.JET_TABLEID();
404    ESE.OpenTableW(this._sessionId, this._dbId, tableName, null,
405                   0, 4 /* JET_bitTableReadOnly */, tableId.address());
406    return tableId;
407  },
408
409  _getBufferForColumn(column) {
410    let buffer;
411    if (column.type == "string") {
412      let wchar_tArray = ctypes.ArrayType(ctypes.char16_t);
413      // size on the column is in bytes, 2 bytes to a wchar, so:
414      let charCount = column.dbSize >> 1;
415      buffer = new wchar_tArray(charCount);
416    } else if (column.type == "boolean") {
417      buffer = new ctypes.uint8_t();
418    } else if (column.type == "date") {
419      buffer = new KERNEL.FILETIME();
420    } else if (column.type == "guid") {
421      let byteArray = ctypes.ArrayType(ctypes.uint8_t);
422      buffer = new byteArray(column.dbSize);
423    } else {
424      throw new Error("Unknown type " + column.type);
425    }
426    return [buffer, buffer.constructor.size];
427  },
428
429  _convertResult(column, buffer, err) {
430    if (err != 0) {
431      if (err == 1004) {
432        // Deal with null values:
433        buffer = null;
434      } else {
435        Cu.reportError("Unexpected JET error: " + err + ";" + " retrieving value for column " + column.name);
436        throw new Error(convertESEError(err));
437      }
438    }
439    if (column.type == "string") {
440      return buffer ? buffer.readString() : "";
441    }
442    if (column.type == "boolean") {
443      return buffer ? (buffer.value == 255) : false;
444    }
445    if (column.type == "guid") {
446      if (buffer.length != 16) {
447        Cu.reportError("Buffer size for guid field " + column.id + " should have been 16!");
448        return "";
449      }
450      let rv = "{";
451      for (let i = 0; i < 16; i++) {
452        if (i == 4 || i == 6 || i == 8 || i == 10) {
453          rv += "-";
454        }
455        let byteValue = buffer.addressOfElement(i).contents;
456        // Ensure there's a leading 0
457        rv += ("0" + byteValue.toString(16)).substr(-2);
458      }
459      return rv + "}";
460    }
461    if (column.type == "date") {
462      if (!buffer) {
463        return null;
464      }
465      let systemTime = new KERNEL.SYSTEMTIME();
466      let result = KERNEL.FileTimeToSystemTime(buffer.address(), systemTime.address());
467      if (result == 0) {
468        throw new Error(ctypes.winLastError);
469      }
470
471      // System time is in UTC, so we use Date.UTC to get milliseconds from epoch,
472      // then divide by 1000 to get seconds, and round down:
473      return new Date(Date.UTC(systemTime.wYear,
474                                 systemTime.wMonth - 1,
475                                 systemTime.wDay,
476                                 systemTime.wHour,
477                                 systemTime.wMinute,
478                                 systemTime.wSecond,
479                                 systemTime.wMilliseconds));
480    }
481    return undefined;
482  },
483
484  _getColumnInfo(tableName, columns) {
485    let rv = [];
486    for (let column of columns) {
487      let columnInfoFromDB = new ESE.JET_COLUMNDEF();
488      ESE.GetColumnInfoW(this._sessionId, this._dbId, tableName, column.name,
489                         columnInfoFromDB.address(), ESE.JET_COLUMNDEF.size, 0 /* JET_ColInfo */);
490      let dbType = parseInt(columnInfoFromDB.coltyp.toString(10), 10);
491      let dbSize = parseInt(columnInfoFromDB.cbMax.toString(10), 10);
492      if (column.type == "string") {
493        if (dbType != COLUMN_TYPES.JET_coltypLongText &&
494            dbType != COLUMN_TYPES.JET_coltypText) {
495          throw new Error("Invalid column type for column " + column.name +
496                          "; expected text type, got type " + getColTypeName(dbType));
497        }
498        if (dbSize > MAX_STR_LENGTH) {
499          throw new Error("Column " + column.name + " has more than 64k data in it. This API is not designed to handle data that large.");
500        }
501      } else if (column.type == "boolean") {
502        if (dbType != COLUMN_TYPES.JET_coltypBit) {
503          throw new Error("Invalid column type for column " + column.name +
504                          "; expected bit type, got type " + getColTypeName(dbType));
505        }
506      } else if (column.type == "date") {
507        if (dbType != COLUMN_TYPES.JET_coltypLongLong) {
508          throw new Error("Invalid column type for column " + column.name +
509                          "; expected long long type, got type " + getColTypeName(dbType));
510        }
511      } else if (column.type == "guid") {
512        if (dbType != COLUMN_TYPES.JET_coltypGUID) {
513          throw new Error("Invalid column type for column " + column.name +
514                          "; expected guid type, got type " + getColTypeName(dbType));
515        }
516      } else if (column.type) {
517        throw new Error("Unknown column type " + column.type + " requested for column " +
518                        column.name + ", don't know what to do.");
519      }
520
521      rv.push({name: column.name, id: columnInfoFromDB.columnid, type: column.type, dbSize, dbType});
522    }
523    return rv;
524  },
525
526  _closeTable(tableId) {
527    ESE.FailSafeCloseTable(this._sessionId, tableId);
528  },
529
530  _close() {
531    this._internalClose();
532    gOpenDBs.delete(this.dbPath);
533  },
534
535  _internalClose() {
536    if (this._opened) {
537      log.debug("close db");
538      ESE.FailSafeCloseDatabase(this._sessionId, this._dbId, 0);
539      log.debug("finished close db");
540      this._opened = false;
541    }
542    if (this._attached) {
543      log.debug("detach db");
544      ESE.FailSafeDetachDatabaseW(this._sessionId, this.dbPath);
545      this._attached = false;
546    }
547    if (this._sessionCreated) {
548      log.debug("end session");
549      ESE.FailSafeEndSession(this._sessionId, 0);
550      this._sessionCreated = false;
551    }
552    if (this._instanceCreated) {
553      log.debug("term");
554      ESE.FailSafeTerm(this._instanceId);
555      this._instanceCreated = false;
556    }
557  },
558
559  incrementReferenceCounter() {
560    this._references++;
561  },
562
563  decrementReferenceCounter() {
564    this._references--;
565    if (this._references <= 0) {
566      this._close();
567    }
568  },
569};
570
571let ESEDBReader = {
572  openDB(rootDir, dbFile, logDir) {
573    let dbFilePath = dbFile.path;
574    if (gOpenDBs.has(dbFilePath)) {
575      let db = gOpenDBs.get(dbFilePath);
576      db.incrementReferenceCounter();
577      return db;
578    }
579    // ESE is really picky about the trailing slashes according to the docs,
580    // so we do as we're told and ensure those are there:
581    return new ESEDB(rootDir.path + "\\", dbFilePath, logDir.path + "\\");
582  },
583
584  closeDB(db) {
585    db.decrementReferenceCounter();
586  },
587
588  COLUMN_TYPES,
589};
590
591