1--- 2-- Library methods for handling MongoDB, creating and parsing packets. 3-- 4-- @author Martin Holst Swende 5-- @copyright Same as Nmap--See https://nmap.org/book/man-legal.html 6-- 7-- @args mongodb.db - the database to use for authentication 8 9-- Created 01/13/2010 - v0.1 - created by Martin Holst Swende <martin@swende.se> 10-- Revised 01/03/2012 - v0.2 - added authentication support <patrik@cqure.net> 11 12local nmap = require "nmap" 13local stdnse = require "stdnse" 14local string = require "string" 15local table = require "table" 16local openssl = stdnse.silent_require "openssl" 17_ENV = stdnse.module("mongodb", stdnse.seeall) 18 19 20-- this is not yet widely implemented but at least used for authentication 21-- ideally, it would be used to set the database against which operations, 22-- that do not require a specific database, should run 23local arg_DB = stdnse.get_script_args("mongodb.db") 24 25-- Some lazy shortcuts 26 27local function dbg(str,...) 28 stdnse.debug3("MngoDb:"..str, ...) 29end 30--local dbg =stdnse.debug1 31 32local err =stdnse.debug1 33 34---------------------------------------------------------------------- 35-- First of all comes a Bson parsing library. This can easily be moved out into a separate library should other 36-- services start to use Bson 37---------------------------------------------------------------------- 38-- Library methods for handling the BSON format 39-- 40-- For more documentation about the BSON format, 41---and more details about its implementations, check out the 42-- python BSON implementation which is available at 43-- http://github.com/mongodb/mongo-python-driver/blob/master/pymongo/bson.py 44-- and licensed under the Apache License, Version 2.0 http://www.apache.org/licenses/LICENSE-2.0) 45-- 46-- @author Martin Holst Swende 47-- @copyright Same as Nmap--See https://nmap.org/book/man-legal.html 48-- 49-- Version 0.1 50 51-- Created 01/13/2010 - v0.1 - created by Martin Holst Swende <martin@swende.se> 52--module("bson", package.seeall) 53local function dbg_err(str,...) 54 stdnse.debug1("Bson-ERR:"..str, ...) 55end 56--local err =stdnse.log_error 57 58--Converts an element (key, value) into bson binary data 59--@param key the key name, must *NOT* contain . (period) or start with $ 60--@param value, the element value 61--@return status : true if ok, false if error 62--@return result : the packed binary data OR error message 63local function _element_to_bson(key, value) 64 65 --Some constraints-checking 66 if type(key) ~= 'string' then 67 return false, "Documents must have only string keys, key was " .. type(key) 68 end 69 if key:sub(1,1) == "$" then 70 return false, "key must not start with $: ".. key 71 end 72 if key:find("%.") then 73 return false, ("key %r must not contain '.'"):format(tostring(key)) 74 end 75 76 local name = string.pack("z", key) -- null-terminated string 77 if type(value) == 'string' then 78 local cstring = string.pack("z", value) -- null-terminated string 79 local length = string.pack("<i4", cstring:len()) 80 local op = "\x02" 81 return true, op .. name .. length .. cstring 82 elseif type(value) =='table' then 83 return true, "\x02" .. name .. toBson(value) 84 elseif type(value)== 'boolean' then 85 return true, "\x08" .. name .. (value and '\x01' or '\0') 86 elseif type(value) == 'number' then 87 return true, '\x01' .. name .. string.pack("<d", value) 88 end 89 90 local _ = ("cannot convert value of type %s to bson"):format(type(value)) 91 return false, _ 92end 93 94--Converts a table of elements to binary bson format 95--@param dict the table 96--@return status : true if ok, false if error 97--@return result : a string of binary data OR error message 98function toBson(dict) 99 100 local elements = "" 101 --Put id first 102 if dict._id then 103 local status,res = _element_to_bson("_id", dict._id) 104 if not status then return false, res end 105 elements = elements..res 106 elseif ( dict._cmd ) then 107 for k, v in pairs(dict._cmd) do 108 local status,res = _element_to_bson(k, v) 109 if not status then return false, res end 110 elements = elements..res 111 end 112 end 113 --Concatenate binary values 114 for key, value in pairs( dict ) do 115 if key ~= "_id" and key ~= "_cmd" then 116 dbg("dictionary to bson : key,value =(%s,%s)",key,value) 117 local status,res = _element_to_bson(key,value) 118 if not status then return false, res end 119 elements = elements..res 120 end 121 end 122 -- Get length 123 local length = #elements + 5 124 125 if length > 4 * 1024 * 1024 then 126 return false, "document too large - BSON documents are limited to 4 MB" 127 end 128 dbg("Packet length is %d",length) 129 --Final pack 130 return true, string.pack("<I4", length) .. elements .. "\0" 131end 132 133-- Reads a null-terminated string. If length is supplied, it is just cut 134-- out from the data, otherwise the data is scanned for at null-char. 135--@param data the data which starts with a c-string 136--@param length optional length of the string 137--@return the string 138--@return the remaining data (*without* null-char) 139local function get_c_string(data,length) 140 if not length then 141 local index = data:find('\0') 142 if index == nil then 143 error({code="C-string did not contain NULL char"}) 144 end 145 length = index 146 end 147 local value = data:sub(1,length-1) 148 149 --dbg("Found char at pos %d, data is %s c-string is %s",length, data, value) 150 151 return value, data:sub(length+1) 152end 153 154-- Element parser. Parse data elements 155-- @param data String containing binary data 156-- @return Position in the data string where parsing stopped 157-- @return Unpacked value 158-- @return error string if error occurred 159local function parse(code,data) 160 if 1 == code then -- double 161 local v, pos = string.unpack("<d", data) 162 return pos, v 163 elseif 2 == code then -- string 164 -- data length = first four bytes 165 local len = string.unpack("<i4",data) 166 -- string data = data[5] --> 167 local value = get_c_string(data:sub(5), len) 168 -- Count position as header (=4) + length of string (=len)+ null char (=1) 169 return 4+len+1,value 170 elseif 3 == code or 4 == code then -- table or array 171 local object, err 172 173 -- Need to know the length, to return later 174 local obj_size = string.unpack("<i4", data) 175 -- Now, get the data object 176 dbg("Recursing into bson array") 177 object, data, err = fromBson(data) 178 dbg("Recurse finished, got data object") 179 -- And return where the parsing stopped 180 return obj_size+1, object 181 --6 = _get_null 182 --7 = _get_oid 183 elseif 8 == code then -- Boolean 184 return 2, data:byte(1) == 1 185 elseif 9 == code then -- int64, UTC datetime 186 local v, pos = string.unpack("<i8", data) 187 return pos, v 188 elseif 10 == code then -- nullvalue 189 return 0,nil 190 --11= _get_regex 191 --12= _get_ref 192 --13= _get_string, # code 193 --14= _get_string, # symbol 194 --15= _get_code_w_scope 195 elseif 16 == code then -- 4 byte integer 196 local v, pos = string.unpack("<i4", data) 197 return pos, v 198 --17= _get_timestamp 199 elseif 18 == code then -- long 200 local v, pos = string.unpack("<i8", data) 201 return pos, v 202 end 203 local err = ("Getter for %d not implemented"):format(code) 204 return 0, data, err 205end 206 207 208-- Reads an element from binary to BSon 209--@param data a string of data to convert 210--@return Name of the element 211--@return Value of the element 212--@return Residual data not used 213--@return any error that occurred 214local function _element_to_dict(data) 215 local element_type, element_name, err, pos, value 216 --local element_size = data:byte(1) 217 element_type = data:byte(1) 218 element_name, data = get_c_string(data:sub(2)) 219 220 dbg(" Read element name '%s' (type:%s), data left: %d",element_name, element_type,data:len()) 221 --pos,value,err = parsers.get(element_type)(data) 222 pos,value,err = parse(element_type,data) 223 if(err ~= nil) then 224 dbg_err(err) 225 return nil,nil, data, err 226 end 227 228 data=data:sub(pos) 229 230 dbg(" Read element value '%s', data left: %d",tostring(value), data:len()) 231 return element_name, value, data 232end 233 234--Reads all elements from binary to BSon 235--@param data the data to read from 236--@return the resulting table 237local function _elements_to_dict(data) 238 local result = {} 239 local key,value 240 while data and data:len() > 1 do 241 key, value, data = _element_to_dict(data) 242 dbg("Parsed (%s='%s'), data left : %d", tostring(key),tostring(value), data:len()) 243 if type(value) ~= 'table' then value=tostring(value) end 244 result[key] = value 245 end 246 return result 247end 248 249--Checks if enough data to parse the result is captured 250--@data binary bson data read from socket 251--@return true if the full bson table is contained in the data, false if data is incomplete 252--@return required size of packet, if known, otherwise nil 253function isPacketComplete(data) 254 -- First, we check that the header is complete 255 if data:len() < 4 then 256 local err_msg = "Not enough data in buffer, at least 4 bytes header info expected" 257 return false 258 end 259 260 local obj_size = string.unpack("<i4", data) 261 262 dbg("BSon packet size is %s", obj_size) 263 264 -- Check that all data is read and the packet is complete 265 if data:len() < obj_size then 266 return false,obj_size 267 end 268 return true,obj_size 269end 270 271-- Converts bson binary data read from socket into a table 272-- of elements 273--@param data: binary data 274--@return table containing elements 275--@return remaining data 276--@return error message if not enough data was in packet 277function fromBson(data) 278 279 dbg("Decoding, got %s bytes of data", data:len()) 280 local complete, object_size = isPacketComplete(data) 281 282 if not complete then 283 local err_msg = ("Not enough data in buffer, expected %s but only has %d"):format(object_size or "?", data:len()) 284 dbg(err_msg) 285 return {},data, err_msg 286 end 287 288 local element_portion = data:sub(5,object_size) 289 local remainder = data:sub(object_size+1) 290 return _elements_to_dict(element_portion), remainder 291end 292 293 294---------------------------------------------------------------------------------- 295-- Test-code for debugging purposes below 296---------------------------------------------------------------------------------- 297function testBson() 298 local p = toBson({hello="world", test="ok"}) 299 300 print( "Encoded something ok") 301 local orig = fromBson(p) 302 print(" Decoded something else ok") 303 for i,v in pairs(orig) do 304 print(i,v) 305 end 306end 307--testBson() 308-------------------------------------------------------------------------------------------------------------- 309--- END of BSON part 310-------------------------------------------------------------------------------------------------------------- 311 312 313--[[ MongoDB wire protocol format 314 315Standard message header : 316struct { 317 int32 messageLength; // total size of the message, including the 4 bytes of length 318 int32 requestID; // client or database-generated identifier for this message 319 int32 responseTo; // requestID from the original request (used in responses from db) 320 int32 opCode; // request type - see table below 321} 322 323Opcodes : 324OP_REPLY 1 Reply to a client request. responseTo is set 325OP_MSG 1000 generic msg command followed by a string 326OP_UPDATE 2001 update document 327OP_INSERT 2002 insert new document 328OP_GET_BY_OID 2003 is this used? 329OP_QUERY 2004 query a collection 330OP_GET_MORE 2005 Get more data from a query. See Cursors 331OP_DELETE 2006 Delete documents 332OP_KILL_CURSORS 2007 Tell database client is done with a cursor 333 334Query message : 335struct { 336 MsgHeader header; // standard message header 337 int32 opts; // query options. See below for details. 338 cstring fullCollectionName; // "dbname.collectionname" 339 int32 numberToSkip; // number of documents to skip when returning results 340 int32 numberToReturn; // number of documents to return in the first OP_REPLY 341 BSON query ; // query object. See below for details. 342 [ BSON returnFieldSelector; ] // OPTIONAL : selector indicating the fields to return. See below for details. 343} 344 345For more info about the MongoDB wire protocol, see http://www.mongodb.org/display/DOCS/Mongo+Wire+Protocol 346 347--]] 348 349-- DIY lua-class to create Mongo packets 350--@usage call MongoData:new({opCode=MongoData.OP.QUERY}) to create query object 351MongoData ={ 352 uniqueRequestId = 12345, 353 -- Opcodes used by Mongo db 354 OP = { 355 REPLY = 1, 356 MSG = 1000, 357 UPDATE = 2001, 358 INSERT = 2002, 359 GET_BY_IOD = 2003, 360 QUERY = 2004, 361 GET_MORE = 2005, 362 DELETE = 2006, 363 KILL_CURSORS = 2007, 364 }, 365 -- Lua-DIY constructor 366 new = function (self,o,opCode,responseTo) 367 o = o or {} -- create object if user does not provide one 368 setmetatable(o, self) -- DIY inheritance a'la javascript 369 self.__index = self 370 self.valueString = '' 371 self.requestID = MongoData.uniqueRequestId -- Create unique id for message 372 MongoData.uniqueRequestId = MongoData.uniqueRequestId +1 373 return o 374 end 375} 376--Adds signed int32 to the message body 377--@param value the value to add 378function MongoData:addInt32(value) 379 self.valueString = self.valueString..string.pack("<i4",value) 380end 381-- Adds a string to the message body 382--@param value the string to add 383function MongoData:addString(value) 384 self.valueString = self.valueString..string.pack('z',value) 385end 386-- Add a table as a BSon object to the body 387--@param dict the table to be converted to BSon 388--@return status : true if ok, false if error occurred 389--@return Error message if error occurred 390function MongoData:addBSON(dict) 391 -- status, res = bson.toBson(dict) 392 local status, res = toBson(dict) 393 if not status then 394 dbg(res) 395 return status,res 396 end 397 398 self.valueString = self.valueString..res 399 return true 400end 401-- Returns the data in this packet in a raw string format to be sent on socket 402-- This method creates necessary header information and puts it with the body 403function MongoData:data() 404 local header = MongoData:new() 405 header:addInt32( self.valueString:len()+4+4+4+4) 406 header:addInt32( self.requestID) 407 header:addInt32( self.responseTo or -1) 408 header:addInt32( self.opCode) 409 return header.valueString .. self.valueString 410end 411-- Creates a query 412-- @param collectionName string specifying the collection to run query against 413-- @param a table containing the query 414--@return status : true for OK, false for error 415--@return packet data OR error message 416local function createQuery(collectionName, query) 417 local packet = MongoData:new({opCode=MongoData.OP.QUERY}) 418 packet:addInt32(0); -- options 419 packet:addString(collectionName); 420 packet:addInt32(0) -- number to skip 421 -- NB: Using value of -1 for "no limit" below is suspect. The protocol 422 -- interprets -1 as requesting only one document, not all documents. 423 -- https://docs.mongodb.com/manual/reference/mongodb-wire-protocol/#wire-op-query 424 packet:addInt32(-1) -- number to return : no limit 425 local status, error = packet:addBSON(query) 426 427 if not status then 428 return status, error 429 end 430 431 return true, packet:data() 432end 433-- Creates a get last error query 434-- @param responseTo optional identifier this packet is a response to 435--@return status : true for OK, false for error 436--@return packet data OR error message 437function lastErrorQuery(responseTo) 438 local collectionName = "test.$cmd" 439 local query = {getlasterror=1} 440 return createQuery(collectionName, query) 441end 442-- Creates a server status query 443-- @param responseTo optional identifier this packet is a response to 444--@return status : true for OK, false for error 445--@return packet data OR error message 446function serverStatusQuery(responseTo) 447 local collectionName = "test.$cmd" 448 local query = {serverStatus = 1} 449 return createQuery(collectionName, query) 450end 451-- Creates a optime query 452-- @param responseTo optional identifier this packet is a response to 453--@return status : true for OK, false for error 454--@return packet data OR error message 455function opTimeQuery(responseTo) 456 local collectionName = "test.$cmd" 457 local query = {getoptime = 1} 458 return createQuery(collectionName, query) 459end 460-- Creates a list databases query 461-- @param responseTo optional identifier this packet is a response to 462--@return status : true for OK, false for error 463--@return packet data OR error message 464function listDbQuery(responseTo) 465 local collectionName = "admin.$cmd" 466 local query = {listDatabases = 1} 467 return createQuery(collectionName, query) 468end 469-- Creates a build info query 470-- @param responseTo optional identifier this packet is a response to 471--@return status : true for OK, false for error 472--@return packet data OR error message 473--@return status : true for OK, false for error 474--@return packet data OR error message 475function buildInfoQuery(responseTo) 476 local collectionName = "admin.$cmd" 477 local query = {buildinfo = 1} 478 return createQuery(collectionName, query) 479end 480--Reads an int32 from data 481--@return int32 value 482--@return data unread 483local function parseInt32(data) 484 local val, pos = string.unpack("<i4", data) 485 return val, data:sub(pos) 486end 487local function parseInt64(data) 488 local val, pos = string.unpack("<i8", data) 489 return val, data:sub(pos) 490end 491-- Parses response header 492-- The response header looks like this : 493--[[ 494struct { 495 MsgHeader header; // standard message header 496 int32 responseFlag; // normally zero, non-zero on query failure 497 int64 cursorID; // id of the cursor created for this query response 498 int32 startingFrom; // indicates where in the cursor this reply is starting 499 int32 numberReturned; // number of documents in the reply 500 BSON[] documents; // documents 501} 502--]] 503--@param the data from socket 504--@return a table containing the header data 505local function parseResponseHeader(data) 506 local response= {} 507 local hdr, rflag, cID, sfrom, nRet, docs 508 509 -- First std message header 510 hdr ={} 511 hdr["messageLength"], data = parseInt32(data) 512 hdr["requestID"], data = parseInt32(data) 513 hdr["responseTo"], data = parseInt32(data) 514 hdr["opCode"], data = parseInt32(data) 515 response["header"] = hdr 516 -- Some additional fields 517 response["responseFlag"] ,data = parseInt32(data) 518 response["cursorID"] ,data = parseInt64(data) 519 response["startingFrom"] ,data = parseInt32(data) 520 response["numberReturned"] ,data = parseInt32(data) 521 response["bson"] = data 522 return response 523end 524--Checks if enough data to parse the result is captured 525--@data binary mongodb data read from socket 526--@return true if the full mongodb packet is contained in the data, false if data is incomplete 527--@return required size of packet, if known, otherwise nil 528function isPacketComplete(data) 529 -- First, we check that the header is complete 530 if data:len() < 4 then 531 local err_msg = "Not enough data in buffer, at least 4 bytes header info expected" 532 return false 533 end 534 535 local obj_size = string.unpack("<i4", data) 536 537 dbg("MongoDb Packet size is %s, (got %d)", obj_size,data:len()) 538 539 -- Check that all data is read and the packet is complete 540 if data:len() < obj_size then 541 return false,obj_size 542 end 543 return true,obj_size 544end 545 546-- Sends a packet over a socket, reads the response 547-- and parses it into a table 548--@return status : true if ok; false if bad 549--@return result : table of status ok, error msg if bad 550--@return if status ok : remaining data read from socket but not used 551function query(socket, data) 552 --Create an error handler 553 local catch = function() 554 socket:close() 555 stdnse.debug1("Query failed") 556 end 557 local try = nmap.new_try(catch) 558 559 try( socket:send( data ) ) 560 561 local data = "" 562 local result = {} 563 local err_msg 564 local isComplete, pSize 565 while not isComplete do 566 dbg("mongo: waiting for data from socket, got %d bytes so far...",data:len()) 567 data = data .. try( socket:receive() ) 568 isComplete, pSize = isPacketComplete(data) 569 end 570 -- All required data should be read now 571 local packetData = data:sub(1,pSize) 572 local residualData = data:sub(pSize+1) 573 local responseHeader = parseResponseHeader(packetData) 574 575 if responseHeader["responseFlag"] ~= 0 then 576 dbg("Response flag not zero : %d, some error occurred", responseHeader["responseFlag"]) 577 end 578 579 local bsonData = responseHeader["bson"] 580 if #bsonData == 0 then 581 dbg("No BSon data returned ") 582 return false, "No Bson data returned" 583 end 584 585 -- result, data, err_msg = bson.fromBson(bsonData) 586 result, data, err_msg = fromBson(bsonData) 587 588 if err_msg then 589 dbg("Got error converting from bson: %s" , err_msg) 590 return false, ("Got error converting from bson: %s"):format(err_msg) 591 end 592 return true,result, residualData 593end 594 595function login(socket, db, username, password) 596 597 local collectionName = ("%s.$cmd"):format(arg_DB or db) 598 local q = { getnonce = 1 } 599 local status, packet = createQuery(collectionName, q) 600 local response 601 status, response = query(socket, packet) 602 if ( not(status) or not(response.nonce) ) then 603 return false, "Failed to retrieve nonce" 604 end 605 606 local nonce = response.nonce 607 local pwdigest = stdnse.tohex(openssl.md5(username .. ':mongo:' ..password)) 608 local digest = stdnse.tohex(openssl.md5(nonce .. username .. pwdigest)) 609 610 q = { user = username, nonce = nonce, key = digest } 611 q._cmd = { authenticate = 1 } 612 613 local status, packet = createQuery(collectionName, q) 614 status, response = query(socket, packet) 615 if ( not(status) ) then 616 return status, response 617 elseif ( response.errmsg == "auth fails" ) then 618 return false, "Authentication failed" 619 elseif ( response.errmsg ) then 620 return false, response.errmsg 621 end 622 return status, response 623end 624 625 626--- Converts a query result as received from MongoDB query into nmap "result" table 627-- @param resultTable table as returned from a query 628-- @return table suitable for <code>stdnse.format_output</code> 629function queryResultToTable( resultTable ) 630 631 local result = {} 632 for k,v in pairs( resultTable ) do 633 if type(v) == 'table' then 634 table.insert(result,k) 635 table.insert(result,queryResultToTable(v)) 636 else 637 table.insert(result,(("%s = %s"):format(tostring(k), tostring(v)))) 638 end 639 end 640 return result 641 642end 643 644return _ENV; 645