1----------------------------------------------------------------------------- 2-- HTTP/1.1 client support for the Lua language. 3-- LuaSocket toolkit. 4-- Author: Diego Nehab 5----------------------------------------------------------------------------- 6 7----------------------------------------------------------------------------- 8-- Declare module and import dependencies 9------------------------------------------------------------------------------- 10local socket = require("socket") 11local url = require("socket.url") 12local ltn12 = require("ltn12") 13local mime = require("mime") 14local string = require("string") 15local headers = require("socket.headers") 16local base = _G 17local table = require("table") 18module("socket.http") 19 20----------------------------------------------------------------------------- 21-- Program constants 22----------------------------------------------------------------------------- 23-- connection timeout in seconds 24TIMEOUT = 60 25-- default port for document retrieval 26PORT = 80 27-- user agent field sent in request 28USERAGENT = socket._VERSION 29 30----------------------------------------------------------------------------- 31-- Reads MIME headers from a connection, unfolding where needed 32----------------------------------------------------------------------------- 33local function receiveheaders(sock, headers) 34 local line, name, value, err 35 headers = headers or {} 36 -- get first line 37 line, err = sock:receive() 38 if err then return nil, err end 39 -- headers go until a blank line is found 40 while line ~= "" do 41 -- get field-name and value 42 name, value = socket.skip(2, string.find(line, "^(.-):%s*(.*)")) 43 if not (name and value) then return nil, "malformed reponse headers" end 44 name = string.lower(name) 45 -- get next line (value might be folded) 46 line, err = sock:receive() 47 if err then return nil, err end 48 -- unfold any folded values 49 while string.find(line, "^%s") do 50 value = value .. line 51 line = sock:receive() 52 if err then return nil, err end 53 end 54 -- save pair in table 55 if headers[name] then headers[name] = headers[name] .. ", " .. value 56 else headers[name] = value end 57 end 58 return headers 59end 60 61----------------------------------------------------------------------------- 62-- Extra sources and sinks 63----------------------------------------------------------------------------- 64socket.sourcet["http-chunked"] = function(sock, headers) 65 return base.setmetatable({ 66 getfd = function() return sock:getfd() end, 67 dirty = function() return sock:dirty() end 68 }, { 69 __call = function() 70 -- get chunk size, skip extention 71 local line, err = sock:receive() 72 if err then return nil, err end 73 local size = base.tonumber(string.gsub(line, ";.*", ""), 16) 74 if not size then return nil, "invalid chunk size" end 75 -- was it the last chunk? 76 if size > 0 then 77 -- if not, get chunk and skip terminating CRLF 78 local chunk, err, part = sock:receive(size) 79 if chunk then sock:receive() end 80 return chunk, err 81 else 82 -- if it was, read trailers into headers table 83 headers, err = receiveheaders(sock, headers) 84 if not headers then return nil, err end 85 end 86 end 87 }) 88end 89 90socket.sinkt["http-chunked"] = function(sock) 91 return base.setmetatable({ 92 getfd = function() return sock:getfd() end, 93 dirty = function() return sock:dirty() end 94 }, { 95 __call = function(self, chunk, err) 96 if not chunk then return sock:send("0\r\n\r\n") end 97 local size = string.format("%X\r\n", string.len(chunk)) 98 return sock:send(size .. chunk .. "\r\n") 99 end 100 }) 101end 102 103----------------------------------------------------------------------------- 104-- Low level HTTP API 105----------------------------------------------------------------------------- 106local metat = { __index = {} } 107 108function open(host, port, create) 109 -- create socket with user connect function, or with default 110 local c = socket.try((create or socket.tcp)()) 111 local h = base.setmetatable({ c = c }, metat) 112 -- create finalized try 113 h.try = socket.newtry(function() h:close() end) 114 -- set timeout before connecting 115 h.try(c:settimeout(TIMEOUT)) 116 h.try(c:connect(host, port or PORT)) 117 -- here everything worked 118 return h 119end 120 121function metat.__index:sendrequestline(method, uri) 122 local reqline = string.format("%s %s HTTP/1.1\r\n", method or "GET", uri) 123 return self.try(self.c:send(reqline)) 124end 125 126function metat.__index:sendheaders(tosend) 127 local canonic = headers.canonic 128 local h = "\r\n" 129 for f, v in base.pairs(tosend) do 130 h = (canonic[f] or f) .. ": " .. v .. "\r\n" .. h 131 end 132 self.try(self.c:send(h)) 133 return 1 134end 135 136function metat.__index:sendbody(headers, source, step) 137 source = source or ltn12.source.empty() 138 step = step or ltn12.pump.step 139 -- if we don't know the size in advance, send chunked and hope for the best 140 local mode = "http-chunked" 141 if headers["content-length"] then mode = "keep-open" end 142 return self.try(ltn12.pump.all(source, socket.sink(mode, self.c), step)) 143end 144 145function metat.__index:receivestatusline() 146 local status = self.try(self.c:receive(5)) 147 -- identify HTTP/0.9 responses, which do not contain a status line 148 -- this is just a heuristic, but is what the RFC recommends 149 if status ~= "HTTP/" then return nil, status end 150 -- otherwise proceed reading a status line 151 status = self.try(self.c:receive("*l", status)) 152 local code = socket.skip(2, string.find(status, "HTTP/%d*%.%d* (%d%d%d)")) 153 return self.try(base.tonumber(code), status) 154end 155 156function metat.__index:receiveheaders() 157 return self.try(receiveheaders(self.c)) 158end 159 160function metat.__index:receivebody(headers, sink, step) 161 sink = sink or ltn12.sink.null() 162 step = step or ltn12.pump.step 163 local length = base.tonumber(headers["content-length"]) 164 local t = headers["transfer-encoding"] -- shortcut 165 local mode = "default" -- connection close 166 if t and t ~= "identity" then mode = "http-chunked" 167 elseif base.tonumber(headers["content-length"]) then mode = "by-length" end 168 return self.try(ltn12.pump.all(socket.source(mode, self.c, length), 169 sink, step)) 170end 171 172function metat.__index:receive09body(status, sink, step) 173 local source = ltn12.source.rewind(socket.source("until-closed", self.c)) 174 source(status) 175 return self.try(ltn12.pump.all(source, sink, step)) 176end 177 178function metat.__index:close() 179 return self.c:close() 180end 181 182----------------------------------------------------------------------------- 183-- High level HTTP API 184----------------------------------------------------------------------------- 185local function adjusturi(reqt) 186 local u = reqt 187 -- if there is a proxy, we need the full url. otherwise, just a part. 188 if not reqt.proxy and not PROXY then 189 u = { 190 path = socket.try(reqt.path, "invalid path 'nil'"), 191 params = reqt.params, 192 query = reqt.query, 193 fragment = reqt.fragment 194 } 195 end 196 return url.build(u) 197end 198 199local function adjustproxy(reqt) 200 local proxy = reqt.proxy or PROXY 201 if proxy then 202 proxy = url.parse(proxy) 203 return proxy.host, proxy.port or 3128 204 else 205 return reqt.host, reqt.port 206 end 207end 208 209local function adjustheaders(reqt) 210 -- default headers 211 local lower = { 212 ["user-agent"] = USERAGENT, 213 ["host"] = reqt.host, 214 ["connection"] = "close, TE", 215 ["te"] = "trailers" 216 } 217 -- if we have authentication information, pass it along 218 if reqt.user and reqt.password then 219 lower["authorization"] = 220 "Basic " .. (mime.b64(reqt.user .. ":" .. reqt.password)) 221 end 222 -- override with user headers 223 for i,v in base.pairs(reqt.headers or lower) do 224 lower[string.lower(i)] = v 225 end 226 return lower 227end 228 229-- default url parts 230local default = { 231 host = "", 232 port = PORT, 233 path ="/", 234 scheme = "http" 235} 236 237local function adjustrequest(reqt) 238 -- parse url if provided 239 local nreqt = reqt.url and url.parse(reqt.url, default) or {} 240 -- explicit components override url 241 for i,v in base.pairs(reqt) do nreqt[i] = v end 242 if nreqt.port == "" then nreqt.port = 80 end 243 socket.try(nreqt.host and nreqt.host ~= "", 244 "invalid host '" .. base.tostring(nreqt.host) .. "'") 245 -- compute uri if user hasn't overriden 246 nreqt.uri = reqt.uri or adjusturi(nreqt) 247 -- ajust host and port if there is a proxy 248 nreqt.host, nreqt.port = adjustproxy(nreqt) 249 -- adjust headers in request 250 nreqt.headers = adjustheaders(nreqt) 251 return nreqt 252end 253 254local function shouldredirect(reqt, code, headers) 255 return headers.location and 256 string.gsub(headers.location, "%s", "") ~= "" and 257 (reqt.redirect ~= false) and 258 (code == 301 or code == 302 or code == 303 or code == 307) and 259 (not reqt.method or reqt.method == "GET" or reqt.method == "HEAD") 260 and (not reqt.nredirects or reqt.nredirects < 5) 261end 262 263local function shouldreceivebody(reqt, code) 264 if reqt.method == "HEAD" then return nil end 265 if code == 204 or code == 304 then return nil end 266 if code >= 100 and code < 200 then return nil end 267 return 1 268end 269 270-- forward declarations 271local trequest, tredirect 272 273function tredirect(reqt, location) 274 local result, code, headers, status = trequest { 275 -- the RFC says the redirect URL has to be absolute, but some 276 -- servers do not respect that 277 url = url.absolute(reqt.url, location), 278 source = reqt.source, 279 sink = reqt.sink, 280 headers = reqt.headers, 281 proxy = reqt.proxy, 282 nredirects = (reqt.nredirects or 0) + 1, 283 create = reqt.create 284 } 285 -- pass location header back as a hint we redirected 286 headers = headers or {} 287 headers.location = headers.location or location 288 return result, code, headers, status 289end 290 291function trequest(reqt) 292 -- we loop until we get what we want, or 293 -- until we are sure there is no way to get it 294 local nreqt = adjustrequest(reqt) 295 local h = open(nreqt.host, nreqt.port, nreqt.create) 296 -- send request line and headers 297 h:sendrequestline(nreqt.method, nreqt.uri) 298 h:sendheaders(nreqt.headers) 299 -- if there is a body, send it 300 if nreqt.source then 301 h:sendbody(nreqt.headers, nreqt.source, nreqt.step) 302 end 303 local code, status = h:receivestatusline() 304 -- if it is an HTTP/0.9 server, simply get the body and we are done 305 if not code then 306 h:receive09body(status, nreqt.sink, nreqt.step) 307 return 1, 200 308 end 309 local headers 310 -- ignore any 100-continue messages 311 while code == 100 do 312 headers = h:receiveheaders() 313 code, status = h:receivestatusline() 314 end 315 headers = h:receiveheaders() 316 -- at this point we should have a honest reply from the server 317 -- we can't redirect if we already used the source, so we report the error 318 if shouldredirect(nreqt, code, headers) and not nreqt.source then 319 h:close() 320 return tredirect(reqt, headers.location) 321 end 322 -- here we are finally done 323 if shouldreceivebody(nreqt, code) then 324 h:receivebody(headers, nreqt.sink, nreqt.step) 325 end 326 h:close() 327 return 1, code, headers, status 328end 329 330local function srequest(u, b) 331 local t = {} 332 local reqt = { 333 url = u, 334 sink = ltn12.sink.table(t) 335 } 336 if b then 337 reqt.source = ltn12.source.string(b) 338 reqt.headers = { 339 ["content-length"] = string.len(b), 340 ["content-type"] = "application/x-www-form-urlencoded" 341 } 342 reqt.method = "POST" 343 end 344 local code, headers, status = socket.skip(1, trequest(reqt)) 345 return table.concat(t), code, headers, status 346end 347 348request = socket.protect(function(reqt, body) 349 if base.type(reqt) == "string" then return srequest(reqt, body) 350 else return trequest(reqt) end 351end) 352