1-----------------------------------------------------------------------------
2-- HTTP/1.1 client support for the Lua language.
3-- LuaSocket toolkit.
4-- Author: Diego Nehab
5-----------------------------------------------------------------------------
6
7-----------------------------------------------------------------------------
8-- Declare module and import dependencies
9-------------------------------------------------------------------------------
10local socket = require("socket")
11local url = require("socket.url")
12local ltn12 = require("ltn12")
13local mime = require("mime")
14local string = require("string")
15local headers = require("socket.headers")
16local base = _G
17local table = require("table")
18module("socket.http")
19
20-----------------------------------------------------------------------------
21-- Program constants
22-----------------------------------------------------------------------------
23-- connection timeout in seconds
24TIMEOUT = 60
25-- default port for document retrieval
26PORT = 80
27-- user agent field sent in request
28USERAGENT = socket._VERSION
29
30-----------------------------------------------------------------------------
31-- Reads MIME headers from a connection, unfolding where needed
32-----------------------------------------------------------------------------
33local function receiveheaders(sock, headers)
34    local line, name, value, err
35    headers = headers or {}
36    -- get first line
37    line, err = sock:receive()
38    if err then return nil, err end
39    -- headers go until a blank line is found
40    while line ~= "" do
41        -- get field-name and value
42        name, value = socket.skip(2, string.find(line, "^(.-):%s*(.*)"))
43        if not (name and value) then return nil, "malformed reponse headers" end
44        name = string.lower(name)
45        -- get next line (value might be folded)
46        line, err  = sock:receive()
47        if err then return nil, err end
48        -- unfold any folded values
49        while string.find(line, "^%s") do
50            value = value .. line
51            line = sock:receive()
52            if err then return nil, err end
53        end
54        -- save pair in table
55        if headers[name] then headers[name] = headers[name] .. ", " .. value
56        else headers[name] = value end
57    end
58    return headers
59end
60
61-----------------------------------------------------------------------------
62-- Extra sources and sinks
63-----------------------------------------------------------------------------
64socket.sourcet["http-chunked"] = function(sock, headers)
65    return base.setmetatable({
66        getfd = function() return sock:getfd() end,
67        dirty = function() return sock:dirty() end
68    }, {
69        __call = function()
70            -- get chunk size, skip extention
71            local line, err = sock:receive()
72            if err then return nil, err end
73            local size = base.tonumber(string.gsub(line, ";.*", ""), 16)
74            if not size then return nil, "invalid chunk size" end
75            -- was it the last chunk?
76            if size > 0 then
77                -- if not, get chunk and skip terminating CRLF
78                local chunk, err, part = sock:receive(size)
79                if chunk then sock:receive() end
80                return chunk, err
81            else
82                -- if it was, read trailers into headers table
83                headers, err = receiveheaders(sock, headers)
84                if not headers then return nil, err end
85            end
86        end
87    })
88end
89
90socket.sinkt["http-chunked"] = function(sock)
91    return base.setmetatable({
92        getfd = function() return sock:getfd() end,
93        dirty = function() return sock:dirty() end
94    }, {
95        __call = function(self, chunk, err)
96            if not chunk then return sock:send("0\r\n\r\n") end
97            local size = string.format("%X\r\n", string.len(chunk))
98            return sock:send(size ..  chunk .. "\r\n")
99        end
100    })
101end
102
103-----------------------------------------------------------------------------
104-- Low level HTTP API
105-----------------------------------------------------------------------------
106local metat = { __index = {} }
107
108function open(host, port, create)
109    -- create socket with user connect function, or with default
110    local c = socket.try((create or socket.tcp)())
111    local h = base.setmetatable({ c = c }, metat)
112    -- create finalized try
113    h.try = socket.newtry(function() h:close() end)
114    -- set timeout before connecting
115    h.try(c:settimeout(TIMEOUT))
116    h.try(c:connect(host, port or PORT))
117    -- here everything worked
118    return h
119end
120
121function metat.__index:sendrequestline(method, uri)
122    local reqline = string.format("%s %s HTTP/1.1\r\n", method or "GET", uri)
123    return self.try(self.c:send(reqline))
124end
125
126function metat.__index:sendheaders(tosend)
127    local canonic = headers.canonic
128    local h = "\r\n"
129    for f, v in base.pairs(tosend) do
130        h = (canonic[f] or f) .. ": " .. v .. "\r\n" .. h
131    end
132    self.try(self.c:send(h))
133    return 1
134end
135
136function metat.__index:sendbody(headers, source, step)
137    source = source or ltn12.source.empty()
138    step = step or ltn12.pump.step
139    -- if we don't know the size in advance, send chunked and hope for the best
140    local mode = "http-chunked"
141    if headers["content-length"] then mode = "keep-open" end
142    return self.try(ltn12.pump.all(source, socket.sink(mode, self.c), step))
143end
144
145function metat.__index:receivestatusline()
146    local status = self.try(self.c:receive(5))
147    -- identify HTTP/0.9 responses, which do not contain a status line
148    -- this is just a heuristic, but is what the RFC recommends
149    if status ~= "HTTP/" then return nil, status end
150    -- otherwise proceed reading a status line
151    status = self.try(self.c:receive("*l", status))
152    local code = socket.skip(2, string.find(status, "HTTP/%d*%.%d* (%d%d%d)"))
153    return self.try(base.tonumber(code), status)
154end
155
156function metat.__index:receiveheaders()
157    return self.try(receiveheaders(self.c))
158end
159
160function metat.__index:receivebody(headers, sink, step)
161    sink = sink or ltn12.sink.null()
162    step = step or ltn12.pump.step
163    local length = base.tonumber(headers["content-length"])
164    local t = headers["transfer-encoding"] -- shortcut
165    local mode = "default" -- connection close
166    if t and t ~= "identity" then mode = "http-chunked"
167    elseif base.tonumber(headers["content-length"]) then mode = "by-length" end
168    return self.try(ltn12.pump.all(socket.source(mode, self.c, length),
169        sink, step))
170end
171
172function metat.__index:receive09body(status, sink, step)
173    local source = ltn12.source.rewind(socket.source("until-closed", self.c))
174    source(status)
175    return self.try(ltn12.pump.all(source, sink, step))
176end
177
178function metat.__index:close()
179    return self.c:close()
180end
181
182-----------------------------------------------------------------------------
183-- High level HTTP API
184-----------------------------------------------------------------------------
185local function adjusturi(reqt)
186    local u = reqt
187    -- if there is a proxy, we need the full url. otherwise, just a part.
188    if not reqt.proxy and not PROXY then
189        u = {
190           path = socket.try(reqt.path, "invalid path 'nil'"),
191           params = reqt.params,
192           query = reqt.query,
193           fragment = reqt.fragment
194        }
195    end
196    return url.build(u)
197end
198
199local function adjustproxy(reqt)
200    local proxy = reqt.proxy or PROXY
201    if proxy then
202        proxy = url.parse(proxy)
203        return proxy.host, proxy.port or 3128
204    else
205        return reqt.host, reqt.port
206    end
207end
208
209local function adjustheaders(reqt)
210    -- default headers
211    local lower = {
212        ["user-agent"] = USERAGENT,
213        ["host"] = reqt.host,
214        ["connection"] = "close, TE",
215        ["te"] = "trailers"
216    }
217    -- if we have authentication information, pass it along
218    if reqt.user and reqt.password then
219        lower["authorization"] =
220            "Basic " ..  (mime.b64(reqt.user .. ":" .. reqt.password))
221    end
222    -- override with user headers
223    for i,v in base.pairs(reqt.headers or lower) do
224        lower[string.lower(i)] = v
225    end
226    return lower
227end
228
229-- default url parts
230local default = {
231    host = "",
232    port = PORT,
233    path ="/",
234    scheme = "http"
235}
236
237local function adjustrequest(reqt)
238    -- parse url if provided
239    local nreqt = reqt.url and url.parse(reqt.url, default) or {}
240    -- explicit components override url
241    for i,v in base.pairs(reqt) do nreqt[i] = v end
242    if nreqt.port == "" then nreqt.port = 80 end
243    socket.try(nreqt.host and nreqt.host ~= "",
244        "invalid host '" .. base.tostring(nreqt.host) .. "'")
245    -- compute uri if user hasn't overriden
246    nreqt.uri = reqt.uri or adjusturi(nreqt)
247    -- ajust host and port if there is a proxy
248    nreqt.host, nreqt.port = adjustproxy(nreqt)
249    -- adjust headers in request
250    nreqt.headers = adjustheaders(nreqt)
251    return nreqt
252end
253
254local function shouldredirect(reqt, code, headers)
255    return headers.location and
256           string.gsub(headers.location, "%s", "") ~= "" and
257           (reqt.redirect ~= false) and
258           (code == 301 or code == 302 or code == 303 or code == 307) and
259           (not reqt.method or reqt.method == "GET" or reqt.method == "HEAD")
260           and (not reqt.nredirects or reqt.nredirects < 5)
261end
262
263local function shouldreceivebody(reqt, code)
264    if reqt.method == "HEAD" then return nil end
265    if code == 204 or code == 304 then return nil end
266    if code >= 100 and code < 200 then return nil end
267    return 1
268end
269
270-- forward declarations
271local trequest, tredirect
272
273function tredirect(reqt, location)
274    local result, code, headers, status = trequest {
275        -- the RFC says the redirect URL has to be absolute, but some
276        -- servers do not respect that
277        url = url.absolute(reqt.url, location),
278        source = reqt.source,
279        sink = reqt.sink,
280        headers = reqt.headers,
281        proxy = reqt.proxy,
282        nredirects = (reqt.nredirects or 0) + 1,
283        create = reqt.create
284    }
285    -- pass location header back as a hint we redirected
286    headers = headers or {}
287    headers.location = headers.location or location
288    return result, code, headers, status
289end
290
291function trequest(reqt)
292    -- we loop until we get what we want, or
293    -- until we are sure there is no way to get it
294    local nreqt = adjustrequest(reqt)
295    local h = open(nreqt.host, nreqt.port, nreqt.create)
296    -- send request line and headers
297    h:sendrequestline(nreqt.method, nreqt.uri)
298    h:sendheaders(nreqt.headers)
299    -- if there is a body, send it
300    if nreqt.source then
301        h:sendbody(nreqt.headers, nreqt.source, nreqt.step)
302    end
303    local code, status = h:receivestatusline()
304    -- if it is an HTTP/0.9 server, simply get the body and we are done
305    if not code then
306        h:receive09body(status, nreqt.sink, nreqt.step)
307        return 1, 200
308    end
309    local headers
310    -- ignore any 100-continue messages
311    while code == 100 do
312        headers = h:receiveheaders()
313        code, status = h:receivestatusline()
314    end
315    headers = h:receiveheaders()
316    -- at this point we should have a honest reply from the server
317    -- we can't redirect if we already used the source, so we report the error
318    if shouldredirect(nreqt, code, headers) and not nreqt.source then
319        h:close()
320        return tredirect(reqt, headers.location)
321    end
322    -- here we are finally done
323    if shouldreceivebody(nreqt, code) then
324        h:receivebody(headers, nreqt.sink, nreqt.step)
325    end
326    h:close()
327    return 1, code, headers, status
328end
329
330local function srequest(u, b)
331    local t = {}
332    local reqt = {
333        url = u,
334        sink = ltn12.sink.table(t)
335    }
336    if b then
337        reqt.source = ltn12.source.string(b)
338        reqt.headers = {
339            ["content-length"] = string.len(b),
340            ["content-type"] = "application/x-www-form-urlencoded"
341        }
342        reqt.method = "POST"
343    end
344    local code, headers, status = socket.skip(1, trequest(reqt))
345    return table.concat(t), code, headers, status
346end
347
348request = socket.protect(function(reqt, body)
349    if base.type(reqt) == "string" then return srequest(reqt, body)
350    else return trequest(reqt) end
351end)
352