1 /***********************************************************************************************************************************
2 HTTP Response
3 ***********************************************************************************************************************************/
4 #include "build.auto.h"
5
6 #include "common/debug.h"
7 #include "common/io/http/client.h"
8 #include "common/io/http/common.h"
9 #include "common/io/http/request.h"
10 #include "common/io/http/response.h"
11 #include "common/io/io.h"
12 #include "common/io/read.h"
13 #include "common/log.h"
14 #include "common/stat.h"
15 #include "common/wait.h"
16
17 /***********************************************************************************************************************************
18 HTTP constants
19 ***********************************************************************************************************************************/
20 #define HTTP_HEADER_CONNECTION "connection"
21 STRING_STATIC(HTTP_HEADER_CONNECTION_STR, HTTP_HEADER_CONNECTION);
22 #define HTTP_HEADER_TRANSFER_ENCODING "transfer-encoding"
23 STRING_STATIC(HTTP_HEADER_TRANSFER_ENCODING_STR, HTTP_HEADER_TRANSFER_ENCODING);
24
25 #define HTTP_VALUE_CONNECTION_CLOSE "close"
26 STRING_STATIC(HTTP_VALUE_CONNECTION_CLOSE_STR, HTTP_VALUE_CONNECTION_CLOSE);
27 #define HTTP_VALUE_TRANSFER_ENCODING_CHUNKED "chunked"
28 STRING_STATIC(HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED);
29
30 /***********************************************************************************************************************************
31 Object type
32 ***********************************************************************************************************************************/
33 struct HttpResponse
34 {
35 HttpResponsePub pub; // Publicly accessible variables
36 HttpSession *session; // HTTP session
37 bool contentChunked; // Is the response content chunked?
38 uint64_t contentSize; // Content size (ignored for chunked)
39 uint64_t contentRemaining; // Content remaining (per chunk if chunked)
40 bool closeOnContentEof; // Will server close after content is sent?
41 bool contentExists; // Does content exist?
42 bool contentEof; // Has all content been read?
43 Buffer *content; // Caches content once requested
44 };
45
46 /***********************************************************************************************************************************
47 When response is done close/reuse the connection
48 ***********************************************************************************************************************************/
49 static void
httpResponseDone(HttpResponse * this)50 httpResponseDone(HttpResponse *this)
51 {
52 FUNCTION_LOG_BEGIN(logLevelTrace);
53 FUNCTION_LOG_PARAM(HTTP_RESPONSE, this);
54 FUNCTION_LOG_END();
55
56 ASSERT(this != NULL);
57 ASSERT(this->session != NULL);
58
59 // If close was requested by the server then free the session
60 if (this->closeOnContentEof)
61 {
62 httpSessionFree(this->session);
63
64 // Only update the close stats after a successful response so it is not counted if there was an error/retry
65 statInc(HTTP_STAT_CLOSE_STR);
66 }
67 // Else return it to the client so it can be reused
68 else
69 httpSessionDone(this->session);
70
71 this->session = NULL;
72
73 FUNCTION_LOG_RETURN_VOID();
74 }
75
76 /***********************************************************************************************************************************
77 Read content
78 ***********************************************************************************************************************************/
79 static size_t
httpResponseRead(THIS_VOID,Buffer * buffer,bool block)80 httpResponseRead(THIS_VOID, Buffer *buffer, bool block)
81 {
82 THIS(HttpResponse);
83
84 FUNCTION_LOG_BEGIN(logLevelTrace);
85 FUNCTION_LOG_PARAM(HTTP_RESPONSE, this);
86 FUNCTION_LOG_PARAM(BUFFER, buffer);
87 FUNCTION_LOG_PARAM(BOOL, block);
88 FUNCTION_LOG_END();
89
90 ASSERT(this != NULL);
91 ASSERT(buffer != NULL);
92 ASSERT(!bufFull(buffer));
93 ASSERT(this->contentEof || this->session != NULL);
94
95 // Read if EOF has not been reached
96 size_t actualBytes = 0;
97
98 if (!this->contentEof)
99 {
100 MEM_CONTEXT_TEMP_BEGIN()
101 {
102 IoRead *rawRead = httpSessionIoRead(this->session);
103
104 // If close was requested and no content specified then the server may send content up until the eof
105 if (this->closeOnContentEof && !this->contentChunked && this->contentSize == 0)
106 {
107 ioRead(rawRead, buffer);
108 this->contentEof = ioReadEof(rawRead);
109 }
110 // Else read using specified encoding or size
111 else
112 {
113 do
114 {
115 // If chunked content and no content remaining
116 if (this->contentChunked && this->contentRemaining == 0)
117 {
118 // Read length of next chunk
119 this->contentRemaining = cvtZToUInt64Base(strZ(strTrim(ioReadLine(rawRead))), 16);
120
121 // If content remaining is still zero then eof
122 if (this->contentRemaining == 0)
123 this->contentEof = true;
124 }
125
126 // Read if there is content remaining
127 if (this->contentRemaining > 0)
128 {
129 // If the buffer is larger than the content that needs to be read then limit the buffer size so the read
130 // won't block or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is
131 // defined as less than 2^32 so content remaining can't be more than that.
132 if (bufRemains(buffer) > this->contentRemaining)
133 bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining));
134
135 actualBytes = bufRemains(buffer);
136 this->contentRemaining -= ioRead(rawRead, buffer);
137
138 // Error if EOF but content read is not complete
139 if (ioReadEof(rawRead))
140 THROW(FileReadError, "unexpected EOF reading HTTP content");
141
142 // Clear limit (this works even if the limit was not set and it is easier than checking)
143 bufLimitClear(buffer);
144 }
145
146 // If no content remaining
147 if (this->contentRemaining == 0)
148 {
149 // If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back
150 // around to check.
151 if (this->contentChunked)
152 {
153 ioReadLine(rawRead);
154 }
155 // If total content size was provided then this is eof
156 else
157 this->contentEof = true;
158 }
159 }
160 while (!bufFull(buffer) && !this->contentEof);
161 }
162
163 // If all content has been read
164 if (this->contentEof)
165 httpResponseDone(this);
166 }
167 MEM_CONTEXT_TEMP_END();
168 }
169
170 FUNCTION_LOG_RETURN(SIZE, (size_t)actualBytes);
171 }
172
173 /***********************************************************************************************************************************
174 Has all content been read?
175 ***********************************************************************************************************************************/
176 static bool
httpResponseEof(THIS_VOID)177 httpResponseEof(THIS_VOID)
178 {
179 THIS(HttpResponse);
180
181 FUNCTION_LOG_BEGIN(logLevelTrace);
182 FUNCTION_LOG_PARAM(HTTP_RESPONSE, this);
183 FUNCTION_LOG_END();
184
185 ASSERT(this != NULL);
186
187 FUNCTION_LOG_RETURN(BOOL, this->contentEof);
188 }
189
190 /**********************************************************************************************************************************/
191 HttpResponse *
httpResponseNew(HttpSession * session,const String * verb,bool contentCache)192 httpResponseNew(HttpSession *session, const String *verb, bool contentCache)
193 {
194 FUNCTION_LOG_BEGIN(logLevelDebug)
195 FUNCTION_LOG_PARAM(HTTP_SESSION, session);
196 FUNCTION_LOG_PARAM(STRING, verb);
197 FUNCTION_LOG_PARAM(BOOL, contentCache);
198 FUNCTION_LOG_END();
199
200 ASSERT(session != NULL);
201 ASSERT(verb != NULL);
202
203 HttpResponse *this = NULL;
204
205 MEM_CONTEXT_NEW_BEGIN("HttpResponse")
206 {
207 this = memNew(sizeof(HttpResponse));
208
209 *this = (HttpResponse)
210 {
211 .pub =
212 {
213 .memContext = MEM_CONTEXT_NEW(),
214 .header = httpHeaderNew(NULL),
215 },
216 .session = httpSessionMove(session, memContextCurrent()),
217 };
218
219 MEM_CONTEXT_TEMP_BEGIN()
220 {
221 // Read status
222 String *status = ioReadLine(httpSessionIoRead(this->session));
223
224 // Check status ends with a CR and remove it to make error formatting easier and more accurate
225 if (!strEndsWith(status, CR_STR))
226 THROW_FMT(FormatError, "HTTP response status '%s' should be CR-terminated", strZ(status));
227
228 status = strSubN(status, 0, strSize(status) - 1);
229
230 // Check status is at least the minimum required length to avoid harder to interpret errors later on
231 if (strSize(status) < sizeof(HTTP_VERSION) + 4)
232 THROW_FMT(FormatError, "HTTP response '%s' has invalid length", strZ(strTrim(status)));
233
234 // If HTTP/1.0 then the connection will be closed on content eof since connections are not reused by default
235 if (strBeginsWith(status, HTTP_VERSION_10_STR))
236 {
237 this->closeOnContentEof = true;
238 }
239 // Else check that the version is the default (1.1)
240 else if (!strBeginsWith(status, HTTP_VERSION_STR))
241 THROW_FMT(FormatError, "HTTP version of response '%s' must be " HTTP_VERSION " or " HTTP_VERSION_10, strZ(status));
242
243 // Read status code
244 status = strSub(status, sizeof(HTTP_VERSION));
245
246 int spacePos = strChr(status, ' ');
247
248 if (spacePos != 3)
249 THROW_FMT(FormatError, "response status '%s' must have a space after the status code", strZ(status));
250
251 this->pub.code = cvtZToUInt(strZ(strSubN(status, 0, (size_t)spacePos)));
252
253 // Read reason phrase. A missing reason phrase will be represented as an empty string.
254 MEM_CONTEXT_BEGIN(this->pub.memContext)
255 {
256 this->pub.reason = strSub(status, (size_t)spacePos + 1);
257 }
258 MEM_CONTEXT_END();
259
260 // Read headers
261 do
262 {
263 // Read the next header
264 String *header = strTrim(ioReadLine(httpSessionIoRead(this->session)));
265
266 // If the header is empty then we have reached the end of the headers
267 if (strSize(header) == 0)
268 break;
269
270 // Split the header and store it
271 int colonPos = strChr(header, ':');
272
273 if (colonPos < 0)
274 THROW_FMT(FormatError, "header '%s' missing colon", strZ(strTrim(header)));
275
276 String *headerKey = strLower(strTrim(strSubN(header, 0, (size_t)colonPos)));
277 String *headerValue = strTrim(strSub(header, (size_t)colonPos + 1));
278
279 httpHeaderAdd(this->pub.header, headerKey, headerValue);
280
281 // Read transfer encoding (only chunked is supported)
282 if (strEq(headerKey, HTTP_HEADER_TRANSFER_ENCODING_STR))
283 {
284 // Error if transfer encoding is not chunked
285 if (!strEq(headerValue, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR))
286 {
287 THROW_FMT(
288 FormatError, "only '%s' is supported for '%s' header", HTTP_VALUE_TRANSFER_ENCODING_CHUNKED,
289 HTTP_HEADER_TRANSFER_ENCODING);
290 }
291
292 this->contentChunked = true;
293 }
294
295 // Read content size
296 if (strEq(headerKey, HTTP_HEADER_CONTENT_LENGTH_STR))
297 {
298 this->contentSize = cvtZToUInt64(strZ(headerValue));
299 this->contentRemaining = this->contentSize;
300 }
301
302 // If the server notified of a closed connection then close the client connection after reading content. This
303 // prevents doing a retry on the next request when using the closed connection.
304 if (strEq(headerKey, HTTP_HEADER_CONNECTION_STR) && strEq(headerValue, HTTP_VALUE_CONNECTION_CLOSE_STR))
305 this->closeOnContentEof = true;
306 }
307 while (1);
308
309 // Error if transfer encoding and content length are both set
310 if (this->contentChunked && this->contentSize > 0)
311 {
312 THROW_FMT(
313 FormatError, "'%s' and '%s' headers are both set", HTTP_HEADER_TRANSFER_ENCODING,
314 HTTP_HEADER_CONTENT_LENGTH);
315 }
316
317 // Was content returned in the response? HEAD will report content but not actually return any.
318 this->contentExists =
319 (this->contentChunked || this->contentSize > 0 || this->closeOnContentEof) && !strEq(verb, HTTP_VERB_HEAD_STR);
320 this->contentEof = !this->contentExists;
321
322 // Create an io object, even if there is no content. This makes the logic for readers easier -- they can just check eof
323 // rather than also checking if the io object exists.
324 MEM_CONTEXT_BEGIN(this->pub.memContext)
325 {
326 this->pub.contentRead = ioReadNewP(this, .eof = httpResponseEof, .read = httpResponseRead);
327 ioReadOpen(httpResponseIoRead(this));
328 }
329 MEM_CONTEXT_END();
330
331 // If there is no content then we are done with the client
332 if (!this->contentExists)
333 {
334 httpResponseDone(this);
335 }
336 // Else cache content when requested or on error
337 else if (contentCache || !httpResponseCodeOk(this))
338 {
339 MEM_CONTEXT_BEGIN(this->pub.memContext)
340 {
341 httpResponseContent(this);
342 }
343 MEM_CONTEXT_END();
344 }
345 }
346 MEM_CONTEXT_TEMP_END();
347 }
348 MEM_CONTEXT_NEW_END();
349
350 FUNCTION_LOG_RETURN(HTTP_RESPONSE, this);
351 }
352
353 /**********************************************************************************************************************************/
354 const Buffer *
httpResponseContent(HttpResponse * this)355 httpResponseContent(HttpResponse *this)
356 {
357 FUNCTION_TEST_BEGIN();
358 FUNCTION_TEST_PARAM(HTTP_RESPONSE, this);
359 FUNCTION_TEST_END();
360
361 ASSERT(this != NULL);
362
363 if (this->content == NULL)
364 {
365 this->content = bufNew(0);
366
367 if (this->contentExists)
368 {
369 do
370 {
371 bufResize(this->content, bufSize(this->content) + ioBufferSize());
372 httpResponseRead(this, this->content, true);
373 }
374 while (!httpResponseEof(this));
375
376 bufResize(this->content, bufUsed(this->content));
377 }
378 }
379
380 FUNCTION_TEST_RETURN(this->content);
381 }
382
383 /**********************************************************************************************************************************/
384 String *
httpResponseToLog(const HttpResponse * this)385 httpResponseToLog(const HttpResponse *this)
386 {
387 return strNewFmt(
388 "{code: %u, reason: %s, header: %s, contentChunked: %s, contentSize: %" PRIu64 ", contentRemaining: %" PRIu64
389 ", closeOnContentEof: %s, contentExists: %s, contentEof: %s, contentCached: %s}",
390 httpResponseCode(this), strZ(httpResponseReason(this)), strZ(httpHeaderToLog(httpResponseHeader(this))),
391 cvtBoolToConstZ(this->contentChunked), this->contentSize, this->contentRemaining, cvtBoolToConstZ(this->closeOnContentEof),
392 cvtBoolToConstZ(this->contentExists), cvtBoolToConstZ(this->contentEof), cvtBoolToConstZ(this->content != NULL));
393 }
394