1 /***********************************************************************************************************************************
2 HTTP Request
3 ***********************************************************************************************************************************/
4 #include "build.auto.h"
5
6 #include "common/debug.h"
7 #include "common/io/http/common.h"
8 #include "common/io/http/request.h"
9 #include "common/log.h"
10 #include "common/stat.h"
11 #include "common/wait.h"
12 #include "version.h"
13
14 /***********************************************************************************************************************************
15 HTTP constants
16 ***********************************************************************************************************************************/
17 STRING_EXTERN(HTTP_VERSION_STR, HTTP_VERSION);
18 STRING_EXTERN(HTTP_VERSION_10_STR, HTTP_VERSION_10);
19
20 STRING_EXTERN(HTTP_VERB_DELETE_STR, HTTP_VERB_DELETE);
21 STRING_EXTERN(HTTP_VERB_GET_STR, HTTP_VERB_GET);
22 STRING_EXTERN(HTTP_VERB_HEAD_STR, HTTP_VERB_HEAD);
23 STRING_EXTERN(HTTP_VERB_POST_STR, HTTP_VERB_POST);
24 STRING_EXTERN(HTTP_VERB_PUT_STR, HTTP_VERB_PUT);
25
26 STRING_EXTERN(HTTP_HEADER_AUTHORIZATION_STR, HTTP_HEADER_AUTHORIZATION);
27 STRING_EXTERN(HTTP_HEADER_CONTENT_LENGTH_STR, HTTP_HEADER_CONTENT_LENGTH);
28 STRING_EXTERN(HTTP_HEADER_CONTENT_MD5_STR, HTTP_HEADER_CONTENT_MD5);
29 STRING_EXTERN(HTTP_HEADER_CONTENT_RANGE_STR, HTTP_HEADER_CONTENT_RANGE);
30 STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_STR, HTTP_HEADER_CONTENT_TYPE);
31 STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL_STR, HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL);
32 STRING_EXTERN(HTTP_HEADER_ETAG_STR, HTTP_HEADER_ETAG);
33 STRING_EXTERN(HTTP_HEADER_DATE_STR, HTTP_HEADER_DATE);
34 STRING_EXTERN(HTTP_HEADER_HOST_STR, HTTP_HEADER_HOST);
35 STRING_EXTERN(HTTP_HEADER_LAST_MODIFIED_STR, HTTP_HEADER_LAST_MODIFIED);
36 #define HTTP_HEADER_USER_AGENT "user-agent"
37
38 // 5xx errors that should always be retried
39 #define HTTP_RESPONSE_CODE_RETRY_CLASS 5
40
41 /***********************************************************************************************************************************
42 Object type
43 ***********************************************************************************************************************************/
44 struct HttpRequest
45 {
46 HttpRequestPub pub; // Publicly accessible variables
47 HttpClient *client; // HTTP client
48 const Buffer *content; // HTTP content
49
50 HttpSession *session; // Session for async requests
51 };
52
53 /***********************************************************************************************************************************
54 Process the request
55 ***********************************************************************************************************************************/
56 static HttpResponse *
httpRequestProcess(HttpRequest * this,bool waitForResponse,bool contentCache)57 httpRequestProcess(HttpRequest *this, bool waitForResponse, bool contentCache)
58 {
59 FUNCTION_LOG_BEGIN(logLevelDebug)
60 FUNCTION_LOG_PARAM(HTTP_REQUEST, this);
61 FUNCTION_LOG_PARAM(BOOL, waitForResponse);
62 FUNCTION_LOG_PARAM(BOOL, contentCache);
63 FUNCTION_LOG_END();
64
65 ASSERT(this != NULL);
66
67 // HTTP Response
68 HttpResponse *result = NULL;
69
70 MEM_CONTEXT_TEMP_BEGIN()
71 {
72 bool retry;
73 Wait *wait = waitNew(httpClientTimeout(this->client));
74
75 do
76 {
77 // Assume there will be no retry
78 retry = false;
79
80 TRY_BEGIN()
81 {
82 MEM_CONTEXT_TEMP_BEGIN()
83 {
84 HttpSession *session = NULL;
85
86 // If a session is saved then the request was already successfully sent
87 if (this->session != NULL)
88 {
89 session = httpSessionMove(this->session, memContextCurrent());
90 this->session = NULL;
91 }
92 // Else the request has not been sent yet or this is a retry
93 else
94 {
95 session = httpClientOpen(this->client);
96
97 // Format the request and user agent
98 String *requestStr =
99 strNewFmt(
100 "%s %s%s%s " HTTP_VERSION CRLF_Z HTTP_HEADER_USER_AGENT ":" PROJECT_NAME "/" PROJECT_VERSION CRLF_Z,
101 strZ(httpRequestVerb(this)), strZ(httpRequestPath(this)), httpRequestQuery(this) == NULL ? "" : "?",
102 httpRequestQuery(this) == NULL ? "" : strZ(httpQueryRenderP(httpRequestQuery(this))));
103
104 // Add headers
105 const StringList *headerList = httpHeaderList(httpRequestHeader(this));
106
107 for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
108 {
109 const String *headerKey = strLstGet(headerList, headerIdx);
110
111 strCatFmt(
112 requestStr, "%s:%s" CRLF_Z, strZ(headerKey),
113 strZ(httpHeaderGet(httpRequestHeader(this), headerKey)));
114 }
115
116 // Add blank line to end of headers and write the request as a buffer so secrets do not show up in logs
117 strCat(requestStr, CRLF_STR);
118 ioWrite(httpSessionIoWrite(session), BUFSTR(requestStr));
119
120 // Write out content if any
121 if (this->content != NULL)
122 ioWrite(httpSessionIoWrite(session), this->content);
123
124 // Flush all writes
125 ioWriteFlush(httpSessionIoWrite(session));
126
127 // If not waiting for the response then move the session to the object context
128 if (!waitForResponse)
129 this->session = httpSessionMove(session, this->pub.memContext);
130 }
131
132 // Wait for response
133 if (waitForResponse)
134 {
135 result = httpResponseNew(session, httpRequestVerb(this), contentCache);
136
137 // Retry when response code is 5xx. These errors generally represent a server error for a request that
138 // looks valid. There are a few errors that might be permanently fatal but they are rare and it seems best
139 // not to try and pick and choose errors in this class to retry.
140 if (httpResponseCode(result) / 100 == HTTP_RESPONSE_CODE_RETRY_CLASS)
141 THROW_FMT(ServiceError, "[%u] %s", httpResponseCode(result), strZ(httpResponseReason(result)));
142
143 // Move response to outer temp context
144 httpResponseMove(result, memContextPrior());
145 }
146 }
147 MEM_CONTEXT_TEMP_END();
148 }
149 CATCH_ANY()
150 {
151 // Sleep and then retry unless the total wait time has expired
152 if (waitMore(wait))
153 {
154 LOG_DEBUG_FMT("retry %s: %s", errorTypeName(errorType()), errorMessage());
155 retry = true;
156
157 statInc(HTTP_STAT_RETRY_STR);
158 }
159 else
160 RETHROW();
161 }
162 TRY_END();
163 }
164 while (retry);
165
166 // Move response to calling context
167 httpResponseMove(result, memContextPrior());
168 }
169 MEM_CONTEXT_TEMP_END();
170
171 FUNCTION_LOG_RETURN(HTTP_RESPONSE, result);
172 }
173
174 /**********************************************************************************************************************************/
175 HttpRequest *
httpRequestNew(HttpClient * client,const String * verb,const String * path,HttpRequestNewParam param)176 httpRequestNew(HttpClient *client, const String *verb, const String *path, HttpRequestNewParam param)
177 {
178 FUNCTION_LOG_BEGIN(logLevelDebug)
179 FUNCTION_LOG_PARAM(HTTP_CLIENT, client);
180 FUNCTION_LOG_PARAM(STRING, verb);
181 FUNCTION_LOG_PARAM(STRING, path);
182 FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
183 FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
184 FUNCTION_LOG_PARAM(BUFFER, param.content);
185 FUNCTION_LOG_END();
186
187 ASSERT(verb != NULL);
188 ASSERT(path != NULL);
189
190 HttpRequest *this = NULL;
191
192 MEM_CONTEXT_NEW_BEGIN("HttpRequest")
193 {
194 this = memNew(sizeof(HttpRequest));
195
196 *this = (HttpRequest)
197 {
198 .pub =
199 {
200 .memContext = MEM_CONTEXT_NEW(),
201 .verb = strDup(verb),
202 .path = strDup(path),
203 .query = httpQueryDupP(param.query),
204 .header = param.header == NULL ? httpHeaderNew(NULL) : httpHeaderDup(param.header, NULL),
205 },
206 .client = client,
207 .content = param.content == NULL ? NULL : bufDup(param.content),
208 };
209
210 // Send the request
211 httpRequestProcess(this, false, false);
212 statInc(HTTP_STAT_REQUEST_STR);
213 }
214 MEM_CONTEXT_NEW_END();
215
216 FUNCTION_LOG_RETURN(HTTP_REQUEST, this);
217 }
218
219 /**********************************************************************************************************************************/
220 HttpResponse *
httpRequestResponse(HttpRequest * this,bool contentCache)221 httpRequestResponse(HttpRequest *this, bool contentCache)
222 {
223 FUNCTION_LOG_BEGIN(logLevelDebug)
224 FUNCTION_LOG_PARAM(HTTP_REQUEST, this);
225 FUNCTION_LOG_PARAM(BOOL, contentCache);
226 FUNCTION_LOG_END();
227
228 ASSERT(this != NULL);
229
230 FUNCTION_LOG_RETURN(HTTP_RESPONSE, httpRequestProcess(this, true, contentCache));
231 }
232
233 /**********************************************************************************************************************************/
234 void
httpRequestError(const HttpRequest * this,HttpResponse * response)235 httpRequestError(const HttpRequest *this, HttpResponse *response)
236 {
237 FUNCTION_LOG_BEGIN(logLevelTrace)
238 FUNCTION_LOG_PARAM(HTTP_REQUEST, this);
239 FUNCTION_LOG_PARAM(HTTP_RESPONSE, response);
240 FUNCTION_LOG_END();
241
242 ASSERT(this != NULL);
243 ASSERT(response != NULL);
244
245 // Error code
246 String *error = strNewFmt("HTTP request failed with %u", httpResponseCode(response));
247
248 // Add reason when present
249 if (strSize(httpResponseReason(response)) > 0)
250 strCatFmt(error, " (%s)", strZ(httpResponseReason(response)));
251
252 // Output path/query
253 strCatZ(error, ":\n*** Path/Query ***:");
254
255 strCatFmt(error, "\n%s", strZ(httpRequestPath(this)));
256
257 if (httpRequestQuery(this) != NULL)
258 strCatFmt(error, "?%s", strZ(httpQueryRenderP(httpRequestQuery(this), .redact = true)));
259
260 // Output request headers
261 const StringList *requestHeaderList = httpHeaderList(httpRequestHeader(this));
262
263 if (!strLstEmpty(requestHeaderList))
264 {
265 strCatZ(error, "\n*** Request Headers ***:");
266
267 for (unsigned int requestHeaderIdx = 0; requestHeaderIdx < strLstSize(requestHeaderList); requestHeaderIdx++)
268 {
269 const String *key = strLstGet(requestHeaderList, requestHeaderIdx);
270
271 strCatFmt(
272 error, "\n%s: %s", strZ(key),
273 httpHeaderRedact(httpRequestHeader(this), key) ? "<redacted>" : strZ(httpHeaderGet(httpRequestHeader(this), key)));
274 }
275 }
276
277 // Output response headers
278 const HttpHeader *responseHeader = httpResponseHeader(response);
279 const StringList *responseHeaderList = httpHeaderList(responseHeader);
280
281 if (!strLstEmpty(responseHeaderList))
282 {
283 strCatZ(error, "\n*** Response Headers ***:");
284
285 for (unsigned int responseHeaderIdx = 0; responseHeaderIdx < strLstSize(responseHeaderList); responseHeaderIdx++)
286 {
287 const String *key = strLstGet(responseHeaderList, responseHeaderIdx);
288 strCatFmt(error, "\n%s: %s", strZ(key), strZ(httpHeaderGet(responseHeader, key)));
289 }
290 }
291
292 // Add response content, if any
293 if (!bufEmpty(httpResponseContent(response)))
294 {
295 strCatZ(error, "\n*** Response Content ***:\n");
296 strCat(error, strNewBuf(httpResponseContent(response)));
297 }
298
299 THROW(ProtocolError, strZ(error));
300 }
301
302 /**********************************************************************************************************************************/
303 String *
httpRequestToLog(const HttpRequest * this)304 httpRequestToLog(const HttpRequest *this)
305 {
306 return strNewFmt(
307 "{verb: %s, path: %s, query: %s, header: %s, contentSize: %zu}", strZ(httpRequestVerb(this)), strZ(httpRequestPath(this)),
308 httpRequestQuery(this) == NULL ? "null" : strZ(httpQueryToLog(httpRequestQuery(this))),
309 strZ(httpHeaderToLog(httpRequestHeader(this))), this->content == NULL ? 0 : bufUsed(this->content));
310 }
311