1 // Licensed to the .NET Foundation under one or more agreements. 2 // The .NET Foundation licenses this file to you under the MIT license. 3 // See the LICENSE file in the project root for more information. 4 5 using System.Buffers; 6 using System.Collections.Generic; 7 using System.Diagnostics; 8 using System.IO; 9 using System.Net.Http.Headers; 10 using System.Runtime.CompilerServices; 11 using System.Runtime.InteropServices; 12 using System.Threading; 13 using System.Threading.Tasks; 14 15 using CURLAUTH = Interop.Http.CURLAUTH; 16 using CURLcode = Interop.Http.CURLcode; 17 using CURLoption = Interop.Http.CURLoption; 18 using CurlProtocols = Interop.Http.CurlProtocols; 19 using CURLProxyType = Interop.Http.curl_proxytype; 20 using SafeCurlHandle = Interop.Http.SafeCurlHandle; 21 using SafeCurlSListHandle = Interop.Http.SafeCurlSListHandle; 22 using SafeCallbackHandle = Interop.Http.SafeCallbackHandle; 23 using SeekCallback = Interop.Http.SeekCallback; 24 using ReadWriteCallback = Interop.Http.ReadWriteCallback; 25 using ReadWriteFunction = Interop.Http.ReadWriteFunction; 26 using SslCtxCallback = Interop.Http.SslCtxCallback; 27 using DebugCallback = Interop.Http.DebugCallback; 28 29 namespace System.Net.Http 30 { 31 internal partial class CurlHandler : HttpMessageHandler 32 { 33 /// <summary>Provides all of the state associated with a single request/response, referred to as an "easy" request in libcurl parlance.</summary> 34 private sealed class EasyRequest : TaskCompletionSource<HttpResponseMessage> 35 { 36 /// <summary>Maximum content length where we'll use COPYPOSTFIELDS to let libcurl dup the content.</summary> 37 private const int InMemoryPostContentLimit = 32 * 1024; // arbitrary limit; could be tweaked in the future based on experimentation 38 /// <summary>Debugging flag used to enable CURLOPT_VERBOSE to dump to stderr when not redirecting it to the event source.</summary> 39 private static readonly bool s_curlDebugLogging = Environment.GetEnvironmentVariable("CURLHANDLER_DEBUG_VERBOSE") == "true"; 40 41 internal readonly CurlHandler _handler; 42 internal readonly MultiAgent _associatedMultiAgent; 43 internal readonly HttpRequestMessage _requestMessage; 44 internal readonly CurlResponseMessage _responseMessage; 45 internal readonly CancellationToken _cancellationToken; 46 internal Stream _requestContentStream; 47 internal long? _requestContentStreamStartingPosition; 48 internal bool _inMemoryPostContent; 49 50 internal SafeCurlHandle _easyHandle; 51 private SafeCurlSListHandle _requestHeaders; 52 53 internal SendTransferState _sendTransferState; 54 internal StrongToWeakReference<EasyRequest> _selfStrongToWeakReference; 55 56 private SafeCallbackHandle _callbackHandle; 57 EasyRequest(CurlHandler handler, MultiAgent agent, HttpRequestMessage requestMessage, CancellationToken cancellationToken)58 public EasyRequest(CurlHandler handler, MultiAgent agent, HttpRequestMessage requestMessage, CancellationToken cancellationToken) : 59 base(TaskCreationOptions.RunContinuationsAsynchronously) 60 { 61 Debug.Assert(handler != null, $"Expected non-null {nameof(handler)}"); 62 Debug.Assert(agent != null, $"Expected non-null {nameof(agent)}"); 63 Debug.Assert(requestMessage != null, $"Expected non-null {nameof(requestMessage)}"); 64 65 _handler = handler; 66 _associatedMultiAgent = agent; 67 _requestMessage = requestMessage; 68 69 _cancellationToken = cancellationToken; 70 _responseMessage = new CurlResponseMessage(this); 71 } 72 73 /// <summary> 74 /// Initialize the underlying libcurl support for this EasyRequest. 75 /// This is separated out of the constructor so that we can take into account 76 /// any additional configuration needed based on the request message 77 /// after the EasyRequest is configured and so that error handling 78 /// can be better handled in the caller. 79 /// </summary> InitializeCurl()80 internal void InitializeCurl() 81 { 82 // Create the underlying easy handle 83 SafeCurlHandle easyHandle = Interop.Http.EasyCreate(); 84 if (easyHandle.IsInvalid) 85 { 86 throw new OutOfMemoryException(); 87 } 88 _easyHandle = easyHandle; 89 90 EventSourceTrace("Configuring request."); 91 92 // Before setting any other options, turn on curl's debug tracing 93 // if desired. CURLOPT_VERBOSE may also be set subsequently if 94 // EventSource tracing is enabled. 95 if (s_curlDebugLogging) 96 { 97 SetCurlOption(CURLoption.CURLOPT_VERBOSE, 1L); 98 } 99 100 // Before actually configuring the handle based on the state of the request, 101 // do any necessary cleanup of the request object. 102 SanitizeRequestMessage(); 103 104 // Configure the handle 105 SetUrl(); 106 SetNetworkingOptions(); 107 SetMultithreading(); 108 SetTimeouts(); 109 SetRedirection(); 110 SetVerb(); 111 SetVersion(); 112 SetDecompressionOptions(); 113 SetProxyOptions(_requestMessage.RequestUri); 114 SetCredentialsOptions(_handler._useDefaultCredentials ? GetDefaultCredentialAndAuth() : _handler.GetCredentials(_requestMessage.RequestUri)); 115 SetCookieOption(_requestMessage.RequestUri); 116 SetRequestHeaders(); 117 SetSslOptions(); 118 119 EventSourceTrace("Done configuring request."); 120 } 121 EnsureResponseMessagePublished()122 public void EnsureResponseMessagePublished() 123 { 124 // If the response message hasn't been published yet, do any final processing of it before it is. 125 if (!Task.IsCompleted) 126 { 127 EventSourceTrace("Publishing response message"); 128 129 // On Windows, if the response was automatically decompressed, Content-Encoding and Content-Length 130 // headers are removed from the response. Do the same thing here. 131 DecompressionMethods dm = _handler.AutomaticDecompression; 132 if (dm != DecompressionMethods.None) 133 { 134 HttpContentHeaders contentHeaders = _responseMessage.Content.Headers; 135 IEnumerable<string> encodings; 136 if (contentHeaders.TryGetValues(HttpKnownHeaderNames.ContentEncoding, out encodings)) 137 { 138 foreach (string encoding in encodings) 139 { 140 if (((dm & DecompressionMethods.GZip) != 0 && string.Equals(encoding, EncodingNameGzip, StringComparison.OrdinalIgnoreCase)) || 141 ((dm & DecompressionMethods.Deflate) != 0 && string.Equals(encoding, EncodingNameDeflate, StringComparison.OrdinalIgnoreCase))) 142 { 143 contentHeaders.Remove(HttpKnownHeaderNames.ContentEncoding); 144 contentHeaders.Remove(HttpKnownHeaderNames.ContentLength); 145 break; 146 } 147 } 148 } 149 } 150 } 151 152 // Now ensure it's published. 153 bool completedTask = TrySetResult(_responseMessage); 154 Debug.Assert(completedTask || Task.IsCompletedSuccessfully, 155 "If the task was already completed, it should have been completed successfully; " + 156 "we shouldn't be completing as successful after already completing as failed."); 157 158 // If we successfully transitioned it to be completed, we also handed off lifetime ownership 159 // of the response to the owner of the task. Transition our reference on the EasyRequest 160 // to be weak instead of strong, so that we don't forcibly keep it alive. 161 if (completedTask) 162 { 163 Debug.Assert(_selfStrongToWeakReference != null, "Expected non-null wrapper"); 164 _selfStrongToWeakReference.MakeWeak(); 165 } 166 } 167 CleanupAndFailRequest(Exception error)168 public void CleanupAndFailRequest(Exception error) 169 { 170 try 171 { 172 Cleanup(); 173 } 174 catch (Exception exc) 175 { 176 // This would only happen in an aggregious case, such as a Stream failing to seek when 177 // it claims to be able to, but in case something goes very wrong, make sure we don't 178 // lose the exception information. 179 error = new AggregateException(error, exc); 180 } 181 finally 182 { 183 FailRequest(error); 184 } 185 } 186 FailRequest(Exception error)187 public void FailRequest(Exception error) 188 { 189 Debug.Assert(error != null, "Expected non-null exception"); 190 EventSourceTrace("Failing request: {0}", error); 191 192 var oce = error as OperationCanceledException; 193 if (oce != null) 194 { 195 TrySetCanceled(oce.CancellationToken); 196 } 197 else 198 { 199 if (error is InvalidOperationException || error is IOException || error is CurlException || error == null) 200 { 201 error = CreateHttpRequestException(error); 202 } 203 TrySetException(error); 204 } 205 // There's not much we can reasonably assert here about the result of TrySet*. 206 // It's possible that the task wasn't yet completed (e.g. a failure while initiating the request), 207 // it's possible that the task was already completed as success (e.g. a failure sending back the response), 208 // and it's possible that the task was already completed as failure (e.g. we handled the exception and 209 // faulted the task, but then tried to fault it again while finishing processing in the main loop). 210 211 // Make sure the exception is available on the response stream so that it's propagated 212 // from any attempts to read from the stream. 213 _responseMessage.ResponseStream.SignalComplete(error); 214 } 215 Cleanup()216 public void Cleanup() // not called Dispose because the request may still be in use after it's cleaned up 217 { 218 // Don't dispose of the ResponseMessage.ResponseStream as it may still be in use 219 // by code reading data stored in the stream. Also don't dispose of the request content 220 // stream; that'll be handled by the disposal of the request content by the HttpClient, 221 // and doing it here prevents reuse by an intermediate handler sitting between the client 222 // and this handler. 223 224 // However, if we got an original position for the request stream, we seek back to that position, 225 // for the corner case where the stream does get reused before it's disposed by the HttpClient 226 // (if the same request object is used multiple times from an intermediate handler, we'll be using 227 // ReadAsStreamAsync, which on the same request object will return the same stream object, which 228 // we've already advanced). 229 if (_requestContentStream != null && _requestContentStream.CanSeek) 230 { 231 Debug.Assert(_requestContentStreamStartingPosition.HasValue, "The stream is seekable, but we don't have a starting position?"); 232 _requestContentStream.Position = _requestContentStreamStartingPosition.GetValueOrDefault(); 233 } 234 235 // Dispose of the underlying easy handle. We're no longer processing it. 236 _easyHandle?.Dispose(); 237 238 // Dispose of the request headers if we had any. We had to keep this handle 239 // alive as long as the easy handle was using it. We didn't need to do any 240 // ref counting on the safe handle, though, as the only processing happens 241 // in Process, which ensures the handle will be rooted while libcurl is 242 // doing any processing that assumes it's valid. 243 _requestHeaders?.Dispose(); 244 245 // Dispose native callback resources 246 _callbackHandle?.Dispose(); 247 248 // Release any send transfer state, which will return its buffer to the pool 249 _sendTransferState?.Dispose(); 250 } 251 SanitizeRequestMessage()252 private void SanitizeRequestMessage() 253 { 254 // Make sure Transfer-Encoding and Content-Length make sense together. 255 if (_requestMessage.Content != null) 256 { 257 SetChunkedModeForSend(_requestMessage); 258 } 259 } 260 SetUrl()261 private void SetUrl() 262 { 263 Uri requestUri = _requestMessage.RequestUri; 264 265 long scopeId; 266 if (IsLinkLocal(requestUri, out scopeId)) 267 { 268 // Uri.AbsoluteUri doesn't include the ScopeId/ZoneID, so if it is link-local, 269 // we separately pass the scope to libcurl. 270 EventSourceTrace("ScopeId: {0}", scopeId); 271 SetCurlOption(CURLoption.CURLOPT_ADDRESS_SCOPE, scopeId); 272 } 273 274 EventSourceTrace("Url: {0}", requestUri); 275 string idnHost = requestUri.IdnHost; 276 string url = requestUri.Host == idnHost ? 277 requestUri.AbsoluteUri : 278 new UriBuilder(requestUri) { Host = idnHost }.Uri.AbsoluteUri; 279 280 SetCurlOption(CURLoption.CURLOPT_URL, url); 281 SetCurlOption(CURLoption.CURLOPT_PROTOCOLS, (long)(CurlProtocols.CURLPROTO_HTTP | CurlProtocols.CURLPROTO_HTTPS)); 282 } 283 IsLinkLocal(Uri url, out long scopeId)284 private static bool IsLinkLocal(Uri url, out long scopeId) 285 { 286 IPAddress ip; 287 if (IPAddress.TryParse(url.DnsSafeHost, out ip) && ip.IsIPv6LinkLocal) 288 { 289 scopeId = ip.ScopeId; 290 return true; 291 } 292 293 scopeId = 0; 294 return false; 295 } 296 SetNetworkingOptions()297 private void SetNetworkingOptions() 298 { 299 // Disable the TCP Nagle algorithm. It's disabled by default starting with libcurl 7.50.2, 300 // and when enabled has a measurably negative impact on latency in key scenarios 301 // (e.g. POST'ing small-ish data). 302 SetCurlOption(CURLoption.CURLOPT_TCP_NODELAY, 1L); 303 } 304 SetMultithreading()305 private void SetMultithreading() 306 { 307 SetCurlOption(CURLoption.CURLOPT_NOSIGNAL, 1L); 308 } 309 SetTimeouts()310 private void SetTimeouts() 311 { 312 // Set timeout limit on the connect phase. 313 SetCurlOption(CURLoption.CURLOPT_CONNECTTIMEOUT_MS, int.MaxValue); 314 315 // Override the default DNS cache timeout. libcurl defaults to a 1 minute 316 // timeout, but we extend that to match the Windows timeout of 10 minutes. 317 const int DnsCacheTimeoutSeconds = 10 * 60; 318 SetCurlOption(CURLoption.CURLOPT_DNS_CACHE_TIMEOUT, DnsCacheTimeoutSeconds); 319 } 320 SetRedirection()321 private void SetRedirection() 322 { 323 if (!_handler._automaticRedirection) 324 { 325 return; 326 } 327 328 SetCurlOption(CURLoption.CURLOPT_FOLLOWLOCATION, 1L); 329 330 CurlProtocols redirectProtocols = string.Equals(_requestMessage.RequestUri.Scheme, UriSchemeHttps, StringComparison.OrdinalIgnoreCase) ? 331 CurlProtocols.CURLPROTO_HTTPS : // redirect only to another https 332 CurlProtocols.CURLPROTO_HTTP | CurlProtocols.CURLPROTO_HTTPS; // redirect to http or to https 333 SetCurlOption(CURLoption.CURLOPT_REDIR_PROTOCOLS, (long)redirectProtocols); 334 335 SetCurlOption(CURLoption.CURLOPT_MAXREDIRS, _handler._maxAutomaticRedirections); 336 EventSourceTrace("Max automatic redirections: {0}", _handler._maxAutomaticRedirections); 337 } 338 339 /// <summary> 340 /// When a Location header is received along with a 3xx status code, it's an indication 341 /// that we're likely to redirect. Prepare the easy handle in case we do. 342 /// </summary> SetPossibleRedirectForLocationHeader(string location)343 internal void SetPossibleRedirectForLocationHeader(string location) 344 { 345 // Reset cookies in case we redirect. Below we'll set new cookies for the 346 // new location if we have any. 347 if (_handler._useCookies) 348 { 349 SetCurlOption(CURLoption.CURLOPT_COOKIE, IntPtr.Zero); 350 } 351 352 // Parse the location string into a relative or absolute Uri, then combine that 353 // with the current request Uri to get the new location. 354 var updatedCredentials = default(KeyValuePair<NetworkCredential, CURLAUTH>); 355 Uri newUri; 356 if (Uri.TryCreate(_requestMessage.RequestUri, location.Trim(), out newUri)) 357 { 358 // Just as with WinHttpHandler, for security reasons, we drop the server credential if it is 359 // anything other than a CredentialCache. We allow credentials in a CredentialCache since they 360 // are specifically tied to URIs. 361 updatedCredentials = _handler._useDefaultCredentials ? 362 GetDefaultCredentialAndAuth() : 363 GetCredentials(newUri, _handler.Credentials as CredentialCache, s_orderedAuthTypes); 364 365 // Reset proxy - it is possible that the proxy has different credentials for the new URI 366 SetProxyOptions(newUri); 367 368 // Set up new cookies 369 if (_handler._useCookies) 370 { 371 SetCookieOption(newUri); 372 } 373 } 374 375 // Set up the new credentials, either for the new Uri if we were able to get it, 376 // or to empty creds if we couldn't. 377 SetCredentialsOptions(updatedCredentials); 378 379 // Set the headers again. This is a workaround for libcurl's limitation in handling 380 // headers with empty values. 381 SetRequestHeaders(); 382 } 383 SetContentLength(CURLoption lengthOption)384 private void SetContentLength(CURLoption lengthOption) 385 { 386 Debug.Assert(lengthOption == CURLoption.CURLOPT_POSTFIELDSIZE || lengthOption == CURLoption.CURLOPT_INFILESIZE); 387 388 if (_requestMessage.Content == null) 389 { 390 // Tell libcurl there's no data to be sent. 391 SetCurlOption(lengthOption, 0L); 392 return; 393 } 394 395 long? contentLengthOpt = _requestMessage.Content.Headers.ContentLength; 396 if (contentLengthOpt != null) 397 { 398 long contentLength = contentLengthOpt.GetValueOrDefault(); 399 if (contentLength <= int.MaxValue) 400 { 401 // Tell libcurl how much data we expect to send. 402 SetCurlOption(lengthOption, contentLength); 403 } 404 else 405 { 406 // Similarly, tell libcurl how much data we expect to send. However, 407 // as the amount is larger than a 32-bit value, switch to the "_LARGE" 408 // equivalent libcurl options. 409 SetCurlOption( 410 lengthOption == CURLoption.CURLOPT_INFILESIZE ? CURLoption.CURLOPT_INFILESIZE_LARGE : CURLoption.CURLOPT_POSTFIELDSIZE_LARGE, 411 contentLength); 412 } 413 EventSourceTrace("Set content length: {0}", contentLength); 414 return; 415 } 416 417 // There is content but we couldn't determine its size. Don't set anything. 418 } 419 SetVerb()420 private void SetVerb() 421 { 422 EventSourceTrace<string>("Verb: {0}", _requestMessage.Method.Method); 423 424 if (_requestMessage.Method == HttpMethod.Put) 425 { 426 SetCurlOption(CURLoption.CURLOPT_UPLOAD, 1L); 427 SetContentLength(CURLoption.CURLOPT_INFILESIZE); 428 } 429 else if (_requestMessage.Method == HttpMethod.Head) 430 { 431 SetCurlOption(CURLoption.CURLOPT_NOBODY, 1L); 432 } 433 else if (_requestMessage.Method == HttpMethod.Post) 434 { 435 SetCurlOption(CURLoption.CURLOPT_POST, 1L); 436 437 // Set the content length if we have one available. We must set POSTFIELDSIZE before setting 438 // COPYPOSTFIELDS, as the setting of COPYPOSTFIELDS uses the size to know how much data to read 439 // out; if POSTFIELDSIZE is not done before, COPYPOSTFIELDS will look for a null terminator, and 440 // we don't necessarily have one. 441 SetContentLength(CURLoption.CURLOPT_POSTFIELDSIZE); 442 443 // For most content types and most HTTP methods, we use a callback that lets libcurl 444 // get data from us if/when it wants it. However, as an optimization, for POSTs that 445 // use content already known to be entirely in memory, we hand that data off to libcurl 446 // ahead of time. This not only saves on costs associated with all of the async transfer 447 // between the content and libcurl, it also lets libcurl do larger writes that can, for 448 // example, enable fewer packets to be sent on the wire. 449 var inMemContent = _requestMessage.Content as ByteArrayContent; 450 ArraySegment<byte> contentSegment; 451 if (inMemContent != null && 452 inMemContent.TryGetBuffer(out contentSegment) && 453 contentSegment.Count <= InMemoryPostContentLimit) // skip if we'd be forcing libcurl to allocate/copy a large buffer 454 { 455 // Only pre-provide the content if the content still has its ContentLength 456 // and if that length matches the segment. If it doesn't, something has been overridden, 457 // and we should rely on reading from the content stream to get the data. 458 long? contentLength = inMemContent.Headers.ContentLength; 459 if (contentLength.HasValue && contentLength.GetValueOrDefault() == contentSegment.Count) 460 { 461 _inMemoryPostContent = true; 462 463 // Debug double-check array segment; this should all have been validated by the ByteArrayContent 464 Debug.Assert(contentSegment.Array != null, "Expected non-null byte content array"); 465 Debug.Assert(contentSegment.Count >= 0, $"Expected non-negative byte content count {contentSegment.Count}"); 466 Debug.Assert(contentSegment.Offset >= 0, $"Expected non-negative byte content offset {contentSegment.Offset}"); 467 Debug.Assert(contentSegment.Array.Length - contentSegment.Offset >= contentSegment.Count, 468 $"Expected offset {contentSegment.Offset} + count {contentSegment.Count} to be within array length {contentSegment.Array.Length}"); 469 470 // Hand the data off to libcurl with COPYPOSTFIELDS for it to copy out the data. (The alternative 471 // is to use POSTFIELDS, which would mean we'd need to pin the array in the ByteArrayContent for the 472 // duration of the request. Often with a ByteArrayContent, the data will be small and the copy cheap.) 473 unsafe 474 { 475 fixed (byte* inMemContentPtr = contentSegment.Array) 476 { 477 SetCurlOption(CURLoption.CURLOPT_COPYPOSTFIELDS, new IntPtr(inMemContentPtr + contentSegment.Offset)); 478 EventSourceTrace("Set post fields rather than using send content callback"); 479 } 480 } 481 } 482 } 483 } 484 else if (_requestMessage.Method == HttpMethod.Trace) 485 { 486 SetCurlOption(CURLoption.CURLOPT_CUSTOMREQUEST, _requestMessage.Method.Method); 487 SetCurlOption(CURLoption.CURLOPT_NOBODY, 1L); 488 } 489 else 490 { 491 SetCurlOption(CURLoption.CURLOPT_CUSTOMREQUEST, _requestMessage.Method.Method); 492 if (_requestMessage.Content != null) 493 { 494 SetCurlOption(CURLoption.CURLOPT_UPLOAD, 1L); 495 SetContentLength(CURLoption.CURLOPT_INFILESIZE); 496 } 497 } 498 } 499 SetVersion()500 private void SetVersion() 501 { 502 Version v = _requestMessage.Version; 503 if (v != null) 504 { 505 // Try to use the requested version, if a known version was explicitly requested. 506 // If an unknown version was requested, we simply use libcurl's default. 507 var curlVersion = 508 (v.Major == 1 && v.Minor == 1) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_1_1 : 509 (v.Major == 1 && v.Minor == 0) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_1_0 : 510 (v.Major == 2 && v.Minor == 0) ? Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_2_0 : 511 Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_NONE; 512 513 if (curlVersion != Interop.Http.CurlHttpVersion.CURL_HTTP_VERSION_NONE) 514 { 515 // Ask libcurl to use the specified version if possible. 516 CURLcode c = Interop.Http.EasySetOptionLong(_easyHandle, CURLoption.CURLOPT_HTTP_VERSION, (long)curlVersion); 517 if (c == CURLcode.CURLE_OK) 518 { 519 // Success. The requested version will be used. 520 EventSourceTrace("HTTP version: {0}", v); 521 } 522 else if (c == CURLcode.CURLE_UNSUPPORTED_PROTOCOL) 523 { 524 // The requested version is unsupported. Fall back to using the default version chosen by libcurl. 525 EventSourceTrace("Unsupported protocol: {0}", v); 526 } 527 else 528 { 529 // Some other error. Fail. 530 ThrowIfCURLEError(c); 531 } 532 } 533 } 534 } 535 SetDecompressionOptions()536 private void SetDecompressionOptions() 537 { 538 if (!_handler.SupportsAutomaticDecompression) 539 { 540 return; 541 } 542 543 DecompressionMethods autoDecompression = _handler.AutomaticDecompression; 544 bool gzip = (autoDecompression & DecompressionMethods.GZip) != 0; 545 bool deflate = (autoDecompression & DecompressionMethods.Deflate) != 0; 546 if (gzip || deflate) 547 { 548 string encoding = (gzip && deflate) ? EncodingNameGzip + "," + EncodingNameDeflate : 549 gzip ? EncodingNameGzip : 550 EncodingNameDeflate; 551 SetCurlOption(CURLoption.CURLOPT_ACCEPT_ENCODING, encoding); 552 EventSourceTrace<string>("Encoding: {0}", encoding); 553 } 554 } 555 SetProxyOptions(Uri requestUri)556 internal void SetProxyOptions(Uri requestUri) 557 { 558 if (!_handler._useProxy) 559 { 560 // Explicitly disable the use of a proxy. This will prevent libcurl from using 561 // any proxy, including ones set via environment variable. 562 SetCurlOption(CURLoption.CURLOPT_PROXY, string.Empty); 563 EventSourceTrace("UseProxy false, disabling proxy"); 564 return; 565 } 566 567 if (_handler.Proxy == null) 568 { 569 // UseProxy was true, but Proxy was null. Let libcurl do its default handling, 570 // which includes checking the http_proxy environment variable. 571 EventSourceTrace("UseProxy true, Proxy null, using default proxy"); 572 573 // Since that proxy set in an environment variable might require a username and password, 574 // use the default proxy credentials if there are any. Currently only NetworkCredentials 575 // are used, as we can't query by the proxy Uri, since we don't know it. 576 SetProxyCredentials(_handler.DefaultProxyCredentials as NetworkCredential); 577 578 return; 579 } 580 581 // Custom proxy specified. 582 Uri proxyUri; 583 try 584 { 585 // Should we bypass a proxy for this URI? 586 if (_handler.Proxy.IsBypassed(requestUri)) 587 { 588 SetCurlOption(CURLoption.CURLOPT_PROXY, string.Empty); 589 EventSourceTrace("Proxy's IsBypassed returned true, bypassing proxy"); 590 return; 591 } 592 593 // Get the proxy Uri for this request. 594 proxyUri = _handler.Proxy.GetProxy(requestUri); 595 if (proxyUri == null) 596 { 597 EventSourceTrace("GetProxy returned null, using default."); 598 return; 599 } 600 } 601 catch (PlatformNotSupportedException) 602 { 603 // WebRequest.DefaultWebProxy throws PlatformNotSupportedException, 604 // in which case we should use the default rather than the custom proxy. 605 EventSourceTrace("PlatformNotSupportedException from proxy, using default"); 606 return; 607 } 608 609 // Configure libcurl with the gathered proxy information 610 611 // uri.AbsoluteUri/ToString() omit IPv6 scope IDs. SerializationInfoString ensures these details 612 // are included, but does not properly handle international hosts. As a workaround we check whether 613 // the host is a link-local IP address, and based on that either return the SerializationInfoString 614 // or the AbsoluteUri. (When setting the request Uri itself, we instead use CURLOPT_ADDRESS_SCOPE to 615 // set the scope id and the url separately, avoiding these issues and supporting versions of libcurl 616 // prior to v7.37 that don't support parsing scope IDs out of the url's host. As similar feature 617 // doesn't exist for proxies.) 618 IPAddress ip; 619 string proxyUrl = IPAddress.TryParse(proxyUri.DnsSafeHost, out ip) && ip.IsIPv6LinkLocal ? 620 proxyUri.GetComponents(UriComponents.SerializationInfoString, UriFormat.UriEscaped) : 621 proxyUri.AbsoluteUri; 622 623 EventSourceTrace<string>("Proxy: {0}", proxyUrl); 624 SetCurlOption(CURLoption.CURLOPT_PROXYTYPE, (long)CURLProxyType.CURLPROXY_HTTP); 625 SetCurlOption(CURLoption.CURLOPT_PROXY, proxyUrl); 626 SetCurlOption(CURLoption.CURLOPT_PROXYPORT, proxyUri.Port); 627 628 KeyValuePair<NetworkCredential, CURLAUTH> credentialScheme = GetCredentials( 629 proxyUri, _handler.Proxy.Credentials, s_orderedAuthTypes); 630 SetProxyCredentials(credentialScheme.Key); 631 } 632 SetProxyCredentials(NetworkCredential credentials)633 private void SetProxyCredentials(NetworkCredential credentials) 634 { 635 if (credentials == CredentialCache.DefaultCredentials) 636 { 637 EventSourceTrace("DefaultCredentials set for proxy. Skipping."); 638 } 639 else if (credentials != null) 640 { 641 if (string.IsNullOrEmpty(credentials.UserName)) 642 { 643 throw new ArgumentException(SR.net_http_argument_empty_string, "UserName"); 644 } 645 646 // Unlike normal credentials, proxy credentials are URL decoded by libcurl, so we URL encode 647 // them in order to allow, for example, a colon in the username. 648 string credentialText = string.IsNullOrEmpty(credentials.Domain) ? 649 WebUtility.UrlEncode(credentials.UserName) + ":" + WebUtility.UrlEncode(credentials.Password) : 650 string.Format("{2}\\{0}:{1}", WebUtility.UrlEncode(credentials.UserName), WebUtility.UrlEncode(credentials.Password), WebUtility.UrlEncode(credentials.Domain)); 651 652 EventSourceTrace("Proxy credentials set."); 653 SetCurlOption(CURLoption.CURLOPT_PROXYUSERPWD, credentialText); 654 } 655 } 656 SetCredentialsOptions(KeyValuePair<NetworkCredential, CURLAUTH> credentialSchemePair)657 internal void SetCredentialsOptions(KeyValuePair<NetworkCredential, CURLAUTH> credentialSchemePair) 658 { 659 if (credentialSchemePair.Key == null) 660 { 661 EventSourceTrace("Credentials cleared."); 662 SetCurlOption(CURLoption.CURLOPT_USERNAME, IntPtr.Zero); 663 SetCurlOption(CURLoption.CURLOPT_PASSWORD, IntPtr.Zero); 664 return; 665 } 666 667 NetworkCredential credentials = credentialSchemePair.Key; 668 CURLAUTH authScheme = credentialSchemePair.Value; 669 string userName = string.IsNullOrEmpty(credentials.Domain) ? 670 credentials.UserName : 671 credentials.Domain + "\\" + credentials.UserName; 672 673 SetCurlOption(CURLoption.CURLOPT_USERNAME, userName); 674 SetCurlOption(CURLoption.CURLOPT_HTTPAUTH, (long)authScheme); 675 if (credentials.Password != null) 676 { 677 SetCurlOption(CURLoption.CURLOPT_PASSWORD, credentials.Password); 678 } 679 680 EventSourceTrace("Credentials set."); 681 } 682 GetDefaultCredentialAndAuth()683 private static KeyValuePair<NetworkCredential, CURLAUTH> GetDefaultCredentialAndAuth() => 684 new KeyValuePair<NetworkCredential, CURLAUTH>(CredentialCache.DefaultNetworkCredentials, CURLAUTH.Negotiate); 685 SetCookieOption(Uri uri)686 internal void SetCookieOption(Uri uri) 687 { 688 if (!_handler._useCookies) 689 { 690 return; 691 } 692 693 string cookieValues = _handler.CookieContainer.GetCookieHeader(uri); 694 if (!string.IsNullOrEmpty(cookieValues)) 695 { 696 SetCurlOption(CURLoption.CURLOPT_COOKIE, cookieValues); 697 EventSourceTrace<string>("Cookies: {0}", cookieValues); 698 } 699 } 700 SetRequestHeaders()701 internal void SetRequestHeaders() 702 { 703 var slist = new SafeCurlSListHandle(); 704 705 bool suppressContentType; 706 if (_requestMessage.Content != null) 707 { 708 // Add content request headers 709 AddRequestHeaders(_requestMessage.Content.Headers, slist); 710 suppressContentType = _requestMessage.Content.Headers.ContentType == null; 711 } 712 else 713 { 714 suppressContentType = true; 715 } 716 717 if (suppressContentType) 718 { 719 // Remove the Content-Type header libcurl adds by default. 720 ThrowOOMIfFalse(Interop.Http.SListAppend(slist, NoContentType)); 721 } 722 723 // Add request headers 724 AddRequestHeaders(_requestMessage.Headers, slist); 725 726 // Since libcurl always adds a Transfer-Encoding header, we need to explicitly block 727 // it if caller specifically does not want to set the header 728 if (_requestMessage.Headers.TransferEncodingChunked.HasValue && 729 !_requestMessage.Headers.TransferEncodingChunked.Value) 730 { 731 ThrowOOMIfFalse(Interop.Http.SListAppend(slist, NoTransferEncoding)); 732 } 733 734 // Since libcurl adds an Expect header if it sees enough post data, we need to explicitly block 735 // it unless the caller has explicitly opted-in to it. 736 if (!_requestMessage.Headers.ExpectContinue.GetValueOrDefault()) 737 { 738 ThrowOOMIfFalse(Interop.Http.SListAppend(slist, NoExpect)); 739 } 740 741 if (!slist.IsInvalid) 742 { 743 SafeCurlSListHandle prevList = _requestHeaders; 744 _requestHeaders = slist; 745 SetCurlOption(CURLoption.CURLOPT_HTTPHEADER, slist); 746 prevList?.Dispose(); 747 } 748 else 749 { 750 slist.Dispose(); 751 } 752 } 753 SetSslOptions()754 private void SetSslOptions() 755 { 756 // SSL Options should be set regardless of the type of the original request, 757 // in case an http->https redirection occurs. 758 // 759 // While this does slow down the theoretical best path of the request the code 760 // to decide that we need to register the callback is more complicated than, and 761 // potentially more expensive than, just always setting the callback. 762 SslProvider.SetSslOptions(this, _handler.ClientCertificateOptions); 763 } 764 765 internal bool ServerCertificateValidationCallbackAcceptsAll => ReferenceEquals( 766 _handler.ServerCertificateCustomValidationCallback, 767 HttpClientHandler.DangerousAcceptAnyServerCertificateValidator); 768 SetCurlCallbacks( IntPtr easyGCHandle, ReadWriteCallback receiveHeadersCallback, ReadWriteCallback sendCallback, SeekCallback seekCallback, ReadWriteCallback receiveBodyCallback, DebugCallback debugCallback)769 internal void SetCurlCallbacks( 770 IntPtr easyGCHandle, 771 ReadWriteCallback receiveHeadersCallback, 772 ReadWriteCallback sendCallback, 773 SeekCallback seekCallback, 774 ReadWriteCallback receiveBodyCallback, 775 DebugCallback debugCallback) 776 { 777 if (_callbackHandle == null) 778 { 779 _callbackHandle = new SafeCallbackHandle(); 780 } 781 782 // Add callback for processing headers 783 Interop.Http.RegisterReadWriteCallback( 784 _easyHandle, 785 ReadWriteFunction.Header, 786 receiveHeadersCallback, 787 easyGCHandle, 788 ref _callbackHandle); 789 ThrowOOMIfInvalid(_callbackHandle); 790 791 // If we're sending data as part of the request and it wasn't already added as 792 // in-memory data, add callbacks for sending request data. 793 if (!_inMemoryPostContent && _requestMessage.Content != null) 794 { 795 Interop.Http.RegisterReadWriteCallback( 796 _easyHandle, 797 ReadWriteFunction.Read, 798 sendCallback, 799 easyGCHandle, 800 ref _callbackHandle); 801 Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers"); 802 803 Interop.Http.RegisterSeekCallback( 804 _easyHandle, 805 seekCallback, 806 easyGCHandle, 807 ref _callbackHandle); 808 Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers"); 809 } 810 811 // If we're expecting any data in response, add a callback for receiving body data 812 if (_requestMessage.Method != HttpMethod.Head) 813 { 814 Interop.Http.RegisterReadWriteCallback( 815 _easyHandle, 816 ReadWriteFunction.Write, 817 receiveBodyCallback, 818 easyGCHandle, 819 ref _callbackHandle); 820 Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers"); 821 } 822 823 if (NetEventSource.IsEnabled) 824 { 825 SetCurlOption(CURLoption.CURLOPT_VERBOSE, 1L); 826 CURLcode curlResult = Interop.Http.RegisterDebugCallback( 827 _easyHandle, 828 debugCallback, 829 easyGCHandle, 830 ref _callbackHandle); 831 Debug.Assert(!_callbackHandle.IsInvalid, $"Should have been allocated (or failed) when originally adding handlers"); 832 if (curlResult != CURLcode.CURLE_OK) 833 { 834 EventSourceTrace("Failed to register debug callback."); 835 } 836 } 837 } 838 SetSslCtxCallback(SslCtxCallback callback, IntPtr userPointer)839 internal CURLcode SetSslCtxCallback(SslCtxCallback callback, IntPtr userPointer) 840 { 841 if (_callbackHandle == null) 842 { 843 _callbackHandle = new SafeCallbackHandle(); 844 } 845 846 return Interop.Http.RegisterSslCtxCallback(_easyHandle, callback, userPointer, ref _callbackHandle); 847 } 848 AddRequestHeaders(HttpHeaders headers, SafeCurlSListHandle handle)849 private static void AddRequestHeaders(HttpHeaders headers, SafeCurlSListHandle handle) 850 { 851 foreach (KeyValuePair<string, IEnumerable<string>> header in headers) 852 { 853 if (string.Equals(header.Key, HttpKnownHeaderNames.ContentLength, StringComparison.OrdinalIgnoreCase)) 854 { 855 // avoid overriding libcurl's handling via INFILESIZE/POSTFIELDSIZE 856 continue; 857 } 858 859 string headerKeyAndValue; 860 string[] values = header.Value as string[]; 861 Debug.Assert(values != null, "Implementation detail, but expected Value to be a string[]"); 862 if (values != null && values.Length < 2) 863 { 864 // 0 or 1 values 865 headerKeyAndValue = values.Length == 0 || string.IsNullOrEmpty(values[0]) ? 866 header.Key + ";" : // semicolon used by libcurl to denote empty value that should be sent 867 header.Key + ": " + values[0]; 868 } 869 else 870 { 871 // Either Values wasn't a string[], or it had 2 or more items. Both are handled by GetHeaderString. 872 string headerValue = headers.GetHeaderString(header.Key); 873 headerKeyAndValue = string.IsNullOrEmpty(headerValue) ? 874 header.Key + ";" : // semicolon needed by libcurl; see above 875 header.Key + ": " + headerValue; 876 } 877 878 ThrowOOMIfFalse(Interop.Http.SListAppend(handle, headerKeyAndValue)); 879 } 880 } 881 SetCurlOption(CURLoption option, string value)882 internal void SetCurlOption(CURLoption option, string value) 883 { 884 ThrowIfCURLEError(Interop.Http.EasySetOptionString(_easyHandle, option, value)); 885 } 886 TrySetCurlOption(CURLoption option, string value)887 internal CURLcode TrySetCurlOption(CURLoption option, string value) 888 { 889 return Interop.Http.EasySetOptionString(_easyHandle, option, value); 890 } 891 SetCurlOption(CURLoption option, long value)892 internal void SetCurlOption(CURLoption option, long value) 893 { 894 ThrowIfCURLEError(Interop.Http.EasySetOptionLong(_easyHandle, option, value)); 895 } 896 SetCurlOption(CURLoption option, IntPtr value)897 internal void SetCurlOption(CURLoption option, IntPtr value) 898 { 899 ThrowIfCURLEError(Interop.Http.EasySetOptionPointer(_easyHandle, option, value)); 900 } 901 SetCurlOption(CURLoption option, SafeHandle value)902 internal void SetCurlOption(CURLoption option, SafeHandle value) 903 { 904 ThrowIfCURLEError(Interop.Http.EasySetOptionPointer(_easyHandle, option, value)); 905 } 906 ThrowOOMIfFalse(bool appendResult)907 private static void ThrowOOMIfFalse(bool appendResult) 908 { 909 if (!appendResult) 910 { 911 ThrowOOM(); 912 } 913 } 914 ThrowOOMIfInvalid(SafeHandle handle)915 private static void ThrowOOMIfInvalid(SafeHandle handle) 916 { 917 if (handle.IsInvalid) 918 { 919 ThrowOOM(); 920 } 921 } 922 ThrowOOM()923 private static void ThrowOOM() 924 { 925 throw CreateHttpRequestException(new CurlException((int)CURLcode.CURLE_OUT_OF_MEMORY, isMulti: false)); 926 } 927 928 internal sealed class SendTransferState : IDisposable 929 { 930 internal byte[] Buffer { get; private set; } 931 internal int Offset { get; set; } 932 internal int Count { get; set; } 933 internal Task<int> Task { get; private set; } 934 SendTransferState(int bufferLength)935 public SendTransferState(int bufferLength) 936 { 937 Debug.Assert(bufferLength > 0 && bufferLength <= MaxRequestBufferSize, $"Expected 0 < bufferLength <= {MaxRequestBufferSize}, got {bufferLength}"); 938 Buffer = ArrayPool<byte>.Shared.Rent(bufferLength); 939 } 940 Dispose()941 public void Dispose() 942 { 943 byte[] b = Buffer; 944 if (b != null) 945 { 946 Buffer = null; 947 ArrayPool<byte>.Shared.Return(b); 948 } 949 } 950 SetTaskOffsetCount(Task<int> task, int offset, int count)951 public void SetTaskOffsetCount(Task<int> task, int offset, int count) 952 { 953 Debug.Assert(offset >= 0, "Offset should never be negative"); 954 Debug.Assert(count >= 0, "Count should never be negative"); 955 Debug.Assert(offset <= count, "Offset should never be greater than count"); 956 957 Task = task; 958 Offset = offset; 959 Count = count; 960 } 961 } 962 StoreLastEffectiveUri()963 internal void StoreLastEffectiveUri() 964 { 965 IntPtr urlCharPtr; // do not free; will point to libcurl private memory 966 CURLcode urlResult = Interop.Http.EasyGetInfoPointer(_easyHandle, Interop.Http.CURLINFO.CURLINFO_EFFECTIVE_URL, out urlCharPtr); 967 if (urlResult == CURLcode.CURLE_OK && urlCharPtr != IntPtr.Zero) 968 { 969 string url = Marshal.PtrToStringAnsi(urlCharPtr); 970 if (url != _requestMessage.RequestUri.OriginalString) 971 { 972 Uri finalUri; 973 if (Uri.TryCreate(url, UriKind.Absolute, out finalUri)) 974 { 975 _requestMessage.RequestUri = finalUri; 976 } 977 } 978 return; 979 } 980 981 Debug.Fail("Expected to be able to get the last effective Uri from libcurl"); 982 } 983 EventSourceTrace(string formatMessage, TArg0 arg0, [CallerMemberName] string memberName = null)984 private void EventSourceTrace<TArg0>(string formatMessage, TArg0 arg0, [CallerMemberName] string memberName = null) 985 { 986 CurlHandler.EventSourceTrace(formatMessage, arg0, easy: this, memberName: memberName); 987 } 988 EventSourceTrace(string message, [CallerMemberName] string memberName = null)989 private void EventSourceTrace(string message, [CallerMemberName] string memberName = null) 990 { 991 CurlHandler.EventSourceTrace(message, easy: this, memberName: memberName); 992 } 993 } 994 } 995 } 996