1<?php 2/** 3 * Matomo - free/libre analytics platform 4 * 5 * @link https://matomo.org 6 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later 7 * 8 */ 9namespace Piwik; 10 11use Composer\CaBundle\CaBundle; 12use Exception; 13use Piwik\Container\StaticContainer; 14 15/** 16 * Contains HTTP client related helper methods that can retrieve content from remote servers 17 * and optionally save to a local file. 18 * 19 * Used to check for the latest Piwik version and download updates. 20 * 21 */ 22class Http 23{ 24 /** 25 * Returns the "best" available transport method for {@link sendHttpRequest()} calls. 26 * 27 * @return string|null Either curl, fopen, socket or null if no method is supported. 28 * @api 29 */ 30 public static function getTransportMethod() 31 { 32 $method = 'curl'; 33 if (!self::isCurlEnabled()) { 34 $method = 'fopen'; 35 if (@ini_get('allow_url_fopen') != '1') { 36 $method = 'socket'; 37 if (!self::isSocketEnabled()) { 38 return null; 39 } 40 } 41 } 42 return $method; 43 } 44 45 protected static function isSocketEnabled() 46 { 47 return function_exists('fsockopen'); 48 } 49 50 protected static function isCurlEnabled() 51 { 52 return function_exists('curl_init') && function_exists('curl_exec'); 53 } 54 55 /** 56 * Sends an HTTP request using best available transport method. 57 * 58 * @param string $aUrl The target URL. 59 * @param int $timeout The number of seconds to wait before aborting the HTTP request. 60 * @param string|null $userAgent The user agent to use. 61 * @param string|null $destinationPath If supplied, the HTTP response will be saved to the file specified by 62 * this path. 63 * @param int|null $followDepth Internal redirect count. Should always pass `null` for this parameter. 64 * @param bool $acceptLanguage The value to use for the `'Accept-Language'` HTTP request header. 65 * @param array|bool $byteRange For `Range:` header. Should be two element array of bytes, eg, `array(0, 1024)` 66 * Doesn't work w/ `fopen` transport method. 67 * @param bool $getExtendedInfo If true returns the status code, headers & response, if false just the response. 68 * @param string $httpMethod The HTTP method to use. Defaults to `'GET'`. 69 * @param string $httpUsername HTTP Auth username 70 * @param string $httpPassword HTTP Auth password 71 * @param bool $checkHostIsAllowed whether we should check if the target host is allowed or not. This should only 72 * be set to false when using a hardcoded URL. 73 * 74 * @throws Exception if the response cannot be saved to `$destinationPath`, if the HTTP response cannot be sent, 75 * if there are more than 5 redirects or if the request times out. 76 * @return bool|string If `$destinationPath` is not specified the HTTP response is returned on success. `false` 77 * is returned on failure. 78 * If `$getExtendedInfo` is `true` and `$destinationPath` is not specified an array with 79 * the following information is returned on success: 80 * 81 * - **status**: the HTTP status code 82 * - **headers**: the HTTP headers 83 * - **data**: the HTTP response data 84 * 85 * `false` is still returned on failure. 86 * @api 87 */ 88 public static function sendHttpRequest($aUrl, 89 $timeout, 90 $userAgent = null, 91 $destinationPath = null, 92 $followDepth = 0, 93 $acceptLanguage = false, 94 $byteRange = false, 95 $getExtendedInfo = false, 96 $httpMethod = 'GET', 97 $httpUsername = null, 98 $httpPassword = null, 99 $checkHostIsAllowed = true) 100 { 101 // create output file 102 $file = self::ensureDestinationDirectoryExists($destinationPath); 103 104 $acceptLanguage = $acceptLanguage ? 'Accept-Language: ' . $acceptLanguage : ''; 105 return self::sendHttpRequestBy(self::getTransportMethod(), $aUrl, $timeout, $userAgent, $destinationPath, $file, 106 $followDepth, $acceptLanguage, $acceptInvalidSslCertificate = false, $byteRange, $getExtendedInfo, $httpMethod, 107 $httpUsername, $httpPassword, null, [], null, $checkHostIsAllowed); 108 } 109 110 public static function ensureDestinationDirectoryExists($destinationPath) 111 { 112 if ($destinationPath) { 113 Filesystem::mkdir(dirname($destinationPath)); 114 if (($file = @fopen($destinationPath, 'wb')) === false || !is_resource($file)) { 115 throw new Exception('Error while creating the file: ' . $destinationPath); 116 } 117 118 return $file; 119 } 120 121 return null; 122 } 123 124 private static function convertWildcardToPattern($wildcardHost) 125 { 126 $flexibleStart = $flexibleEnd = false; 127 if (strpos($wildcardHost, '*.') === 0) { 128 $flexibleStart = true; 129 $wildcardHost = substr($wildcardHost, 2); 130 } 131 if (Common::stringEndsWith($wildcardHost, '.*')) { 132 $flexibleEnd = true; 133 $wildcardHost = substr($wildcardHost, 0, -2); 134 } 135 $pattern = preg_quote($wildcardHost); 136 137 if ($flexibleStart) { 138 $pattern = '.*\.' . $pattern; 139 } 140 141 if ($flexibleEnd) { 142 $pattern .= '\..*'; 143 } 144 145 return '/^' . $pattern . '$/i'; 146 } 147 148 /** 149 * Sends an HTTP request using the specified transport method. 150 * 151 * @param string $method 152 * @param string $aUrl 153 * @param int $timeout in seconds 154 * @param string $userAgent 155 * @param string $destinationPath 156 * @param resource $file 157 * @param int $followDepth 158 * @param bool|string $acceptLanguage Accept-language header 159 * @param bool $acceptInvalidSslCertificate Only used with $method == 'curl'. If set to true (NOT recommended!) the SSL certificate will not be checked 160 * @param array|bool $byteRange For Range: header. Should be two element array of bytes, eg, array(0, 1024) 161 * Doesn't work w/ fopen method. 162 * @param bool $getExtendedInfo True to return status code, headers & response, false if just response. 163 * @param string $httpMethod The HTTP method to use. Defaults to `'GET'`. 164 * @param string $httpUsername HTTP Auth username 165 * @param string $httpPassword HTTP Auth password 166 * @param array|string $requestBody If $httpMethod is 'POST' this may accept an array of variables or a string that needs to be posted 167 * @param array $additionalHeaders List of additional headers to set for the request 168 * @param bool $checkHostIsAllowed whether we should check if the target host is allowed or not. This should only 169 * be set to false when using a hardcoded URL. 170 * 171 * @return string|array true (or string/array) on success; false on HTTP response error code (1xx or 4xx) 172 *@throws Exception 173 */ 174 public static function sendHttpRequestBy( 175 $method, 176 $aUrl, 177 $timeout, 178 $userAgent = null, 179 $destinationPath = null, 180 $file = null, 181 $followDepth = 0, 182 $acceptLanguage = false, 183 $acceptInvalidSslCertificate = false, 184 $byteRange = false, 185 $getExtendedInfo = false, 186 $httpMethod = 'GET', 187 $httpUsername = null, 188 $httpPassword = null, 189 $requestBody = null, 190 $additionalHeaders = array(), 191 $forcePost = null, 192 $checkHostIsAllowed = true 193 ) { 194 if ($followDepth > 5) { 195 throw new Exception('Too many redirects (' . $followDepth . ')'); 196 } 197 198 $aUrl = preg_replace('/[\x00-\x1F\x7F]/', '', trim($aUrl)); 199 $parsedUrl = @parse_url($aUrl); 200 201 if (empty($parsedUrl['scheme'])) { 202 throw new Exception('Missing scheme in given url'); 203 } 204 205 $allowedProtocols = Config::getInstance()->General['allowed_outgoing_protocols']; 206 $isAllowed = false; 207 208 foreach (explode(',', $allowedProtocols) as $protocol) { 209 if (strtolower($parsedUrl['scheme']) === strtolower(trim($protocol))) { 210 $isAllowed = true; 211 break; 212 } 213 } 214 215 if (!$isAllowed) { 216 throw new Exception(sprintf( 217 'Protocol %s not in list of allowed protocols: %s', 218 $parsedUrl['scheme'], 219 $allowedProtocols 220 )); 221 } 222 223 if ($checkHostIsAllowed) { 224 $disallowedHosts = StaticContainer::get('http.blocklist.hosts'); 225 226 $isBlocked = false; 227 228 foreach ($disallowedHosts as $host) { 229 if (preg_match(self::convertWildcardToPattern($host), $parsedUrl['host']) === 1) { 230 $isBlocked = true; 231 break; 232 } 233 } 234 235 if ($isBlocked) { 236 throw new Exception(sprintf( 237 'Hostname %s is in list of disallowed hosts', 238 $parsedUrl['host'] 239 )); 240 } 241 } 242 243 $contentLength = 0; 244 $fileLength = 0; 245 246 if ( !empty($requestBody ) && is_array($requestBody )) { 247 $requestBodyQuery = self::buildQuery($requestBody ); 248 } else { 249 $requestBodyQuery = $requestBody; 250 } 251 252 // Piwik services behave like a proxy, so we should act like one. 253 $xff = 'X-Forwarded-For: ' 254 . (isset($_SERVER['HTTP_X_FORWARDED_FOR']) && !empty($_SERVER['HTTP_X_FORWARDED_FOR']) ? $_SERVER['HTTP_X_FORWARDED_FOR'] . ',' : '') 255 . IP::getIpFromHeader(); 256 257 if (empty($userAgent)) { 258 $userAgent = self::getUserAgent(); 259 } 260 261 $via = 'Via: ' 262 . (isset($_SERVER['HTTP_VIA']) && !empty($_SERVER['HTTP_VIA']) ? $_SERVER['HTTP_VIA'] . ', ' : '') 263 . Version::VERSION . ' ' 264 . ($userAgent ? " ($userAgent)" : ''); 265 266 // range header 267 $rangeBytes = ''; 268 $rangeHeader = ''; 269 if (!empty($byteRange)) { 270 $rangeBytes = $byteRange[0] . '-' . $byteRange[1]; 271 $rangeHeader = 'Range: bytes=' . $rangeBytes . "\r\n"; 272 } 273 274 [$proxyHost, $proxyPort, $proxyUser, $proxyPassword] = self::getProxyConfiguration($aUrl); 275 276 // other result data 277 $status = null; 278 $headers = array(); 279 $response = null; 280 281 $httpAuthIsUsed = !empty($httpUsername) || !empty($httpPassword); 282 283 $httpAuth = ''; 284 if ($httpAuthIsUsed) { 285 $httpAuth = 'Authorization: Basic ' . base64_encode($httpUsername.':'.$httpPassword) . "\r\n"; 286 } 287 288 $httpEventParams = array( 289 'httpMethod' => $httpMethod, 290 'body' => $requestBody, 291 'userAgent' => $userAgent, 292 'timeout' => $timeout, 293 'headers' => array_map('trim', array_filter(array_merge(array( 294 $rangeHeader, $via, $xff, $httpAuth, $acceptLanguage 295 ), $additionalHeaders))), 296 'verifySsl' => !$acceptInvalidSslCertificate, 297 'destinationPath' => $destinationPath 298 ); 299 300 /** 301 * Triggered to send an HTTP request. Allows plugins to resolve the HTTP request themselves or to find out 302 * when an HTTP request is triggered to log this information for example to a monitoring tool. 303 * 304 * @param string $url The URL that needs to be requested 305 * @param array $params HTTP params like 306 * - 'httpMethod' (eg GET, POST, ...), 307 * - 'body' the request body if the HTTP method needs to be posted 308 * - 'userAgent' 309 * - 'timeout' After how many seconds a request should time out 310 * - 'headers' An array of header strings like array('Accept-Language: en', '...') 311 * - 'verifySsl' A boolean whether SSL certificate should be verified 312 * - 'destinationPath' If set, the response of the HTTP request should be saved to this file 313 * @param string &$response A plugin listening to this event should assign the HTTP response it received to this variable, for example "{value: true}" 314 * @param string &$status A plugin listening to this event should assign the HTTP status code it received to this variable, for example "200" 315 * @param array &$headers A plugin listening to this event should assign the HTTP headers it received to this variable, eg array('Content-Length' => '5') 316 */ 317 Piwik::postEvent('Http.sendHttpRequest', array($aUrl, $httpEventParams, &$response, &$status, &$headers)); 318 319 if ($response !== null || $status !== null || !empty($headers)) { 320 // was handled by event above... 321 /** 322 * described below 323 * @ignore 324 */ 325 Piwik::postEvent('Http.sendHttpRequest.end', array($aUrl, $httpEventParams, &$response, &$status, &$headers)); 326 327 if ($destinationPath && file_exists($destinationPath)) { 328 return true; 329 } 330 if ($getExtendedInfo) { 331 return array( 332 'status' => $status, 333 'headers' => $headers, 334 'data' => $response 335 ); 336 } else { 337 return trim($response); 338 } 339 } 340 341 if ($method == 'socket') { 342 if (!self::isSocketEnabled()) { 343 // can be triggered in tests 344 throw new Exception("HTTP socket support is not enabled (php function fsockopen is not available) "); 345 } 346 // initialization 347 $url = @parse_url($aUrl); 348 if ($url === false || !isset($url['scheme'])) { 349 throw new Exception('Malformed URL: ' . $aUrl); 350 } 351 352 if ($url['scheme'] != 'http' && $url['scheme'] != 'https') { 353 throw new Exception('Invalid protocol/scheme: ' . $url['scheme']); 354 } 355 $host = $url['host']; 356 $port = isset($url['port']) ? $url['port'] : ('https' == $url['scheme'] ? 443 : 80); 357 $path = isset($url['path']) ? $url['path'] : '/'; 358 if (isset($url['query'])) { 359 $path .= '?' . $url['query']; 360 } 361 $errno = null; 362 $errstr = null; 363 364 if ((!empty($proxyHost) && !empty($proxyPort)) 365 || !empty($byteRange) 366 ) { 367 $httpVer = '1.1'; 368 } else { 369 $httpVer = '1.0'; 370 } 371 372 $proxyAuth = null; 373 if (!empty($proxyHost) && !empty($proxyPort)) { 374 $connectHost = $proxyHost; 375 $connectPort = $proxyPort; 376 if (!empty($proxyUser) && !empty($proxyPassword)) { 377 $proxyAuth = 'Proxy-Authorization: Basic ' . base64_encode("$proxyUser:$proxyPassword") . "\r\n"; 378 } 379 $requestHeader = "$httpMethod $aUrl HTTP/$httpVer\r\n"; 380 } else { 381 $connectHost = $host; 382 $connectPort = $port; 383 $requestHeader = "$httpMethod $path HTTP/$httpVer\r\n"; 384 385 if ('https' == $url['scheme']) { 386 $connectHost = 'ssl://' . $connectHost; 387 } 388 } 389 390 // connection attempt 391 if (($fsock = @fsockopen($connectHost, $connectPort, $errno, $errstr, $timeout)) === false || !is_resource($fsock)) { 392 if (is_resource($file)) { 393 @fclose($file); 394 } 395 throw new Exception("Error while connecting to: $host. Please try again later. $errstr"); 396 } 397 398 // send HTTP request header 399 $requestHeader .= 400 "Host: $host" . ($port != 80 && ('https' == $url['scheme'] && $port != 443) ? ':' . $port : '') . "\r\n" 401 . ($httpAuth ? $httpAuth : '') 402 . ($proxyAuth ? $proxyAuth : '') 403 . 'User-Agent: ' . $userAgent . "\r\n" 404 . ($acceptLanguage ? $acceptLanguage . "\r\n" : '') 405 . $xff . "\r\n" 406 . $via . "\r\n" 407 . $rangeHeader 408 . (!empty($additionalHeaders) ? implode("\r\n", $additionalHeaders) . "\r\n" : '') 409 . "Connection: close\r\n"; 410 fwrite($fsock, $requestHeader); 411 412 if (strtolower($httpMethod) === 'post' && !empty($requestBodyQuery )) { 413 fwrite($fsock, self::buildHeadersForPost($requestBodyQuery )); 414 fwrite($fsock, "\r\n"); 415 fwrite($fsock, $requestBodyQuery ); 416 } else { 417 fwrite($fsock, "\r\n"); 418 } 419 420 $streamMetaData = array('timed_out' => false); 421 @stream_set_blocking($fsock, true); 422 423 if (function_exists('stream_set_timeout')) { 424 @stream_set_timeout($fsock, $timeout); 425 } elseif (function_exists('socket_set_timeout')) { 426 @socket_set_timeout($fsock, $timeout); 427 } 428 429 // process header 430 $status = null; 431 432 while (!feof($fsock)) { 433 $line = fgets($fsock, 4096); 434 435 $streamMetaData = @stream_get_meta_data($fsock); 436 if ($streamMetaData['timed_out']) { 437 if (is_resource($file)) { 438 @fclose($file); 439 } 440 @fclose($fsock); 441 throw new Exception('Timed out waiting for server response'); 442 } 443 444 // a blank line marks the end of the server response header 445 if (rtrim($line, "\r\n") == '') { 446 break; 447 } 448 449 // parse first line of server response header 450 if (!$status) { 451 // expect first line to be HTTP response status line, e.g., HTTP/1.1 200 OK 452 if (!preg_match('~^HTTP/(\d\.\d)\s+(\d+)(\s*.*)?~', $line, $m)) { 453 if (is_resource($file)) { 454 @fclose($file); 455 } 456 @fclose($fsock); 457 throw new Exception('Expected server response code. Got ' . rtrim($line, "\r\n")); 458 } 459 460 $status = (integer)$m[2]; 461 462 // Informational 1xx or Client Error 4xx 463 if ($status < 200 || $status >= 400) { 464 if (is_resource($file)) { 465 @fclose($file); 466 } 467 @fclose($fsock); 468 469 if (!$getExtendedInfo) { 470 return false; 471 } else { 472 return array('status' => $status); 473 } 474 } 475 476 continue; 477 } 478 479 // handle redirect 480 if (preg_match('/^Location:\s*(.+)/', rtrim($line, "\r\n"), $m)) { 481 if (is_resource($file)) { 482 @fclose($file); 483 } 484 @fclose($fsock); 485 // Successful 2xx vs Redirect 3xx 486 if ($status < 300) { 487 throw new Exception('Unexpected redirect to Location: ' . rtrim($line) . ' for status code ' . $status); 488 } 489 return self::sendHttpRequestBy( 490 $method, 491 trim($m[1]), 492 $timeout, 493 $userAgent, 494 $destinationPath, 495 $file, 496 $followDepth + 1, 497 $acceptLanguage, 498 $acceptInvalidSslCertificate = false, 499 $byteRange, 500 $getExtendedInfo, 501 $httpMethod, 502 $httpUsername, 503 $httpPassword, 504 $requestBodyQuery, 505 $additionalHeaders 506 ); 507 } 508 509 // save expected content length for later verification 510 if (preg_match('/^Content-Length:\s*(\d+)/', $line, $m)) { 511 $contentLength = (integer)$m[1]; 512 } 513 514 self::parseHeaderLine($headers, $line); 515 } 516 517 if (feof($fsock) 518 && $httpMethod != 'HEAD' 519 ) { 520 throw new Exception('Unexpected end of transmission'); 521 } 522 523 // process content/body 524 $response = ''; 525 526 while (!feof($fsock)) { 527 $line = fread($fsock, 8192); 528 529 $streamMetaData = @stream_get_meta_data($fsock); 530 if ($streamMetaData['timed_out']) { 531 if (is_resource($file)) { 532 @fclose($file); 533 } 534 @fclose($fsock); 535 throw new Exception('Timed out waiting for server response'); 536 } 537 538 $fileLength += strlen($line); 539 540 if (is_resource($file)) { 541 // save to file 542 fwrite($file, $line); 543 } else { 544 // concatenate to response string 545 $response .= $line; 546 } 547 } 548 549 // determine success or failure 550 @fclose(@$fsock); 551 } elseif ($method == 'fopen') { 552 $response = false; 553 554 // we make sure the request takes less than a few seconds to fail 555 // we create a stream_context (works in php >= 5.2.1) 556 // we also set the socket_timeout (for php < 5.2.1) 557 $default_socket_timeout = @ini_get('default_socket_timeout'); 558 @ini_set('default_socket_timeout', $timeout); 559 560 $ctx = null; 561 if (function_exists('stream_context_create')) { 562 $stream_options = array( 563 'http' => array( 564 'header' => 'User-Agent: ' . $userAgent . "\r\n" 565 . ($httpAuth ? $httpAuth : '') 566 . ($acceptLanguage ? $acceptLanguage . "\r\n" : '') 567 . $xff . "\r\n" 568 . $via . "\r\n" 569 . (!empty($additionalHeaders) ? implode("\r\n", $additionalHeaders) . "\r\n" : '') 570 . $rangeHeader, 571 'max_redirects' => 5, // PHP 5.1.0 572 'timeout' => $timeout, // PHP 5.2.1 573 ) 574 ); 575 576 if (!empty($proxyHost) && !empty($proxyPort)) { 577 $stream_options['http']['proxy'] = 'tcp://' . $proxyHost . ':' . $proxyPort; 578 $stream_options['http']['request_fulluri'] = true; // required by squid proxy 579 if (!empty($proxyUser) && !empty($proxyPassword)) { 580 $stream_options['http']['header'] .= 'Proxy-Authorization: Basic ' . base64_encode("$proxyUser:$proxyPassword") . "\r\n"; 581 } 582 } 583 584 if (strtolower($httpMethod) === 'post' && !empty($requestBodyQuery )) { 585 $postHeader = self::buildHeadersForPost($requestBodyQuery ); 586 $postHeader .= "\r\n"; 587 $stream_options['http']['method'] = 'POST'; 588 $stream_options['http']['header'] .= $postHeader; 589 $stream_options['http']['content'] = $requestBodyQuery; 590 } 591 592 $ctx = stream_context_create($stream_options); 593 } 594 595 // save to file 596 if (is_resource($file)) { 597 if (!($handle = fopen($aUrl, 'rb', false, $ctx))) { 598 throw new Exception("Unable to open $aUrl"); 599 } 600 while (!feof($handle)) { 601 $response = fread($handle, 8192); 602 $fileLength += strlen($response); 603 fwrite($file, $response); 604 } 605 fclose($handle); 606 } else { 607 $response = @file_get_contents($aUrl, 0, $ctx); 608 609 // try to get http status code from response headers 610 if (isset($http_response_header) && preg_match('~^HTTP/(\d\.\d)\s+(\d+)(\s*.*)?~', implode("\n", $http_response_header), $m)) { 611 $status = (int)$m[2]; 612 } 613 614 if (!$status && $response === false) { 615 $error = ErrorHandler::getLastError(); 616 throw new \Exception($error); 617 } 618 $fileLength = strlen($response); 619 } 620 621 foreach ($http_response_header as $line) { 622 self::parseHeaderLine($headers, $line); 623 } 624 625 // restore the socket_timeout value 626 if (!empty($default_socket_timeout)) { 627 @ini_set('default_socket_timeout', $default_socket_timeout); 628 } 629 } elseif ($method == 'curl') { 630 if (!self::isCurlEnabled()) { 631 // can be triggered in tests 632 throw new Exception("CURL is not enabled in php.ini, but is being used."); 633 } 634 $ch = @curl_init(); 635 636 if (!empty($proxyHost) && !empty($proxyPort)) { 637 @curl_setopt($ch, CURLOPT_PROXY, $proxyHost . ':' . $proxyPort); 638 if (!empty($proxyUser) && !empty($proxyPassword)) { 639 // PROXYAUTH defaults to BASIC 640 @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyUser . ':' . $proxyPassword); 641 } 642 } 643 644 $curl_options = array( 645 // internal to ext/curl 646 CURLOPT_BINARYTRANSFER => is_resource($file), 647 648 // curl options (sorted oldest to newest) 649 CURLOPT_URL => $aUrl, 650 CURLOPT_USERAGENT => $userAgent, 651 CURLOPT_HTTPHEADER => array_merge(array( 652 $xff, 653 $via, 654 $acceptLanguage 655 ), $additionalHeaders), 656 // only get header info if not saving directly to file 657 CURLOPT_HEADER => is_resource($file) ? false : true, 658 CURLOPT_CONNECTTIMEOUT => $timeout, 659 CURLOPT_TIMEOUT => $timeout, 660 ); 661 662 if ($rangeBytes) { 663 curl_setopt($ch, CURLOPT_RANGE, $rangeBytes); 664 } else { 665 // see https://github.com/matomo-org/matomo/pull/17009 for more info 666 // NOTE: we only do this when CURLOPT_RANGE is not being used, because when using both the 667 // response is empty. 668 $curl_options[CURLOPT_ENCODING] = ""; 669 } 670 671 // Case core:archive command is triggering archiving on https:// and the certificate is not valid 672 if ($acceptInvalidSslCertificate) { 673 $curl_options += array( 674 CURLOPT_SSL_VERIFYHOST => false, 675 CURLOPT_SSL_VERIFYPEER => false, 676 ); 677 } 678 @curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $httpMethod); 679 if ($httpMethod == 'HEAD') { 680 @curl_setopt($ch, CURLOPT_NOBODY, true); 681 } 682 683 if (strtolower($httpMethod) === 'post' && !empty($requestBodyQuery )) { 684 curl_setopt($ch, CURLOPT_POST, 1); 685 curl_setopt($ch, CURLOPT_POSTFIELDS, $requestBodyQuery ); 686 } 687 688 if (!empty($httpUsername) && !empty($httpPassword)) { 689 $curl_options += array( 690 CURLOPT_USERPWD => $httpUsername . ':' . $httpPassword, 691 ); 692 } 693 694 @curl_setopt_array($ch, $curl_options); 695 self::configCurlCertificate($ch); 696 697 /* 698 * as of php 5.2.0, CURLOPT_FOLLOWLOCATION can't be set if 699 * in safe_mode or open_basedir is set 700 */ 701 if ((string)ini_get('safe_mode') == '' && ini_get('open_basedir') == '') { 702 $protocols = 0; 703 704 foreach (explode(',', $allowedProtocols) as $protocol) { 705 if (defined('CURLPROTO_' . strtoupper(trim($protocol)))) { 706 $protocols |= constant('CURLPROTO_' . strtoupper(trim($protocol))); 707 } 708 } 709 710 $curl_options = array( 711 // curl options (sorted oldest to newest) 712 CURLOPT_FOLLOWLOCATION => true, 713 CURLOPT_REDIR_PROTOCOLS => $protocols, 714 CURLOPT_MAXREDIRS => 5, 715 ); 716 if ($forcePost) { 717 $curl_options[CURLOPT_POSTREDIR] = CURL_REDIR_POST_ALL; 718 } 719 @curl_setopt_array($ch, $curl_options); 720 } 721 722 if (is_resource($file)) { 723 // write output directly to file 724 @curl_setopt($ch, CURLOPT_FILE, $file); 725 } else { 726 // internal to ext/curl 727 @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); 728 } 729 730 ob_start(); 731 $response = @curl_exec($ch); 732 ob_end_clean(); 733 734 if ($response === true) { 735 $response = ''; 736 } elseif ($response === false) { 737 $errstr = curl_error($ch); 738 if ($errstr != '') { 739 throw new Exception('curl_exec: ' . $errstr 740 . '. Hostname requested was: ' . UrlHelper::getHostFromUrl($aUrl)); 741 } 742 $response = ''; 743 } else { 744 $header = ''; 745 // redirects are included in the output html, so we look for the last line that starts w/ HTTP/... 746 // to split the response 747 while (substr($response, 0, 5) == "HTTP/") { 748 $split = explode("\r\n\r\n", $response, 2); 749 750 if(count($split) == 2) { 751 [$header, $response] = $split; 752 } else { 753 $response = ''; 754 $header = $split; 755 } 756 } 757 758 foreach (explode("\r\n", $header) as $line) { 759 self::parseHeaderLine($headers, $line); 760 } 761 } 762 763 $contentLength = @curl_getinfo($ch, CURLINFO_CONTENT_LENGTH_DOWNLOAD); 764 $fileLength = is_resource($file) ? @curl_getinfo($ch, CURLINFO_SIZE_DOWNLOAD) : strlen($response); 765 $status = @curl_getinfo($ch, CURLINFO_HTTP_CODE); 766 767 @curl_close($ch); 768 unset($ch); 769 } else { 770 throw new Exception('Invalid request method: ' . $method); 771 } 772 773 if (is_resource($file)) { 774 fflush($file); 775 @fclose($file); 776 777 $fileSize = filesize($destinationPath); 778 if ($contentLength > 0 779 && $fileSize != $contentLength 780 ) { 781 throw new Exception('File size error: ' . $destinationPath . '; expected ' . $contentLength . ' bytes; received ' . $fileLength . ' bytes; saved ' . $fileSize . ' bytes to file'); 782 } 783 return true; 784 } 785 786 /** 787 * Triggered when an HTTP request finished. A plugin can for example listen to this and alter the response, 788 * status code, or finish a timer in case the plugin is measuring how long it took to execute the request 789 * 790 * @param string $url The URL that needs to be requested 791 * @param array $params HTTP params like 792 * - 'httpMethod' (eg GET, POST, ...), 793 * - 'body' the request body if the HTTP method needs to be posted 794 * - 'userAgent' 795 * - 'timeout' After how many seconds a request should time out 796 * - 'headers' An array of header strings like array('Accept-Language: en', '...') 797 * - 'verifySsl' A boolean whether SSL certificate should be verified 798 * - 'destinationPath' If set, the response of the HTTP request should be saved to this file 799 * @param string &$response The response of the HTTP request, for example "{value: true}" 800 * @param string &$status The returned HTTP status code, for example "200" 801 * @param array &$headers The returned headers, eg array('Content-Length' => '5') 802 */ 803 Piwik::postEvent('Http.sendHttpRequest.end', array($aUrl, $httpEventParams, &$response, &$status, &$headers)); 804 805 if (!$getExtendedInfo) { 806 return trim($response); 807 } else { 808 return array( 809 'status' => $status, 810 'headers' => $headers, 811 'data' => $response 812 ); 813 } 814 } 815 816 public static function buildQuery($params) 817 { 818 return http_build_query($params, '', '&'); 819 } 820 821 private static function buildHeadersForPost($requestBody) 822 { 823 $postHeader = "Content-Type: application/x-www-form-urlencoded\r\n"; 824 $postHeader .= "Content-Length: " . strlen($requestBody) . "\r\n"; 825 826 return $postHeader; 827 } 828 829 /** 830 * Downloads the next chunk of a specific file. The next chunk's byte range 831 * is determined by the existing file's size and the expected file size, which 832 * is stored in the option table before starting a download. The expected 833 * file size is obtained through a `HEAD` HTTP request. 834 * 835 * _Note: this function uses the **Range** HTTP header to accomplish downloading in 836 * parts. Not every server supports this header._ 837 * 838 * The proper use of this function is to call it once per request. The browser 839 * should continue to send requests to Piwik which will in turn call this method 840 * until the file has completely downloaded. In this way, the user can be informed 841 * of a download's progress. 842 * 843 * **Example Usage** 844 * 845 * ``` 846 * // browser JavaScript 847 * var downloadFile = function (isStart) { 848 * var ajax = new ajaxHelper(); 849 * ajax.addParams({ 850 * module: 'MyPlugin', 851 * action: 'myAction', 852 * isStart: isStart ? 1 : 0 853 * }, 'post'); 854 * ajax.setCallback(function (response) { 855 * var progress = response.progress 856 * // ...update progress... 857 * 858 * downloadFile(false); 859 * }); 860 * ajax.send(); 861 * } 862 * 863 * downloadFile(true); 864 * ``` 865 * 866 * ``` 867 * // PHP controller action 868 * public function myAction() 869 * { 870 * $outputPath = PIWIK_INCLUDE_PATH . '/tmp/averybigfile.zip'; 871 * $isStart = Common::getRequestVar('isStart', 1, 'int'); 872 * Http::downloadChunk("http://bigfiles.com/averybigfile.zip", $outputPath, $isStart == 1); 873 * } 874 * ``` 875 * 876 * @param string $url The url to download from. 877 * @param string $outputPath The path to the file to save/append to. 878 * @param bool $isContinuation `true` if this is the continuation of a download, 879 * or if we're starting a fresh one. 880 * @throws Exception if the file already exists and we're starting a new download, 881 * if we're trying to continue a download that never started 882 * @return array 883 * @api 884 */ 885 public static function downloadChunk($url, $outputPath, $isContinuation) 886 { 887 // make sure file doesn't already exist if we're starting a new download 888 if (!$isContinuation 889 && file_exists($outputPath) 890 ) { 891 throw new Exception( 892 Piwik::translate('General_DownloadFail_FileExists', "'" . $outputPath . "'") 893 . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting')); 894 } 895 896 // if we're starting a download, get the expected file size & save as an option 897 $downloadOption = $outputPath . '_expectedDownloadSize'; 898 if (!$isContinuation) { 899 $expectedFileSizeResult = Http::sendHttpRequest( 900 $url, 901 $timeout = 300, 902 $userAgent = null, 903 $destinationPath = null, 904 $followDepth = 0, 905 $acceptLanguage = false, 906 $byteRange = false, 907 $getExtendedInfo = true, 908 $httpMethod = 'HEAD' 909 ); 910 911 $expectedFileSize = 0; 912 if (isset($expectedFileSizeResult['headers']['Content-Length'])) { 913 $expectedFileSize = (int)$expectedFileSizeResult['headers']['Content-Length']; 914 } 915 916 if ($expectedFileSize == 0) { 917 Log::info("HEAD request for '%s' failed, got following: %s", $url, print_r($expectedFileSizeResult, true)); 918 throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail')); 919 } 920 921 Option::set($downloadOption, $expectedFileSize); 922 } else { 923 $expectedFileSize = (int)Option::get($downloadOption); 924 if ($expectedFileSize === false) { // sanity check 925 throw new Exception("Trying to continue a download that never started?! That's not supposed to happen..."); 926 } 927 } 928 929 // if existing file is already big enough, then fail so we don't accidentally overwrite 930 // existing DB 931 $existingSize = file_exists($outputPath) ? filesize($outputPath) : 0; 932 if ($existingSize >= $expectedFileSize) { 933 throw new Exception( 934 Piwik::translate('General_DownloadFail_FileExistsContinue', "'" . $outputPath . "'") 935 . ' ' . Piwik::translate('General_DownloadPleaseRemoveExisting')); 936 } 937 938 // download a chunk of the file 939 $result = Http::sendHttpRequest( 940 $url, 941 $timeout = 300, 942 $userAgent = null, 943 $destinationPath = null, 944 $followDepth = 0, 945 $acceptLanguage = false, 946 $byteRange = array($existingSize, min($existingSize + 1024 * 1024 - 1, $expectedFileSize)), 947 $getExtendedInfo = true 948 ); 949 950 if ($result === false 951 || $result['status'] < 200 952 || $result['status'] > 299 953 ) { 954 $result['data'] = self::truncateStr($result['data'], 1024); 955 Log::info("Failed to download range '%s-%s' of file from url '%s'. Got result: %s", 956 $byteRange[0], $byteRange[1], $url, print_r($result, true)); 957 958 throw new Exception(Piwik::translate('General_DownloadFail_HttpRequestFail')); 959 } 960 961 // write chunk to file 962 $f = fopen($outputPath, 'ab'); 963 fwrite($f, $result['data']); 964 fclose($f); 965 966 clearstatcache($clear_realpath_cache = true, $outputPath); 967 return array( 968 'current_size' => filesize($outputPath), 969 'expected_file_size' => $expectedFileSize, 970 ); 971 } 972 973 /** 974 * Will configure CURL handle $ch 975 * to use local list of Certificate Authorities, 976 */ 977 public static function configCurlCertificate(&$ch) 978 { 979 $general = Config::getInstance()->General; 980 if (!empty($general['custom_cacert_pem'])) { 981 $cacertPath = $general['custom_cacert_pem']; 982 } else { 983 $cacertPath = CaBundle::getBundledCaBundlePath(); 984 } 985 @curl_setopt($ch, CURLOPT_CAINFO, $cacertPath); 986 } 987 988 public static function getUserAgent() 989 { 990 return !empty($_SERVER['HTTP_USER_AGENT']) 991 ? $_SERVER['HTTP_USER_AGENT'] 992 : 'Matomo/' . Version::VERSION; 993 } 994 995 /** 996 * Fetches a file located at `$url` and saves it to `$destinationPath`. 997 * 998 * @param string $url The URL of the file to download. 999 * @param string $destinationPath The path to download the file to. 1000 * @param int $tries (deprecated) 1001 * @param int $timeout The amount of seconds to wait before aborting the HTTP request. 1002 * @throws Exception if the response cannot be saved to `$destinationPath`, if the HTTP response cannot be sent, 1003 * if there are more than 5 redirects or if the request times out. 1004 * @return bool `true` on success, throws Exception on failure 1005 * @api 1006 */ 1007 public static function fetchRemoteFile($url, $destinationPath = null, $tries = 0, $timeout = 10) 1008 { 1009 @ignore_user_abort(true); 1010 SettingsServer::setMaxExecutionTime(0); 1011 return self::sendHttpRequest($url, $timeout, 'Update', $destinationPath); 1012 } 1013 1014 /** 1015 * Utility function, parses an HTTP header line into key/value & sets header 1016 * array with them. 1017 * 1018 * @param array $headers 1019 * @param string $line 1020 */ 1021 private static function parseHeaderLine(&$headers, $line) 1022 { 1023 $parts = explode(':', $line, 2); 1024 if (count($parts) == 1) { 1025 return; 1026 } 1027 1028 [$name, $value] = $parts; 1029 $name = trim($name); 1030 $headers[$name] = trim($value); 1031 1032 /** 1033 * With HTTP/2 Cloudflare is passing headers in lowercase (e.g. 'content-type' instead of 'Content-Type') 1034 * which breaks any code which uses the header data. 1035 */ 1036 if (version_compare(PHP_VERSION, '5.5.16', '>=')) { 1037 // Passing a second arg to ucwords is not supported by older versions of PHP 1038 $camelName = ucwords($name, '-'); 1039 if ($camelName !== $name) { 1040 $headers[$camelName] = trim($value); 1041 } 1042 } 1043 } 1044 1045 /** 1046 * Utility function that truncates a string to an arbitrary limit. 1047 * 1048 * @param string $str The string to truncate. 1049 * @param int $limit The maximum length of the truncated string. 1050 * @return string 1051 */ 1052 private static function truncateStr($str, $limit) 1053 { 1054 if (strlen($str) > $limit) { 1055 return substr($str, 0, $limit) . '...'; 1056 } 1057 return $str; 1058 } 1059 1060 /** 1061 * Returns the If-Modified-Since HTTP header if it can be found. If it cannot be 1062 * found, an empty string is returned. 1063 * 1064 * @return string 1065 */ 1066 public static function getModifiedSinceHeader() 1067 { 1068 $modifiedSince = ''; 1069 if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { 1070 $modifiedSince = $_SERVER['HTTP_IF_MODIFIED_SINCE']; 1071 1072 // strip any trailing data appended to header 1073 if (false !== ($semicolonPos = strpos($modifiedSince, ';'))) { 1074 $modifiedSince = substr($modifiedSince, 0, $semicolonPos); 1075 } 1076 } 1077 return $modifiedSince; 1078 } 1079 1080 /** 1081 * Returns Proxy to use for connecting via HTTP to given URL 1082 * 1083 * @param string $url 1084 * @return array 1085 */ 1086 private static function getProxyConfiguration($url) 1087 { 1088 $hostname = UrlHelper::getHostFromUrl($url); 1089 1090 if (Url::isLocalHost($hostname)) { 1091 return array(null, null, null, null); 1092 } 1093 1094 // proxy configuration 1095 $proxyHost = Config::getInstance()->proxy['host']; 1096 $proxyPort = Config::getInstance()->proxy['port']; 1097 $proxyUser = Config::getInstance()->proxy['username']; 1098 $proxyPassword = Config::getInstance()->proxy['password']; 1099 1100 return array($proxyHost, $proxyPort, $proxyUser, $proxyPassword); 1101 } 1102} 1103