1<?php 2 3declare(strict_types=1); 4 5/** 6 * @author Christoph Wurst <christoph@winzerhof-wurst.at> 7 * 8 * Mail 9 * 10 * This code is free software: you can redistribute it and/or modify 11 * it under the terms of the GNU Affero General Public License, version 3, 12 * as published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU Affero General Public License for more details. 18 * 19 * You should have received a copy of the GNU Affero General Public License, version 3, 20 * along with this program. If not, see <http://www.gnu.org/licenses/> 21 * 22 */ 23 24namespace OCA\Mail\IMAP; 25 26use Horde_Imap_Client; 27use Horde_Imap_Client_Base; 28use Horde_Imap_Client_Data_Fetch; 29use Horde_Imap_Client_Exception; 30use Horde_Imap_Client_Fetch_Query; 31use Horde_Imap_Client_Search_Query; 32use Horde_Imap_Client_Ids; 33use Horde_Imap_Client_Socket; 34use Horde_Mime_Mail; 35use Horde_Mime_Part; 36use OCA\Mail\Db\Mailbox; 37use OCA\Mail\Exception\ServiceException; 38use OCA\Mail\Model\IMAPMessage; 39use OCP\AppFramework\Db\DoesNotExistException; 40use Psr\Log\LoggerInterface; 41use function array_filter; 42use function array_map; 43use function count; 44use function in_array; 45use function iterator_to_array; 46use function max; 47use function min; 48use function reset; 49use function sprintf; 50 51class MessageMapper { 52 53 /** @var LoggerInterface */ 54 private $logger; 55 56 public function __construct(LoggerInterface $logger) { 57 $this->logger = $logger; 58 } 59 60 /** 61 * @return IMAPMessage 62 * @throws DoesNotExistException 63 * @throws Horde_Imap_Client_Exception 64 */ 65 public function find(Horde_Imap_Client_Base $client, 66 string $mailbox, 67 int $id, 68 bool $loadBody = false): IMAPMessage { 69 $result = $this->findByIds($client, $mailbox, [$id], $loadBody); 70 71 if (count($result) === 0) { 72 throw new DoesNotExistException("Message does not exist"); 73 } 74 75 return $result[0]; 76 } 77 78 /** 79 * @param Horde_Imap_Client_Socket $client 80 * @param string $mailbox 81 * 82 * @param int $maxResults 83 * @param int $highestKnownUid 84 * 85 * @return array 86 * @throws Horde_Imap_Client_Exception 87 */ 88 public function findAll(Horde_Imap_Client_Socket $client, 89 string $mailbox, 90 int $maxResults, 91 int $highestKnownUid): array { 92 /** 93 * To prevent memory exhaustion, we don't want to just ask for a list of 94 * all UIDs and limit them client-side. Instead we can (hopefully 95 * efficiently) query the min and max UID as well as the number of 96 * messages. Based on that we assume that UIDs are somewhat distributed 97 * equally and build a page to fetch. 98 * 99 * This logic might return fewer or more results than $maxResults 100 */ 101 102 $metaResults = $client->search( 103 $mailbox, 104 null, 105 [ 106 'results' => [ 107 Horde_Imap_Client::SEARCH_RESULTS_MIN, 108 Horde_Imap_Client::SEARCH_RESULTS_MAX, 109 Horde_Imap_Client::SEARCH_RESULTS_COUNT, 110 ] 111 ] 112 ); 113 /** @var int $min */ 114 $min = (int) $metaResults['min']; 115 /** @var int $max */ 116 $max = (int) $metaResults['max']; 117 /** @var int $total */ 118 $total = (int) $metaResults['count']; 119 120 if ($total === 0) { 121 // Nothing to fetch for this mailbox 122 return [ 123 'messages' => [], 124 'all' => true, 125 'total' => $total, 126 ]; 127 } 128 129 // The inclusive range of UIDs 130 $totalRange = $max - $min + 1; 131 // Here we assume somewhat equally distributed UIDs 132 // +1 is added to fetch all messages with the rare case of strictly 133 // continuous UIDs and fractions 134 $estimatedPageSize = (int)(($totalRange / $total) * $maxResults) + 1; 135 // Determine min UID to fetch, but don't exceed the known maximum 136 $lower = max( 137 $min, 138 $highestKnownUid + 1 139 ); 140 // Determine max UID to fetch, but don't exceed the known maximum 141 $upper = min( 142 $max, 143 $lower + $estimatedPageSize 144 ); 145 $this->logger->debug("Built range for findAll: min=$min max=$max total=$total totalRange=$totalRange estimatedPageSize=$estimatedPageSize lower=$lower upper=$upper highestKnownUid=$highestKnownUid"); 146 147 $query = new Horde_Imap_Client_Fetch_Query(); 148 $query->uid(); 149 $fetchResult = $client->fetch( 150 $mailbox, 151 $query, 152 [ 153 'ids' => new Horde_Imap_Client_Ids($lower . ':' . $upper) 154 ] 155 ); 156 if (count($fetchResult) === 0) { 157 /* 158 * There were no messages in this range. 159 * This means we should try again until there is a 160 * page that actually returns at least one message 161 * 162 * We take $upper as the lowest known UID as we just found out that 163 * there is nothing to fetch in $highestKnownUid:$upper 164 */ 165 $this->logger->debug("Range for findAll did not find any messages. Trying again with a succeeding range"); 166 return $this->findAll($client, $mailbox, $maxResults, $upper); 167 } 168 $uidCandidates = array_filter( 169 array_map( 170 function (Horde_Imap_Client_Data_Fetch $data) { 171 return $data->getUid(); 172 }, 173 iterator_to_array($fetchResult) 174 ), 175 176 function (int $uid) use ($highestKnownUid) { 177 // Don't load the ones we already know 178 return $uid > $highestKnownUid; 179 } 180 ); 181 $uidsToFetch = array_slice( 182 $uidCandidates, 183 0, 184 $maxResults 185 ); 186 $highestUidToFetch = $uidsToFetch[count($uidsToFetch) - 1]; 187 $this->logger->debug(sprintf("Range for findAll min=$min max=$max found %d messages, %d left after filtering. Highest UID to fetch is %d", count($uidCandidates), count($uidsToFetch), $highestUidToFetch)); 188 if ($highestUidToFetch === $max) { 189 $this->logger->debug("All messages of mailbox $mailbox have been fetched"); 190 } else { 191 $this->logger->debug("Mailbox $mailbox has more messages to fetch"); 192 } 193 return [ 194 'messages' => $this->findByIds( 195 $client, 196 $mailbox, 197 $uidsToFetch 198 ), 199 'all' => $highestUidToFetch === $max, 200 'total' => $total, 201 ]; 202 } 203 204 /** 205 * @return IMAPMessage[] 206 * @throws Horde_Imap_Client_Exception 207 */ 208 public function findByIds(Horde_Imap_Client_Base $client, 209 string $mailbox, 210 array $ids, 211 bool $loadBody = false): array { 212 $query = new Horde_Imap_Client_Fetch_Query(); 213 $query->envelope(); 214 $query->flags(); 215 $query->uid(); 216 $query->imapDate(); 217 $query->headerText( 218 [ 219 'cache' => true, 220 'peek' => true, 221 ] 222 ); 223 224 /** @var Horde_Imap_Client_Data_Fetch[] $fetchResults */ 225 $fetchResults = iterator_to_array($client->fetch($mailbox, $query, [ 226 'ids' => new Horde_Imap_Client_Ids($ids), 227 ]), false); 228 229 if (empty($fetchResults)) { 230 $this->logger->debug("findByIds in $mailbox got " . count($ids) . " UIDs but found none"); 231 } else { 232 $minRequested = $ids[0]; 233 $maxRequested = $ids[count($ids) - 1]; 234 $minFetched = $fetchResults[0]->getUid(); 235 $maxFetched = $fetchResults[count($fetchResults) - 1]->getUid(); 236 $this->logger->debug("findByIds in $mailbox got " . count($ids) . " UIDs ($minRequested:$maxRequested) and found " . count($fetchResults) . ". minFetched=$minFetched maxFetched=$maxFetched"); 237 } 238 239 return array_map(function (Horde_Imap_Client_Data_Fetch $fetchResult) use ($client, $mailbox, $loadBody) { 240 if ($loadBody) { 241 return new IMAPMessage( 242 $client, 243 $mailbox, 244 $fetchResult->getUid(), 245 null, 246 $loadBody 247 ); 248 } else { 249 return new IMAPMessage( 250 $client, 251 $mailbox, 252 $fetchResult->getUid(), 253 $fetchResult 254 ); 255 } 256 }, $fetchResults); 257 } 258 259 /** 260 * @param Horde_Imap_Client_Base $client 261 * @param string $sourceFolderId 262 * @param int $messageId 263 * @param string $destFolderId 264 */ 265 public function move(Horde_Imap_Client_Base $client, 266 string $sourceFolderId, 267 int $messageId, 268 string $destFolderId): void { 269 try { 270 $client->copy($sourceFolderId, $destFolderId, 271 [ 272 'ids' => new Horde_Imap_Client_Ids($messageId), 273 'move' => true, 274 ]); 275 } catch (Horde_Imap_Client_Exception $e) { 276 $this->logger->debug($e->getMessage(), 277 [ 278 'exception' => $e, 279 ] 280 ); 281 282 throw new ServiceException( 283 "Could not move message $$messageId from $sourceFolderId to $destFolderId", 284 0, 285 $e 286 ); 287 } 288 } 289 290 public function markAllRead(Horde_Imap_Client_Base $client, 291 string $mailbox): void { 292 $client->store($mailbox, [ 293 'add' => [ 294 Horde_Imap_Client::FLAG_SEEN, 295 ], 296 ]); 297 } 298 299 /** 300 * @throws ServiceException 301 */ 302 public function expunge(Horde_Imap_Client_Base $client, 303 string $mailbox, 304 int $id): void { 305 try { 306 $client->expunge( 307 $mailbox, 308 [ 309 'ids' => new Horde_Imap_Client_Ids([$id]), 310 'delete' => true, 311 ]); 312 } catch (Horde_Imap_Client_Exception $e) { 313 $this->logger->debug($e->getMessage(), 314 [ 315 'exception' => $e, 316 ] 317 ); 318 319 throw new ServiceException("Could not expunge message $id", 0, $e); 320 } 321 322 $this->logger->info("Message expunged: $id from mailbox $mailbox"); 323 } 324 325 /** 326 * @throws Horde_Imap_Client_Exception 327 */ 328 public function save(Horde_Imap_Client_Socket $client, 329 Mailbox $mailbox, 330 Horde_Mime_Mail $mail, 331 array $flags = []): int { 332 $flags = array_merge([ 333 Horde_Imap_Client::FLAG_SEEN, 334 ], $flags); 335 336 $uids = $client->append( 337 $mailbox->getName(), 338 [ 339 [ 340 'data' => $mail->getRaw(), 341 'flags' => $flags, 342 ] 343 ] 344 ); 345 346 return (int)$uids->current(); 347 } 348 349 /** 350 * @throws Horde_Imap_Client_Exception 351 */ 352 public function addFlag(Horde_Imap_Client_Socket $client, 353 Mailbox $mailbox, 354 array $uids, 355 string $flag): void { 356 $client->store( 357 $mailbox->getName(), 358 [ 359 'ids' => new Horde_Imap_Client_Ids($uids), 360 'add' => [$flag], 361 ] 362 ); 363 } 364 365 /** 366 * @throws Horde_Imap_Client_Exception 367 */ 368 public function removeFlag(Horde_Imap_Client_Socket $client, 369 Mailbox $mailbox, 370 array $uids, 371 string $flag): void { 372 $client->store( 373 $mailbox->getName(), 374 [ 375 'ids' => new Horde_Imap_Client_Ids($uids), 376 'remove' => [$flag], 377 ] 378 ); 379 } 380 381 /** 382 * @param Horde_Imap_Client_Socket $client 383 * @param Mailbox $mailbox 384 * @param string $flag 385 * @return int[] 386 * 387 * @throws Horde_Imap_Client_Exception 388 */ 389 public function getFlagged(Horde_Imap_Client_Socket $client, 390 Mailbox $mailbox, 391 string $flag): array { 392 $query = new Horde_Imap_Client_Search_Query(); 393 $query->flag($flag, true); 394 $messages = $client->search($mailbox->getName(), $query); 395 return $messages['match']->ids ?? []; 396 } 397 398 /** 399 * @param Horde_Imap_Client_Socket $client 400 * @param string $mailbox 401 * @param int $uid 402 * 403 * @return string|null 404 * @throws ServiceException 405 */ 406 public function getFullText(Horde_Imap_Client_Socket $client, 407 string $mailbox, 408 int $uid): ?string { 409 $query = new Horde_Imap_Client_Fetch_Query(); 410 $query->uid(); 411 $query->fullText([ 412 'peek' => true, 413 ]); 414 415 try { 416 $result = iterator_to_array($client->fetch($mailbox, $query, [ 417 'ids' => new Horde_Imap_Client_Ids($uid), 418 ]), false); 419 } catch (Horde_Imap_Client_Exception $e) { 420 throw new ServiceException( 421 "Could not fetch message source: " . $e->getMessage(), 422 (int) $e->getCode(), 423 $e 424 ); 425 } 426 427 $msg = array_map(function (Horde_Imap_Client_Data_Fetch $result) { 428 return $result->getFullMsg(); 429 }, $result); 430 431 if (empty($msg)) { 432 return null; 433 } 434 435 return reset($msg); 436 } 437 438 public function getHtmlBody(Horde_Imap_Client_Socket $client, 439 string $mailbox, 440 int $uid): ?string { 441 $messageQuery = new Horde_Imap_Client_Fetch_Query(); 442 $messageQuery->envelope(); 443 $messageQuery->structure(); 444 445 $result = $client->fetch($mailbox, $messageQuery, [ 446 'ids' => new Horde_Imap_Client_Ids([$uid]), 447 ]); 448 449 if (($message = $result->first()) === null) { 450 throw new DoesNotExistException('Message does not exist'); 451 } 452 453 $structure = $message->getStructure(); 454 $htmlPartId = $structure->findBody('html'); 455 if ($htmlPartId === null) { 456 // No HTML part 457 return null; 458 } 459 $partsQuery = $this->buildAttachmentsPartsQuery($structure, [$htmlPartId]); 460 461 $parts = $client->fetch($mailbox, $partsQuery, [ 462 'ids' => new Horde_Imap_Client_Ids([$uid]), 463 ]); 464 465 foreach ($parts as $part) { 466 /** @var Horde_Imap_Client_Data_Fetch $part */ 467 $body = $part->getBodyPart($htmlPartId); 468 if ($body !== null) { 469 $mimeHeaders = $part->getMimeHeader($htmlPartId, Horde_Imap_Client_Data_Fetch::HEADER_PARSE); 470 if ($enc = $mimeHeaders->getValue('content-transfer-encoding')) { 471 $structure->setTransferEncoding($enc); 472 } 473 $structure->setContents($body); 474 return $structure->getContents(); 475 } 476 } 477 478 return null; 479 } 480 481 public function getRawAttachments(Horde_Imap_Client_Socket $client, 482 string $mailbox, 483 int $uid, 484 ?array $attachmentIds = []): array { 485 $messageQuery = new Horde_Imap_Client_Fetch_Query(); 486 $messageQuery->structure(); 487 488 $result = $client->fetch($mailbox, $messageQuery, [ 489 'ids' => new Horde_Imap_Client_Ids([$uid]), 490 ]); 491 492 if (($structureResult = $result->first()) === null) { 493 throw new DoesNotExistException('Message does not exist'); 494 } 495 496 $structure = $structureResult->getStructure(); 497 $partsQuery = $this->buildAttachmentsPartsQuery($structure, $attachmentIds); 498 499 $parts = $client->fetch($mailbox, $partsQuery, [ 500 'ids' => new Horde_Imap_Client_Ids([$uid]), 501 ]); 502 if (($messageData = $parts->first()) === null) { 503 throw new DoesNotExistException('Message does not exist'); 504 } 505 506 $attachments = []; 507 foreach ($structure->partIterator() as $key => $part) { 508 /** @var Horde_Mime_Part $part */ 509 510 if (!$part->isAttachment()) { 511 continue; 512 } 513 514 $stream = $messageData->getBodyPart($key, true); 515 $mimeHeaders = $messageData->getMimeHeader($key, Horde_Imap_Client_Data_Fetch::HEADER_PARSE); 516 if ($enc = $mimeHeaders->getValue('content-transfer-encoding')) { 517 $part->setTransferEncoding($enc); 518 } 519 $part->setContents($stream, [ 520 'usestream' => true, 521 ]); 522 $decoded = $part->getContents(); 523 524 $attachments[] = $decoded; 525 } 526 return $attachments; 527 } 528 529 /** 530 * Get Attachments with size, content and name properties 531 * 532 * @param Horde_Imap_Client_Socket $client 533 * @param string $mailbox 534 * @param integer $uid 535 * @param array|null $attachmentIds 536 * @return array[] 537 */ 538 public function getAttachments(Horde_Imap_Client_Socket $client, 539 string $mailbox, 540 int $uid, 541 ?array $attachmentIds = []): array { 542 $messageQuery = new Horde_Imap_Client_Fetch_Query(); 543 $messageQuery->structure(); 544 545 $result = $client->fetch($mailbox, $messageQuery, [ 546 'ids' => new Horde_Imap_Client_Ids([$uid]), 547 ]); 548 549 if (($structureResult = $result->first()) === null) { 550 throw new DoesNotExistException('Message does not exist'); 551 } 552 553 $structure = $structureResult->getStructure(); 554 $partsQuery = $this->buildAttachmentsPartsQuery($structure, $attachmentIds); 555 556 $parts = $client->fetch($mailbox, $partsQuery, [ 557 'ids' => new Horde_Imap_Client_Ids([$uid]), 558 ]); 559 if (($messageData = $parts->first()) === null) { 560 throw new DoesNotExistException('Message does not exist'); 561 } 562 563 $attachments = []; 564 foreach ($structure->partIterator() as $key => $part) { 565 /** @var Horde_Mime_Part $part */ 566 567 if (!$part->isAttachment()) { 568 continue; 569 } 570 571 $stream = $messageData->getBodyPart($key, true); 572 $mimeHeaders = $messageData->getMimeHeader($key, Horde_Imap_Client_Data_Fetch::HEADER_PARSE); 573 if ($enc = $mimeHeaders->getValue('content-transfer-encoding')) { 574 $part->setTransferEncoding($enc); 575 } 576 $part->setContents($stream, [ 577 'usestream' => true, 578 ]); 579 $attachments[] = [ 580 'content' => $part->getContents(), 581 'name' => $part->getName(), 582 'size' => $part->getSize() 583 ]; 584 } 585 return $attachments; 586 } 587 588 /** 589 * Build the parts query for attachments 590 * 591 * @param $structure 592 * @param array $attachmentIds 593 * @return Horde_Imap_Client_Fetch_Query 594 */ 595 private function buildAttachmentsPartsQuery($structure, array $attachmentIds) : Horde_Imap_Client_Fetch_Query { 596 $partsQuery = new Horde_Imap_Client_Fetch_Query(); 597 $partsQuery->fullText(); 598 foreach ($structure->partIterator() as $part) { 599 /** @var Horde_Mime_Part $part */ 600 if ($part->getMimeId() === '0') { 601 // Ignore message header 602 continue; 603 } 604 if (!empty($attachmentIds) && !in_array($part->getMIMEId(), $attachmentIds, true)) { 605 // We are looking for specific parts only and this is not one of them 606 continue; 607 } 608 609 $partsQuery->bodyPart($part->getMimeId(), [ 610 'peek' => true, 611 ]); 612 $partsQuery->mimeHeader($part->getMimeId(), [ 613 'peek' => true 614 ]); 615 $partsQuery->bodyPartSize($part->getMimeId()); 616 } 617 return $partsQuery; 618 } 619 620 /** 621 * @param Horde_Imap_Client_Socket $client 622 * @param int[] $uids 623 * 624 * @return MessageStructureData[] 625 * @throws Horde_Imap_Client_Exception 626 */ 627 public function getBodyStructureData(Horde_Imap_Client_Socket $client, 628 string $mailbox, 629 array $uids): array { 630 $structureQuery = new Horde_Imap_Client_Fetch_Query(); 631 $structureQuery->structure(); 632 633 $structures = $client->fetch($mailbox, $structureQuery, [ 634 'ids' => new Horde_Imap_Client_Ids($uids), 635 ]); 636 637 return array_map(function (Horde_Imap_Client_Data_Fetch $fetchData) use ($mailbox, $client) { 638 $hasAttachments = false; 639 $text = ''; 640 641 $structure = $fetchData->getStructure(); 642 foreach ($structure as $part) { 643 if ($part instanceof Horde_Mime_Part && $part->isAttachment()) { 644 $hasAttachments = true; 645 break; 646 } 647 } 648 649 $textBodyId = $structure->findBody('text'); 650 // $htmlBodyId = $structure->findBody('html'); 651 // $htmlBody = $data->getBodyPart($htmlBodyId); 652 653 $partsQuery = new Horde_Imap_Client_Fetch_Query(); 654 if ($textBodyId === null) { 655 return new MessageStructureData($hasAttachments, $text); 656 } 657 $partsQuery->bodyPart($textBodyId, [ 658 'decode' => true, 659 'peek' => true, 660 ]); 661 $partsQuery->mimeHeader($textBodyId, [ 662 'peek' => true 663 ]); 664 $parts = $client->fetch($mailbox, $partsQuery, [ 665 'ids' => new Horde_Imap_Client_Ids([$fetchData->getUid()]), 666 ]); 667 /** @var Horde_Imap_Client_Data_Fetch $part */ 668 $part = $parts[$fetchData->getUid()]; 669 $body = $part->getBodyPart($textBodyId); 670 671 if (!empty($body)) { 672 $mimeHeaders = $fetchData->getMimeHeader($textBodyId, Horde_Imap_Client_Data_Fetch::HEADER_PARSE); 673 if ($enc = $mimeHeaders->getValue('content-transfer-encoding')) { 674 $structure->setTransferEncoding($enc); 675 } 676 $structure->setContents($body); 677 /** @var string $text */ 678 $text = $structure->getContents(); 679 } 680 681 return new MessageStructureData($hasAttachments, $text); 682 }, iterator_to_array($structures->getIterator())); 683 } 684} 685