1# Copyright 2011 - 2021 Patrick Ulbrich <zulu99@gmx.net>
2# Copyright 2020 Andreas Angerer
3# Copyright 2016, 2018 Timo Kankare <timo.kankare@iki.fi>
4# Copyright 2011 Leighton Earl <leighton.earl@gmx.com>
5# Copyright 2011 Ralf Hersel <ralf.hersel@gmx.net>
6# Copyright 2019 razer <razerraz@free.fr>
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation; either version 2 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License
19# along with this program; if not, write to the Free Software
20# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
21# MA 02110-1301, USA.
22#
23
24import time
25import email
26import os
27import logging
28import hashlib
29
30from email.header import decode_header, make_header
31from Mailnag.common.i18n import _
32from Mailnag.common.config import cfg_folder
33
34
35#
36# Mail class
37#
38class Mail:
39	def __init__(self, datetime, subject, sender, id, account, flags):
40		self.datetime = datetime
41		self.subject = subject
42		self.sender = sender
43		self.account = account
44		self.id = id
45		self.flags = flags
46
47
48#
49# MailCollector class
50#
51class MailCollector:
52	def __init__(self, cfg, accounts):
53		self._cfg = cfg
54		self._accounts = accounts
55
56
57	def collect_mail(self, sort = True):
58		mail_list = []
59		mail_ids = {}
60
61		for acc in self._accounts:
62			# open mailbox for this account
63			try:
64				if not acc.is_open():
65					acc.open()
66			except Exception as ex:
67				logging.error("Failed to open mailbox for account '%s' (%s)." % (acc.name, ex))
68				continue
69
70			try:
71				for folder, msg, flags in acc.list_messages():
72					sender, subject, datetime, msgid = self._get_header(msg)
73					id = self._get_id(msgid, acc, folder, sender, subject, datetime)
74
75					# Discard mails with identical IDs (caused
76					# by mails with a non-unique fallback ID,
77					# i.e. mails received in the same folder with
78					# identical sender and subject but *no datetime*,
79					# see _get_id()).
80					# Also filter duplicates caused by Gmail labels.
81					if id not in mail_ids:
82						mail_list.append(Mail(datetime, subject, \
83							sender, id, acc, flags))
84						mail_ids[id] = None
85			except Exception as ex:
86				# Catch exceptions here, so remaining accounts will still be checked
87				# if a specific account has issues.
88				#
89				# Re-throw the exception for accounts that support notifications (i.e. imap IDLE),
90				# so the calling idler thread can handle the error and reset the connection if needed (see idlers.py).
91				# NOTE: Idler threads always check single accounts (i.e. len(self._accounts) == 1),
92				#       so there are no remaining accounts to be checked for now.
93				if acc.supports_notifications():
94					raise
95				else:
96					logging.error("An error occured while processing mails of account '%s' (%s)." % (acc.name, ex))
97			finally:
98				# leave account with notifications open, so that it can
99				# send notifications about new mails
100				if not acc.supports_notifications():
101					# disconnect from Email-Server
102					acc.close()
103
104
105		# sort mails
106		if sort:
107			mail_list.sort(key = lambda m: m.datetime, reverse = True)
108
109		return mail_list
110
111
112	def _get_header(self, msg_dict):
113		# Get sender
114		sender = ('', '')
115
116		try:
117			content = self._get_header_field(msg_dict, 'From')
118			# get the two parts of the sender
119			addr = email.utils.parseaddr(content)
120
121			if len(addr) != 2:
122				logging.warning('Malformed sender field in message.')
123			else:
124				sender_real = self._convert(addr[0])
125				sender_addr = self._convert(addr[1])
126
127				sender = (sender_real, sender_addr)
128		except:
129			pass
130
131		# Get subject
132		try:
133			content = self._get_header_field(msg_dict, 'Subject')
134			subject = self._convert(content)
135		except:
136			subject = _('No subject')
137
138		# Get date
139		try:
140			content = self._get_header_field(msg_dict, 'Date')
141
142			# make a 10-tupel (UTC)
143			parsed_date = email.utils.parsedate_tz(content)
144			# convert 10-tupel to seconds incl. timezone shift
145			datetime = email.utils.mktime_tz(parsed_date)
146		except:
147			logging.warning('Email date set to zero.')
148			datetime = 0
149
150		# Get message id
151		try:
152			msgid = self._get_header_field(msg_dict, 'Message-ID')
153		except:
154			msgid = ''
155
156		return (sender, subject, datetime, msgid)
157
158
159	def _get_header_field(self, msg_dict, key):
160		if key in msg_dict:
161			value = msg_dict[key]
162		elif key.lower() in msg_dict:
163			value = msg_dict[key.lower()]
164		else:
165			logging.debug("Couldn't get %s from message." % key)
166			raise KeyError
167
168		return value
169
170
171	# return utf-8 decoded string from multi-part/multi-charset header text
172	def _convert(self, text):
173		decoded = decode_header(text)
174		if not decoded[0][1] or 'unknown' in decoded[0][1]:
175			decoded = [(decoded[0][0], 'latin-1')]
176		return str(make_header(decoded))
177
178
179	def _get_id(self, msgid, acc, folder, sender, subject, datetime):
180		if len(msgid) > 0:
181			id = hashlib.md5(msgid.encode('utf-8')).hexdigest()
182		else:
183			# Fallback ID.
184			# Note: mails received on the same server,
185			# in the same folder with identical sender and
186			# subject but *no datetime* will have the same hash id,
187			# i.e. only the first mail is notified.
188			# (Should happen very rarely).
189			id = hashlib.md5((acc.get_id() + folder +
190				sender[1] + subject + str(datetime))
191				.encode('utf-8')).hexdigest()
192
193		return id
194
195
196#
197# MailSyncer class
198#
199class MailSyncer:
200	def __init__(self, cfg):
201		self._cfg = cfg
202		self._mails_by_account = {}
203		self._mail_list = []
204
205
206	def sync(self, accounts):
207		needs_rebuild = False
208
209		# collect mails from given accounts
210		rcv_lst = MailCollector(self._cfg, accounts).collect_mail(sort = False)
211
212		# group received mails by account
213		tmp = {}
214		for acc in accounts:
215			tmp[acc.get_id()] = {}
216		for mail in rcv_lst:
217			tmp[mail.account.get_id()][mail.id] = mail
218
219		# compare current mails against received mails
220		# and remove those that are gone (probably opened in mail client).
221		for acc_id in self._mails_by_account.keys():
222			if acc_id in tmp:
223				del_ids = []
224				for mail_id in self._mails_by_account[acc_id].keys():
225					if not (mail_id in tmp[acc_id]):
226						del_ids.append(mail_id)
227						needs_rebuild = True
228				for mail_id in del_ids:
229					del self._mails_by_account[acc_id][mail_id]
230
231		# compare received mails against current mails
232		# and add new mails.
233		for acc_id in tmp:
234			if not (acc_id in self._mails_by_account):
235				self._mails_by_account[acc_id] = {}
236			for mail_id in tmp[acc_id]:
237				if not (mail_id in self._mails_by_account[acc_id]):
238					self._mails_by_account[acc_id][mail_id] = tmp[acc_id][mail_id]
239					needs_rebuild = True
240
241		# rebuild and sort mail list
242		if needs_rebuild:
243			self._mail_list = []
244			for acc_id in self._mails_by_account:
245				for mail_id in self._mails_by_account[acc_id]:
246					self._mail_list.append(self._mails_by_account[acc_id][mail_id])
247			self._mail_list.sort(key = lambda m: m.datetime, reverse = True)
248
249		return self._mail_list
250
251
252#
253# Memorizer class
254#
255class Memorizer(dict):
256	def __init__(self):
257		dict.__init__(self)
258		self._changed = False
259
260
261	def load(self):
262		self.clear()
263		self._changed = False
264
265		# load last known messages from mailnag.dat
266		dat_file = os.path.join(cfg_folder, 'mailnag.dat')
267
268		if os.path.exists(dat_file):
269			with open(dat_file, 'r') as f:
270				for line in f:
271					# remove CR at the end
272					stripedline = line.strip()
273					# get all items from one line in a list: [mailid, seen_flag]
274					pair = stripedline.split(',')
275					# add to dict [id : flag]
276					self[pair[0]] = pair[1]
277
278
279	# save mail ids to a file
280	def save(self, force = False):
281		if (not self._changed) and (not force):
282			return
283
284		if not os.path.exists(cfg_folder):
285			os.makedirs(cfg_folder)
286
287		dat_file = os.path.join(cfg_folder, 'mailnag.dat')
288		with open(dat_file, 'w') as f:
289			for id, seen_flag in list(self.items()):
290				line = id + ',' + seen_flag + '\n'
291				f.write(line)
292
293		self._changed = False
294
295
296	def sync(self, mail_list):
297		for m in mail_list:
298			if not m.id in self:
299				# new mail is not yet known to the memorizer
300				self[m.id] = '0'
301				self._changed = True
302
303		for id in list(self.keys()):
304			found = False
305			for m in mail_list:
306				if id == m.id:
307					found = True
308					# break inner for loop
309					break
310			if not found:
311				del self[id]
312				self._changed = True
313
314
315	# check if mail id is in the memorizer list
316	def contains(self, id):
317		return (id in self)
318
319
320	# set seen flag for this email
321	def set_to_seen(self, id):
322		self[id] = '1'
323		self._changed = True
324
325
326	def is_unseen(self, id):
327		if id in self:
328			flag = self[id]
329			return (flag == '0')
330		else:
331			return True
332