1# Copyright (C) 2011-2020 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
17
18"""Test the bounce runner."""
19
20import unittest
21
22from datetime import timedelta
23from mailman.app.bounces import send_probe
24from mailman.app.lifecycle import create_list
25from mailman.config import config
26from mailman.database.transaction import transaction
27from mailman.interfaces.bounce import (
28    BounceContext, IBounceProcessor, UnrecognizedBounceDisposition)
29from mailman.interfaces.member import DeliveryStatus, MemberRole
30from mailman.interfaces.styles import IStyle, IStyleManager
31from mailman.interfaces.usermanager import IUserManager
32from mailman.runners.bounce import BounceRunner
33from mailman.testing.helpers import (
34    LogFileMark, get_queue_messages, make_testable_runner,
35    specialized_message_from_string as message_from_string)
36from mailman.testing.layers import ConfigLayer
37from mailman.utilities.datetime import now
38from zope.component import getUtility
39from zope.interface import implementer
40
41
42class TestBounceRunner(unittest.TestCase):
43    """Test the bounce runner."""
44
45    layer = ConfigLayer
46
47    def setUp(self):
48        self._mlist = create_list('test@example.com')
49        self._mlist.send_welcome_message = False
50        self._bounceq = config.switchboards['bounces']
51        self._runner = make_testable_runner(BounceRunner, 'bounces')
52        self._anne = getUtility(IUserManager).create_address(
53            'anne@example.com')
54        self._member = self._mlist.subscribe(self._anne, MemberRole.member)
55        self._msg = message_from_string("""\
56From: mail-daemon@example.com
57To: test-bounces+anne=example.com@example.com
58Message-Id: <first>
59
60""")
61        self._msgdata = dict(listid='test.example.com')
62        self._processor = getUtility(IBounceProcessor)
63        config.push('site owner', """
64        [mailman]
65        site_owner: postmaster@example.com
66        """)
67        self.addCleanup(config.pop, 'site owner')
68
69    def test_does_no_processing(self):
70        # If the mailing list does no bounce processing, the messages are
71        # simply discarded.
72        self._mlist.process_bounces = False
73        self._bounceq.enqueue(self._msg, self._msgdata)
74        self._runner.run()
75        get_queue_messages('bounces', expected_count=0)
76        self.assertEqual(len(list(self._processor.events)), 0)
77
78    def test_verp_detection(self):
79        # When we get a VERPd bounce, and we're doing processing, a bounce
80        # event will be registered.
81        self._bounceq.enqueue(self._msg, self._msgdata)
82        self._runner.run()
83        get_queue_messages('bounces', expected_count=0)
84        events = list(self._processor.events)
85        self.assertEqual(len(events), 1)
86        self.assertEqual(events[0].email, 'anne@example.com')
87        self.assertEqual(events[0].list_id, 'test.example.com')
88        self.assertEqual(events[0].message_id, '<first>')
89        self.assertEqual(events[0].context, BounceContext.normal)
90        self.assertEqual(events[0].processed, True)
91
92    def test_nonfatal_verp_detection(self):
93        # A VERPd bounce was received, but the error was nonfatal.
94        nonfatal = message_from_string("""\
95From: mail-daemon@example.com
96To: test-bounces+anne=example.com@example.com
97Message-Id: <first>
98Content-Type: multipart/report; report-type=delivery-status; boundary=AAA
99MIME-Version: 1.0
100
101--AAA
102Content-Type: message/delivery-status
103
104Action: delayed
105Original-Recipient: rfc822; somebody@example.com
106
107--AAA--
108""")
109        self._bounceq.enqueue(nonfatal, self._msgdata)
110        self._runner.run()
111        get_queue_messages('bounces', expected_count=0)
112        events = list(self._processor.events)
113        self.assertEqual(len(events), 0)
114
115    def test_verp_probe_bounce(self):
116        # A VERP probe bounced.  The primary difference here is that the
117        # registered bounce event will have a different context.  The
118        # Message-Id will be different too, because of the way we're
119        # simulating the probe bounce.
120        #
121        # Start be simulating a probe bounce.
122        send_probe(self._member, self._msg)
123        items = get_queue_messages('virgin', expected_count=1)
124        message = items[0].msg
125        bounce = message_from_string("""\
126To: {0}
127From: mail-daemon@example.com
128Message-Id: <second>
129
130""".format(message['From']))
131        self._bounceq.enqueue(bounce, self._msgdata)
132        self._runner.run()
133        get_queue_messages('bounces', expected_count=0)
134        events = list(self._processor.events)
135        self.assertEqual(len(events), 1)
136        self.assertEqual(events[0].email, 'anne@example.com')
137        self.assertEqual(events[0].list_id, 'test.example.com')
138        self.assertEqual(events[0].message_id, '<second>')
139        self.assertEqual(events[0].context, BounceContext.probe)
140        self.assertEqual(events[0].processed, True)
141
142    def test_nonverp_detectable_fatal_bounce(self):
143        # Here's a bounce that is not VERPd, but which has a bouncing address
144        # that can be parsed from a known bounce format.  DSN is as good as
145        # any, but we'll make the parsed address different for the fun of it.
146        dsn = message_from_string("""\
147From: mail-daemon@example.com
148To: test-bounces@example.com
149Message-Id: <first>
150Content-Type: multipart/report; report-type=delivery-status; boundary=AAA
151MIME-Version: 1.0
152
153--AAA
154Content-Type: message/delivery-status
155
156Action: fail
157Original-Recipient: rfc822; bart@example.com
158
159--AAA--
160""")
161        self._bounceq.enqueue(dsn, self._msgdata)
162        self._runner.run()
163        get_queue_messages('bounces', expected_count=0)
164        events = list(self._processor.events)
165        self.assertEqual(len(events), 1)
166        self.assertEqual(events[0].email, 'bart@example.com')
167        self.assertEqual(events[0].list_id, 'test.example.com')
168        self.assertEqual(events[0].message_id, '<first>')
169        self.assertEqual(events[0].context, BounceContext.normal)
170        self.assertEqual(events[0].processed, True)
171
172    def test_nonverp_detectable_nonfatal_bounce(self):
173        # Here's a bounce that is not VERPd, but which has a bouncing address
174        # that can be parsed from a known bounce format.  The bounce is
175        # non-fatal so no bounce event is registered and the bounce is not
176        # reported as unrecognized.
177        self._mlist.forward_unrecognized_bounces_to = (
178            UnrecognizedBounceDisposition.site_owner)
179        dsn = message_from_string("""\
180From: mail-daemon@example.com
181To: test-bounces@example.com
182Message-Id: <first>
183Content-Type: multipart/report; report-type=delivery-status; boundary=AAA
184MIME-Version: 1.0
185
186--AAA
187Content-Type: message/delivery-status
188
189Action: delayed
190Original-Recipient: rfc822; bart@example.com
191
192--AAA--
193""")
194        self._bounceq.enqueue(dsn, self._msgdata)
195        mark = LogFileMark('mailman.bounce')
196        self._runner.run()
197        get_queue_messages('bounces', expected_count=0)
198        events = list(self._processor.events)
199        self.assertEqual(len(events), 0)
200        # There should be nothing in the 'virgin' queue.
201        get_queue_messages('virgin', expected_count=0)
202        # There should be log event in the log file.
203        log_lines = mark.read().splitlines()
204        self.assertTrue(len(log_lines) > 0)
205
206    def test_no_detectable_bounce_addresses(self):
207        # A bounce message was received, but no addresses could be detected.
208        # A message will be logged in the bounce log though, and the message
209        # can be forwarded to someone who can do something about it.
210        self._mlist.forward_unrecognized_bounces_to = (
211            UnrecognizedBounceDisposition.site_owner)
212        bogus = message_from_string("""\
213From: mail-daemon@example.com
214To: test-bounces@example.com
215Message-Id: <third>
216
217""")
218        self._bounceq.enqueue(bogus, self._msgdata)
219        mark = LogFileMark('mailman.bounce')
220        self._runner.run()
221        get_queue_messages('bounces', expected_count=0)
222        events = list(self._processor.events)
223        self.assertEqual(len(events), 0)
224        line = mark.readline()
225        self.assertEqual(
226            line[-51:-1],
227            'Bounce message w/no discernable addresses: <third>')
228        # Here's the forwarded message to the site owners.
229        items = get_queue_messages('virgin', expected_count=1)
230        self.assertEqual(items[0].msg['to'], 'postmaster@example.com')
231
232
233# Create a style for the mailing list which sets the absolute minimum
234# attributes.  In particular, this will not set the bogus `bounce_processing`
235# attribute which the default style set (before LP: #876774 was fixed).
236
237@implementer(IStyle)
238class TestStyle:
239    """See `IStyle`."""
240
241    name = 'test'
242    description = 'A test style.'
243
244    def apply(self, mailing_list):
245        """See `IStyle`."""
246        mailing_list.preferred_language = 'en'
247
248
249class TestBounceRunnerBug876774(unittest.TestCase):
250    """Test LP: #876774.
251
252    Quoting:
253
254    It seems that bounce_processing is defined in src/mailman/styles/default.py
255    The style are applied at mailing-list creation, but bounce_processing
256    attribute is not persisted, the src/mailman/database/mailman.sql file
257    doesn't define it.
258    """
259    layer = ConfigLayer
260
261    def setUp(self):
262        self._style = TestStyle()
263        self._style_manager = getUtility(IStyleManager)
264        self._style_manager.register(self._style)
265        self.addCleanup(self._style_manager.unregister, self._style)
266        # Now we can create the mailing list.
267        self._mlist = create_list('test@example.com', style_name='test')
268        self._bounceq = config.switchboards['bounces']
269        self._processor = getUtility(IBounceProcessor)
270        self._runner = make_testable_runner(BounceRunner, 'bounces')
271
272    def test_bug876774(self):
273        # LP: #876774, see above.
274        bounce = message_from_string("""\
275From: mail-daemon@example.com
276To: test-bounces+anne=example.com@example.com
277Message-Id: <first>
278
279""")
280        self._bounceq.enqueue(bounce, dict(listid='test.example.com'))
281        self.assertEqual(len(self._bounceq.files), 1)
282        self._runner.run()
283        get_queue_messages('bounces', expected_count=0)
284        events = list(self._processor.events)
285        self.assertEqual(len(events), 0)
286
287
288class TestBounceRunnerPeriodicRun(unittest.TestCase):
289    """Test the bounce runner's periodic function.."""
290
291    layer = ConfigLayer
292
293    def setUp(self):
294        self._mlist = create_list('test@example.com')
295        self._mlist.send_welcome_message = False
296        self._runner = make_testable_runner(BounceRunner, 'bounces')
297        self._anne = getUtility(IUserManager).create_address(
298            'anne@example.com')
299        self._member = self._mlist.subscribe(self._anne, MemberRole.member)
300        self._msg = message_from_string("""\
301From: mail-daemon@example.com
302To: test-bounces+anne=example.com@example.com
303Message-Id: <first>
304
305""")
306        self._msgdata = dict(listid='test.example.com')
307        self._processor = getUtility(IBounceProcessor)
308        config.push('site owner', """
309        [mailman]
310        site_owner: postmaster@example.com
311        """)
312        self.addCleanup(config.pop, 'site owner')
313
314    def _subscribe_and_add_bounce_event(
315            self, addr, subscribe=True, create=True, context=None, count=1):
316        user_mgr = getUtility(IUserManager)
317        with transaction():
318            if create:
319                anne = user_mgr.create_address(addr)
320            else:
321                anne = user_mgr.get_address(addr)
322            if subscribe:
323                self._mlist.subscribe(anne)
324            self._processor.register(
325                self._mlist, addr, self._msg, where=context)
326        return self._mlist.members.get_member(addr)
327
328    def test_periodic_bounce_event_processing(self):
329        anne = self._subscribe_and_add_bounce_event(
330            'anne@example.com', subscribe=False, create=False)
331        bart = self._subscribe_and_add_bounce_event('bart@example.com')
332        # Since MailingList has process_bounces set to False, nothing happens
333        # with the events.
334        self._runner.run()
335        self.assertEqual(anne.bounce_score, 1.0)
336        self.assertEqual(bart.bounce_score, 1.0)
337        for event in self._processor.events:
338            self.assertEqual(event.processed, True)
339
340    def test_events_disable_delivery(self):
341        self._mlist.bounce_score_threshold = 3
342        anne = self._subscribe_and_add_bounce_event(
343            'anne@example.com', subscribe=False, create=False)
344        anne.bounce_score = 2
345        anne.last_bounce_received = now() - timedelta(days=2)
346        self._runner.run()
347        self.assertEqual(anne.bounce_score, 3)
348        self.assertEqual(
349            anne.preferences.delivery_status, DeliveryStatus.by_bounces)
350        # There should also be a pending notification for the the list
351        # administrator.
352        items = get_queue_messages('virgin', expected_count=2)
353        if items[0].msg['to'] == 'test-owner@example.com':
354            owner_notif, disable_notice = items
355        else:
356            disable_notice, owner_notif = items
357        self.assertEqual(owner_notif.msg['Subject'],
358                         "anne@example.com's subscription disabled on Test")
359
360        self.assertEqual(disable_notice.msg['to'], 'anne@example.com')
361        self.assertEqual(
362            str(disable_notice.msg['subject']),
363            'Your subscription for Test mailing list has been disabled')
364
365    def test_events_send_warning(self):
366        self._mlist.bounce_you_are_disabled_warnings = 3
367        self._mlist.bounce_you_are_disabled_warnings_interval = timedelta(
368            days=2)
369
370        anne = self._mlist.members.get_member(self._anne.email)
371        anne.preferences.delivery_status = DeliveryStatus.by_bounces
372        anne.total_warnings_sent = 1
373        anne.last_warning_sent = now() - timedelta(days=3)
374
375        self._runner.run()
376        items = get_queue_messages('virgin', expected_count=1)
377        self.assertEqual(str(items[0].msg['to']), 'anne@example.com')
378        self.assertEqual(
379            str(items[0].msg['subject']),
380            'Your subscription for Test mailing list has been disabled')
381        self.assertEqual(anne.total_warnings_sent, 2)
382        self.assertEqual(anne.last_warning_sent.day, now().day)
383
384    def test_events_bounce_already_disabled(self):
385        # A bounce received for an already disabled member is only logged.
386        anne = self._subscribe_and_add_bounce_event(
387            'anne@example.com', subscribe=False, create=False)
388        self._mlist.bounce_score_threshold = 3
389        anne.bounce_score = 3
390        anne.preferences.delivery_status = DeliveryStatus.by_bounces
391        anne.total_warnings_sent = 1
392        anne.last_warning_sent = now() - timedelta(days=3)
393        mark = LogFileMark('mailman.bounce')
394        self._runner.run()
395        get_queue_messages('virgin', expected_count=0)
396        self.assertEqual(anne.total_warnings_sent, 1)
397        self.assertIn(
398           'Residual bounce received for member anne@example.com '
399           'on list test.example.com.', mark.read()
400           )
401
402    def test_events_membership_removal(self):
403        self._mlist.bounce_notify_owner_on_removal = True
404        self._mlist.bounce_you_are_disabled_warnings = 3
405        self._mlist.bounce_you_are_disabled_warnings_interval = timedelta(
406            days=2)
407
408        anne = self._mlist.members.get_member(self._anne.email)
409        anne.preferences.delivery_status = DeliveryStatus.by_bounces
410        anne.total_warnings_sent = 3
411        # Don't remove immediately.
412        anne.last_warning_sent = now() - timedelta(days=2)
413
414        self._runner.run()
415        items = get_queue_messages('virgin', expected_count=2)
416        if items[0].msg['to'] == 'test-owner@example.com':
417            owner_notif, user_notif = items
418        else:
419            user_notif, owner_notif = items
420        self.assertEqual(user_notif.msg['to'], 'anne@example.com')
421        self.assertEqual(
422            user_notif.msg['subject'],
423            'You have been unsubscribed from the Test mailing list')
424
425        self.assertEqual(
426            str(owner_notif.msg['subject']),
427            'anne@example.com unsubscribed from Test mailing '
428            'list due to bounces')
429        # The membership should no longer exist.
430        self.assertIsNone(
431            self._mlist.members.get_member(self._anne.email))
432
433    def test_events_membership_removal_no_warnings(self):
434        self._mlist.bounce_notify_owner_on_removal = True
435        self._mlist.bounce_you_are_disabled_warnings = 0
436        self._mlist.bounce_you_are_disabled_warnings_interval = timedelta(
437            days=2)
438
439        anne = self._mlist.members.get_member(self._anne.email)
440        anne.preferences.delivery_status = DeliveryStatus.by_bounces
441        anne.total_warnings_sent = 0
442        # Remove immediately.
443        anne.last_warning_sent = now()
444
445        self._runner.run()
446        items = get_queue_messages('virgin', expected_count=2)
447        if items[0].msg['to'] == 'test-owner@example.com':
448            owner_notif, user_notif = items
449        else:
450            user_notif, owner_notif = items
451        self.assertEqual(user_notif.msg['to'], 'anne@example.com')
452        self.assertEqual(
453            user_notif.msg['subject'],
454            'You have been unsubscribed from the Test mailing list')
455
456        self.assertEqual(
457            str(owner_notif.msg['subject']),
458            'anne@example.com unsubscribed from Test mailing '
459            'list due to bounces')
460        # The membership should no longer exist.
461        self.assertIsNone(
462            self._mlist.members.get_member(self._anne.email))
463
464    def test_events_membership_removal_not_immediate(self):
465        self._mlist.bounce_notify_owner_on_removal = True
466        self._mlist.bounce_you_are_disabled_warnings = 3
467        self._mlist.bounce_you_are_disabled_warnings_interval = timedelta(
468            days=2)
469
470        anne = self._mlist.members.get_member(self._anne.email)
471        anne.preferences.delivery_status = DeliveryStatus.by_bounces
472        anne.total_warnings_sent = 3
473        # Don't remove immediately.
474        anne.last_warning_sent = now()
475
476        self._runner.run()
477        get_queue_messages('virgin', expected_count=0)
478        # The membership should still exist.
479        self.assertIsNotNone(
480            self._mlist.members.get_member(self._anne.email))
481