1# -*- indent-tabs-mode: nil; -*-
2# vim:ft=perl:et:sw=4
3# $Id$
4
5# Sympa - SYsteme de Multi-Postage Automatique
6#
7# Copyright (c) 1997, 1998, 1999 Institut Pasteur & Christophe Wolfhugel
8# Copyright (c) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
9# 2006, 2007, 2008, 2009, 2010, 2011 Comite Reseau des Universites
10# Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017 GIP RENATER
11# Copyright 2017, 2019 The Sympa Community. See the AUTHORS.md file at
12# the top-level directory of this distribution and at
13# <https://github.com/sympa-community/sympa.git>.
14#
15# This program is free software; you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation; either version 2 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program.  If not, see <http://www.gnu.org/licenses/>.
27
28package Sympa::Spindle::ProcessIncoming;
29
30use strict;
31use warnings;
32use File::Copy qw();
33
34use Sympa;
35use Conf;
36use Sympa::Language;
37use Sympa::List;
38use Sympa::Log;
39use Sympa::Mailer;
40use Sympa::Process;
41use Sympa::Spool::Listmaster;
42use Sympa::Tools::Data;
43
44use base qw(Sympa::Spindle);
45
46my $language = Sympa::Language->instance;
47my $log      = Sympa::Log->instance;
48my $mailer   = Sympa::Mailer->instance;
49
50use constant _distaff => 'Sympa::Spool::Incoming';
51
52sub _init {
53    my $self  = shift;
54    my $state = shift;
55
56    if ($state == 0) {
57        $self->{_loop_info}     = {};
58        $self->{_msgid}         = {};
59        $self->{_msgid_cleanup} = time;
60    } elsif ($state == 1) {
61        # Process grouped notifications.
62        Sympa::Spool::Listmaster->instance->flush;
63
64        # Cleanup in-memory msgid table, only in a while.
65        if (time > $self->{_msgid_cleanup} +
66            $Conf::Conf{'msgid_table_cleanup_frequency'}) {
67            $self->_clean_msgid_table();
68            $self->{_msgid_cleanup} = time;
69        }
70
71        # Clear "quiet" flag set by AuthorizeMessage spindle.
72        delete $self->{quiet};
73    }
74
75    1;
76}
77
78sub _on_success {
79    my $self    = shift;
80    my $message = shift;
81    my $handle  = shift;
82
83    if ($self->{keepcopy}) {
84        unless (
85            File::Copy::copy(
86                $self->{distaff}->{directory} . '/' . $handle->basename,
87                $self->{keepcopy} . '/' . $handle->basename
88            )
89        ) {
90            $log->syslog(
91                'notice',
92                'Could not rename %s/%s to %s/%s: %m',
93                $self->{distaff}->{directory},
94                $handle->basename,
95                $self->{keepcopy},
96                $handle->basename
97            );
98        }
99    }
100
101    $self->SUPER::_on_success($message, $handle);
102}
103
104# Old name: process_message() in sympa_msg.pl.
105sub _twist {
106    my $self    = shift;
107    my $message = shift;
108
109    unless (defined $message->{'message_id'}
110        and length $message->{'message_id'}) {
111        $log->syslog('err', 'Message %s has no message ID', $message);
112        $log->db_log(
113            #'robot'        => $robot,
114            #'list'         => $listname,
115            'action'       => 'process_message',
116            'parameters'   => $message->get_id,
117            'target_email' => "",
118            'msg_id'       => "",
119            'status'       => 'error',
120            'error_type'   => 'no_message_id',
121            'user_email'   => $message->{'sender'}
122        );
123        return undef;
124    }
125
126    my $msg_id = $message->{message_id};
127
128    $language->set_lang($self->{lang}, $Conf::Conf{'lang'}, 'en');
129
130    # Compatibility: Message with checksum by Sympa <=6.2a.40
131    # They should be migrated.
132    if ($message and $message->{checksum}) {
133        $log->syslog('err',
134            '%s: Message with old format.  Run upgrade_send_spool.pl',
135            $message);
136        return 0;    # Skip
137    }
138
139    $log->syslog(
140        'notice',
141        'Processing %s; envelope_sender=%s; message_id=%s; sender=%s',
142        $message,
143        $message->{envelope_sender},
144        $message->{message_id},
145        $message->{sender}
146    );
147
148    my $robot;
149    my $listname;
150
151    if (ref $message->{context} eq 'Sympa::List') {
152        $robot = $message->{context}->{'domain'};
153    } elsif ($message->{context} and $message->{context} ne '*') {
154        $robot = $message->{context};
155    } else {
156        # Older "sympa" alias may not have "@domain" in argument of queue
157        # program.
158        $robot = $Conf::Conf{'domain'};
159    }
160    $listname = $message->{'listname'};
161
162    ## Ignoring messages with no sender
163    my $sender = $message->{'sender'};
164    unless ($message->{'md5_check'} or $sender) {
165        $log->syslog('err', 'No sender found in message %s', $message);
166        $log->db_log(
167            'robot'        => $robot,
168            'list'         => $listname,
169            'action'       => 'process_message',
170            'parameters'   => "",
171            'target_email' => "",
172            'msg_id'       => $msg_id,
173            'status'       => 'error',
174            'error_type'   => 'no_sender',
175            'user_email'   => $sender
176        );
177        return undef;
178    }
179
180    # Unknown robot.
181    unless ($message->{'md5_check'} or Conf::valid_robot($robot)) {
182        $log->syslog('err', 'Robot %s does not exist', $robot);
183        Sympa::send_dsn('*', $message, {}, '5.1.2');
184        $log->db_log(
185            'robot'        => $robot,
186            'list'         => $listname,
187            'action'       => 'process_message',
188            'parameters'   => "",
189            'target_email' => "",
190            'msg_id'       => $msg_id,
191            'status'       => 'error',
192            'error_type'   => 'unknown_robot',
193            'user_email'   => $sender
194        );
195        return undef;
196    }
197
198    $language->set_lang(Conf::get_robot_conf($robot, 'lang'));
199
200    # Load spam status.
201    $message->check_spam_status;
202    # Check DKIM signatures.
203    $message->check_dkim_signature;
204    # Check ARC seals
205    $message->check_arc_chain;
206    # Check S/MIME signature.
207    $message->check_smime_signature;
208    # Decrypt message.  On success, check nested S/MIME signature.
209    if ($message->smime_decrypt and not $message->{'smime_signed'}) {
210        $message->check_smime_signature;
211    }
212
213    # *** Now message content may be altered. ***
214
215    # Enable SMTP logging if required.
216    $mailer->{log_smtp} = $self->{log_smtp}
217        || Sympa::Tools::Data::smart_eq(
218        Conf::get_robot_conf($robot, 'log_smtp'), 'on');
219    # Setting log_level using conf unless it is set by calling option.
220    $log->{level} =
221        (defined $self->{log_level})
222        ? $self->{log_level}
223        : Conf::get_robot_conf($robot, 'log_level');
224
225    ## Strip of the initial X-Sympa-To and X-Sympa-Checksum internal headers
226    delete $message->{'rcpt'};
227    delete $message->{'checksum'};
228
229    my $list =
230        (ref $message->{context} eq 'Sympa::List')
231        ? $message->{context}
232        : undef;
233
234    my $list_address;
235    if ($message->{'listtype'} and $message->{'listtype'} eq 'sympaowner') {
236        # Discard messages for sympa-request address to avoid loop caused by
237        # misconfiguration.
238        $log->syslog('err',
239            'Don\'t forward sympa-request to Sympa. Check configuration of MTA'
240        );
241        return undef;
242    } elsif ($message->{'listtype'}
243        and $message->{'listtype'} eq 'listmaster') {
244        $list_address = Sympa::get_address($robot, 'listmaster');
245    } elsif ($message->{'listtype'} and $message->{'listtype'} eq 'sympa') {
246        $list_address = Sympa::get_address($robot);
247    } else {
248        unless (ref $list eq 'Sympa::List') {
249            $log->syslog('err', 'List %s does not exist', $listname);
250            Sympa::send_dsn($message->{context} || '*', $message, {},
251                '5.1.1');
252            $log->db_log(
253                'robot'        => $robot,
254                'list'         => $listname,
255                'action'       => 'process_message',
256                'parameters'   => "",
257                'target_email' => "",
258                'msg_id'       => $msg_id,
259                'status'       => 'error',
260                'error_type'   => 'unknown_list',
261                'user_email'   => $sender
262            );
263            return undef;
264        }
265        $list_address = Sympa::get_address($list, $message->{listtype})
266            || Sympa::get_address($list);
267    }
268
269    ## Loop prevention
270    if (ref $list eq 'Sympa::List'
271        and Sympa::Tools::Data::smart_eq(
272            $list->{'admin'}{'reject_mail_from_automates_feature'}, 'on'
273        )
274    ) {
275        my $conf_loop_prevention_regex;
276        $conf_loop_prevention_regex =
277            $list->{'admin'}{'loop_prevention_regex'};
278        $conf_loop_prevention_regex ||=
279            Conf::get_robot_conf($robot, 'loop_prevention_regex');
280        if ($sender =~ /^($conf_loop_prevention_regex)(\@|$)/mi) {
281            $log->syslog(
282                'err',
283                'Ignoring message which would cause a loop, sent by %s; matches loop_prevention_regex',
284                $sender
285            );
286            return undef;
287        }
288
289        ## Ignore messages that would cause a loop
290        ## Content-Identifier: Auto-replied is generated by some non standard
291        ## X400 mailer
292        if (grep {/Auto-replied/i} $message->get_header('Content-Identifier')
293            or grep {/Auto Reply to/i}
294            $message->get_header('X400-Content-Identifier')
295            or grep { !/^no$/i } $message->get_header('Auto-Submitted')) {
296            $log->syslog('err',
297                "Ignoring message which would cause a loop; message appears to be an auto-reply"
298            );
299            return undef;
300        }
301    }
302
303    # Loop prevention.
304    foreach my $loop ($message->get_header('X-Loop')) {
305        $log->syslog('debug3', 'X-Loop: %s', $loop);
306        if ($loop and $loop eq $list_address) {
307            $log->syslog('err',
308                'Ignoring message which would cause a loop (X-Loop: %s)',
309                $loop);
310            return undef;
311        }
312    }
313
314    # Anti-virus
315    my $rc =
316        $message->check_virus_infection(debug => $self->{debug_virus_check});
317    if ($rc) {
318        my $antivirus_notify =
319            Conf::get_robot_conf($robot, 'antivirus_notify') || 'none';
320        if ($antivirus_notify eq 'sender') {
321            Sympa::send_file(
322                $robot,
323                'your_infected_msg',
324                $sender,
325                {   'virus_name'     => $rc,
326                    'recipient'      => $list_address,
327                    'sender'         => $message->{sender},
328                    'lang'           => Conf::get_robot_conf($robot, 'lang'),
329                    'auto_submitted' => 'auto-replied'
330                }
331            );
332        } elsif ($antivirus_notify eq 'delivery_status') {
333            Sympa::send_dsn(
334                $message->{context},
335                $message,
336                {   'virus_name' => $rc,
337                    'recipient'  => $list_address,
338                    'sender'     => $message->{sender}
339                },
340                '5.7.0'
341            );
342        }
343        $log->syslog('notice',
344            "Message for %s from %s ignored, virus %s found",
345            $list_address, $sender, $rc);
346        $log->db_log(
347            'robot'        => $robot,
348            'list'         => $listname,
349            'action'       => 'process_message',
350            'parameters'   => "",
351            'target_email' => "",
352            'msg_id'       => $msg_id,
353            'status'       => 'error',
354            'error_type'   => 'virus',
355            'user_email'   => $sender
356        );
357        return undef;
358    } elsif (!defined($rc)) {
359        Sympa::send_notify_to_listmaster(
360            $robot,
361            'antivirus_failed',
362            [   sprintf
363                    "Could not scan message %s; The message has been saved as BAD.",
364                $message->get_id
365            ]
366        );
367
368        return undef;
369    }
370
371    # Route messages to appropriate handlers.
372    if (    $message->{listtype}
373        and $message->{listtype} eq 'owner'
374        and $message->{'decoded_subject'}
375        and $message->{'decoded_subject'} =~
376        /\A\s*(subscribe|unsubscribe)(\s*$listname)?\s*\z/i) {
377        # Simulate Smartlist behaviour with command in subject.
378        $message->{listtype} = lc $1;
379    }
380    return [$self->_splicing_to($message)];
381}
382
383# Private subroutines.
384
385# Cleanup the msgid_table every 'msgid_table_cleanup_frequency' seconds.
386# Removes all entries older than 'msgid_table_cleanup_ttl' seconds.
387# Old name: clean_msgid_table() in sympa_msg.pl.
388sub _clean_msgid_table {
389    my $self = shift;
390
391    foreach my $rcpt (keys %{$self->{_msgid}}) {
392        foreach my $msgid (keys %{$self->{_msgid}{$rcpt}}) {
393            if (time > $self->{_msgid}{$rcpt}{$msgid} +
394                $Conf::Conf{'msgid_table_cleanup_ttl'}) {
395                delete $self->{_msgid}{$rcpt}{$msgid};
396            }
397        }
398    }
399
400    return 1;
401}
402
403sub _splicing_to {
404    my $self    = shift;
405    my $message = shift;
406
407    return {
408        editor      => 'Sympa::Spindle::DoForward',
409        listmaster  => 'Sympa::Spindle::DoForward',
410        owner       => 'Sympa::Spindle::DoForward',    # -request
411        return_path => 'Sympa::Spindle::DoForward',    # -owner
412        subscribe   => 'Sympa::Spindle::DoCommand',
413        sympa       => 'Sympa::Spindle::DoCommand',
414        unsubscribe => 'Sympa::Spindle::DoCommand',
415    }->{$message->{listtype} || ''}
416        || 'Sympa::Spindle::DoMessage';
417}
418
4191;
420__END__
421
422=encoding utf-8
423
424=head1 NAME
425
426Sympa::Spindle::ProcessIncoming - Workflow of processing incoming messages
427
428=head1 SYNOPSIS
429
430  use Sympa::Spindle::ProcessIncoming;
431
432  my $spindle = Sympa::Spindle::ProcessIncoming->new;
433  $spindle->spin;
434
435=head1 DESCRIPTION
436
437L<Sympa::Spindle::ProcessIncoming> defines workflow to process incoming
438messages.
439
440When spin() method is invoked, it reads the messages in incoming spool and
441rejects, quarantines or modifies them.
442Processing are done in the following order:
443
444=over
445
446=item *
447
448Checks if message has message ID and sender, and if not, quarantines it.
449Because such messages will be source of various troubles.
450
451=item *
452
453Checks if robot which message is bound for exists, and if not, rejects it.
454
455=item *
456
457Checks spam status, DKIM signature and S/MIME signature,
458and decrypts message if possible.
459Result of these checks are stored in message object and used in succeeding
460process.
461
462=item *
463
464If message is bound for the list, checks if the list exists, and if not,
465rejects it.
466
467=item *
468
469Loop prevention.  If loop is detected, ignores message.
470
471=item *
472
473Virus checking, if enabled by configuration.
474And if malware is detected, rejects or discards message.
475
476=item *
477
478Splices message to appropriate class according to the type of message:
479L<Sympa::Spindle::DoCommand> for command message;
480L<Sympa::Spindle::DoForward> for message bound for administrator;
481L<Sympa::Spindle::DoMessage> for ordinal post.
482
483=back
484
485Order to process messages in source spool are controlled by modification time
486of files and delivery date.
487Some messages are skipped according to these priorities
488(See L<Sympa::Spool::Incoming>):
489
490=over
491
492=item *
493
494Messages with lowest priority (C<z> or C<Z>) are skipped.
495
496=item *
497
498Messages with possibly higher priority are chosen.
499This is done by skipping messages with lower priority than those already
500found.
501
502=back
503
504=head2 Public methods
505
506See also L<Sympa::Spindle/"Public methods">.
507
508=over
509
510=item new ( [ keepcopy =E<gt> $directory ], [ lang =E<gt> $lang ],
511[ log_level =E<gt> $level ],
512[ log_smtp =E<gt> 0|1 ] )
513
514=item spin ( )
515
516new() may take following options:
517
518=over
519
520=item keepcopy =E<gt> $directory
521
522spin() keeps copy of successfully processed messages in $directory.
523
524=item lang =E<gt> $lang
525
526Overwrites lang parameter in configuration.
527
528=item log_level =E<gt> $level
529
530Overwrites log_level parameter in configuration.
531
532=item log_smtp =E<gt> 0|1
533
534Overwrites log_smtp parameter in configuration.
535
536=back
537
538=back
539
540=head2 Properties
541
542See also L<Sympa::Spindle/"Properties">.
543
544=over
545
546=item {distaff}
547
548Instance of L<Sympa::Spool::Incoming> class.
549
550=back
551
552=head1 SEE ALSO
553
554L<Sympa::Message>,
555L<Sympa::Spindle>, L<Sympa::Spindle::DoCommand>, L<Sympa::Spindle::DoForward>,
556L<Sympa::Spindle::DoMessage>,
557L<Sympa::Spool::Incoming>.
558
559=head1 HISTORY
560
561L<Sympa::Spindle::ProcessIncoming> appeared on Sympa 6.2.13.
562
563=cut
564