1#!/usr/bin/perl 2 3# 4# Create Mailman list(s) from Majordomo list configuration files. 5# 6# main() is fully commented and provides a good outline of this script. 7# 8# LIMITATIONS: 9# - Archives are not currently handled. 10# - A few Majordomo configuration values are ignored (documented in the comment 11# above the getMailmanConfig() function) because they are either inactive, 12# system/constant settings, or don't tranlsate into Mailman. 13# - This script was tested against Majordomo 1.94.4/5 and Mailman 2.1.14-1. 14# Different versions of Majordomo or Mailman may not work with this script. 15# However, some legacy settings for Majordomo are handled. 16# 17# REQUIREMENTS/ASSUMPTIONS: 18# - Mailman is installed so that its bin/* scripts can be called. 19# - Majordomo has all of its list configurations in a single, local directory. 20# - Majordomo's aliases file exists locally. 21# - $DOMO_INACTIVITY_LIMIT set to zero or the output of Majordomo's 22# consistency_check 23# command is stored locally. 24# - Run this script as root. 25# 26# BEFORE RUNNING THIS SCRIPT: 27# - Change the "ENVIRONMENT-SPECIFIC VALUES" below to match your system. 28# - It is recommended to run this script with the --stats option first to get 29# a sense of your data. Fields with many 'other' or 'no value' values, or 30# fields that don't get imported (e.g. message_headers) that have many 31# 'has value' values probably need to be considered more closely. 32# 33# TODO: IMPORT ARCHIVE OPTION 34# - One solution: get all archives inot a 'Unix mbox' file and then use the 35# bin/arch tool. bin/cleanarch can sanity check the mbox before running 36# bin/arch. 37# 38 39use strict; 40use warnings; 41 42use Getopt::Long; 43use Log::Handler; 44use File::Temp qw(tempfile); 45use Email::Simple; 46use Email::Sender::Simple qw(try_to_sendmail); 47use Data::Dump qw(dump); 48 49 50#----------------------- ENVIRONMENT-SPECIFIC VALUES --------------------------# 51my $DOMO_PATH = '/opt/majordomo'; 52my $DOMO_LIST_DIR = "$DOMO_PATH/lists"; 53my $MM_PATH = '/usr/local/mailman'; 54my $DOMO_ALIASES = "$MM_PATH/majordomo/aliases"; 55my $DOMO_CHECK_CONSISTENCY = "$MM_PATH/majordomo/check_consistency.txt"; 56my $BOUNCED_OWNERS = "/opt/mailman-2.1.14-1/uo/majordomo/" . 57 "email_addresses_that_bounced.txt"; 58my $TMP_DIR = '/tmp'; 59# Only import lists that have been active in the last N days. 60my $DOMO_INACTIVITY_LIMIT = 548; # Optional. 548 days = 18 months. 61# If set, overwrite Majordomo's "resend_host" and thus Mailman's "host_name". 62my $NEW_HOSTNAME = ''; # Optional 63my $LANGUAGE = 'en'; # Preferred language for all Mailman lists 64my $MAX_MSG_SIZE = 20000; # In KB. Used for the Mailman config. 65#------------------------------------------------------------------------------# 66 67# 68# Global constants 69# 70my $MM_LIST_DIR = "$MM_PATH/lists"; 71my $MM_LIST_LISTS = "$MM_PATH/bin/list_lists"; 72my $MM_NEWLIST = "$MM_PATH/bin/newlist"; 73my $MM_CONFIGLIST = "$MM_PATH/bin/config_list"; 74my $MM_ADDMEMBERS = "$MM_PATH/bin/add_members"; 75my $MM_CHECK_PERMS = "$MM_PATH/bin/check_perms"; 76my $SCRIPT_NAME = $0 =~ /\/?(\b\w+\b)\.pl$/ ? $1 : '<script name>'; 77my $LOG_FILE = "$TMP_DIR/$SCRIPT_NAME.log"; 78 79# 80# Global namespace 81# 82my $log = Log::Handler->new(); 83my $domoStats = { 84 'general_stats' => { 85 'Lists without owner in aliases' => 0, 86 'Total lists' => 0 87 } 88}; 89 90 91# 92# Main program execution 93# 94main(); 95 96 97# 98# Functions 99# 100sub main { 101 # Verify the environment. 102 preImportChecks(); 103 # Get the CLI options. 104 my $opts = getCLIOpts(); 105 # Set up logging. 106 addLogHandler($opts); 107 # Get lists to import. 108 my @domoListNames = getDomoListsToImport($opts); 109 # Get a mapping of list names to list owners. 110 my $listOwnerMap = getListToOwnerMap(); 111 # Get lists that already exist in Mailman. 112 my %existingLists = getExistingLists(); 113 # Get all lists that have been inactive longer than the specified limit. 114 my $inactiveLists = getInactiveLists(); 115 116 if ($opts->{'email_notify'}) { 117 # Email Majordomo list owners about the upcoming migration to Mailman. 118 sendCustomEmailsToDomoOwners($listOwnerMap, $inactiveLists, 1); 119 exit; 120 } 121 122 if ($opts->{'email_test'}) { 123 # Email Majordomo list owners about the upcoming migration to Mailman. 124 sendCustomEmailsToDomoOwners($listOwnerMap, $inactiveLists); 125 exit; 126 } 127 128 # Iterate through every list, collecting stats, and possibly importing. 129 for my $listName (@domoListNames) { 130 $log->info("Starting list $listName..."); 131 132 if (not exists $listOwnerMap->{$listName}) { 133 $log->warning("List $listName has no owner in aliases. Skipping..."); 134 $domoStats->{'general_stats'}->{'Lists without owner in aliases'} += 1; 135 next; 136 } 137 138 # Don't import lists that are pending deletion. This is a University of 139 # Oregon customization, but it should be harmless for others. 140 if (-e "$DOMO_LIST_DIR/$listName.pendel") { 141 $log->info("List $listName has a .pendel file. Skipping..."); 142 $domoStats->{'general_stats'}->{'Lists pending deletion'} += 1; 143 next; 144 } 145 146 # Don't import the list if it's been inactive beyond the specified limit. 147 if (exists $inactiveLists->{$listName}) { 148 $log->notice("List $listName has been inactive for " . 149 "$inactiveLists->{$listName} days. Skipping..."); 150 $domoStats->{'general_stats'}->{'Lists inactive for more than ' . 151 "$DOMO_INACTIVITY_LIMIT days"} += 1; 152 next; 153 } 154 155 # Get the Majordomo configuration. 156 $log->info("Getting Majordomo config for list $listName..."); 157 my %domoConfig = getDomoConfig($listName, $listOwnerMap); 158 if (not %domoConfig) { 159 $log->debug("No config returned by getDomoConfig(). Skipping..."); 160 next; 161 } 162 163 # Add this list to the stats data structure and then skip if --stats. 164 $log->debug("Appending this list's data into the stats structure..."); 165 appendDomoStats(%domoConfig); 166 if ($opts->{'stats'}) { 167 next; 168 } 169 170 # Don't import this list if it already exists in Mailman. 171 if (exists $existingLists{$listName}) { 172 $log->notice("$listName already exists. Skipping..."); 173 next; 174 } 175 176 # Get a hash of Mailman config values mapped from Majordomo. 177 my %mmConfig = getMailmanConfig(%domoConfig); 178 179 # Create the template configuration file for this list. 180 my $mmConfigFilePath = 181 createMailmanConfigFile($domoConfig{'approve_passwd'}, %mmConfig); 182 183 # Create the Mailman list. 184 createMailmanList($listName, 185 $mmConfig{'owner'}, 186 $domoConfig{'admin_passwd'}); 187 188 # Apply the configuration template to the list. 189 configureMailmanList($listName, $mmConfigFilePath); 190 191 # Add members to the list. 192 if ($opts->{'subscribers'}) { 193 # Create files of digest and non-digest member emails to be used 194 # when calling Mailman's bin/config_list. 195 my $membersFilePath = 196 createMailmanMembersList($domoConfig{'subscribers'}); 197 my $digestMembersFilePath = 198 createMailmanMembersList($domoConfig{'digest_subscribers'}); 199 200 # Subscribe the member emails to the Mailman list. 201 if ($membersFilePath or $digestMembersFilePath) { 202 addMembersToMailmanList($listName, 203 $membersFilePath, 204 $digestMembersFilePath); 205 } 206 } 207 } 208 209 # Output stats if requested or else the resuls of the import. 210 if ($opts->{'stats'}) { 211 printStats(); 212 } else { 213 cleanUp(); 214 print "Import complete! " . 215 "$domoStats->{'general_stats'}->{'Total lists'} lists imported.\n"; 216 } 217 print "Complete log: $LOG_FILE\n"; 218} 219 220# Environment/system/setting checks before modifying state 221sub preImportChecks { 222 # User "mailman" is required because of various calls to the mailman/bin/* 223 # scripts. 224 my $script_executor = $ENV{LOGNAME} || $ENV{USER} || getpwuid($<); 225 if ($script_executor !~ /^(mailman|root)$/) { 226 die "Error: Please run this script as user mailman (or root).\n"; 227 } 228 # Check that the Majordomo and Mailman list directories exist. 229 for my $dir ($DOMO_LIST_DIR, $MM_LIST_DIR) { 230 if (not $dir or not -d $dir) { 231 die "Error: Lists directory does not exist: $dir\n"; 232 } 233 } 234 # Check that the Mailman binaries exist. 235 for my $bin ($MM_LIST_LISTS, $MM_NEWLIST, $MM_CONFIGLIST, $MM_ADDMEMBERS) { 236 if (not $bin or not -e $bin) { 237 die "Error: Mailman binary doesn't exist: $bin\n"; 238 } 239 } 240 # Check the path of $DOMO_CHECK_CONSISTENCY. 241 if ($DOMO_CHECK_CONSISTENCY and not -e $DOMO_CHECK_CONSISTENCY) { 242 die "Error: \$DOMO_CHECK_CONSISTENCY does not exist: " . 243 "$DOMO_CHECK_CONSISTENCY\nCorrect the value or set it to ''.\n"; 244 } 245 # If $DOMO_CHECK_CONSISTENCY exists, then so must $DOMO_ACTIVITY_LIMIT. 246 if ($DOMO_CHECK_CONSISTENCY and not $DOMO_INACTIVITY_LIMIT) { 247 die "Error: \$DOMO_CHECK_CONSISTENCY exists but " . 248 "\$DOMO_INACTIVITY_LIMIT does not.\nPlease set this value.\n"; 249 } 250 # $LANGUAGE must be present and should only contain a-z. 251 if (not $LANGUAGE or $LANGUAGE !~ /[a-z]+/i) { 252 die "Error: \$LANGUAGE was not set or invalid: $LANGUAGE\n"; 253 } 254 # $MAX_MSG_SIZE must be present and should really be above a minimum size. 255 if (not $MAX_MSG_SIZE or $MAX_MSG_SIZE < 5) { 256 die "Error: \$MAX_MSG_SIZE was not set or less than 5KB: $MAX_MSG_SIZE\n"; 257 } 258} 259 260# Get CLI options. 261sub getCLIOpts { 262 my $opts = {}; 263 GetOptions('list=s' => \$opts->{'list'}, 264 'all' => \$opts->{'all'}, 265 'subscribers' => \$opts->{'subscribers'}, 266 'stats' => \$opts->{'stats'}, 267 'email-notify' => \$opts->{'email_notify'}, 268 'email-test' => \$opts->{'email_test'}, 269 'loglevel=s' => \$opts->{'loglevel'}, 270 'help' => \$opts->{'help'}, 271 ); 272 273 if ($opts->{'help'}) { 274 help(); 275 } 276 277 # If --all or --list was not specified, get stats for all lists. 278 if (($opts->{'stats'} or $opts->{'email_notify'} or $opts->{'email_test'}) 279 and not ($opts->{'all'} or $opts->{'list'})) { 280 $opts->{'all'} = 1; 281 } 282 283 # Validate --loglevel. 284 if ($opts->{'loglevel'}) { 285 if ($opts->{'loglevel'} !~ /^(debug|info|notice|warning|error)$/) { 286 print "ERROR: invalid --loglevel value: $opts->{'loglevel'}\n"; 287 help(); 288 } 289 } else { 290 $opts->{'loglevel'} = 'error'; 291 } 292 293 return $opts; 294} 295 296sub addLogHandler { 297 my $opts = shift; 298 $log->add(file => { filename => $LOG_FILE, 299 #mode => 'trunc', 300 maxlevel => 'debug', 301 minlevel => 'emerg' }, 302 screen => { log_to => 'STDOUT', 303 maxlevel => $opts->{'loglevel'}, 304 minlevel => 'emerg' } 305 ); 306} 307 308# Return an array of all list names in Majordomo that have a <list>.config file. 309sub getDomoListsToImport { 310 my $opts = shift; 311 my @domoListNames = (); 312 # If only one list was specified, validate and return that list. 313 if ($opts->{'list'}) { 314 my $listConfig = $opts->{'list'} . '.config'; 315 my $listPath = "$DOMO_LIST_DIR/$listConfig"; 316 if (not -e $listPath) { 317 $log->die(crit => "Majordomo list config does not exist: $listPath"); 318 } 319 @domoListNames = ($opts->{'list'}); 320 # If all lists were requested, grab all list names from .config files in the 321 # $DOMO_LIST_DIR, ignoring digest lists (i.e. *-digest.config files). 322 } elsif ($opts->{'all'}) { 323 $log->info("Collecting all Majordomo list config files..."); 324 opendir DIR, $DOMO_LIST_DIR or 325 $log->die("Can't open dir $DOMO_LIST_DIR: $!"); 326 # Don't get digest lists because these are not separate lists in Mailman. 327 @domoListNames = grep !/\-digest$/, 328 map { /^([a-zA-Z0-9_\-]+)\.config$/ } 329 readdir DIR; 330 closedir DIR; 331 if (not @domoListNames) { 332 $log->die(crit => "No Majordomo configs found in $DOMO_LIST_DIR"); 333 } 334 # If we're here, --list or --all was not used, so exit. 335 } else { 336 $log->error("--list=NAME or --all was not used. Nothing to do."); 337 help(); 338 } 339 return @domoListNames; 340} 341 342# Find all list owners from aliases and create a map of lists to aliases. 343sub getListToOwnerMap { 344 my %listOwnerMap = (); 345 open ALIASES, $DOMO_ALIASES or $log->die("Can't open $DOMO_ALIASES: $!"); 346 while (my $line = <ALIASES>) { 347 if ($line =~ /^owner\-([^:]+):\s*(.*)$/) { 348 my ($listName, $listOwners) = (strip($1), strip($2)); 349 # Some lists in Majordomo's aliases file have the same email listed 350 # twice as the list owner (e.g. womenlaw). 351 # Converting the listed owners into a hash prevents duplicates. 352 my %ownersHash = map { $_ => 1 } split /,/, $listOwners; 353 $listOwnerMap{$listName} = 354 "'" . (join "', '", keys %ownersHash) . "'"; 355 } 356 } 357 close ALIASES or $log->die("Can't close $DOMO_ALIASES: $!"); 358 359 return \%listOwnerMap; 360} 361 362# Return a hash of all lists that already exist in Mailman. 363sub getExistingLists { 364 my $cmd = "$MM_LIST_LISTS -b"; 365 $log->debug("Calling $cmd..."); 366 my %lists = map { strip($_) => 1 } `$cmd` or $log->die("Command failed: $!"); 367 return %lists; 368} 369 370# By parsing the output of Majordomo's "consistency_check" command, get a list 371# of all Majordomo lists inactive beyond the specified $DOMO_INACTIVITY_LIMIT. 372sub getInactiveLists { 373 my %lists = (); 374 if ($DOMO_CHECK_CONSISTENCY) { 375 for my $line (split /\n/, getFileTxt($DOMO_CHECK_CONSISTENCY)) { 376 377 if ($line =~ /(\S+) has been inactive for (\d+) days/) { 378 if ($2 >= $DOMO_INACTIVITY_LIMIT) { 379 $lists{$1} = $2; 380 } 381 } 382 } 383 } 384 385 return \%lists; 386} 387 388sub getBouncedOwners { 389 my @bouncedOwners = (); 390 for my $line (split /\n/, getFileTxt($BOUNCED_OWNERS)) { 391 if ($line =~ /Failed to send mail to (\S+)\./) { 392 push @bouncedOwners, $1; 393 } 394 } 395 396 return @bouncedOwners; 397} 398 399sub getOwnerListMap { 400 my ($listOwnerMap, $inactiveLists) = @_; 401 my $ownerListMap = {}; 402 403 for my $list (keys %$listOwnerMap) { 404 for my $owner (split /,/, $listOwnerMap->{$list}) { 405 my $type = exists $inactiveLists->{$list} ? 'inactive' : 'active'; 406 my $owner = strip($owner, { full => 1 }); 407 push @{$ownerListMap->{$owner}->{$type}}, $list; 408 } 409 } 410 411 return $ownerListMap; 412} 413 414# Send an individualized email to each Majordomo list owner that details 415# the upcoming migration procedure and their active and inactive lists. 416sub sendCustomEmailsToDomoOwners { 417 my ($listOwnerMap, $inactiveLists, $notify) = @_; 418 $notify = 0 if not defined $notify; 419 420 print "Send email to all Majordomo list owners?\n"; 421 my $answer = ''; 422 while ($answer !~ /^(yes|no)$/) { 423 print "Please type 'yes' or 'no': "; 424 $answer = <>; 425 chomp $answer; 426 } 427 428 if ($answer ne "yes") { 429 print "No emails were sent.\n"; 430 return; 431 } 432 433 # Create the body of the email for each owner 434 my $ownerListMap = getOwnerListMap($listOwnerMap, $inactiveLists); 435 for my $owner (keys %$ownerListMap) { 436 my $body = $notify ? getEmailNotifyText('top') : getEmailTestText('top'); 437 438 # Append the active and inactive lists section to the email body 439 my $listsTxt = ''; 440 for my $listType (qw(active inactive)) { 441 if ($ownerListMap->{$owner}->{$listType} and 442 @{$ownerListMap->{$owner}->{$listType}}) { 443 $listsTxt .= $notify ? 444 getEmailNotifyText($listType, 445 $ownerListMap->{$owner}->{$listType}, 446 $inactiveLists) : 447 getEmailTestText($listType, 448 $ownerListMap->{$owner}->{$listType}, 449 $inactiveLists); 450 } 451 } 452 453 if (not $listsTxt) { 454 $log->warning("No active or inactive lists found for owner $owner"); 455 next; 456 } 457 458 $body .= $listsTxt; 459 $body .= $notify ? getEmailNotifyText('bottom') : 460 getEmailTestText('bottom'); 461 462 # Create and send the email. 463 my $email = Email::Simple->create( 464 header => [ 'From' => 'listmaster@lists-test.uoregon.edu', 465 'To' => $owner, 466 'Reply-To' => 'listmaster@lists.uoregon.edu', 467 'Subject' => 'Mailman (Majordomo replacement) ready ' . 468 'for testing' ], 469 body => $body, 470 ); 471 $log->debug("Sending notification email to $owner..."); 472 my $result = try_to_sendmail($email); 473 if (not defined $result) { 474 $log->notice("Failed to send mail to $owner."); 475 } 476 } 477} 478 479# Return various sections, as requested, of a notification email for 480# current Majordomo list owners. 481sub getEmailNotifyText { 482 my ($section, $lists, $inactiveLists) = @_; 483 484 if ($section eq 'top') { 485 return <<EOF; 486Greetings; 487 488You are receiving this email because you have been identified as the owner 489of one or more Majordomo lists. We're in the process of informing all 490current Majordomo list owners of an impending change in the list 491processing software used by the University. 492 493Majordomo, the current list processing software in use by the University, 494hasn't been updated by its developers since January of 2000. The versions 495of the underlying software as well as the operating system required by 496Majordomo are no longer supported or maintained. As a result, we are 497implementing a new list processing software for the UO campus. 498 499Mailman (http://www.gnu.org/software/mailman/index.html) has been 500identified as a robust replacement for Majordomo. Mailman has a Web-based 501interface as well as a set of commands issued through email. There is a 502wide array of configuration options for the individual lists. Connections 503to the Mailman services through the Web are secured with SSL, and 504authentication into the server will be secured with LDAP, tied to the 505DuckID information. 506 507Information Services is currently in the process of configuring the 508Mailman server and testing the list operations. Our plan is to have the 509service ready for use very soon, with existing lists showing activity 510within the last 18 months being migrated seamlessly onto the new service. 511 512EOF 513 } elsif ($section eq 'active') { 514 my $listTxt = join "\n", sort @$lists; 515 return <<EOF; 516Our records indicate that you are the owner of the following lists: 517 518$listTxt 519 520These lists have had activity in the last 18 months, indicating that they 521may still be active. Unless you contact us, these active lists will be 522automatically migrated to Mailman. 523 524If you no longer want to keep any of the lists shown above, please send 525email to email indicating the name(s) of the list, and 526that you'd like the list(s) ignored during our migration procedure. You 527could also go to the Web page http://lists.uoregon.edu/delap.html and delete the 528list prior to the migration. 529 530EOF 531 } elsif ($section eq 'inactive') { 532 my $listTxt = join "\n", 533 map { my $years = int($inactiveLists->{$_} / 365); 534 $years = $years == 1 ? "$years year" : "$years years"; 535 my $days = $inactiveLists->{$_} % 365; 536 $days = $days == 1 ? "$days day" : "$days days"; 537 "$_ (inactive $years, $days)" } sort @$lists; 538 return <<EOF; 539The following lists, for which you are the owner, have had no activity in 540the last 18 months: 541 542$listTxt 543 544Lists with no activity within the last 18 months will not be migrated, and 545will be unavailable after the migration to Mailman. 546 547If you want to retain one of the lists detailed above, you can either send 548a message to the list (which will update its activity), or send an email 549to email with an explanation as to why this 550inactive list should be migrated and maintained. 551 552EOF 553 } elsif ($section eq 'bottom') { 554 return <<EOF; 555There should be only brief interruptions in service during the migration. 556The change to Mailman will be scheduled to take place during the standard 557Tuesday maintenance period between 5am and 7am. If your list has been 558migrated to Mailman, you and your subscribers will be able to send posts 559to your list with the same address, and those messages will be distributed 560in the same way that Majordomo would. 561 562Documentation on using Mailman is in development, and will be available to 563facilitate the list migration. We will be making a test environment 564available to the list owners prior to the migration to the production 565Mailman server, so that you can see the settings for your lists and become 566accustomed to the new Web interface. 567 568If you have any questions or concerns, please contact email, 569or email me directly at email. 570 } 571 572 return ''; 573} 574 575sub getEmailTestText { 576 my ($section, $lists, $inactiveLists) = @_; 577 578 if ($section eq 'top') { 579 return <<EOF; 580Greetings; 581 582Information Services is in the process of configuring Mailman as the UO list 583processing software, replacing Majordomo. You're receiving this email because 584you have been identified as the owner of one or more lists, either active or 585inactive. 586 587The final migration from Majordomo to Mailman is scheduled to take place on 588April 27th, 2012. At that time, your active lists will be migrated to the new 589service, and lists.uoregon.edu will begin using Mailman instead of Majordomo 590for its list processing. This move should be seamless with no downtime. 591 592We have set up a test server and would greatly appreciate your feedback. 593Specifically, we'd like to know: 594- how well the Majordomo settings for your lists were translated into Mailman 595- whether Mailman works as you'd expect and, if not, what didn't work 596- any other thoughts, concerns, responses, etc you have 597 598The test server is at https://domain. You will encounter a 599browser warning about an "unsigned certificate". This is expected: please 600add an exception for the site. Once Mailman goes live, however, there will 601be a signed certificate in place and this issue will not exist. 602 603All of your Majordomo list settings were migrated onto the Mailman test server 604except for subscribers. We did not import subscribers so that you could test 605the email functionality of Mailman for your list. To do so, subscribe your 606email address to your list, and perhaps a few other people who might be 607interested in testing Mailman (fellow moderators?), and send off a few emails. 608 609The links to your specific Mailman lists are listed below. Please feel free 610to change anything you want: it is a test server, after all, and we'd love 611for you to thoroughly test Mailman. Anything you change, however, will not 612show up on the production server when it goes live. The production server 613will create your Majordomo lists in Mailman based on the Majordomo settings 614for your list on April 27, 2012. 615 616The password for these lists will be the same as the adminstrative password 617used on the Majordomo list. If you forgot that password, email 618email and we will reply with the owner information for 619your list, including passwords and a subset of the current list configuration. 620EOF 621 } elsif ($section eq 'active') { 622 my $listTxt = join "\n", 623 map { "https://domain/mailman/listinfo/$_" } 624 sort @$lists; 625 return <<EOF; 626 627---- 628 629Here is the list of your active lists: 630 631$listTxt 632 633EOF 634 } elsif ($section eq 'inactive') { 635 my $listTxt = join "\n", 636 map { my $years = int($inactiveLists->{$_} / 365); 637 $years = $years == 1 ? "$years year" : "$years years"; 638 my $days = $inactiveLists->{$_} % 365; 639 $days = $days == 1 ? "$days day" : "$days days"; 640 "$_ (inactive $years, $days)" } sort @$lists; 641 return <<EOF; 642---- 643 644Here is the list of your inactive lists (no activity in the last 18 months): 645 646$listTxt 647 648Lists with no activity within the last 18 months will not be migrated, and 649will be unavailable after the migration to Mailman. 650 651If you want to retain one of the lists detailed above, you can either send 652a message to the list (which will update its activity), or send an email 653to email with an explanation as to why this 654inactive list should be migrated and maintained. 655 656---- 657 658EOF 659 } elsif ($section eq 'bottom') { 660 return <<EOF; 661We are continuing to develop documentation for Mailman, which you can access 662through the IT Web site. That documentation is still a work in progress; 663the final versions have yet to be published. User and administrator guides for 664Mailman can be found at http://it.uoregon.edu/mailman. We'll provide further 665UO-specific documentation as it becomes available. 666 667It is our intention that this transition proceed smoothly, with a minimum of 668disruption in services for you. Please report any problems or concerns to 669email, and we will address your concerns as quickly as 670possible. 671EOF 672 } 673} 674 675 676# Parse all text configuration files for a Majordomo list and return that 677# info in the %config hash with fields as keys and field values 678# as hash values. Example: {'subscribe_policy' => 'open'}. 679# Note that every text configuration file is parsed, not just <listname>.config. 680# So, for example, <listname>.post is added to %config as 681# {'restrict_post_emails': 'email1,email2,...'}. The following files 682# are examined: listname, listname.info, listname.intro, listname.config, 683# listname.post, listname.moderator, listname-digest, listname-digest.config, 684# listname.closed, listname.private, listname.auto, listname.passwd, 685# listname.strip, and listname.hidden. 686sub getDomoConfig { 687 my ($listName, $listOwnerMap) = @_; 688 my $listPath = "$DOMO_LIST_DIR/$listName"; 689 # All of these values come from <listname>.config unless a comment 690 # says otherwise. 691 my %config = ( 692 'admin_passwd' => '', # from the list config or passwd files 693 'administrivia' => '', 694 'advertise' => '', 695 'aliases_owner' => $listOwnerMap->{$listName}, 696 'announcements' => 'yes', 697 'approve_passwd' => '', 698 'description' => "$listName Mailing List", 699 'digest_subscribers' => '', 700 'get_access' => '', 701 'index_access' => '', 702 'info_access' => '', 703 'intro_access' => '', 704 'info' => '', # from the <listname>.info file 705 'intro' => '', # from the <listname>.intro file 706 'list_name' => $listName, 707 'message_footer' => '', 708 'message_footer_digest' => '', # from the <listname>-digest.config file 709 'message_fronter' => '', 710 'message_fronter_digest' => '', # from the <listname>-digest.config file 711 'message_headers' => '', 712 'moderate' => 'no', 713 'moderator' => '', 714 'moderators' => '', # from the emails in <listname>.moderator 715 'noadvertise' => '', 716 'post_access' => '', 717 'reply_to' => '', 718 'resend_host' => '', 719 'restrict_post' => '', 720 'restrict_post_emails' => '', # from the emails in restrict_post files 721 'strip' => '', 722 'subject_prefix' => '', 723 'subscribe_policy' => '', 724 'subscribers' => '', # from the emails in the <listname> file 725 'taboo_body' => '', 726 'taboo_headers' => '', 727 'unsubscribe_policy' => '', 728 'welcome' => 'yes', 729 'which_access' => '', 730 'who_access' => '', 731 ); 732 733 # Parse <listname>.config for list configuration options 734 my $configPath = "$listPath.config"; 735 open CONFIG, $configPath or $log->die("Can't open $configPath: $!"); 736 while (my $line = <CONFIG>) { 737 # Pull out the config field and its value. 738 if ($line =~ /^\s*([^#\s]+)\s*=\s*(.+)\s*$/) { 739 my ($option, $value) = ($1, $2); 740 $config{$option} = $value; 741 # Some config option values span multiple lines. 742 } elsif ($line =~ /^\s*([^#\s]+)\s*<<\s*(\b\S+\b)\s*$/) { 743 my ($option, $heredocTag) = ($1, $2); 744 while (my $line = <CONFIG>) { 745 last if $line =~ /^$heredocTag\s*$/; 746 $config{$option} .= $line; 747 } 748 } 749 } 750 751 # Parse <listname> for subscribers 752 my @subscribers = getFileEmails($listPath); 753 $config{'subscribers'} = join ',', @subscribers; 754 755 # Parse <listname>-digest for digest subscribers 756 my @digestSubscribers = getFileEmails("$listPath-digest"); 757 $config{'digest_subscribers'} = join ',', @digestSubscribers; 758 759 # Parse filenames listed in restrict_post for emails with post permissions 760 if ($config{'restrict_post'}) { 761 my @postPermissions = (); 762 for my $restrictPostFilename (split /[\s:]/, $config{'restrict_post'}) { 763 # No need to be explicit in Mailman about letting members post to the 764 # list because it is the default behavior. 765 if ($restrictPostFilename eq $listName) { 766 next; 767 } 768 769 # If posting is restricted to another list, use Mailman's shortcut 770 # reference of '@<listname>' instead of adding those emails 771 # individually. 772 if ($restrictPostFilename !~ /\-digest$/ and 773 exists $listOwnerMap->{$restrictPostFilename}) { 774 $log->info("Adding '\@$restrictPostFilename' shortcut list " . 775 "reference to restrict_post_emails..."); 776 push @postPermissions, "\@$restrictPostFilename"; 777 } else { 778 my @emails = getFileEmails("$DOMO_LIST_DIR/$restrictPostFilename"); 779 if (@emails) { 780 push @postPermissions, @emails; 781 } 782 } 783 } 784 $config{'restrict_post_emails'} = 785 "'" . (join "','", @postPermissions) . "'"; 786 } else { 787 # If restrict_post is empty, then anyone can post to it. This can be set 788 # in Mailman with a regex that matches everything. Mailman requires 789 # regexes in the accept_these_nonmembers field to begin with a caret. 790 # TODO: test this setting! 791 $config{'restrict_post_emails'} = "'^.*'"; 792 } 793 794 # Parse <listname>.moderator for moderators 795 my @moderators = getFileEmails("$listPath.moderator"); 796 if (defined $config{'moderator'} and $config{'moderator'} and 797 not $config{'moderator'} ~~ @moderators) { 798 push @moderators, $config{'moderator'}; 799 } 800 if (@moderators) { 801 $config{'moderators'} = "'" . (join "', '", @moderators) . "'"; 802 } 803 804 $config{'info'} = getFileTxt("$listPath.info", ('skip_dates' => 1)); 805 $config{'intro'} = getFileTxt("$listPath.intro", ('skip_dates' => 1)); 806 807 # 808 # Overwrite some config values if legacy files/settings exist. 809 # 810 if (-e "$listPath.private") { 811 for my $option (qw/get_access index_access which_access who_access/) { 812 $config{$option} = "closed"; 813 } 814 } 815 816 if (-e "$listPath.closed") { 817 $config{'subscribe_policy'} = "closed"; 818 $config{'unsubscribe_policy'} = "closed"; 819 if (-e "$listPath.auto") { 820 $log->warning("$listName.auto and $listName.closed exist. Setting " . 821 "the list as closed."); 822 } 823 } elsif (-e "$listPath.auto") { 824 $config{'subscribe_policy'} = "auto"; 825 $config{'unsubscribe_policy'} = "auto"; 826 } 827 828 $config{'strip'} = 1 if -e "$listPath.strip"; 829 $config{'noadvertise'} = '/.*/' if -e "$listPath.hidden"; 830 831 # Password precedence: 832 # (1) $DOMO_LIST_DIR/$config{(admin|approve)_passwd} file 833 # (2) The (admin|approve)_passwd value itself in <listname>.config 834 # (3) <listname>.passwd file 835 for my $passwdOption (qw/admin_passwd approve_passwd/) { 836 my $passwdFile = "$DOMO_LIST_DIR/$config{$passwdOption}"; 837 if (-e $passwdFile) { 838 $config{$passwdOption} = getFileTxt($passwdFile, 839 ('first_line_only' => 1)); 840 } elsif (not $config{$passwdOption} and -e "$listPath.passwd") { 841 $config{$passwdOption} = getFileTxt("$listPath.passwd", 842 ('first_line_only' => 1)); 843 } 844 } 845 846 # admin_password is required to non-interactively run Mailman's bin/newlist. 847 if (not $config{'admin_passwd'}) { 848 $log->warning("No admin_passwd or $listName.passwd file. Skipping..."); 849 $domoStats->{'general_stats'}->{'Lists without admin_passwd'} += 1; 850 return; 851 } 852 853 # Munge Majordomo text that references Majordomo-specific commands, etc 854 for my $field (qw/info intro message_footer message_fronter 855 message_headers/) { 856 # Convert references from the majordomo@ admin email to the Mailman one. 857 $config{$field} =~ s/majordomo\@/$listName-request\@/mgi; 858 # Change owner-<listname>@... to <listname>-owner@... 859 $config{$field} =~ s/owner-$listName\@/$listName-owner\@/mgi; 860 # Remove the mailing list name from the Majordomo commands. 861 $config{$field} =~ 862 s/(subscribe|unsubscribe)\s*$listName(\-digest)?/$1/mgi; 863 # Remove the "end" on a single line listed after all Majordomo commands. 864 $config{$field} =~ s/(\s+)end(\s+|\s*\n|$)/$1 $2/mgi; 865 } 866 $log->debug("Majordomo config for list $listName:\n" . dump(\%config) . 867 "\n"); 868 869 return %config; 870} 871 872# Create and return a hash of Mailman configuration options and values. 873# The hash is initialized to the most common default values and then modified 874# based on the Majordomo list configuration. 875# 876# **** The following Majordomo configuration options are not imported. **** 877# archive_dir - dead option in Majordomo, so safe to ignore. 878# comments - notes section for list admin; safe to ignore. 879# date_info - puts a datetime header at top of info file; very safe to ignore. 880# date_intro - puts a datetime header at top of intro file; very safe to ignore. 881# debug - only useful for the Majordomo admin; very safe to ignore. 882# digest_* - digest options don't match up well in Mailman; semi-safe to ignore. 883# get_access - who can retrieve files from archives. Safe to ignore because the 884# "index_access" is consulted to determine archive access. 885# message_headers - email headers. Not in Mailman, and probably important for 886# some lists. 887# mungedomain - not recommended to be set in Majordomo, so safe to ignore. 888# precedence - mailman handles precedence internally, so safe to ignore. 889# purge_received - majordomo recommends not setting this, so safe to ignore. 890# resend_host - system setting that never changes, so safe to ignore. 891# sender - system setting that never changes, so safe to ignore. 892# strip - whether to strip everything but the email address. Not in Mailman. 893# taboo_body - message body filtering; roughly used below. Not in Mailman. 894# which_access - get the lists an email is subscribed to. Not in Mailman. 895# 896# Additionally, the message_fronter and message_footer of the digest version of 897# the list is not imported because of the difficulty in parsing and translating 898# that text and because Mailman by default includes its own message header. 899sub getMailmanConfig { 900 my (%domoConfig) = @_; 901 my $listName = $domoConfig{'list_name'}; 902 903 # Set default Mailman list configuration values 904 my %mmConfig = ( 905 'accept_these_nonmembers' => "[$domoConfig{'restrict_post_emails'}]", 906 'admin_immed_notify' => 1, 907 'admin_notify_mchanges' => 908 $domoConfig{'announcements'} =~ /y/i ? 1 : 0, 909 'administrivia' => 'True', 910 'advertised' => 1, 911 'anonymous_list' => 'False', 912 'from_is_list' => 'False', 913 # NOTE: some may wish to map some Majordomo setting, such as index_access 914 # to Mailman's archive. As is, all archiving is turned off for imported 915 # lists. 916 'archive' => 'False', # This doesn't change below 917 'archive_private' => 918 $domoConfig{'index_access'} =~ /open/ ? 0 : 1, 919 'bounce_processing' => 1, # This doesn't change below 920 'default_member_moderation' => 0, 921 'description' => "'''$domoConfig{'description'}'''", 922 'digest_header' => 923 "'''$domoConfig{'message_fronter_digest'}'''", 924 'digest_footer' => 925 "'''$domoConfig{'message_footer_digest'}'''", 926 'digest_is_default' => 'False', 927 'digestable' => 'True', 928 'filter_content' => 'False', 929 'forward_auto_discards' => 1, # This doesn't change below 930 'generic_nonmember_action' => 3, # 3: discard 931 'goodbye_msg' => '', 932 'header_filter_rules' => '[]', 933 'host_name' => "'$NEW_HOSTNAME'", 934 'info' => '', 935 'max_message_size' => 100, # KB (40 is Mailman's default) 936 'moderator' => "[$domoConfig{'moderators'}]", 937 'msg_header' => '', 938 'msg_footer' => '', 939 'nondigestable' => 1, 940 'obscure_addresses' => 1, # This doesn't change below 941 'owner' => "[$domoConfig{'aliases_owner'}]", 942 'personalize' => 0, 943 'preferred_language' => "'$LANGUAGE'", 944 'private_roster' => 2, # 0: open; 1: members; 2: admin 945 'real_name' => "'$listName'", 946 'reply_goes_to_list' => 0, # 0: poster, 1: list, 2: address 947 'reply_to_address' => '', 948 'respond_to_post_requests' => 1, 949 'send_reminders' => 'False', 950 'send_welcome_msg' => $domoConfig{'welcome'} =~ /y/i ? 1 : 0, 951 'subject_prefix' => "'$domoConfig{'subject_prefix'}'", 952 'subscribe_policy' => 3, # 1: confirm; 3: confirm and approval 953 'unsubscribe_policy' => 0, # 0: can unsubscribe; 1: can not 954 'welcome_msg' => '', 955 ); 956 957 # Majordomo's "who_access" => Mailman's "private_roster" 958 if ($domoConfig{'who_access'} =~ /list/i) { 959 $mmConfig{'private_roster'} = 1; 960 } elsif ($domoConfig{'who_access'} =~ /open/i) { 961 $mmConfig{'private_roster'} = 0; 962 } 963 964 # Majordomo's "administrivia" => Mailman's "administrivia" 965 if ($domoConfig{'administrivia'} =~ /no/i) { 966 $mmConfig{'administrivia'} = 'False'; 967 } 968 969 # Majordomo's "resend_host" => Mailman's "host_name" 970 if ($domoConfig{'resend_host'} and not $NEW_HOSTNAME) { 971 $mmConfig{'host_name'} = "'$domoConfig{'resend_host'}'"; 972 } 973 974 # Majordomo's "message_fronter" => Mailman's "msg_header" 975 # Majordomo's "message_footer" => Mailman's "msg_footer" 976 for my $fieldsArray (['message_fronter', 'msg_header'], 977 ['message_footer', 'msg_footer']) { 978 my ($domoOption, $mmOption) = @$fieldsArray; 979 if ($domoConfig{$domoOption}) { 980 $mmConfig{$mmOption} = "'''$domoConfig{$domoOption}'''"; 981 } 982 } 983 984 # Majordomo's "maxlength" (# chars) => Mailman's "max_message_size" (KB) 985 if ($domoConfig{'maxlength'}) { 986 my $charsInOneKB = 500; # 1KB = 500 characters 987 $mmConfig{'max_message_size'} = $domoConfig{'maxlength'} / $charsInOneKB; 988 if ($mmConfig{'max_message_size'} > $MAX_MSG_SIZE) { 989 $mmConfig{'max_message_size'} = $MAX_MSG_SIZE; 990 } 991 } 992 993 # Majordomo's "taboo_headers" => Mailman's "header_filter_rules" 994 if ($domoConfig{'taboo_headers'}) { 995 my @rules = split /\n/, $domoConfig{'taboo_headers'}; 996 $mmConfig{'header_filter_rules'} = "[('" . (join '\r\n', @rules) . 997 "', 3, False)]"; 998 } 999 1000 # Majordomo's "taboo_body" and "taboo_headers" => Mailman's "filter_content" 1001 # 1002 # NOTE: This is a very rough mapping. What we're doing here is turning on 1003 # default content filtering in Mailman if there was *any* header or body 1004 # filtering in Majordomo. The regexes in the taboo_* fields in Majordomo are 1005 # too varied for pattern-parsing. This blunt method is a paranoid, 1006 # conservative approach. 1007 if ($domoConfig{'taboo_headers'} or $domoConfig{'taboo_body'}) { 1008 $mmConfig{'filter_content'} = "True"; 1009 } 1010 1011 # Majordomo's "subscribe_policy" => Mailman's "subscribe_policy" 1012 if ($domoConfig{'subscribe_policy'} =~ /open(\+confirm)?/i) { 1013 $mmConfig{'subscribe_policy'} = 1; 1014 } 1015 1016 # Majordomo's "unsubscribe_policy" => Mailman's "unsubscribe_policy" 1017 if ($domoConfig{'unsubscribe_policy'} =~ /closed/i) { 1018 $mmConfig{'unsubscribe_policy'} = 1; 1019 } 1020 1021 # Majordomo's "moderate" => Mailman's "default_member_moderation" 1022 if ($domoConfig{'moderate'} =~ /yes/i) { 1023 $mmConfig{'default_member_moderation'} = 1; 1024 } 1025 1026 # Majordomo's "advertise", "noadvertise", "intro_access", "info_access", 1027 # and "subscribe_policy" => Mailman's "advertised" 1028 # 1029 # NOTE: '(no)?advertise' in Majordomo contain regexes, which would be 1030 # difficult to parse accurately, so just be extra safe here by considering 1031 # the existence of anything in '(no)?advertise' to mean that the list should 1032 # be hidden. Also hide the list if intro_access, info_access, or 1033 # subscribe_policy are at all locked down. This is an appropriate setting 1034 # for organizations with sensitive data policies (e.g. SOX, FERPA, etc), but 1035 # not ideal for open organizations with their Mailman instance hidden from 1036 # the Internet. 1037 if ($domoConfig{'advertise'} or 1038 $domoConfig{'noadvertise'} or 1039 $domoConfig{'intro_access'} =~ /(list|closed)/i or 1040 $domoConfig{'info_access'} =~ /(list|closed)/i or 1041 $domoConfig{'subscribe_policy'} =~ /close/i) { 1042 $mmConfig{'advertised'} = 0; 1043 } 1044 1045 # Majordomo's "reply_to" => Mailman's "reply_goes_to_list" and 1046 # "reply_to_address" 1047 if ($domoConfig{'reply_to'} =~ /\$sender/i) { 1048 $mmConfig{'reply_goes_to_list'} = 0; 1049 } elsif ($domoConfig{'reply_to'} =~ /(\$list|$listName@)/i) { 1050 $domoConfig{'reply_to'} =~ /\$list/i or 1051 $mmConfig{'reply_goes_to_list'} = 1; 1052 } elsif ($domoConfig{'reply_to'} =~ /\s*[^@]+@[^@]+\s*/) { 1053 $mmConfig{'reply_goes_to_list'} = 2; 1054 $mmConfig{'reply_to_address'} = "'" . strip($domoConfig{'reply_to'}) . 1055 "'"; 1056 } 1057 1058 # Majordomo's "subject_prefix" => Mailman's "subject_prefix" 1059 if ($mmConfig{'subject_prefix'}) { 1060 $mmConfig{'subject_prefix'} =~ s/\$list/$listName/i; 1061 } 1062 1063 # Majordomo's "welcome to the list" message for new subscribers exists in 1064 # <listname>.intro or <listname>.info. <listname>.intro takes precedence 1065 # so this is checked first. If it doesn't exist, <listname>.info is used, 1066 # if it exists. 1067 if ($domoConfig{'intro'}) { 1068 $mmConfig{'welcome_msg'} = "'''$domoConfig{'intro'}'''"; 1069 } elsif ($domoConfig{'info'}) { 1070 $mmConfig{'welcome_msg'} = "'''$domoConfig{'info'}'''"; 1071 } 1072 1073 if ($domoConfig{'message_headers'}) { 1074 $log->warning("List $listName has message_headers set in Majordomo, " . 1075 "but they can't be imported."); 1076 } 1077 1078 $log->debug("Mailman config for list $listName: " . dump(\%mmConfig) . 1079 "\n"); 1080 1081 return %mmConfig; 1082} 1083 1084# Call $MM_NEWLIST to create a new Mailman list. 1085sub createMailmanList { 1086 my ($listName, $ownerEmail, $listPassword) = @_; 1087 # Any additional owners will be added when configureMailmanList() is called. 1088 $ownerEmail = (split /,/, $ownerEmail)[0]; 1089 $ownerEmail =~ s/['"\[\]]//g; 1090 my $cmd = "$MM_NEWLIST -l $LANGUAGE -q $listName $ownerEmail " . 1091 "'$listPassword' >> $LOG_FILE 2>&1"; 1092 $log->debug("Calling $cmd..."); 1093 system($cmd) == 0 or $log->die("Command failed: $!"); 1094} 1095 1096# Create a temporary file that contains a list's configuration values that have 1097# been translated from Majordomo into Mailman's list template format. 1098sub createMailmanConfigFile { 1099 my ($domoApprovePasswd, %mmConfig) = @_; 1100 my $configFh = File::Temp->new(SUFFIX => ".mm.cfg", UNLINK => 0); 1101 print $configFh "# coding: utf-8\n"; 1102 for my $cfgField (sort keys %mmConfig) { 1103 if ($mmConfig{$cfgField}) { 1104 print $configFh "$cfgField = $mmConfig{$cfgField}\n"; 1105 } 1106 } 1107 1108 # The moderator password must be set with Python instead of a config setting. 1109 if ($domoApprovePasswd) { 1110 print $configFh <<END; 1111 1112from Mailman.Utils import sha_new 1113mlist.mod_password = sha_new('$domoApprovePasswd').hexdigest() 1114END 1115 } 1116 return $configFh->filename; 1117} 1118 1119# Call $MM_CONFIGLIST to apply the just-created Mailman configuration options 1120# file to a Mailman list. 1121sub configureMailmanList { 1122 my ($listName, $configFilePath) = @_; 1123 # Redirect STDOUT/STDERR to the log file to hide the "attribute 'sha_new' 1124 # ignored" message. This message occurs because Python code to set the 1125 # moderator password exists at the bottom of the Mailman config file that 1126 # this script created. 1127 my $cmd = "$MM_CONFIGLIST -i $configFilePath $listName >> $LOG_FILE 2>&1"; 1128 $log->debug("Calling $cmd..."); 1129 system($cmd) == 0 or $log->die("Command failed: $!"); 1130} 1131 1132# Create a temporary file with a single email address per line to be used later 1133# on to subscribe these emails to a Mailman list. 1134sub createMailmanMembersList { 1135 my $membersString = shift; 1136 if ($membersString) { 1137 my $membersFh = File::Temp->new(SUFFIX => ".mm.members", UNLINK => 0); 1138 for my $memberEmail (split ',', $membersString) { 1139 print $membersFh strip($memberEmail) . "\n"; 1140 } 1141 return $membersFh->filename; 1142 } 1143 return ''; 1144} 1145 1146# Call $MM_ADDMEMBERS to subscribe email addresses to a Mailman list. 1147sub addMembersToMailmanList { 1148 my ($listName, $membersFilePath, $digestMembersFilePath) = @_; 1149 my $cmd = "$MM_ADDMEMBERS -w n -a n"; 1150 $cmd .= " -r $membersFilePath" if $membersFilePath; 1151 $cmd .= " -d $digestMembersFilePath" if $digestMembersFilePath; 1152 $cmd .= " $listName >> $LOG_FILE"; 1153 $log->debug("Calling $cmd..."); 1154 system($cmd) == 0 or $log->die("Command failed: $!"); 1155} 1156 1157# Take the passed in list's Majordomo config and append many of its values to 1158# the global $domoStats hash ref. 1159sub appendDomoStats { 1160 my (%domoConfig) = @_; 1161 my $listName = $domoConfig{'list_name'}; 1162 # Some fields are uninteresting or part of other fields (e.g. 'moderator' 1163 # is in 'moderators'). 1164 my @skipFields = qw/archive_dir comments date_info date_intro date_post 1165 list_name message_footer_digest message_fronter_digest 1166 moderator restrict_post_emails/; 1167 # Some fields are highly variable, so collapse them into 'has value' and 1168 # 'no value' values. 1169 my @yesNoFields = qw/admin_passwd advertise aliases_owner approve_passwd 1170 bounce_text description info intro message_footer 1171 message_fronter message_headers noadvertise 1172 taboo_body taboo_headers/; 1173 1174 # Run through every Majordomo configuration field and count values. 1175 for my $field (keys %domoConfig) { 1176 # Standardize/tidy up the fields and their values. 1177 $field = lc($field); 1178 my $value = lc(strip($domoConfig{$field})); 1179 1180 # Skip unimportant fields 1181 next if $field ~~ @skipFields; 1182 1183 # Handle all of the highly variable fields by collapsing their values into 1184 # one of two choices: does the field have a value or not? 1185 if ($field ~~ @yesNoFields) { 1186 $value = $value ? 'has value' : 'no value'; 1187 $domoStats->{$field}->{$value} += 1; 1188 next; 1189 } 1190 1191 # Some fields are moderately variable, but they are important to know 1192 # about. Handle those fields individually to provide more granular data. 1193 if ($field eq 'restrict_post') { 1194 for my $restriction (split /[\s:]/, $domoConfig{'restrict_post'}) { 1195 if (strip($restriction) eq $listName) { 1196 $domoStats->{$field}->{'list'} += 1; 1197 } elsif (strip($restriction) eq "$listName-digest") { 1198 $domoStats->{$field}->{'list-digest'} += 1; 1199 } elsif (strip($restriction) eq "$listName.post") { 1200 $domoStats->{$field}->{'list.post'} += 1; 1201 } else { 1202 $domoStats->{$field}->{'other'} += 1; 1203 } 1204 } 1205 next; 1206 } elsif ($field eq 'sender') { 1207 if (not $value) { 1208 $value = 'no value'; 1209 } elsif ($value =~ /^owner-$listName/i) { 1210 $value = 'owner-list'; 1211 } elsif ($value =~ /^owner-/ and $value !~ /@/) { 1212 $value = 'owner of another list'; 1213 } else { 1214 $value = 'other'; 1215 } 1216 } elsif ($field eq 'subject_prefix' or $field eq 'digest_name') { 1217 if (not $value) { 1218 $value = 'no value'; 1219 } elsif ($value =~ /^\s*(\$list|\W*$listName\W*)/i) { 1220 $value = 'list'; 1221 } else { 1222 $value = 'other'; 1223 } 1224 } elsif ($field eq 'reply_to') { 1225 if (not $value) { 1226 $value = 'no value'; 1227 } elsif ($value =~ /\$(list|sender)/i) { 1228 $value = $1; 1229 } elsif ($value =~ /^$listName(-list)?/) { 1230 $value = 'list'; 1231 } else { 1232 $value = 'other'; 1233 } 1234 } elsif ($field =~ /^(subscribers|digest_subscribers|moderators)/) { 1235 my $count = () = split /,/, $value, -1; 1236 if (not $count) { 1237 $domoStats->{$field}->{'0'} += 1; 1238 next; 1239 } 1240 $domoStats->{$field}->{'2000+'} += 1 if $count >= 2000; 1241 $domoStats->{$field}->{'1000-2000'} += 1 if $count >= 1000 and $count < 2000; 1242 $domoStats->{$field}->{'500-999'} += 1 if $count >= 500 and $count < 1000; 1243 $domoStats->{$field}->{'101-500'} += 1 if $count <= 500 and 1244 $count > 100; 1245 $domoStats->{$field}->{'26-100'} += 1 if $count <= 100 and $count > 25; 1246 $domoStats->{$field}->{'6-25'} += 1 if $count <= 25 and $count > 5; 1247 $domoStats->{$field}->{'1-5'} += 1 if $count < 5; 1248 $domoStats->{'general_stats'}->{'Total subscribers'} += $count; 1249 next; 1250 } elsif ($field eq 'maxlength') { 1251 $value = 0 if not $value; 1252 $domoStats->{$field}->{'1,000,000+'} += 1 if $value > 1000000; 1253 $domoStats->{$field}->{'100,000-999,999'} += 1 if $value >= 100000 and 1254 $value < 1000000; 1255 $domoStats->{$field}->{'50,000-99,999'} += 1 if $value >= 50000 and 1256 $value < 100000; 1257 $domoStats->{$field}->{'0-49,999'} += 1 if $value < 50000; 1258 next; 1259 } 1260 1261 $value = 'no value' if not $value; 1262 $domoStats->{$field}->{$value} += 1; 1263 1264 } 1265 $domoStats->{'general_stats'}->{'Total lists'} += 1; 1266} 1267 1268sub printStats { 1269 if (not %$domoStats) { 1270 print "No stats were generated.\n"; 1271 return; 1272 } 1273 1274 print <<END; 1275+-----------------+ 1276| Majordomo Stats | 1277+-----------------+ 1278Total Lists: $domoStats->{'general_stats'}->{'Total lists'} 1279 1280Config Options 1281-------------- 1282END 1283 for my $option (sort keys %$domoStats) { 1284 next if $option eq 'general_stats'; 1285 print " * $option: "; 1286 for my $value (sort { $domoStats->{$option}->{$b} <=> 1287 $domoStats->{$option}->{$a} } 1288 keys %{$domoStats->{$option}}) { 1289 print "$value ($domoStats->{$option}->{$value}), "; 1290 } 1291 print "\n"; 1292 } 1293 1294 if ($domoStats and 1295 exists $domoStats->{'general_stats'} and 1296 defined $domoStats->{'general_stats'} and 1297 keys %{$domoStats->{'general_stats'}}) { 1298 print "\nImportant Information" . 1299 "\n---------------------\n"; 1300 for my $field (sort keys %{$domoStats->{'general_stats'}}) { 1301 next if $field eq 'Total lists'; 1302 print " * $field: $domoStats->{'general_stats'}->{$field}\n"; 1303 } 1304 print "\n"; 1305 } 1306} 1307 1308# 1309# Utility functions 1310# 1311 1312# Print the help menu to screen and exit. 1313sub help { 1314 print <<EOF 1315 1316 Usage: $SCRIPT_NAME [--loglevel=LEVEL] [--stats] 1317 [--list=NAME] [--all] [--subscribers] 1318 1319 Examples: 1320 # Print stats about your Majordomo lists 1321 ./$SCRIPT_NAME --stats 1322 1323 # Verbosely import the 'law-school' mailing list and its subscribers 1324 ./$SCRIPT_NAME --loglevel=debug --list=law-school --subscribers 1325 1326 # Import all Majordomo lists and their subscribers 1327 ./$SCRIPT_NAME --all --subscribers 1328 1329 Options: 1330 --all Import all Majordomo lists 1331 --list=NAME Import a single list 1332 --subscribers Import subscribers in addition to creating the list 1333 --stats Print some stats about your Majordomo lists 1334 --email-notify Email Majordomo list owners about the upcoming migration 1335 --email-test Email Majordomo list owners with link to test server 1336 --loglevel Set STDOUT log level. 1337 Possible values: debug, info, notice, warning, error 1338 Note: All log entries still get written to the log file. 1339 --help Print this screen 1340 1341EOF 1342; 1343 exit; 1344} 1345 1346# Slurp a file into a variable, optionally skipping the Majordomo datetime 1347# header or only grabbing the first line. 1348sub getFileTxt { 1349 my $filePath = shift; 1350 my %args = ( 1351 'skip_dates' => 0, 1352 'first_line_only' => 0, 1353 @_ 1354 ); 1355 my $fileTxt = ''; 1356 if (-e $filePath) { 1357 open FILE, $filePath or $log->die("Can't open $filePath: $!"); 1358 while (my $line = <FILE>) { 1359 next if $args{'skip_dates'} and $line =~ /^\s*\[Last updated on:/; 1360 $fileTxt .= $line; 1361 if ($args{'first_line_only'}) { 1362 $fileTxt = strip($fileTxt); 1363 last; 1364 } 1365 } 1366 close FILE or $log->die("Can't close $filePath: $!"); 1367 } 1368 return $fileTxt; 1369} 1370 1371# Given a text file, extract one email per line. Return these emails in a hash. 1372sub getFileEmails { 1373 my $filePath = shift; 1374 my %emails = (); 1375 if (-e $filePath) { 1376 open FILE, $filePath or $log->die("Can't open $filePath: $!"); 1377 while (my $line = <FILE>) { 1378 if ($line =~ /^#/) { 1379 next; 1380 } 1381 if ($line =~ /\b([A-Za-z0-9._-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4})\b/) { 1382 $emails{lc($1)} = 1; 1383 } 1384 } 1385 close FILE or $log->die("Can't close $filePath: $!"); 1386 } 1387 return keys %emails; 1388} 1389 1390# Undo any side-effects of this script (e.g. temporary files, permissions, etc). 1391sub cleanUp { 1392 # Delete temporary config files. 1393 $log->debug("Deleting $TMP_DIR/*.mm.* files..."); 1394 opendir DIR, $TMP_DIR or $log->die("Can't open dir $TMP_DIR: $!"); 1395 my @tmpFiles = grep { /\.mm\.[a-z]+$/i } readdir DIR; 1396 closedir DIR or $log->die("Can't close dir $TMP_DIR: $!"); 1397 for my $tmpFile (@tmpFiles) { 1398 unlink "$TMP_DIR/$tmpFile"; 1399 } 1400 1401 # Fix permissions of newly created Mailman list files. 1402 my $cmd = "$MM_CHECK_PERMS -f >> $LOG_FILE 2>&1"; 1403 $log->debug("Calling $cmd..."); 1404 system($cmd) == 0 or $log->die("Command failed: $!"); 1405 $log->debug("Calling $cmd again for good measure..."); 1406 system($cmd) == 0 or $log->die("Command failed: $!"); 1407} 1408 1409# Strip whitespace from the beginning and end of a string. 1410sub strip { 1411 my $string = shift || ''; 1412 my $args = shift; 1413 $string =~ s/(^\s*|\s*$)//g; 1414 if (exists $args->{'full'}) { 1415 $string =~ s/(^['"]+|['"]+$)//g; 1416 } 1417 return $string; 1418} 1419