Great Circle Associates Majordomo-Users
(April 1995)
 

Indexed By Date: [Previous] [Next] Indexed By Thread: [Previous] [Next]

Subject: Re: large volumes?
From: dwolfe @ risc . sps . mot . com (Dave Wolfe)
Date: Fri, 21 Apr 1995 16:45:06 -0500 (CDT)
To: majordomo-users @ greatcircle . com (Majordomo user's mailing list)
Cc: majordomo-workers @ greatcircle . com (Majordomo developer's mailing list)
Reply-to: David Wolfe <david_wolfe @ risc . sps . mot . com>

[ I previously wrote: ]
> [ Marko Toivanen writes: ]
> > 
> > Responding to a previous article by Deborah A Hamilton:
> > * Also - sorting the large mailing lists by host.domain.name made
> > * our mailer more efficient.
> > 
> > So, anybody got a sort program that would sort beginning from the end of
> > each line where the most significat domain lies, and would also lock the
> > list file to prevent majordomo using while sorting?
> 
> [ re change to majordomo to sort list every subscription ] Since
> there's been some discussion about unsubscribing taking too long
> (it has to rewrite the entire list for each unsubscribe request), I
> realize that this change could also be too time and memory consuming
> to sort the list for every subscription, so decide for yourself.

Here's a stand-alone utility to lock and sort mailing lists. Obviously
not a lot of field time on this one either, so caveat user. Let me know
if you find any problems or have any suggestions.

You can run this daily from cron, for example. I recommend running it
under wrapper so it plays nicely with others, e.g.:

33 3 * * * /usr/local/majordomo/wrapper sortlist lista listb listc

---- Cut Here and feed the following to sh ----
#!/bin/sh
# This is a shell archive (produced by shar 3.49)
# To extract the files from this archive, save it to a file, remove
# everything above the "!/bin/sh" line above, and type "sh file_name".
#
# made 04/21/1995 21:18 UTC by dwolfe@miaow
# Source directory /tmp_mnt/home/dwolfe/local/src
#
# existing files will NOT be overwritten unless -c is specified
#
# This shar contains:
# length  mode       name
# ------ ---------- ------------------------------------------
#   5303 -rwxr-xr-x majordomo/contrib/sortlist
#
# ============= majordomo/contrib/sortlist ==============
if test ! -d 'majordomo'; then
    echo 'x - creating directory majordomo'
    mkdir 'majordomo'
fi
if test ! -d 'majordomo/contrib'; then
    echo 'x - creating directory majordomo/contrib'
    mkdir 'majordomo/contrib'
fi
if test -f 'majordomo/contrib/sortlist' -a X"$1" != X"-c"; then
	echo 'x - skipping majordomo/contrib/sortlist (File already exists)'
else
echo 'x - extracting majordomo/contrib/sortlist (Text)'
sed 's/^X//' << 'SHAR_EOF' > 'majordomo/contrib/sortlist' &&
X#!/usr/local/bin/perl
X
X# Copyright 1995, David A. Wolfe, Sr.  See the Majordomo license agreement
X#     for usage rights.
X#
X# Sort changed mailing lists by domain. This utility is best run by the
X# wrapper so that environment, ownership, and permissions are consistent
X# with majordomo.
X
X# Isolate our name
X$0 =~ s|.*/(.*)|$1|;
X
X# Read and execute the .cf file
X$cf = $ENV{"MAJORDOMO_CF"} || "/etc/majordomo.cf";
Xif ($ARGV[0] eq "-C") {
X    shift(@ARGV); 
X    $cf = shift(@ARGV); 
X}
Xif (! -r $cf) {
X    die("$0: $cf not readable\n");
X}
Xeval(`cat $cf`)  || die "$0: eval of $cf failed\n";
X
Xchdir($homedir) || die("$0: can't chdir($homedir): $!\n");
Xunshift(@INC, $homedir);
Xrequire "getopts.pl";
Xrequire "timelocal.pl";
Xrequire "ctime.pl";
Xrequire "shlock.pl";
Xrequire "majordomo.pl";
X
X&Getopts("f") || die("$0: Getopts() failed: $!\n");
X
Xdie <<END unless @ARGV;
X$0 sorts the specified lists in domain order if the log indicates
Xnew subscriptions.
XUsage: $0 [ -C config-file ] [ -f ] list [, list ...]
X	-C specifies an alternate configuration file
X	-f forces sorting of the specified lists
XEND
X
X%mnum = ( 'Jan', 0,
X	  'Feb', 1,
X	  'Mar', 2,
X	  'Apr', 3,
X	  'May', 4,
X	  'Jun', 5,
X	  'Jul', 6,
X	  'Aug', 7,
X	  'Sep', 8,
X	  'Oct', 9,
X	  'Nov', 10,
X	  'Dec', 11);
X$this_year = (localtime(time))[5];
X
Xchop($hostname = `hostname`);
X&set_log($log, $hostname, $0);
X
X# Validate the lists
Xforeach $list (@ARGV)
X{
X    unless (-e "$listdir/$list")
X    {
X	warn "$0: no such list as $listdir/$list\n";
X	next;
X    }
X    if (-z _)
X    {
X	warn "$0: $listdir/$list is empty\n";
X	next;
X    }
X    unless (-r _)
X    {
X	warn "$0: can't read $listdir/$list\n";
X	next;
X    }
X    push(@lists, $list);
X}
Xexit 1 unless @lists;
X
Xunless ($opt_f)
X{
X    # Build tables of the most recent subscription and sort for each
X    # list (can't assume the log is in ascending time order until queued
X    # file locking is always used on the log file)
X
X    &lopen(LOG, "", $log) || die "$0: can't open $log: $!\n";
X    while (<LOG>)
X    {
X	($mname, $mday, $hms, $cmd, $list) = (split)[0, 1, 2, 6, 7];
X	next unless ($cmd eq 'subscribe' || $cmd eq 'sort') &&
X	    grep($_ eq $list, @lists);
X
X	($hour, $min, $sec) = split(/:/, $hms);
X	$timestamp = &timelocal($sec, $min, $hour, $mday, $mnum{$mname},
X	    $this_year);
X	$timestamp = &timelocal($sec, $min, $hour, $mday, $mnum{$mname},
X	    $this_year - 1) if $timestamp > $^T; # Adjust for last year
X
X	$subscribed{$list} = $timestamp
X	    if $cmd eq 'subscribe' && $subscribed{$list} < $timestamp;
X	$sorted{$list} = $timestamp
X	    if $cmd eq 'sort' && $sorted{$list} < $timestamp;
X    }
X    &lclose(LOG);
X
X    # Has anything changed in the specified lists
X    @ARGV = @lists;
X    @lists = ();
X    foreach $list (@ARGV)
X    {
X	if ($sorted{$list} > $subscribed{$list})
X	{
X	    warn "$0: no new subscriptions in $list since last sort\n";
X	    warn "\t(use -f to force resort)\n";
X	    next;
X	}
X	if ($subscribed{$list} == (stat("$listdir/$list"))[9])
X	{
X	    warn "$0: $list hasn't changed since last subscription\n";
X	    warn "\t(use -f to force resort)\n";
X	    next;
X	}
X	push(@lists, $list);
X    }
X    exit 1 unless @lists;
X}
X
X# Sort the remaining lists
XLIST_SORT:foreach $list (@lists)
X{
X    &lopen(LIST, "", "$listdir/$list") ||
X	&abort_sort("can't open $listdir/$list");
X    $state = 1;
X
X    # Read and canonicalize the list for sorting. Each entry is stripped
X    # to the bare address (user@host[[[.net3].net2].net1]), which
X    # is standarized to net1\0net2\0net3\0host\0user for sorting.
X    # Missing nodes in the address are set to null to keep all levels
X    # consistent.
X
X    while (<LIST>)
X    {
X	$entry = $_;
X	tr/A-Z/a-z/;
X	($_) = &ParseAddrs($_);
X	($user, @nodes) = split(/[.\@]/);
X	@nodes = $#nodes ? ((reverse @nodes), ('') x (3 - $#nodes))
X			 : (('') x 3, @nodes); # just host
X	$canonical{join("\0", @nodes, $user)} = $entry;
X    }
X
X    # Create the sorted list, keeping the file mode and group of the
X    # original
X    (($mode, $uid, $gid) = (stat(LIST))[2,4,5]) ||
X	&abort_sort("can't stat listdir/$list");
X    open(NEW, ">$listdir/$list.new") ||
X	&abort_sort("can't open $listdir/$list.new");
X    $state = 2;
X    chmod($mode, "$listdir/$list.new") ||
X	&abort_sort("chmod($mode, $listdir/$list.new) failed");
X    chown($uid, $gid, "$listdir/$list.new") ||
X	&abort_sort("chown($uid, $gid, $listdir/$list.new) failed");
X    for (sort keys %canonical) {
X	print NEW $canonical{$_};
X    }
X    close(NEW) || &abort_sort("error closing listdir/$list.new");
X    $state = 3;
X
X    # Swap the sorted file for the original and remove the original
X    rename("$listdir/$list", "$listdir/$list.old") ||
X	&abort_sort("rename($listdir/$list, $listdir/$list.old)");
X    $state = 4;
X    rename("$listdir/$list.new", "$listdir/$list") ||
X	&abort_sort("rename($listdir/$list.new, $listdir/$list)");
X    unlink("$listdir/$list.old");
X
X    # Release the lock and log the sort
X    &lclose(LIST);
X    $state = 0;
X    &log("sort $list");
X}
Xexit 0;
X
Xsub abort_sort
X{
X    warn "$0: ", @_, ": $!\n";
X    (rename("$listdir/$list.old", "$listdir/$list") ||
X	warn "$0: rename($listdir/$list.old, $listdir/$list) failed: $!\n")
X					if $state >  3;
X    unlink("$listdir/$list.new")	if $state >= 2;
X    close(NEW)				if $state >  1;
X    &lclose(LIST)			if $state >  0;
X    $state = 0;
X    next LIST_SORT;
X}
SHAR_EOF
chmod 0755 majordomo/contrib/sortlist ||
echo 'restore of majordomo/contrib/sortlist failed'
Wc_c="`wc -c < 'majordomo/contrib/sortlist'`"
test 5303 -eq "$Wc_c" ||
	echo 'majordomo/contrib/sortlist: original size 5303, current size' "$Wc_c"
fi
exit 0

-- 
 Dave Wolfe    *Not a spokesman for Motorola*  (512) 891-3246
 Motorola MMTG  6501 Wm. Cannon Dr. W. OE112  Austin  TX  78735-8598

Indexed By Date Previous: Re: administrivia setting
From: Mike Northam <mbn@SSD.intel.com>
Next: subjectline info
From: Henk Meij <hmeij@ciesin.org>
Indexed By Thread Previous: Re: large volumes?
From: debbie@qsun.ho.att.com (Deborah A Hamilton)
Next: [Low Priority Message]
From: faisal@qks.com (Faisal Jawdat)

Google
 
Search Internet Search www.greatcircle.com