I fixed a small issue with the avoid duplicates patch I sent to the list last week; now admins can set the nodupes flag just like any other in the subscriber options page. This patch also includes a new feature which I would have submitted as a separate patch, but the two are a little bit co-dependant. (I can re-submit it as a separate patch, but it'll take a bit of work to re-tool it for plain Mailman CVS.) With this new feature, admins can specify a value for the user options bitfield per-list, and all new users created will have their options field (hide, ack, nodupes, etc) set from that field. They can still change their values per-user, of course. This is really useful if you want to make all new lists have nodupes enabled; I could have coded in yet another "make X option default" option, but I opted to make a general solution that may help other folks who want global hide, ack, nodupes, or whatever set for all new users per-list. The value is set via a checkbox field in General Options. I tried a scrollable multi-value field box, but it was a pain in the butt to CTRL-click all the values I wanted, so a checkbox field seemed more appropriate. Note that I left out nomail and digest, due to the fact that there's no way to put in a <br> between the checkboxes returned from mm_cfg.Checkbox-style options, and using over 5 options would make the field very long. In addition, mm_cfg.DEFAULT_LIST_OPTIONS contains a value for the user options bitfield that will be used as the default options for all new lists created. I had to bump the DATA_FILE_VERSION to make this new default_options list value stick. By default, on upgrade, it's set to whatever mm_cfg.DEFAULT_LIST_OPTIONS is (which is 0 to keep current behavior). Patch follows, and is against current Mailman CVS. Ben diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/Cgi/admin.py mailman/Mailman/Cgi/admin.py --- mailman-wp/Mailman/Cgi/admin.py Wed Sep 5 12:04:45 2001 +++ mailman/Mailman/Cgi/admin.py Thu Sep 6 16:32:38 2001 @@ -800,7 +800,7 @@ usertable.AddRow([Center(Italic(_('%(allcnt)s members total')))]) usertable.AddCellInfo(usertable.GetCurrentRowIndex(), usertable.GetCurrentCellIndex(), - colspan=9, + colspan=10, bgcolor=mm_cfg.WEB_ADMINITEM_COLOR) # Add the alphabetical links if bucket: @@ -818,16 +818,16 @@ usertable.AddRow([Center(joiner.join(cells))]) usertable.AddCellInfo(usertable.GetCurrentRowIndex(), usertable.GetCurrentCellIndex(), - colspan=9, + colspan=10, bgcolor=mm_cfg.WEB_ADMINITEM_COLOR) usertable.AddRow([Center(h) for h in (_('unsub'), _('member address<br>member name'), _('hide'), _('nomail'), _('ack'), _('not metoo'), - _('digest'), _('plain'), - _('language'))]) + _('nodupes'), _('digest'), + _('plain'), _('language'))]) rowindex = usertable.GetCurrentRowIndex() - for i in range(9): + for i in range(10): usertable.AddCellInfo(rowindex, i, bgcolor=mm_cfg.WEB_ADMINITEM_COLOR) # Find the longest name in the list longest = 0 @@ -848,7 +848,7 @@ name + Hidden('user', urllib.quote(addr)).Format(), ] - for opt in ('hide', 'nomail', 'ack', 'notmetoo'): + for opt in ('hide', 'nomail', 'ack', 'notmetoo', 'nodupes'): if mlist.getMemberOption(addr, MailCommandHandler.option_info[opt]): value = 'on' @@ -898,6 +898,9 @@ _('''<b>not metoo</b> -- Does the member avoid copies of their own posts?''')) legend.AddItem( + _('''<b>nodupes</b> -- Does the member avoid duplicates of the same + message?''')) + legend.AddItem( _('''<b>digest</b> -- Does the member get messages in digests? (otherwise, individual messages)''')) legend.AddItem( @@ -1186,6 +1189,16 @@ mlist.bump_digest_volume() elif property == '_send_digest_now' and value: mlist.send_digest_now() + elif property == 'default_options': + checked_defaults = cgidata.getvalue("default_options") + i = 0 + new_defaults = 0 + for opt in ("hide", "ack", "notmetoo", "plain", "nodupes"): + opt_code = MailCommandHandler.option_info[opt] + if `i` in checked_defaults: + new_defaults = new_defaults | opt_code + i = i + 1 + mlist.default_options = new_defaults elif getattr(mlist, property) <> value: # TBD: Ensure that mlist.real_name differs only in letter # case. Otherwise a security hole can potentially be opened @@ -1329,7 +1342,8 @@ if newlang and newlang <> oldlang: mlist.setMemberLanguage(user, newlang) - for opt in ("hide", "nomail", "ack", "notmetoo", "plain"): + for opt in ("hide", "nomail", "ack", "notmetoo", "plain", + "nodupes"): opt_code = MailCommandHandler.option_info[opt] if cgidata.has_key('%s_%s' % (user, opt)): mlist.setMemberOption(user, opt_code, 1) diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/Cgi/options.py mailman/Mailman/Cgi/options.py --- mailman-wp/Mailman/Cgi/options.py Thu Aug 2 13:14:43 2001 +++ mailman/Mailman/Cgi/options.py Thu Sep 6 13:01:10 2001 @@ -375,6 +375,7 @@ ('conceal', mm_cfg.ConcealSubscription), ('remind', mm_cfg.SuppressPasswordReminder), ('rcvtopic', mm_cfg.ReceiveNonmatchingTopics), + ('nodupes', mm_cfg.DontReceiveDuplicates), ): try: newval = int(cgidata.getvalue(item)) @@ -449,9 +450,18 @@ global_remind = newval break - if global_enable is not None or global_remind is not None: + global_nodupes = None + if cgidata.getvalue('nodupes-globally'): + for flag, newval in newvals: + if flag == mm_cfg.DontReceiveDuplicates: + global_nodupes = newval + break + + if (global_enable is not None or global_remind is not None + or global_nodupes is not None): for gmlist in lists_of_member(mlist.host_name, user): - global_options(gmlist, user, global_enable, global_remind) + global_options(gmlist, user, global_enable, global_remind, + global_nodupes) # Now print the results if cantdigest: @@ -526,6 +536,10 @@ mlist.FormatOptionButton(mm_cfg.ConcealSubscription, 0, user)) replacements['<mm-hide-subscription-button>'] = mlist.FormatOptionButton( mm_cfg.ConcealSubscription, 1, user) + replacements['<mm-dont-receive-duplicates-button>'] = ( + mlist.FormatOptionButton(mm_cfg.DontReceiveDuplicates, 1, user)) + replacements['<mm-receive-duplicates-button>'] = ( + mlist.FormatOptionButton(mm_cfg.DontReceiveDuplicates, 0, user)) replacements['<mm-unsubscribe-button>'] = ( mlist.FormatButton('unsub', _('Unsubscribe')) + '<br>' + CheckBox('unsubconfirm', 1, checked=0).Format() + @@ -555,6 +569,8 @@ CheckBox('deliver-globally', 1, checked=0).Format()) replacements['<mm-global-remind-button>'] = ( CheckBox('remind-globally', 1, checked=0).Format()) + replacements['<mm-global-nodupes-button>'] = ( + CheckBox('nodupes-globally', 1, checked=0).Format()) days = int(mm_cfg.PENDING_REQUEST_LIFE / mm_cfg.days(1)) if days > 1: @@ -741,7 +757,7 @@  -def global_options(mlist, user, global_enable, global_remind): +def global_options(mlist, user, global_enable, global_remind, global_nodupes): def sigterm_handler(signum, frame, mlist=mlist): # Make sure the list gets unlocked... mlist.Unlock() @@ -762,6 +778,10 @@ if global_remind is not None: mlist.setMemberOption(user, mm_cfg.SuppressPasswordReminder, global_remind) + + if global_nodupes is not None: + mlist.setMemberOption(user, mm_cfg.DontReceiveDuplicates, + global_nodupes) mlist.Save() finally: diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/Defaults.py.in mailman/Mailman/Defaults.py.in --- mailman-wp/Mailman/Defaults.py.in Wed Sep 5 12:04:45 2001 +++ mailman/Mailman/Defaults.py.in Thu Sep 6 13:01:10 2001 @@ -286,6 +286,7 @@ 'Hold', 'Tagger', 'CalcRecips', + 'AvoidDuplicates', 'Cleanse', 'CookHeaders', 'ToDigest', @@ -570,6 +571,10 @@ # Make it 1 when it works. DEFAULT_MEMBER_POSTING_ONLY = 0 +# See "Bitfield for user options" below; make this a sum of those +# options, to make all new members of lists start with those options +# flagged. +DEFAULT_LIST_OPTIONS = 0  ##### @@ -731,6 +736,7 @@ TEXTFIELDWIDTH = 40 # Bitfield for user options +# See DEFAULT_LIST_OPTIONS above to set defaults for all new lists Digests = 0 # handled by other mechanism, doesn't need a flag. DisableDelivery = 1 DontReceiveOwnPosts = 2 # Non-digesters only @@ -739,6 +745,7 @@ ConcealSubscription = 16 SuppressPasswordReminder = 32 ReceiveNonmatchingTopics = 64 +DontReceiveDuplicates = 128 # Authentication contexts. # diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/Gui/General.py mailman/Mailman/Gui/General.py --- mailman-wp/Mailman/Gui/General.py Fri Aug 17 14:47:11 2001 +++ mailman/Mailman/Gui/General.py Thu Sep 6 15:29:49 2001 @@ -30,6 +30,21 @@ def GetConfigInfo(self, mlist): WIDTH = mm_cfg.TEXTFIELDWIDTH + # These are for the default_options checkboxes below. + # this should be set in a module somewhere.. + option_info = {'hide' : mm_cfg.ConcealSubscription, + 'ack' : mm_cfg.AcknowledgePosts, + 'notmetoo' : mm_cfg.DontReceiveOwnPosts, + 'plain' : mm_cfg.DisableMime, + 'nodupes' : mm_cfg.DontReceiveDuplicates + } + + options = ['hide', 'ack', 'notmetoo', 'plain', 'nodupes'] + option_values = [] + + for o in options: + option_values.append(mlist.default_options & option_info[o]) + return [ _('''Fundamental list characteristics, including descriptive info and basic behaviors.'''), @@ -252,6 +267,9 @@ ('send_reminders', mm_cfg.Radio, (_('No'), _('Yes')), 0, _('''Send monthly password reminders or no? Overrides the previous option.''')), + + ('default_options', mm_cfg.Checkbox, (options, option_values, 1), + 0, _('''Default options for all members that join this list.''')), ('send_welcome_msg', mm_cfg.Radio, (_('No'), _('Yes')), 0, _('Send welcome message when people subscribe?'), diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/HTMLFormatter.py mailman/Mailman/HTMLFormatter.py --- mailman-wp/Mailman/HTMLFormatter.py Wed Sep 5 12:04:45 2001 +++ mailman/Mailman/HTMLFormatter.py Thu Sep 6 13:01:10 2001 @@ -114,6 +114,7 @@ mm_cfg.ConcealSubscription : 'conceal', mm_cfg.SuppressPasswordReminder : 'remind', mm_cfg.ReceiveNonmatchingTopics : 'rcvtopic', + mm_cfg.DontReceiveDuplicates : 'nodupes', }[type] return '<input type=radio name="%s" value="%d"%s>' % ( name, value, checked) diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/Handlers/AvoidDuplicates.py mailman/Mailman/Handlers/AvoidDuplicates.py --- mailman-wp/Mailman/Handlers/AvoidDuplicates.py Thu Jan 1 09:00:00 1970 +++ mailman/Mailman/Handlers/AvoidDuplicates.py Thu Sep 6 13:01:10 2001 @@ -0,0 +1,118 @@ +# Copyright (C) 1998,1999,2000,2001 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""If the user wishes it, do not send duplicates of the same message. + +This module keeps an in-memory dictionary of Message-ID and recipient +pairs. If a message with an identical Message-ID is about to be sent +to someone who has already received a copy, we either drop the message, +add a duplicate warning header, or pass it through, depending on the +user's preferences. +""" + +import string + +from Mailman import mm_cfg +from Mailman import Utils +from Mailman import Message +from Mailman import Errors +from Mailman.i18n import _ +from mimelib.address import getaddresses + + + +class DuplicateDetected(Errors.DiscardMessage): + """The message would have been sent multiple times to a user who + prefers not to receive duplicates.""" + +# A dictionary of dictionaries, used to store which recipients have received +# which message IDs. +recip_msgids = {} + + + +def process(mlist, msg, msgdata): + + recips = msgdata['recips'] + msgid = msg.get('message-id') + + if not recips or not msgid: + return + + # This dictionary will hold recips who want their mail to have + # the X-Mailman-Duplicate: yes header. + if not msgdata.has_key('add-dupe-header'): + msgdata['add-dupe-header'] = {} + + # Happily borrowed from mimelib.getaddresses() example + tos = msg.getall('to') + ccs = msg.getall('cc') + resent_tos = msg.getall('resent-to') + resent_ccs = msg.getall('resent-cc') + external_recips = getaddresses(tos + ccs + resent_tos + resent_ccs) + + # Anyone mentioned in the to/cc/resent-to/resent-cc headers should + # not get a duplicate of the message. + for (name, email) in external_recips: + + # If getaddresses fails, email could be null. Skip those. + if not email: + continue + + # Initialize the external recipient's msgid hash if this is the + # first email they've received with this message-id. + if not recip_msgids.has_key(email): + recip_msgids[email] = {} + + # We don't do anything except record that that address has + # gotten or will get a copy of this email externally. + recip_msgids[email][msgid] = 1 + + newrecips = [] + + for r in recips: + if not recip_msgids.has_key(r): + recip_msgids[r] = {} + + # If they have received a message with this message-id already, + # see if they don't want duplicates. + if recip_msgids[r].has_key(msgid): + send_duplicate = 1 + + # If the member wants to receive duplicates, or if the recipient + # is not a member at all, just flag the X-Mailman-Duplicate: yes + # header. + try: + if mlist.getMemberOption(r, mm_cfg.DontReceiveDuplicates): + send_duplicate = 0 + except Errors.NotAMemberError: + pass + + # We'll send a duplicate unless the user doesn't wish it. + # If personalization is enabled, the add-dupe-header flag will + # add a X-Mailman-Duplicate: yes header for this user's message. + if send_duplicate: + msgdata['add-dupe-header'][r] = 1 + newrecips.append(r) + + else: + # Otherwise, this is the first time they've been in the recips + # list. Add them to the newrecips list and flag them as having + # received this message. + recip_msgids[r][msgid] = 1 + newrecips.append(r) + + msgdata['recips'] = newrecips diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/Handlers/Personalize.py mailman/Mailman/Handlers/Personalize.py --- mailman-wp/Mailman/Handlers/Personalize.py Sat Aug 18 06:20:58 2001 +++ mailman/Mailman/Handlers/Personalize.py Thu Sep 6 13:01:10 2001 @@ -46,11 +46,23 @@ msg['To'] = '%s (%s)' % (member, name) else: msg['To'] = member + + # We can flag the mail as a duplicate for each member, if + # they've already received that message. (See AvoidDuplicates.py) + if msgdata['add-dupe-header'].has_key(member): + msg['X-Mailman-Duplicate'] = 'yes' + elif msg.has_key('X-Mailman-Duplicate'): + del msg['X-Mailman-Duplicate'] + inq.enqueue(msg, metadatacopy, listname=mlist.internal_name()) # Restore the original To: line del msg['To'] msg['To'] = originalto + + # The original message is not a duplicate. + if msg.has_key('X-Mailman-Duplicate'): + del msg['X-Mailman-Duplicate'] # Don't let the normal ToOutgoing processing actually send the original # copy. diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/MailCommandHandler.py mailman/Mailman/MailCommandHandler.py --- mailman-wp/Mailman/MailCommandHandler.py Fri Aug 17 14:37:15 2001 +++ mailman/Mailman/MailCommandHandler.py Thu Sep 6 13:01:10 2001 @@ -80,27 +80,36 @@ you get digests in MIME format, which are much better if you have a mail reader that supports MIME.""") -option_desc = {'hide' : HIDE, - 'nomail' : NOMAIL, - 'ack' : ACK, - 'notmetoo': NOTMETOO, - 'digest' : DIGEST, - 'plain' : PLAIN, +NODUPES = _("""When turned on, you do *not* receive duplicate mails if mail is +sent to multiple lists that you belong to. This option will let you avoid +duplicate mails; if you turn it on, you will never receive multiple copies +of the same message. Also, if you *and* the list are mentioned explicitly +in the To: or Cc: headers of a message, you will not receive duplicates if +this is turned on.""") + +option_desc = {'hide' : HIDE, + 'nomail' : NOMAIL, + 'ack' : ACK, + 'notmetoo' : NOTMETOO, + 'digest' : DIGEST, + 'plain' : PLAIN, + 'nodupes' : NODUPES, } # jcrey: and then the real one _ = Mailman.i18n._ -option_info = {'hide' : mm_cfg.ConcealSubscription, - 'nomail' : mm_cfg.DisableDelivery, - 'ack' : mm_cfg.AcknowledgePosts, - 'notmetoo': mm_cfg.DontReceiveOwnPosts, - 'digest' : 0, - 'plain' : mm_cfg.DisableMime, +option_info = {'hide' : mm_cfg.ConcealSubscription, + 'nomail' : mm_cfg.DisableDelivery, + 'ack' : mm_cfg.AcknowledgePosts, + 'notmetoo' : mm_cfg.DontReceiveOwnPosts, + 'digest' : 0, + 'plain' : mm_cfg.DisableMime, + 'nodupes' : mm_cfg.DontReceiveDuplicates } # ordered list -options = ('hide', 'nomail', 'ack', 'notmetoo', 'digest', 'plain') +options = ('hide', 'nomail', 'ack', 'notmetoo', 'digest', 'plain', 'nodupes') # strip just the outer layer of quotes quotecre = re.compile(r'["\'`](?P<cmd>.*)["\'`]') diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/MailList.py mailman/Mailman/MailList.py --- mailman-wp/Mailman/MailList.py Wed Sep 5 12:04:45 2001 +++ mailman/Mailman/MailList.py Thu Sep 6 12:51:31 2001 @@ -254,7 +254,8 @@ self.language = {} self.usernames = {} self.passwords = {} - + self.default_options = mm_cfg.DEFAULT_LIST_OPTIONS + # This stuff is configurable self.respond_to_post_requests = 1 self.advertised = mm_cfg.DEFAULT_LIST_ADVERTISED diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/OldStyleMemberships.py mailman/Mailman/OldStyleMemberships.py --- mailman-wp/Mailman/OldStyleMemberships.py Wed Sep 5 12:04:45 2001 +++ mailman/Mailman/OldStyleMemberships.py Thu Sep 6 12:42:19 2001 @@ -176,6 +176,8 @@ self.setMemberLanguage(member, language) if realname: self.setMemberName(member, realname) + if self.__mlist.default_options: + self.__mlist.user_options[member] = self.__mlist.default_options def removeMember(self, member): assert self.__mlist.Locked() diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/Version.py mailman/Mailman/Version.py --- mailman-wp/Mailman/Version.py Fri Aug 17 14:41:03 2001 +++ mailman/Mailman/Version.py Thu Sep 6 12:57:05 2001 @@ -36,7 +36,7 @@ (REL_LEVEL << 4) | (REL_SERIAL << 0)) # config.db schema version number -DATA_FILE_VERSION = 35 +DATA_FILE_VERSION = 36 # qfile/*.db schema version number QFILE_SCHEMA_VERSION = 3 diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/htmlformat.py mailman/Mailman/htmlformat.py diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/Mailman/versions.py mailman/Mailman/versions.py --- mailman-wp/Mailman/versions.py Fri Aug 17 14:40:28 2001 +++ mailman/Mailman/versions.py Thu Sep 6 12:55:00 2001 @@ -211,7 +211,7 @@ add_only_if_missing('one_last_digest', {}) add_only_if_missing('usernames', {}) add_only_if_missing('personalize', 0) - + add_only_if_missing('default_options', mm_cfg.DEFAULT_LIST_OPTIONS)  def UpdateOldUsers(l): diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/templates/en/help.txt mailman/templates/en/help.txt --- mailman-wp/templates/en/help.txt Sat May 19 06:28:54 2001 +++ mailman/templates/en/help.txt Thu Sep 6 13:01:10 2001 @@ -79,6 +79,10 @@ Conceals your address when people look at who is on this list. + nodupes: + Turn this on if you do not want to receive duplicate mail + from the list, in case you are explicitly in the To: or Cc: + fields already or are included in multiple lists in one message. options Show the current values of your list options. diff -x CVS -x *.mo -x *.go -x *.pot -ruN mailman-wp/templates/en/options.html mailman/templates/en/options.html --- mailman-wp/templates/en/options.html Thu Jul 19 06:54:40 2001 +++ mailman/templates/en/options.html Thu Sep 6 13:01:10 2001 @@ -280,6 +280,26 @@ <mm-receive-nonmatching-topics>Yes </td></tr> + <tr><td bgcolor="#cccccc"> + <strong>Avoid duplicate copies of messages?</strong><p> + + When you are listed explicitly in the To: or Cc: headers + of a list message, or a message is sent to multiple lists + that you are a member of, you can opt to not receive another + copy from the mailing list. Select <em>Yes</em> to avoid + receiving duplicate copies from the mailing list; select + <em>No</em> to receive duplicate copies. + + <p>If the list has per-message personalization + enabled, every duplicate mail will have a + <tt>X-Mailman-Duplicate: yes</tt> header added to it. + + </td><td bgcolor="#cccccc"> + <mm-receive-duplicates-button>No<br> + <mm-dont-receive-duplicates-button>Yes<p> + <mm-global-nodupes-button><i>Set globally</i> + </td></tr> + <tr><TD colspan="2"> <center><MM-options-Submit-button></center> </td></tr>