[patch] Spam report functionality

Hi All, The patch attached (created against 2.1.9), will add functionality where the moderator is able to report a moderated message as spam. This message will than either be stored on the disk or will be send -to a in the config- definable email address. Together will a small shell script it will be pretty easy to load this message into a spam learning program like sa-learn. This will make the moderators more happy, cause they are able to contribute to the improvement of the spam filter. I do believe this will gradually lower the amount of false negatives, cause having people to teach the spam filter is great :-) Best regards, /Rick -- http://rickvanderzwet.nl diff -u -x *.pyc -r ./Cgi/admindb.py /usr/local/mailman/Mailman/Cgi/admindb.py --- ./Cgi/admindb.py Wed Aug 30 16:54:22 2006 +++ /usr/local/mailman/Mailman/Cgi/admindb.py Fri Aug 10 01:53:24 2007 @@ -196,6 +196,11 @@ ' ' + _('Discard all messages marked <em>Defer</em>') )) + form.AddItem(Center( + CheckBox('reportspamalldefersp', 0).Format() + + ' ' + + _('Report all messages marked <em>Defer</em> as SPAM') + )) # Add a link back to the overview, if we're not viewing the overview! adminurl = mlist.GetScriptURL('admin', absolute=1) d = {'listname' : mlist.real_name, @@ -244,6 +249,11 @@ ' ' + _('Discard all messages marked <em>Defer</em>') )) + form.AddItem(Center( + CheckBox('reportspamalldefersp', 0).Format() + + ' ' + + _('Report all messages marked <em>Defer</em> as SPAM') + )) form.AddItem(Center(SubmitButton('submit', _('Submit All Data')))) doc.AddItem(mlist.GetMailmanFooter()) print doc.Format() @@ -302,11 +312,13 @@ radio = RadioButtonArray(id, (_('Defer'), _('Approve'), _('Reject'), - _('Discard')), + _('Discard'), + _('Spam')), values=(mm_cfg.DEFER, mm_cfg.SUBSCRIBE, mm_cfg.REJECT, - mm_cfg.DISCARD), + mm_cfg.DISCARD, + mm_cfg.SPAM), checked=0).Format() if addr not in mlist.ban_list: radio += '<br>' + CheckBox('ban-%d' % id, 1).Format() + \ @@ -361,11 +373,13 @@ RadioButtonArray(id, (_('Defer'), _('Approve'), _('Reject'), - _('Discard')), + _('Discard'), + _('Spam')), values=(mm_cfg.DEFER, mm_cfg.UNSUBSCRIBE, mm_cfg.REJECT, - mm_cfg.DISCARD), + mm_cfg.DISCARD, + mm_cfg.SPAM), checked=0), TextBox('comment-%d' % id, size=45) ]) @@ -401,9 +415,11 @@ left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) btns = hacky_radio_buttons( 'senderaction-' + qsender, - (_('Defer'), _('Accept'), _('Reject'), _('Discard')), - (mm_cfg.DEFER, mm_cfg.APPROVE, mm_cfg.REJECT, mm_cfg.DISCARD), - (1, 0, 0, 0)) + (_('Defer'), _('Accept'), _('Reject'), + _('Discard'), _('Spam')), + (mm_cfg.DEFER, mm_cfg.APPROVE, mm_cfg.REJECT, + mm_cfg.DISCARD,mm_cfg.SPAM), + (1, 0, 0, 0, 0)) left.AddRow([btns]) left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2) left.AddRow([ @@ -633,13 +649,15 @@ # We can't use a RadioButtonArray here because horizontal placement can be # confusing to the user and vertical placement takes up too much # real-estate. This is a hack! - buttons = Table(cellspacing="5", cellpadding="0") - buttons.AddRow(map(lambda x, s=' '*5: s+x+s, - (_('Defer'), _('Approve'), _('Reject'), _('Discard')))) + buttons = Table(cellspacing="6", cellpadding="0") + buttons.AddRow(map(lambda x, s=' '*6: s+x+s, + (_('Defer'), _('Approve'), _('Reject'), + _('Discard'), _('Spam')))) buttons.AddRow([Center(RadioButton(id, mm_cfg.DEFER, 1)), Center(RadioButton(id, mm_cfg.APPROVE, 0)), Center(RadioButton(id, mm_cfg.REJECT, 0)), Center(RadioButton(id, mm_cfg.DISCARD, 0)), + Center(RadioButton(id, mm_cfg.SPAM, 0)), ]) t.AddRow([Bold(_('Action:')), buttons]) t.AddCellInfo(row+3, col-1, align='right') @@ -692,6 +710,10 @@ discardalldefersp = cgidata.getvalue('discardalldefersp', 0) except ValueError: discardalldefersp = 0 + try: + reportspamalldefersp = cgidata.getvalue('reportspamalldefersp', 0) + except ValueError: + reportspamalldefersp = 0 for sender in senderactions.keys(): actions = senderactions[sender] # Handle what to do about all this sender's held messages @@ -701,11 +723,19 @@ action = mm_cfg.DEFER if action == mm_cfg.DEFER and discardalldefersp: action = mm_cfg.DISCARD + elif action == mm_cfg.DEFER and reportspamalldefersp: + action = mm_cfg.SPAM if action in (mm_cfg.DEFER, mm_cfg.APPROVE, - mm_cfg.REJECT, mm_cfg.DISCARD): - preserve = actions.get('senderpreserve', 0) - forward = actions.get('senderforward', 0) - forwardaddr = actions.get('senderforwardto', '') + mm_cfg.REJECT, mm_cfg.DISCARD, + mm_cfg.SPAM): + if action == mm_cfg.SPAM: + preserve = mm_cfg.SPAM_STORE + forward = mm_cfg.SPAM_FORWARD + forwardaddr = mm_cfg.SPAM_FORWARD_ADDRESS + else: + preserve = actions.get('senderpreserve', 0) + forward = actions.get('senderforward', 0) + forwardaddr = actions.get('senderforwardto', '') comment = _('No reason given') bysender = helds_by_sender(mlist) for id in bysender.get(sender, []): @@ -759,7 +789,7 @@ continue if v not in (mm_cfg.DEFER, mm_cfg.APPROVE, mm_cfg.REJECT, mm_cfg.DISCARD, mm_cfg.SUBSCRIBE, mm_cfg.UNSUBSCRIBE, - mm_cfg.ACCEPT, mm_cfg.HOLD): + mm_cfg.ACCEPT, mm_cfg.HOLD, mm_cfg.SPAM): continue # Get the action comment and reasons if present. commentkey = 'comment-%d' % request_id diff -u -x *.pyc -r ./Defaults.py /usr/local/mailman/Mailman/Defaults.py --- ./Defaults.py Fri Aug 10 00:30:06 2007 +++ /usr/local/mailman/Mailman/Defaults.py Fri Aug 10 01:48:45 2007 @@ -167,6 +167,12 @@ # -owners address, unless the message is explicitly approved. KNOWN_SPAMMERS = [] +# Spam actions +# Default to store spam messages +SPAM_STORE = 1 +SPAM_FORWARD = 0 +SPAM_FORWARD_ADDRESS = 'spam2trainer@example.com' + ##### @@ -1220,6 +1226,7 @@ UNSUBSCRIBE = 5 ACCEPT = 6 HOLD = 7 +SPAM = 8 # Standard text field width TEXTFIELDWIDTH = 40 diff -u -x *.pyc -r ./ListAdmin.py /usr/local/mailman/Mailman/ListAdmin.py --- ./ListAdmin.py Sat Aug 27 03:40:17 2005 +++ /usr/local/mailman/Mailman/ListAdmin.py Fri Aug 10 00:50:36 2007 @@ -295,10 +295,13 @@ self.__refuse(_('Posting of your message titled "%(subject)s"'), sender, comment or _('[No reason given]'), lang=self.getMemberLanguage(sender)) - else: - assert value == mm_cfg.DISCARD + elif value == mm_cfg.DISCARD: # Discarded rejection = 'Discarded' + else: + assert value == mm_cfg.SPAM + # Spam reported + rejection = 'SpamReported' # Forward the message if forward and addr: # If we've approved the message, we need to be sure to craft a

--On 10 August 2007 02:26:45 +0200 Rick van der Zwet <rick@wzoeterwoude.net> wrote:
That's nice. Don't forget that you need to show good email as well as spam, otherwise SA will believe everything is spam.
Best regards, /Rick
-- Ian Eiloart IT Services, University of Sussex x3148

On 8/10/07 11:31 AM, Ian Eiloart wrote:
Yes true, feeding SA initially with 1000 good (ham) messages and set autolearn to true will do this. But other implementations of spam filtering are also possible, there are plenty systems around.
The patch any provide a way to have moderators report messages identified as spam, it is up the the email administrator what to do with this kind of messages.
/Rick

--On 10 August 2007 02:26:45 +0200 Rick van der Zwet <rick@wzoeterwoude.net> wrote:
That's nice. Don't forget that you need to show good email as well as spam, otherwise SA will believe everything is spam.
Best regards, /Rick
-- Ian Eiloart IT Services, University of Sussex x3148

On 8/10/07 11:31 AM, Ian Eiloart wrote:
Yes true, feeding SA initially with 1000 good (ham) messages and set autolearn to true will do this. But other implementations of spam filtering are also possible, there are plenty systems around.
The patch any provide a way to have moderators report messages identified as spam, it is up the the email administrator what to do with this kind of messages.
/Rick
participants (2)
-
Ian Eiloart
-
Rick van der Zwet