From a425a57d95d085f41e3e65311d8c4b0960bfcea3 Mon Sep 17 00:00:00 2001 From: Stephan Bosch <stephan@rename-it.nl> Date: Tue, 16 Oct 2012 21:33:15 +0200 Subject: [PATCH] lib-sieve: vnd.dovecot.duplicate extension: Added new features to the duplicate test. It is now possible to track duplicates based on arbitrary headers or even arbitrary string values using the new :header and :value arguments respectively. The experation time can be configured using the new :seconds argument. This change is backwards compatible as long as the name argument wasn't used. This is now a :handle <handle> argument. --- doc/rfc/spec-bosch-sieve-duplicate.txt | 286 ++++++++++++++---- doc/rfc/xml/reference.IMAP4FLAGS.xml | 15 + doc/rfc/xml/reference.MAILBOX.xml | 15 + doc/rfc/xml/reference.VACATION.xml | 17 ++ doc/rfc/xml/spec-bosch-sieve-duplicate.xml | 164 +++++++--- .../duplicate/ext-duplicate-common.c | 137 ++++++--- .../duplicate/ext-duplicate-common.h | 11 +- .../vnd.dovecot/duplicate/ext-duplicate.c | 3 +- .../vnd.dovecot/duplicate/tst-duplicate.c | 273 +++++++++++++++-- .../vnd.dovecot/duplicate/errors.svtest | 2 +- .../vnd.dovecot/duplicate/errors/syntax.sieve | 5 +- .../vnd.dovecot/duplicate/execute.svtest | 10 +- 12 files changed, 748 insertions(+), 190 deletions(-) create mode 100644 doc/rfc/xml/reference.IMAP4FLAGS.xml create mode 100644 doc/rfc/xml/reference.MAILBOX.xml create mode 100644 doc/rfc/xml/reference.VACATION.xml diff --git a/doc/rfc/spec-bosch-sieve-duplicate.txt b/doc/rfc/spec-bosch-sieve-duplicate.txt index 95512457f..2656d6e25 100644 --- a/doc/rfc/spec-bosch-sieve-duplicate.txt +++ b/doc/rfc/spec-bosch-sieve-duplicate.txt @@ -2,7 +2,7 @@ Pigeonhole Project S. Bosch - February 25, 2012 + October 16, 2012 Sieve Email Filtering: Detecting Duplicate Deliveries @@ -10,23 +10,23 @@ Pigeonhole Project S. Bosch Abstract This document defines a new vendor-defined test command "duplicate" - for the "Sieve" email filtering language that tests whether an e-mail - message is a duplicate, i.e. whether it was seen before by the - delivery agent. Users can use this new test to remove duplicate - deliveries commonly caused by mailing list subscriptions or mail - account aliases. + for the "Sieve" email filtering language. It can be used to test + whether a particular string value is a duplicate, i.e. whether it was + seen before by the delivery agent that is executing the Sieve script. + The main application for this new test is detecting duplicate message + deliveries commonly caused by mailing list subscriptions or + redirected mail addresses. + + + + + + + + -Table of Contents - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Conventions Used in This Document . . . . . . . . . . . . . . . 2 - 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 2 - 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 3 - 5. Example . . . . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 3 - 7. Normative References . . . . . . . . . . . . . . . . . . . . . 3 - Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 3 @@ -54,24 +54,92 @@ Table of Contents Bosch [Page 1] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . . 3 + 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 4 + 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 5 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 6 + 7. References . . . . . . . . . . . . . . . . . . . . . . . . . . 6 + 7.1. Normative References . . . . . . . . . . . . . . . . . . . 6 + 7.2. Informative References . . . . . . . . . . . . . . . . . . 6 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 7 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Bosch [Page 2] + + Sieve: Detecting Duplicate Deliveries October 2012 1. Introduction This is an extension to the Sieve filtering language defined by RFC - 5228 [SIEVE]. It adds a test to determine whether a message was seen - before by the delivery agent based on the Message-ID header. + 5228 [SIEVE]. It adds a test to determine whether a certain string + value was seen before by the delivery agent in an earlier execution + of the Sieve script. This can be used to detect and handle duplicate + message deliveries. Duplicate deliveries are a common side-effect of being subscribed to - a mailing list. If a member of the list decides to reply to both the - user and the mailing list itself, the user will get a copy of the - message directly and through mailing list. In another scenario, the - user has several aliases for his mail account and one of his contacts - sends the message to multiple addresses that eventually map to the - same account. Using the vnd.dovecot.duplicate extension, users have - the means to detect such duplicates and deal with these - appropriately, e.g. by discarding them. + a mailing list. For example, if a member of the list decides to + reply to both the user and the mailing list itself, the user will get + a copy of the message directly and through mailing list. Also, if + someone cross-posts over several mailing lists to which the user is + subscribed, the user will receive a copy from each of those lists. + In another scenario, the user has several redirected mail addresses + all pointing to his main mail account. If one of the user's contacts + sends the message to more than one of those addresses, the user will + receive more than a single copy. Using the "vnd.dovecot.duplicate" + extension, users have the means to detect and handle such duplicates, + e.g. by discarding them or putting them in a special folder. + + Duplicate messages are normally detected using the Message-ID header + field, which is required to be unique for each message. However, the + "duplicate" test is flexible enough to use different (weaker) + criteria for defining what makes a message a duplicate, for example + based on the subject line. Also, other applications of this new test + command are possible, as long as the tracked value is a string. This extension is specific to the Pigeonhole Sieve implementation for the Dovecot Secure IMAP server. It will therefore most likely not be @@ -89,31 +157,79 @@ Bosch [Page 1] arguments syntax. -3. Test "duplicate" - Usage: "duplicate" [<name: string>] - The "duplicate" test keeps track of which Message-ID values were seen - before by this test in an earlier delivery operation. It evaluates - to "true" when the Message-ID header of the current message was seen - before. If it is not known, the test evaluates to "false" and the - Message-ID is added to a persistent internal tracking list. - Implementations SHOULD limit the number of messages that are tracked - and SHOULD let Message-ID entries expire after some short period of - time. - Using the "name" argument, the duplicate test can be employed for - multiple independent purposes. Only when the Message-ID was seen - before in an earlier script execution by a duplicate test with the -Bosch [Page 2] + +Bosch [Page 3] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 + +3. Test "duplicate" - same "name" argument, it is recognized as a duplicate. + Usage: "duplicate" [":seconds" <timeout: number>] + [":header" <header-name: string> / + ":value" <value: string>] + [":handle" <handle: string>] + + The "duplicate" test keeps track of which values were seen before by + this test in an earlier execution of this Sieve script. In its basic + form, the tested value is the content of the Message-ID header of the + message. This way, this test can be used to detect duplicate + deliveries of the same message. It can also detect duplicate + deliveries based on other message header fields if requested and it + can even use a user-provided string value, e.g. as composed from text + extracted from the message using the "variables" [VARIABLES] + extension. + + The "duplicate" test evaluates to "true" when the provided value was + seen before. If the value is not known, the test evaluates to + "false" and the value is added to an internal value tracking list. + Implementations SHOULD limit the number of values (and thereby + messages) that are tracked. Also, implementations SHOULD let entries + in the value tracking list expire after a short period of time. + + The user can explicitly control the length this expiration time by + means of the ":seconds" argument. If the ":seconds" argument is + omitted, an appropriate default MUST be used. Sites SHOULD impose a + maximum limit on the expiration time. If that limit is exceeded, the + maximum value MUST silently be substituted; exceeding the limit MUST + NOT produce an error. + + By default the tracked value is the content of the message's + Message-ID header field. For more advanced purposes, the content of + another header can be chosen for tracking by specifying the ":header" + argument. The tracked string value can also be specified explicitly + using the ":value" argument. The ":header" and ":value" arguments + are mutually exclusive and specifying both for a single "duplicate" + test command MUST trigger an error at compile time. If the value is + extracted from a header, i.e. when the ":value" argument is not used, + leading and trailing whitespace (see Section 2.2 of RFC 5228 [SIEVE]) + MUST first be trimmed from the value before executing the test. + + Using the ":handle" argument, the duplicate test can be employed for + multiple independent purposes. Only when the tracked value was seen + before in an earlier script execution by a "duplicate" test with the + same ":handle" argument, it is recognized as a duplicate. + + NOTE: The necessary mechanism to track duplicate messages is very + + + +Bosch [Page 4] + + Sieve: Detecting Duplicate Deliveries October 2012 + + + similar to the mechanism that is needed for tracking duplicate + responses for the "vacation" [VACATION] action. One way to implement + the necessary mechanism for the "duplicate" test is therefore to + store a hash of the tracked value and, if provided, the ":handle" + argument. 4. Sieve Capability Strings @@ -122,29 +238,71 @@ Bosch [Page 2] advertise the capability string "vnd.dovecot.duplicate". -5. Example +5. Examples - In this example, duplicate deliveries are stored in a special folder - contained in the user's Trash folder. If the folder does not exist, - it is created. This way, the user has a chance to recover messages - when necessary. Messages that are not recognized as duplicates are - stored in the user's inbox as normal. + In the following basic example, message duplicates are detected by + tracking the Message-ID header. Duplicate deliveries are stored in a + special folder contained in the user's Trash folder. If the folder + does not exist, it is created automatically using the "mailbox" + [MAILBOX] extension. This way, the user has a chance to recover + messages when necessary. Messages that are not recognized as + duplicates are stored in the user's inbox as normal. require ["vnd.dovecot.duplicate", "fileinto", "mailbox"]; if duplicate { - fileinto :create "Trash/Duplicate"; + fileinto :create "Trash/Duplicate"; } + The next example shows a more complex use of the "duplicate" test. + The user gets network alerts from a set of remote automated + monitoring systems. Multiple notifications can be received about the + same event from different monitoring systems. The Message-ID of + these messages is different, because these are all distinct messages + from different senders. To avoid being notified multiple times about + the same event the user writes the following script: + + require ["vnd.dovecot.duplicate", "variables", "imap4flags", + "fileinto"]; + + if header :matches "subject" "ALERT: *" { + if duplicate :seconds 60 :value "${1}" { + setflag "\\seen"; + } + fileinto "Alerts"; + } + + The subjects of the notification message are structured with a + + + +Bosch [Page 5] + + Sieve: Detecting Duplicate Deliveries October 2012 + + + predictable pattern which includes a description of the event. In + the script above the "duplicate" test is used to detect duplicate + alert events. The message subject is matched against a pattern and + the event description is extracted using the "variables" [VARIABLES] + extension. If a message with that event in the subject was received + before, but more than a minute ago, it is not detected as a duplicate + due to the specified ":seconds" argument. In the the event of a + duplicate, the message is marked as seen using the "imap4flags" + [IMAP4FLAGS] extension. All alert messages are put into the "Alerts" + mailbox irrespective of wether those messages are duplicates or not. + 6. Security Considerations - A flood of unique messages could cause the list of tracked Message- - IDs to grow indefinitely. Implementations therefore SHOULD implement - limits on the number and lifespan of entries in that list. + A flood of unique messages could cause the list of tracked values to + grow indefinitely. Implementations therefore SHOULD implement limits + on the number and lifespan of entries in that list. + +7. References -7. Normative References +7.1. Normative References [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate @@ -153,20 +311,30 @@ Bosch [Page 2] [SIEVE] Guenther, P. and T. Showalter, "Sieve: An Email Filtering Language", RFC 5228, January 2008. +7.2. Informative References + [IMAP4FLAGS] + Melnikov, A., "Sieve Email Filtering: Imap4flags + Extension", RFC 5232, January 2008. + [MAILBOX] Melnikov, A., "The Sieve Mail-Filtering Language -- + Extensions for Checking Mailbox Status and Accessing + Mailbox Metadata", RFC 5490, March 2009. + [VACATION] + Showalter, T. and N. Freed, "Sieve Email Filtering: + Vacation Extension", RFC 5230, January 2008. + [VARIABLES] + Homme, K., "Sieve Email Filtering: Variables Extension", + RFC 5229, January 2008. - - - -Bosch [Page 3] +Bosch [Page 6] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 Author's Address @@ -220,5 +388,5 @@ Author's Address -Bosch [Page 4] +Bosch [Page 7] diff --git a/doc/rfc/xml/reference.IMAP4FLAGS.xml b/doc/rfc/xml/reference.IMAP4FLAGS.xml new file mode 100644 index 000000000..65a1cad9d --- /dev/null +++ b/doc/rfc/xml/reference.IMAP4FLAGS.xml @@ -0,0 +1,15 @@ +<?xml version='1.0' encoding='UTF-8'?> + +<reference anchor='IMAP4FLAGS'> + +<front> +<title>Sieve Email Filtering: Imap4flags Extension</title> +<author initials='A.' surname='Melnikov' fullname='A. Melnikov'> +<organization /></author> +<date year='2008' month='January' /> +<abstract> +<t>Recent discussions have shown that it is desirable to set different IMAP (RFC 3501) flags on message delivery. This can be done, for example, by a Sieve interpreter that works as a part of a Mail Delivery Agent.</t><t> This document describes an extension to the Sieve mail filtering language for setting IMAP flags. The extension allows setting of both IMAP system flags and IMAP keywords. [STANDARDS-TRACK]</t></abstract></front> + +<seriesInfo name='RFC' value='5232' /> +<format type='TXT' octets='21964' target='http://www.rfc-editor.org/rfc/rfc5232.txt' /> +</reference> diff --git a/doc/rfc/xml/reference.MAILBOX.xml b/doc/rfc/xml/reference.MAILBOX.xml new file mode 100644 index 000000000..4ac609e32 --- /dev/null +++ b/doc/rfc/xml/reference.MAILBOX.xml @@ -0,0 +1,15 @@ +<?xml version='1.0' encoding='UTF-8'?> + +<reference anchor='MAILBOX'> + +<front> +<title>The Sieve Mail-Filtering Language -- Extensions for Checking Mailbox Status and Accessing Mailbox Metadata</title> +<author initials='A.' surname='Melnikov' fullname='A. Melnikov'> +<organization /></author> +<date year='2009' month='March' /> +<abstract> +<t>This memo defines an extension to the Sieve mail filtering language (RFC 5228) for accessing mailbox and server annotations, checking for mailbox existence, and controlling mailbox creation on "fileinto" action. [STANDARDS-TRACK]</t></abstract></front> + +<seriesInfo name='RFC' value='5490' /> +<format type='TXT' octets='16065' target='http://www.rfc-editor.org/rfc/rfc5490.txt' /> +</reference> diff --git a/doc/rfc/xml/reference.VACATION.xml b/doc/rfc/xml/reference.VACATION.xml new file mode 100644 index 000000000..639187941 --- /dev/null +++ b/doc/rfc/xml/reference.VACATION.xml @@ -0,0 +1,17 @@ +<?xml version='1.0' encoding='UTF-8'?> + +<reference anchor='VACATION'> + +<front> +<title>Sieve Email Filtering: Vacation Extension</title> +<author initials='T.' surname='Showalter' fullname='T. Showalter'> +<organization /></author> +<author initials='N.' surname='Freed' fullname='N. Freed'> +<organization /></author> +<date year='2008' month='January' /> +<abstract> +<t>This document describes an extension to the Sieve email filtering language for an autoresponder similar to that of the Unix "vacation" command for replying to messages. Various safety features are included to prevent problems such as message loops. [STANDARDS-TRACK]</t></abstract></front> + +<seriesInfo name='RFC' value='5230' /> +<format type='TXT' octets='29822' target='http://www.rfc-editor.org/rfc/rfc5230.txt' /> +</reference> diff --git a/doc/rfc/xml/spec-bosch-sieve-duplicate.xml b/doc/rfc/xml/spec-bosch-sieve-duplicate.xml index 063e94393..9fe53ff1b 100644 --- a/doc/rfc/xml/spec-bosch-sieve-duplicate.xml +++ b/doc/rfc/xml/spec-bosch-sieve-duplicate.xml @@ -41,13 +41,12 @@ Sieve Email Filtering: Detecting Duplicate Deliveries <keyword>duplicate deliveries</keyword> <abstract> -<t> -This document defines a new vendor-defined test command "duplicate" for the -"Sieve" email filtering language that tests whether an e-mail message is a -duplicate, i.e. whether it was seen before by the delivery agent. Users can use -this new test to remove duplicate deliveries commonly caused by mailing list -subscriptions or mail account aliases. -</t> +<t>This document defines a new vendor-defined test command "duplicate" for the +"Sieve" email filtering language. It can be used to test whether a particular +string value is a duplicate, i.e. whether it was seen before by the delivery +agent that is executing the Sieve script. The main application for this new test +is detecting duplicate message deliveries commonly caused by mailing list +subscriptions or redirected mail addresses.</t> </abstract> </front> @@ -56,22 +55,32 @@ subscriptions or mail account aliases. <section title="Introduction"> <t>This is an extension to the Sieve filtering language defined by <xref target="SIEVE">RFC 5228</xref>. It adds a test to determine whether a -message was seen before by the delivery agent based on the Message-ID header. -</t> +certain string value was seen before by the delivery agent in an earlier +execution of the Sieve script. This can be used to detect and handle duplicate +message deliveries.</t> <t>Duplicate deliveries are a common side-effect of being subscribed to a -mailing list. If a member of the list decides to reply to both the user and the -mailing list itself, the user will get a copy of the message directly and -through mailing list. In another scenario, the user has several aliases for his -mail account and one of his contacts sends the message to multiple addresses -that eventually map to the same account. Using the vnd.dovecot.duplicate -extension, users have the means to detect such duplicates and deal with these -appropriately, e.g. by discarding them.</t> +mailing list. For example, if a member of the list decides to reply to both the +user and the mailing list itself, the user will get a copy of the message +directly and through mailing list. Also, if someone cross-posts over several +mailing lists to which the user is subscribed, the user will receive a copy from +each of those lists. In another scenario, the user has several redirected +mail addresses all pointing to his main mail account. If one of the user's +contacts sends the message to more than one of those addresses, the user will +receive more than a single copy. Using the "vnd.dovecot.duplicate" extension, +users have the means to detect and handle such duplicates, e.g. by discarding +them or putting them in a special folder.</t> + +<t>Duplicate messages are normally detected using the Message-ID header field, +which is required to be unique for each message. However, the "duplicate" test +is flexible enough to use different (weaker) criteria for defining what makes +a message a duplicate, for example based on the subject line. Also, other +applications of this new test command are possible, as long as the tracked +value is a string.</t> <t>This extension is specific to the Pigeonhole Sieve implementation for the Dovecot Secure IMAP server. It will therefore most likely not be supported by -web interfaces and GUI-based Sieve editors. -</t> +web interfaces and GUI-based Sieve editors.</t> </section> <section title="Conventions Used in This Document"> @@ -84,27 +93,60 @@ arguments syntax.</t> </section> <section title="Test "duplicate""> -<?rfc needLines="3" ?> +<?rfc needLines="4" ?> <figure> <artwork><![CDATA[ -Usage: "duplicate" [<name: string>] +Usage: "duplicate" [":seconds" <timeout: number>] + [":header" <header-name: string> / + ":value" <value: string>] + [":handle" <handle: string>] ]]></artwork> </figure> -<t>The "duplicate" test keeps track of which Message-ID values were seen before -by this test in an earlier delivery operation. It evaluates to "true" when the -Message-ID header of the current message was seen before. If it is not known, -the test evaluates to "false" and the Message-ID is added to a persistent -internal tracking list. Implementations SHOULD limit the number of messages that -are tracked and SHOULD let Message-ID entries expire after some short period of -time. -</t> - -<t>Using the "name" argument, the duplicate test can be employed for multiple -independent purposes. Only when the Message-ID was seen before in an earlier -script execution by a duplicate test with the same "name" argument, it is +<t>The "duplicate" test keeps track of which values were seen before by this +test in an earlier execution of this Sieve script. In its basic form, the tested +value is the content of the Message-ID header of the message. This way, this +test can be used to detect duplicate deliveries of the same message. It can also +detect duplicate deliveries based on other message header fields if requested +and it can even use a user-provided string value, e.g. as composed from text +extracted from the message using the "variables" <xref target="VARIABLES"/> +extension.</t> + +<t>The "duplicate" test evaluates to "true" when the provided value was seen +before. If the value is not known, the test evaluates to "false" and the value +is added to an internal value tracking list. Implementations SHOULD limit the +number of values (and thereby messages) that are tracked. Also, implementations +SHOULD let entries in the value tracking list expire after a short period of +time.</t> + +<t>The user can explicitly control the length this expiration time by means of +the ":seconds" argument. If the ":seconds" argument is omitted, an appropriate +default MUST be used. Sites SHOULD impose a maximum limit on the expiration +time. If that limit is exceeded, the maximum value MUST silently be substituted; +exceeding the limit MUST NOT produce an error.</t> + +<t>By default the tracked value is the content of the message's Message-ID +header field. For more advanced purposes, the content of another header can be +chosen for tracking by specifying the ":header" argument. The tracked string +value can also be specified explicitly using the ":value" argument. The +":header" and ":value" arguments are mutually exclusive and specifying both for +a single "duplicate" test command MUST trigger an error at compile time. If the +value is extracted from a header, i.e. when the ":value" argument is not used, +leading and trailing whitespace (see Section 2.2 of +<xref target="SIEVE">RFC 5228</xref>) MUST first be trimmed from the value +before executing the test.</t> + +<t>Using the ":handle" argument, the duplicate test can be employed for multiple +independent purposes. Only when the tracked value was seen before in an earlier +script execution by a "duplicate" test with the same ":handle" argument, it is recognized as a duplicate. </t> + +<t>NOTE: The necessary mechanism to track duplicate messages is very similar to +the mechanism that is needed for tracking duplicate responses for the "vacation" +<xref target="VACATION"/> action. One way to implement the necessary mechanism +for the "duplicate" test is therefore to store a hash of the tracked value and, +if provided, the ":handle" argument.</t> </section> <section title="Sieve Capability Strings"> @@ -113,12 +155,13 @@ will advertise the capability string "vnd.dovecot.duplicate". </t> </section> -<section title="Example"> -<t>In this example, duplicate deliveries are stored in a special folder +<section title="Examples"> +<t>In the following basic example, message duplicates are detected by tracking +the Message-ID header. Duplicate deliveries are stored in a special folder contained in the user's Trash folder. If the folder does not exist, it is -created. This way, the user has a chance to recover messages when necessary. -Messages that are not recognized as duplicates are stored in the user's inbox as -normal. +created automatically using the "mailbox" <xref target="MAILBOX"/> extension. +This way, the user has a chance to recover messages when necessary. Messages +that are not recognized as duplicates are stored in the user's inbox as normal. </t> <?rfc needLines="7" ?> @@ -127,15 +170,49 @@ normal. require ["vnd.dovecot.duplicate", "fileinto", "mailbox"]; if duplicate { - fileinto :create "Trash/Duplicate"; + fileinto :create "Trash/Duplicate"; +} +]]></artwork> +</figure> + +<t>The next example shows a more complex use of the "duplicate" test. The user +gets network alerts from a set of remote automated monitoring systems. Multiple +notifications can be received about the same event from different monitoring +systems. The Message-ID of these messages is different, because these are all +distinct messages from different senders. To avoid being notified multiple times +about the same event the user writes the following script:</t> + +<?rfc needLines="9" ?> +<figure> +<artwork><![CDATA[ +require ["vnd.dovecot.duplicate", "variables", "imap4flags", + "fileinto"]; + +if header :matches "subject" "ALERT: *" { + if duplicate :seconds 60 :value "${1}" { + setflag "\\seen"; + } + fileinto "Alerts"; } ]]></artwork> </figure> +<t>The subjects of the notification message are structured with a predictable +pattern which includes a description of the event. In the script above the +"duplicate" test is used to detect duplicate alert events. The message subject +is matched against a pattern and the event description is extracted using the +"variables" <xref target="VARIABLES"/> extension. If a message with that +event in the subject was received before, but more than a minute ago, it is not +detected as a duplicate due to the specified ":seconds" argument. In the the +event of a duplicate, the message is marked as seen using the "imap4flags" +<xref target="IMAP4FLAGS"/> extension. All alert messages are put into the +"Alerts" mailbox irrespective of wether those messages are duplicates or not. +</t> + </section> <section anchor="Security" title="Security Considerations"> -<t>A flood of unique messages could cause the list of tracked Message-IDs to +<t>A flood of unique messages could cause the list of tracked values to grow indefinitely. Implementations therefore SHOULD implement limits on the number and lifespan of entries in that list.</t> </section> @@ -151,9 +228,12 @@ number and lifespan of entries in that list.</t> <?rfc include="reference.SIEVE.xml"?> </references> -<!--<references title="Informative References"> - -</references> --> +<references title="Informative References"> + <?rfc include="reference.IMAP4FLAGS.xml"?> + <?rfc include="reference.MAILBOX.xml"?> + <?rfc include="reference.VACATION.xml"?> + <?rfc include="reference.VARIABLES.xml"?> +</references> </back> </rfc> diff --git a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c index b24806906..df4d53599 100644 --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c @@ -1,20 +1,20 @@ -/* Copyright (c) 2002-2012 Sieve duplicate Plugin authors, see the included - * COPYING file. +/* Copyright (c) 2002-2012 Pigeonhole authors, see the included COPYING file */ #include "lib.h" #include "md5.h" #include "ioloop.h" #include "str.h" +#include "array.h" #include "sieve-common.h" #include "sieve-settings.h" #include "sieve-error.h" #include "sieve-extensions.h" +#include "sieve-message.h" #include "sieve-code.h" #include "sieve-runtime.h" #include "sieve-actions.h" -#include "sieve-result.h" #include "ext-duplicate-common.h" @@ -24,27 +24,29 @@ #define EXT_DUPLICATE_DEFAULT_PERIOD (1*24*60*60) -struct ext_duplicate_config { - unsigned int period; -}; - bool ext_duplicate_load (const struct sieve_extension *ext, void **context) { struct sieve_instance *svinst = ext->svinst; struct ext_duplicate_config *config; - sieve_number_t period; + sieve_number_t default_period, max_period; if ( *context != NULL ) ext_duplicate_unload(ext); if ( !sieve_setting_get_duration_value - (svinst, "sieve_duplicate_period", &period) ) { - period = EXT_DUPLICATE_DEFAULT_PERIOD; + (svinst, "sieve_duplicate_default_period", &default_period) ) { + default_period = EXT_DUPLICATE_DEFAULT_PERIOD; + } + + if ( !sieve_setting_get_duration_value + (svinst, "sieve_duplicate_max_period", &max_period) ) { + max_period = EXT_DUPLICATE_DEFAULT_PERIOD; } config = i_new(struct ext_duplicate_config, 1); - config->period = period; + config->default_period = default_period; + config->max_period = max_period; *context = (void *) config; return TRUE; @@ -63,58 +65,93 @@ void ext_duplicate_unload * Duplicate checking */ -struct ext_duplicate_context { +struct ext_duplicate_handle { + const char *handle; unsigned int duplicate:1; }; +struct ext_duplicate_context { + ARRAY_DEFINE(handles, struct ext_duplicate_handle); + + unsigned int nohandle_duplicate:1; + unsigned int nohandle_checked:1; +}; + bool ext_duplicate_check -(const struct sieve_runtime_env *renv, string_t *name) +(const struct sieve_runtime_env *renv, string_t *handle, + const char *value, size_t value_len, sieve_number_t period) { const struct sieve_extension *this_ext = renv->oprtn->ext; const struct sieve_script_env *senv = renv->scriptenv; struct ext_duplicate_context *rctx; - pool_t pool; + bool duplicate = FALSE; + pool_t pool = NULL; + static const char *id = "sieve duplicate"; + unsigned char dupl_hash[MD5_RESULTLEN]; + struct md5_context ctx; + + if ( !sieve_action_duplicate_check_available(senv) || value == NULL ) + return FALSE; /* Get context; find out whether duplicate was checked earlier */ rctx = (struct ext_duplicate_context *) - sieve_result_extension_get_context(renv->result, this_ext); - - if ( rctx != NULL ) { - /* Already checked for duplicate */ - return rctx->duplicate; + sieve_message_context_extension_get(renv->msgctx, this_ext); + + if ( rctx == NULL ) { + /* Create context */ + pool = sieve_message_context_pool(renv->msgctx); + rctx = p_new(pool, struct ext_duplicate_context, 1); + sieve_message_context_extension_set(renv->msgctx, this_ext, (void *)rctx); + } else { + if ( handle == NULL ) { + if ( rctx->nohandle_checked ) { + /* Already checked for duplicate */ + return rctx->nohandle_duplicate; + } + } else if ( array_is_created(&rctx->handles) ) { + const struct ext_duplicate_handle *record; + array_foreach (&rctx->handles, record) { + if ( strcmp(record->handle, str_c(handle)) == 0 ) + return record->duplicate; + } + } + } + + /* Create hash */ + md5_init(&ctx); + md5_update(&ctx, id, strlen(id)); + if (handle != NULL) { + md5_update(&ctx, "h-", 2); + md5_update(&ctx, str_c(handle), str_len(handle)); + } else { + md5_update(&ctx, "default", 7); } - - /* Create context */ - pool = sieve_result_pool(renv->result); - rctx = p_new(pool, struct ext_duplicate_context, 1); - sieve_result_extension_set_context(renv->result, this_ext, (void *)rctx); - - /* Lookup duplicate */ - if ( sieve_action_duplicate_check_available(senv) - && renv->msgdata->id != NULL ) { - static const char *id = "sieve duplicate"; - struct ext_duplicate_config *ext_config = - (struct ext_duplicate_config *) this_ext->context; - unsigned char dupl_hash[MD5_RESULTLEN]; - struct md5_context ctx; - - /* Create hash */ - md5_init(&ctx); - md5_update(&ctx, id, strlen(id)); - if (name != NULL) - md5_update(&ctx, str_c(name), str_len(name)); - md5_update(&ctx, renv->msgdata->id, strlen(renv->msgdata->id)); - md5_final(&ctx, dupl_hash); - - /* Check duplicate */ - rctx->duplicate = sieve_action_duplicate_check - (senv, dupl_hash, sizeof(dupl_hash)); - - /* Create/refresh entry */ - sieve_action_duplicate_mark - (senv, dupl_hash, sizeof(dupl_hash), ioloop_time + ext_config->period); + md5_update(&ctx, value, value_len); + md5_final(&ctx, dupl_hash); + + /* Check duplicate */ + duplicate = sieve_action_duplicate_check + (senv, dupl_hash, sizeof(dupl_hash)); + + /* Create/refresh entry */ + sieve_action_duplicate_mark + (senv, dupl_hash, sizeof(dupl_hash), ioloop_time + period); + + if ( handle == NULL ) { + rctx->nohandle_duplicate = duplicate; + rctx->nohandle_checked = TRUE; + } else { + struct ext_duplicate_handle *record; + + if ( pool == NULL ) + pool = sieve_message_context_pool(renv->msgctx); + if ( !array_is_created(&rctx->handles) ) + p_array_init(&rctx->handles, pool, 64); + record = array_append_space(&rctx->handles); + record->handle = p_strdup(pool, str_c(handle)); + record->duplicate = duplicate; } - return rctx->duplicate; + return duplicate; } diff --git a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h index 5fc6f7433..97ab57005 100644 --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h @@ -1,5 +1,4 @@ -/* Copyright (c) 2002-2012 Sieve duplicate Plugin authors, see the included - * COPYING file. +/* Copyright (c) 2002-2012 Pigeonhole authors, see the included COPYING file */ #ifndef __EXT_DUPLICATE_COMMON_H @@ -11,6 +10,11 @@ * Extension */ +struct ext_duplicate_config { + unsigned int default_period; + unsigned int max_period; +}; + bool ext_duplicate_load (const struct sieve_extension *ext, void **context); void ext_duplicate_unload @@ -35,6 +39,7 @@ extern const struct sieve_operation_def tst_duplicate_operation; */ bool ext_duplicate_check - (const struct sieve_runtime_env *renv, string_t *name); + (const struct sieve_runtime_env *renv, string_t *handle, + const char *value, size_t value_len, sieve_number_t period); #endif /* EXT_DUPLICATE_COMMON_H */ diff --git a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate.c b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate.c index a9b2c2320..322313af1 100644 --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate.c +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate.c @@ -1,5 +1,4 @@ -/* Copyright (c) 2002-2012 Sieve duplicate Plugin authors, see the included - * COPYING file. +/* Copyright (c) 2002-2012 Pigeonhole authors, see the included COPYING file */ /* Extension vnd.dovecot.duplicate diff --git a/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c b/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c index 8c5b67290..4f77d5af6 100644 --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c @@ -1,5 +1,4 @@ -/* Copyright (c) 2002-2012 Sieve duplicate Plugin authors, see the included - * COPYING file. +/* Copyright (c) 2002-2012 Pigeonhole authors, see the included COPYING file */ #include "lib.h" @@ -18,27 +17,77 @@ /* Duplicate test * * Syntax: - * "duplicate" [<name: string>] - * + * Usage: "duplicate" [":seconds" <timeout: number>] + * [":header" <header-name: string> / + * ":value" <value: string>] + * [":handle" <handle: string>] */ -static bool tst_duplicate_validate - (struct sieve_validator *valdtr, struct sieve_command *cmd); +static bool tst_duplicate_registered + (struct sieve_validator *valdtr, const struct sieve_extension *ext, + struct sieve_command_registration *cmd_reg); static bool tst_duplicate_generate (const struct sieve_codegen_env *cgenv, struct sieve_command *ctx); const struct sieve_command_def tst_duplicate = { "duplicate", SCT_TEST, - -1, /* We check positional arguments ourselves */ - 0, FALSE, FALSE, - NULL, NULL, - tst_duplicate_validate, - NULL, + 0, 0, FALSE, FALSE, + tst_duplicate_registered, + NULL, NULL, NULL, tst_duplicate_generate, NULL, }; +/* + * Duplicate test tags + */ + +static bool tst_duplicate_validate_number_tag + (struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd); +static bool tst_duplicate_validate_string_tag + (struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd); + +static const struct sieve_argument_def duplicate_seconds_tag = { + "seconds", + NULL, + tst_duplicate_validate_number_tag, + NULL, NULL, NULL, +}; + +static const struct sieve_argument_def duplicate_header_tag = { + "header", + NULL, + tst_duplicate_validate_string_tag, + NULL, NULL, NULL +}; + +static const struct sieve_argument_def duplicate_value_tag = { + "value", + NULL, + tst_duplicate_validate_string_tag, + NULL, NULL, NULL +}; + +static const struct sieve_argument_def duplicate_handle_tag = { + "handle", + NULL, + tst_duplicate_validate_string_tag, + NULL, NULL, NULL +}; + +/* Codes for optional arguments */ + +enum tst_duplicate_optional { + OPT_END, + OPT_SECONDS, + OPT_HEADER, + OPT_VALUE, + OPT_HANDLE +}; + /* * Duplicate operation */ @@ -56,23 +105,104 @@ const struct sieve_operation_def tst_duplicate_operation = { }; /* - * Validation + * Tag validation */ -static bool tst_duplicate_validate -(struct sieve_validator *valdtr, struct sieve_command *cmd) +static bool tst_duplicate_validate_number_tag +(struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd) { - struct sieve_ast_argument *arg = cmd->first_positional; + const struct sieve_extension *ext = sieve_argument_ext(*arg); + const struct ext_duplicate_config *config = + (const struct ext_duplicate_config *) ext->context; + struct sieve_ast_argument *tag = *arg; + sieve_number_t seconds; - if ( arg == NULL ) - return TRUE; + /* Detach the tag itself */ + *arg = sieve_ast_arguments_detach(*arg,1); + + /* Check syntax: + * :seconds number + */ + if ( !sieve_validate_tag_parameter + (valdtr, cmd, tag, *arg, NULL, 0, SAAT_NUMBER, FALSE) ) { + return FALSE; + } + + seconds = sieve_ast_argument_number(*arg); + /* Enforce :days <= max_period */ + if ( config->max_period > 0 && seconds > config->max_period ) { + seconds = config->max_period; + + sieve_argument_validate_warning(valdtr, *arg, + "specified :seconds value '%lu' is over the maximum", + (unsigned long) seconds); + } + + sieve_ast_argument_number_set(*arg, seconds); + + /* Skip parameter */ + *arg = sieve_ast_argument_next(*arg); + + return TRUE; +} + +static bool tst_duplicate_validate_string_tag +(struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd) +{ + struct sieve_ast_argument *tag = *arg; - if ( !sieve_validate_positional_argument - (valdtr, cmd, arg, "name", 1, SAAT_STRING) ) { + /* Detach the tag itself */ + *arg = sieve_ast_arguments_detach(*arg,1); + + /* Check syntax: + * :header <header-name: string> + * :value <value: string> + * :handle <handle: string> + */ + if ( !sieve_validate_tag_parameter + (valdtr, cmd, tag, *arg, NULL, 0, SAAT_STRING, FALSE) ) { return FALSE; } - return sieve_validator_argument_activate(valdtr, cmd, arg, FALSE); + if ((bool)cmd->data == TRUE) { + sieve_argument_validate_error(valdtr, *arg, + "conflicting :header and :value arguments specified " + "for the duplicate test"); + return TRUE; + } + + if ( sieve_argument_is(tag, duplicate_header_tag) ) { + if ( !sieve_command_verify_headers_argument(valdtr, *arg) ) + return FALSE; + cmd->data = (void*)TRUE; + } else if ( sieve_argument_is(tag, duplicate_value_tag) ) { + cmd->data = (void*)TRUE; + } + + /* Skip parameter */ + *arg = sieve_ast_argument_next(*arg); + return TRUE; +} + +/* + * Command registration + */ + +static bool tst_duplicate_registered +(struct sieve_validator *valdtr, const struct sieve_extension *ext, + struct sieve_command_registration *cmd_reg) +{ + sieve_validator_register_tag + (valdtr, cmd_reg, ext, &duplicate_seconds_tag, OPT_SECONDS); + sieve_validator_register_tag + (valdtr, cmd_reg, ext, &duplicate_header_tag, OPT_HEADER); + sieve_validator_register_tag + (valdtr, cmd_reg, ext, &duplicate_value_tag, OPT_VALUE); + sieve_validator_register_tag + (valdtr, cmd_reg, ext, &duplicate_handle_tag, OPT_HANDLE); + return TRUE; } /* @@ -87,10 +217,6 @@ static bool tst_duplicate_generate if ( !sieve_generate_arguments(cgenv, cmd, NULL) ) return FALSE; - /* Emit a placeholder when the <name> argument is missing */ - if ( cmd->first_positional == NULL ) - sieve_opr_omitted_emit(cgenv->sblock); - return TRUE; } @@ -101,10 +227,43 @@ static bool tst_duplicate_generate static bool tst_duplicate_operation_dump (const struct sieve_dumptime_env *denv, sieve_size_t *address) { + int opt_code = 0; + sieve_code_dumpf(denv, "DUPLICATE"); sieve_code_descend(denv); - return sieve_opr_string_dump_ex(denv, address, "name", ""); + /* Dump optional operands */ + + for (;;) { + int opt; + bool opok = TRUE; + + if ( (opt=sieve_opr_optional_dump(denv, address, &opt_code)) < 0 ) + return FALSE; + + if ( opt == 0 ) break; + + switch ( opt_code ) { + case OPT_SECONDS: + opok = sieve_opr_number_dump(denv, address, "seconds"); + break; + case OPT_HEADER: + opok = sieve_opr_string_dump(denv, address, "header"); + break; + case OPT_VALUE: + opok = sieve_opr_string_dump(denv, address, "value"); + break; + case OPT_HANDLE: + opok = sieve_opr_string_dump(denv, address, "handle"); + break; + default: + return FALSE; + } + + if ( !opok ) return FALSE; + } + + return TRUE; } /* @@ -114,7 +273,14 @@ static bool tst_duplicate_operation_dump static int tst_duplicate_operation_execute (const struct sieve_runtime_env *renv, sieve_size_t *address ATTR_UNUSED) { - string_t *name = NULL; + const struct sieve_extension *ext = renv->oprtn->ext; + const struct ext_duplicate_config *config = + (const struct ext_duplicate_config *) ext->context; + int opt_code = 0; + string_t *handle = NULL, *header = NULL, *value; + const char *val = NULL; + size_t val_len = 0; + sieve_number_t seconds = config->default_period; bool duplicate = FALSE; int ret; @@ -122,10 +288,36 @@ static int tst_duplicate_operation_execute * Read operands */ - /* Read rejection reason */ - if ( (ret=sieve_opr_string_read_ex(renv, address, "name", TRUE, &name, NULL)) - <= 0 ) - return ret; + /* Optional operands */ + + for (;;) { + int opt; + + if ( (opt=sieve_opr_optional_read(renv, address, &opt_code)) < 0 ) + return SIEVE_EXEC_BIN_CORRUPT; + + if ( opt == 0 ) break; + + switch ( opt_code ) { + case OPT_SECONDS: + ret = sieve_opr_number_read(renv, address, "seconds", &seconds); + break; + case OPT_HEADER: + ret = sieve_opr_string_read(renv, address, "header", &header); + break; + case OPT_VALUE: + ret = sieve_opr_string_read(renv, address, "value", &value); + break; + case OPT_HANDLE: + ret = sieve_opr_string_read(renv, address, "handle", &handle); + break; + default: + sieve_runtime_trace_error(renv, "unknown optional operand"); + ret = SIEVE_EXEC_BIN_CORRUPT; + } + + if ( ret <= 0 ) return ret; + } /* * Perform operation @@ -135,8 +327,27 @@ static int tst_duplicate_operation_execute sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "duplicate test"); sieve_runtime_trace_descend(renv); + /* Get value */ + if (header != NULL) { + if (mail_get_first_header(renv->msgdata->mail, str_c(header), &val) > 0) + val_len = strlen(val); + } else if (value != NULL) { + val = str_c(value); + val_len = str_len(value); + } else { + val = renv->msgdata->id; + val_len = strlen(renv->msgdata->id); + } + /* Check duplicate */ - if ( (duplicate=ext_duplicate_check(renv, name)) ) { + if (val == NULL) { + duplicate = FALSE; + } else { + duplicate = ext_duplicate_check(renv, handle, val, val_len, seconds); + } + + /* Trace */ + if (duplicate) { sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "message is a duplicate"); } else { diff --git a/tests/extensions/vnd.dovecot/duplicate/errors.svtest b/tests/extensions/vnd.dovecot/duplicate/errors.svtest index 62d7374d4..769ca64d9 100644 --- a/tests/extensions/vnd.dovecot/duplicate/errors.svtest +++ b/tests/extensions/vnd.dovecot/duplicate/errors.svtest @@ -12,7 +12,7 @@ test "Invalid Syntax" { test_fail "compile should have failed"; } - if not test_error :count "eq" :comparator "i;ascii-numeric" "4" { + if not test_error :count "eq" :comparator "i;ascii-numeric" "5" { test_fail "wrong number of errors reported"; } } diff --git a/tests/extensions/vnd.dovecot/duplicate/errors/syntax.sieve b/tests/extensions/vnd.dovecot/duplicate/errors/syntax.sieve index 5ecfb65f4..f62aa2c0e 100644 --- a/tests/extensions/vnd.dovecot/duplicate/errors/syntax.sieve +++ b/tests/extensions/vnd.dovecot/duplicate/errors/syntax.sieve @@ -3,7 +3,10 @@ require "vnd.dovecot.duplicate"; # Used as a command duplicate; -# Used with string argument (not an error) +# Used with no argument (not an error) +if duplicate {} + +# Used with string argument if duplicate "frop" { } # Used with numer argument diff --git a/tests/extensions/vnd.dovecot/duplicate/execute.svtest b/tests/extensions/vnd.dovecot/duplicate/execute.svtest index 00e9cf1bc..386550f87 100644 --- a/tests/extensions/vnd.dovecot/duplicate/execute.svtest +++ b/tests/extensions/vnd.dovecot/duplicate/execute.svtest @@ -6,7 +6,15 @@ test "Run" { test_fail "test erroneously reported a duplicate"; } - if duplicate "name" { + if duplicate :handle "handle" { + test_fail "test with name erroneously reported a duplicate"; + } + + if duplicate { + test_fail "test erroneously reported a duplicate"; + } + + if duplicate :handle "handle" { test_fail "test with name erroneously reported a duplicate"; } } -- GitLab