Issue #1188584 by davideads: Improved message cleaning/filtering options.

Mail Comment 6.x-2.x, 2011-08-30

--------------------------------

#1188584 by davideads: Improved message cleaning/filtering options.

Mail Comment 6.x-2.0-beta2, 2011-08-30

--------------------------------------

package = "Mailhandler"

dependencies[] = notifications

dependencies[] = mailhandler

files[] = tests/mailcomment.test

core = 6.x

  variable_set('mailcomment_mailboxes', array($mailbox));

}

/**

 * Set the new mailcomment_mailboxes variable with the mailcomment_mailbox variable by default.

 */

function mailcomment_update_6001() {

  // Set the new mailcomment_mailboxes variable with the mailcomment_mailbox variable by default.

  $mailbox = variable_get('mailcomment_mailbox', '');

  variable_set('mailcomment_mailboxes', array($mailbox));

  return array(array('success' => TRUE, 'query' => 'Set the mailcomment_mailboxes variable.'));

}

/**

 * Add filtering modes: To mimic old behavior, use basic filtering when mailcomment_reply_text was set by user.

 */

function mailcomment_update_6002() {

  if (variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'))) {

    variable_set('mailcomment_import_filter_mode', MAILCOMMENT_FILTER_BASIC);

    $query = 'Set mailcomment_import_filter_mode variable to use basic filtering.';

  }

  else {

    variable_set('mailcomment_import_filter_mode', MAILCOMMENT_FILTER_NONE);

    $query = 'Set mailcomment_import_filter_mode variable to use no filtering.';

  }

  return array(array('success' => TRUE, 'query' => $query));

}

// Error code, no valid user

define('MAILCOMMENT_ERROR_USER', 4);

// Filter modes

define('MAILCOMMENT_FILTER_NONE', 0);

define('MAILCOMMENT_FILTER_BASIC', 1);

define('MAILCOMMENT_FILTER_AGGRESSIVE', 2);

/**

 * Implementation of hook_menu().

 */

    '#default_value' => variable_get('mailcomment_expiration', 0),

    '#description' => t('Time after which the signature of outgoing e-mails expires and responses won\'t be accepted anymore.'),

  );

  $form['mailcomment_reply_text'] = array(

  $form['filter_options'] = array(

    '#title' => t('Message filtering'),

    '#type' => 'fieldset',

    '#collapsible' => TRUE,

  );

  $form['filter_options']['mailcomment_import_filter_mode'] = array(

    '#title' => t('Filter mode'),

    '#type' => 'radios',

    '#default_value' => variable_get('mailcomment_import_filter_mode', MAILCOMMENT_FILTER_NONE),

    '#options' => array(

      MAILCOMMENT_FILTER_NONE => t('No import filter. Leave messages as-is.'),

      MAILCOMMENT_FILTER_BASIC => t('Basic import filter: Filter signatures and below custom reply separator.'),

      MAILCOMMENT_FILTER_AGGRESSIVE => t('Aggressive: Filter below custom reply separator and common mail client text strings.'),

    ),

    '#description' => t('Should messages be filtered during import? If <em>No import filter</em> is selected, messages will be stored in the database as they are received.'),

  );

  $form['filter_options']['mailcomment_reply_text'] = array(

    '#title' => t('Reply text'),

    '#type' => 'textfield',

    '#default_value' => variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))')),

    '#description' => t('Text to separate reply from the rest of the e-mail. Leave blank for not using this feature.'),

    '#description' => t('Text to separate reply from the rest of the e-mail. Leave blank to disable.'),

  );

  $form['mailcomment_insert_reply_text'] = array(

  $form['filter_options']['mailcomment_insert_reply_text'] = array(

    '#title' => t('Insert reply to text'),

    '#type' => 'checkbox',

    '#default_value' => variable_get('mailcomment_insert_reply_text', 1),

    '#description' => t('If this box is checked, mail comment will insert the text specified as the Reply text at the top of messages sent. You may uncheck this box and insert the text manually into a messaging template (for example, above the footer).'),

  );

  $form['mailcomment_error_bounce'] = array(

    '#title' => t('Bounce rejected emails'),

    '#type' => 'checkbox',

    '#default_value' => variable_get('mailcomment_error_bounce', 1),

    '#description' => t('If this box is checked, a reply will be sent for wrong emails with some information about the cause of rejection.'),

  );

  $form['mailcomment_server_string'] = array(

    '#title' => t('Server string for Message Id'),

    '#type' => 'textfield',

    '#default_value' => variable_get('mailcomment_server_string', $_SERVER['HTTP_HOST']),

    '#description' => t('Server name to be used in Message Id\'s. It will be included in outgoing emails and checked on responses.'),

  );

  $form['mailcomment_alter_subjects'] = array(

    '#type' => 'radios',

    '#title' => t('Use mailing list-style subjects'),

  return md5(implode('-', $params));

}

// Need to clean up body text after the comment is parsed and before it is processed

/**

 * Implementation of hook_feeds_after_parse().

 *

 * Apply filters before writing to database.

 */

function mailcomment_feeds_after_parse(FeedsImporter $importer, FeedsSource $source) {

  // Now trim out the rest of the message if separator text exists

  // @ TODO May fail for html mails

  if ($marker = variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'))) {

  switch (variable_get('mailcomment_import_filter_mode', 0)) {

    case MAILCOMMENT_FILTER_BASIC:

      foreach ($source->batch->items as &$message) {

      $pos = strpos($message['origbody'], $marker);

      if ($pos !== FALSE) {

        $message['origbody'] = drupal_substr($message['origbody'], 0, $pos);

        $message['origbody'] = _mailcomment_filter_basic($message['origbody']);

      }

      break;

    case MAILCOMMENT_FILTER_AGGRESSIVE:

      foreach ($source->batch->items as &$message) {

        $message['origbody'] = _mailcomment_filter_aggressive($message['origbody']);

      }

      break;

  }

}

function mailcomment_filter_prepare($delta = 0, $format = -1, $text = '', $cache_id = 0) {

  switch ($delta) {

    case '0':

      return _mailcomment_filter_basic($text);

    case '1':

      return _mailcomment_filter_aggressive($text);

  }

}

/**

 * Filter a message for signature in a simple way.

 *

 * @param $text

 *   Text to filter.

 * @return

 *   A filtered string.

 */

function _mailcomment_filter_basic($text) {

  // Get the offset of the last two line breaks and substring.

  $search = preg_match_all("/<br\s*\/*><br\s*\/*>|\r\n\r\n|\n\n[^<br\s*\/*><br\s*\/*>|\r\n\r\n|\n\n]+$/m", $text, $matches, PREG_OFFSET_CAPTURE);

  if (!empty($matches[0])) {

    $key = max(array_keys($matches[0]));

    $offset = $matches[0][$key][1];

        $text = drupal_substr($text, 0, $offset);

    $text = trim(drupal_substr($text, 0, $offset));

  }

  if ($marker = variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'))) {

    $search = preg_match_all("/[\> ]*". preg_quote($marker) ."/", $text, $matches, PREG_OFFSET_CAPTURE);

    if (!empty($matches[0])) {

      $key = max(array_keys($matches[0]));

      $offset = $matches[0][$key][1];

      $text = trim(drupal_substr($text, 0, $offset));

    }

  }

  return $text;

    case '1':

      /*

       * Below are what each pattern tries to match.

       * gmail 1 => On 3/10/09,

       * gmail 2 => On Mon, Mar 9, 2009 at 10:14 PM,

       * mac_mail_1 => On Feb 17, 2009, at 2:30 PM, foo@example.com wrote:

       * mac_mail_2 => On Feb 17, 2009, at 2:30 PM,

       * outlook_express => ----- Original Message -----

       * outlook_legacy => From: foo@example.com [foo@example.com]

}

/**

 * Aggressively filter a message for quotes and signatures.

 *

 * @param $text

 *   Text to filter.

 * @return

 *   A filtered string.

 */

      // Build an array of different possible matching expressions

      $expressions = array(

        'gmail_1' => "/On \d{1,2}[\/\-]{1}\d{1,2}[\/\-]{1}\d{1,2},/",

        'gmail_2' => "/On \w{3}, \w{3} \d{1,2}, \d{4} at \d{1,2}:\d{1,2} [AP]M/",

        'mac_mail_1' => "/On [A-Z][a-z]{2} [0-9]{1,2}, [0-9]{4}, at [0-9]{1,2}:[0-9]{2} [AP]M, [a-zA-Z0-9._%-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4} wrote/",

        'mac_mail_2' => "/On [A-Z][a-z]{2} [0-9]{1,2}, [0-9]{4}, at [0-9]{1,2}:[0-9]{2} [AP]M,/", // currently mac mail uses a name rather than the email address

        'outlook_express' => '/-----\s?Original Message\s?-----/',

        'outlook_legacy' => '/From:(\s*)[A-Za-z0-9._%+-]+@[A-Za-z0-9._%+-]+\.[A-Za-z]{2,3}(\s*)\[mailto:[A-Za-z0-9._%+-]+@[A-Za-z0-9._%+-]+\.[A-Za-z]{2,3}\]/',

        'generic_1' => '/From:.*Sent:.*To:.*Subject:/msU',

function _mailcomment_filter_aggressive($text) {

  $delimiters = array();

  $delimiters = array(

    "On \d{1,2}[\/\-]{1}\d{1,2}[\/\-]{1}\d{1,2},", // Gmail form 1

    "On \w{3}, \w{3} \d{1,2}, \d{4} at \d{1,2}:\d{1,2} [AP]M", // Gmail form 1

    "On \w{3}, \w{3} \d{1,2}, \d{4} at \d{1,2}:\d{1,2} [AP]M", // Gmail form 2

    "On \d{1,2}\/\d{1,2}\/\d{4} \d{1,2}:\d{1,2} [AP]M,", // Thunderbird

    "On [A-Z][a-z]{2} [0-9]{1,2}, [0-9]{4}, at [0-9]{1,2}:[0-9]{2} [AP]M, [a-zA-Z0-9._%-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4} wrote", // Mac mail form 1

    "On [A-Z][a-z]{2} [0-9]{1,2}, [0-9]{4}, at [0-9]{1,2}:[0-9]{2} [AP]M,", // Mac mail form 2

    "On \d{1,2} \w{3} \d{4}, at \d{1,2}:\d{2} [AP]M, [a-zA-Z0-9._%-]+@?[a-zA-Z0-9.-]*[\.]?[a-zA-Z]{0,4} wrote:", // Mac mail v4.2

    "-----\s?Original Message\s?-----", // Outlook Express

    "From:(\s*)[A-Za-z0-9._%+-]+@[A-Za-z0-9._%+-]+\.[A-Za-z]{2,3}(\s*)\[mailto:[A-Za-z0-9._%+-]+@[A-Za-z0-9._%+-]+\.[A-Za-z]{2,3}\]", // Outlook legacy

    "[_]*\nFrom:", // Yahoo + Outlook web

    "From:.*Sent:.*To:.*Subject:", // Fallback

  );

      $match = 0;

      foreach ($expressions as $name => $expression) {

        if ($match < 1) {

          $split = preg_split($expression, $text);

          //$split = preg_split("/\>/",$split[0]);

          if (count($split) > 1) {

            // An expression was matched, so break the loop.

            $match = 1;

          }

        }

  if ($marker = variable_get('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'))) {

    $delimiters[] = '[\> ]*'. preg_quote($marker);

  }

      if (!empty($split[1])) {

        $text = $split[0];

  $expression = '/('. implode('|', $delimiters) .')/';

  if (preg_match($expression, $text, $matches, PREG_OFFSET_CAPTURE)) {

    $text = trim(drupal_substr($text, 0, $matches[0][1]));

  }

  return $text;

}

}

<?php

class MailCommentTestCase extends DrupalUnitTestCase {

  /**

   * getInfo() returns properties that are displayed in the test selection form.

   */

  public static function getinfo() {

    return array(

      'name' => 'Mailcomment',

      'description' => 'Test mailcomment message parsing.',

      'group' => 'mailcomment',

    );

  }

  /**

   * setUp() performs any pre-requisite tasks that need to happen. 

   */

  public function setUp() {

    variable_set('mailcomment_reply_text', t('((( Reply ABOVE this LINE to POST a COMMENT )))'));

    // Enable any modules required for the test.

    parent::setUp(

      'ctools',

      'notifications',

      'messaging', 

      'feeds',

      'autoload',

      'token',

      'job_scheduler',

      'messaging_template',

      'mailhandler',

      'mailcomment'

    );

  }

  public static function mailCommentTestMessages() {

    $messages = array();

    // Gmail

    $messages['gmail'] =  "Hello world.

On Fri, Aug 19, 2011 at 9:51 AM, test person <user@hostname.tld> wrote:

> ((( Reply ABOVE this LINE to POST a COMMENT )))

>

> user said:

>

> Courgette lettuce bell pepper asparagus collard greens epazote broccoli rabe. Wattle 

> seed cress welsh onion broccoli rabe pea sprouts tomatillo celery ricebean napa cabbage

> bush tomato daikon horseradish turnip winter purslane. Chickweed radicchio taro tigernut 

> spinach tomatillo black-eyed pea avocado.

-- 

Test person name | http://hostname.tld";

    // Thunderbird

    $messages['thunderbird'] = "Hello world.

On 08/18/2011 04:19 PM, Test Person wrote:

> ((( Reply ABOVE this LINE to POST a COMMENT )))

> 

> user said:

> 

> Mustard scallion yarrow aubergine sierra leone bologi sea lettuce

> chicory broccoli collard greens turnip greens shallot radicchio komatsuna.

> Broccoli celery grape bell pepper kohlrabi radicchio sierra leone bologi.";

    // Mac Mail 4.2

    $messages['mac_mail42'] = "Hello world. 

On 14 Sep 2011, at 4:37 PM, someuser wrote:

> ((( Reply ABOVE this LINE to POST a COMMENT ))) 

>

> Kakadu plum burdock collard greens sorrel dulse seakale winter purslane soko

> beet greens summer purslane radish tomato bunya nuts. Amaranth chickweed onion

> broccoli rabe shallot kombu pea napa cabbage welsh onion soko water chestnut 

> tomatillo bell pepper arugula pea sprouts. Azuki bean seakale scallion turnip 

> greens azuki bean groundnut beetroot cress courgette lentil epazote kale celery 

> garbanzo watercress.";

    // Yahoo mail

    $messages['yahoo'] = "Hello world.

________________________________

From: Firstname Lastname <user@hostname.tld>

To: recipient@hostname.tld

Sent: Tuesday, September 13, 2011 10:14 AM

Subject: Re: Test subject

((( Reply ABOVE this LINE to POST a COMMENT )))

Earthnut pea eggplant brussels sprout bitterleaf quandong desert raisin wattle

seed celtuce bamboo shoot mung bean cauliflower maize salad chickpea celery.

Winter purslane silver beet daikon brussels sprout broccoli rabe lotus root

water chestnut garlic melon lettuce potato pumpkin. Ricebean groundnut potato

celery corn artichoke sea lettuce. No joke.";

    // Outlook web

    $messages['outlook_web'] = "Hello world.

________________________________________

From: user@hostname.tld [user@hostname.tld]

Sent: Thursday, August 18, 2011 4:19 PM

To: Recipient Name

Subject: Re: [Site-name] Subject

((( Reply ABOVE this LINE to POST a COMMENT )))

Test user said:

Pea cress water spinach summer purslane asparagus scallion squash sorrel. Pea

green bean celery chickpea lotus root parsnip celery arugula carrot winter

purslane cress plantain tatsoi pumpkin pea ricebean mung bean lentil.";

    $messages['generic_reply'] = "Hello world.

> ((( Reply ABOVE this LINE to POST a COMMENT )))

> 

> user said:

> 

> Mustard scallion yarrow aubergine sierra leone bologi sea lettuce

> chicory broccoli collard greens turnip greens shallot radicchio komatsuna.

> Broccoli celery grape bell pepper kohlrabi radicchio sierra leone bologi.";

    return $messages;

  }

  public function testMailCommentParseMessage() {

    $messages = $this->mailCommentTestMessages();

    foreach ($messages as $key => $message) {

      $text = _mailcomment_filter_aggressive($message);

      $this->assertEqual($text, 'Hello world.', 'Filtered '. $key);

    }

  }

}

 No newline at end of file