Issue #3088648 by mfb, beeyayjay: Add SHA-512 et al. to list of available hashing algorithms

duplicate files to be detected, and allow copies to be verified against the

original source.

File Hash module generates and stores MD5, SHA-1 and/or SHA-256 hashes for each

file uploaded to the site.

File Hash module generates and stores MD5, SHA-1, SHA-224, SHA-256, SHA-384,

SHA-512/224, SHA-512/256, SHA-512, SHA3-224, SHA3-256, SHA3-384 and/or

SHA3-512 hashes for each file uploaded to the site.

Hash algorithms can be enabled and disabled by the site administrator.

    '#title' => t('Disallow duplicate files'),

    '#type' => 'checkbox',

  );

  return system_settings_form($form);

  $form = system_settings_form($form);

  $form['#submit'][] = 'filehash_settings_submit';

  return $form;

}

/**

 * Settings submit callback.

 */

function filehash_settings_submit() {

  filehash_add_columns();

}

/**

    drupal_set_message(t('An error occurred after processing @processed files.', $variables), 'warning');

  }

}

/**

 * Builds the File Hash clean up form.

 */

function filehash_clean() {

  $columns = filehash_get_disabled_columns_with_data();

  if (!$columns) {

    drupal_set_message(t('All clean! No disabled hash algorithms are installed.'));

    return;

  }

  $form['explanation'] = array(

    '#type' => 'item',

    '#description' => t('Are you sure you want to delete data for disabled hash algorithms? @count hash algorithm columns (@algos) will be purged from the database.', array(

      '@count' => count($columns),

      '@algos' => implode(', ', $columns),

    )),

  );

  $form['submit'] = array(

    '#type' => 'submit',

    '#value' => t('Delete data'),

  );

  return $form;

}

/**

 * Submit callback for File Hash clean up.

 */

function filehash_clean_submit($form, &$form_state) {

  batch_set(array(

    'operations' => array(array('filehash_clean_process', array())),

    'finished' => 'filehash_clean_finished',

    'title' => t('Removing disabled database columns'),

    'init_message' => t('File hash clean up is starting.'),

    'progress_message' => t('Please wait...'),

    'error_message' => t('File hash clean up has encountered an error.'),

    'file' => drupal_get_path('module', 'filehash') . '/filehash.admin.inc',

  ));

}

/**

 * Returns an arry of disabled hash algorithms that have a database column.

 */

function filehash_get_disabled_columns_with_data() {

  $columns = array();

  $enabled = filehash_columns();

  foreach (filehash_names() as $column => $name) {

    if (empty($enabled[$column]) && db_field_exists('filehash', $column)) {

      $columns[$column] = $name;

    }

  }

  return $columns;

}

/**

 * Drops database columns for disabled hash algorithms.

 */

function filehash_clean_process(&$context) {

  $columns = filehash_get_disabled_columns_with_data();

  if (!isset($context['results']['processed'])) {

    $context['results']['processed'] = 0;

    $context['sandbox']['count'] = count($columns);

  }

  foreach ($columns as $column => $name) {

    db_drop_field('filehash', $column);

    $context['results']['processed']++;

    $context['message'] = t('Dropped %name column.', array('%name' => $name));

    break;

  }

  $context['finished'] = $context['sandbox']['count'] ? ($context['results']['processed'] / $context['sandbox']['count']) : 1;

}

/**

 * Batch finish callback.

 */

function filehash_clean_finished($success, $results, $operations) {

  $variables = array('@processed' => $results['processed']);

  if ($success) {

    drupal_set_message(format_plural($results['processed'], 'Processed @processed hash algorithm column.', 'Processed @processed hash algorithm columns.', $variables));

  }

  else {

    drupal_set_message(format_plural($results['processed'], 'An error occurred after processing @processed hash algorithm column.', 'An error occurred after processing @processed hash algorithm columns.', $variables), 'warning');

  }

}

        'unsigned' => TRUE,

        'not null' => TRUE,

      ),

      'md5' => array(

        'description' => 'MD5 hash for this file.',

        'type' => 'char',

        'length' => 32,

        'not null' => FALSE,

      ),

      'sha1' => array(

        'description' => 'SHA-1 hash for this file.',

        'type' => 'char',

        'length' => 40,

        'not null' => FALSE,

      ),

      'sha256' => array(

        'description' => 'SHA-256 hash for this file.',

        'type' => 'char',

        'length' => 64,

        'not null' => FALSE,

      ),

    ),

    'primary key' => array('fid'),

    'indexes' => array(

      'md5_idx' => array('md5'),

      'sha1_idx' => array('sha1'),

      'sha256_idx' => array('sha256'),

    ),

    'foreign keys' => array(

      'fid' => array(

        'table' => 'file_managed',

    'file' => 'filehash.admin.inc',

    'type' => MENU_LOCAL_TASK,

  );

  $items['admin/config/media/filehash/clean'] = array(

    'title' => 'Clean up',

    'description' => 'Delete data for disabled hash algorithms.',

    'page callback' => 'drupal_get_form',

    'page arguments' => array('filehash_clean'),

    'access arguments' => array('administer site configuration'),

    'file' => 'filehash.admin.inc',

    'type' => MENU_LOCAL_TASK,

  );

  return $items;

}

/**

 * Returns array of enabled hash algorithms.

 * Returns array of enabled File Hash algorithm identifiers.

 */

function filehash_columns() {

  $options = filehash_names();

  foreach ($options as $column => $name) {

    $options[$column] = $column;

  }

  return array_intersect_assoc($options, variable_get('filehash_algos', array()));

}

/**

 * Returns array of enabled PHP hash algorithm identifiers.

 */

function filehash_algos() {

  return array_diff(variable_get('filehash_algos', array()), array(0));

  return str_replace(array('sha3_', 'sha512_'), array('sha3-', 'sha512/'), filehash_columns());

}

/**

 * Implements hook_file_load().

 */

function filehash_file_load($files) {

  $algos = filehash_algos();

  $algos = filehash_columns();

  if ($algos) {

    $result = db_select('filehash')

      ->fields('filehash')

      ->execute();

    foreach ($result as $record) {

      foreach ($algos as $algo) {

        $files[$record->fid]->filehash[$algo] = $record->$algo;

        // Handle missing column.

        $files[$record->fid]->filehash[$algo] = property_exists($record, $algo) ? $record->$algo : NULL;

      }

    }

    // Generate hash if it does not already exist for the file.

  $errors = array();

  // Zero size may indicate unreadable remote media source; bypass dedupe.

  if ($file->filesize && variable_get('filehash_dedupe', 0)) {

    foreach (filehash_algos() as $algo) {

      if ($fid = db_query("SELECT fid FROM {filehash} WHERE $algo = :hash LIMIT 1", array(':hash' => hash_file($algo, $file->uri)))->fetchField()) {

    foreach (filehash_algos() as $column => $algo) {

      try {

        $fid = filehash_duplicate_lookup($column, $algo, $file);

      }

      catch (PDOException $e) {

        filehash_add_columns();

        $fid = filehash_duplicate_lookup($column, $algo, $file);

      }

      if ($fid) {

        $duplicate = file_load($fid);

        if (file_download_access($duplicate->uri)) {

          $errors[] = t('This file has already been uploaded as %filename.', array('%filename' => $duplicate->filename));

  return $errors;

}

/**

 * Returns file ID for any duplicates.

 */

function filehash_duplicate_lookup($column, $algo, $file) {

  return db_query("SELECT fid FROM {filehash} WHERE $column = :hash LIMIT 1", array(':hash' => hash_file($algo, $file->uri)))->fetchField();

}

/**

 * Returns array of human-readable hash algorithm names.

 */

function filehash_names() {

  return array('md5' => 'MD5', 'sha1' => 'SHA-1', 'sha256' => 'SHA-256');

  return array(

    'md5' => 'MD5',

    'sha1' => 'SHA-1',

    'sha224' => 'SHA-224',

    'sha256' => 'SHA-256',

    'sha384' => 'SHA-384',

    'sha512_224' => 'SHA-512/224',

    'sha512_256' => 'SHA-512/256',

    'sha512' => 'SHA-512',

    'sha3_224' => 'SHA3-224',

    'sha3_256' => 'SHA3-256',

    'sha3_384' => 'SHA3-384',

    'sha3_512' => 'SHA3-512',

  );

}

/**

 * Calculates and saves the file hashes.

 */

function filehash_save($file) {

  $file->filehash = array_fill_keys(array('md5', 'sha1', 'sha256'), NULL);

  foreach (filehash_algos() as $algo) {

    $file->filehash[$algo] = hash_file($algo, $file->uri);

  $file->filehash = array_fill_keys(filehash_columns(), NULL);

  foreach (filehash_algos() as $column => $algo) {

    $file->filehash[$column] = hash_file($algo, $file->uri);

  }

  try {

    filehash_merge($file);

  }

  catch (PDOException $e) {

    filehash_add_columns();

    filehash_merge($file);

  }

}

/**

 * Merges file hashes into database.

 */

function filehash_merge($file) {

  db_merge('filehash')

    ->key(array('fid' => $file->fid))

    ->fields($file->filehash)

    ->execute();

}

/**

 * Adds any database columns required by configuration.

 */

function filehash_add_columns() {

  foreach (filehash_algos() as $column => $algo) {

    if (!db_field_exists('filehash', $column)) {

      db_add_field('filehash', $column, array(

        'description' => "The $algo hash for this file.",

        'type' => 'char',

        'length' => strlen(hash($algo, '')),

        'not null' => FALSE,

      ));

    }

    if (!db_index_exists('filehash', $column . '_idx')) {

      db_add_index('filehash', $column . '_idx', array($column));

    }

  }

}

/**

 * Implements hook_views_api().

 */

 * Implements hook_field_formatter_info().

 */

function filehash_field_formatter_info() {

  $algos = filehash_algos();

  $algos = filehash_columns();

  return array(

    'filehash_table' => array(

      'label' => t('Table of files with file hashes'),

function filehash_field_formatter_settings_form($field, $instance, $view_mode, $form, &$form_state) {

  $names = filehash_names();

  $options = array();

  foreach (filehash_algos() as $algo) {

  foreach (filehash_columns() as $algo) {

    $options[$algo] = $names[$algo];

  }

  $element['algo'] = array(

function filehash_token_info() {

  $info = array();

  $names = filehash_names();

  foreach (filehash_algos() as $algo) {

    $info['tokens']['file']["filehash-$algo"] = array(

      'name' => t('@algo hash', array('@algo' => $names[$algo])),

      'description' => t('The @algo hash of the file.', array('@algo' => $names[$algo])),

  foreach (filehash_columns() as $column) {

    $info['tokens']['file']["filehash-$column"] = array(

      'name' => t('@algo hash', array('@algo' => $names[$column])),

      'description' => t('The @algo hash of the file.', array('@algo' => $names[$column])),

    );

    $info['tokens']['file']["filehash-$algo-pair-1"] = array(

      'name' => t('@algo hash: Initial pair', array('@algo' => $names[$algo])),

      'description' => t('The @algo hash of the file: first and second characters.', array('@algo' => $names[$algo])),

    $info['tokens']['file']["filehash-$column-pair-1"] = array(

      'name' => t('@algo hash: Initial pair', array('@algo' => $names[$column])),

      'description' => t('The @algo hash of the file: first and second characters.', array('@algo' => $names[$column])),

    );

    $info['tokens']['file']["filehash-$algo-pair-2"] = array(

      'name' => t('@algo hash: Second pair', array('@algo' => $names[$algo])),

      'description' => t('The @algo hash of the file: third and fourth characters.', array('@algo' => $names[$algo])),

    $info['tokens']['file']["filehash-$column-pair-2"] = array(

      'name' => t('@algo hash: Second pair', array('@algo' => $names[$column])),

      'description' => t('The @algo hash of the file: third and fourth characters.', array('@algo' => $names[$column])),

    );

  }

  return $info;

function filehash_tokens($type, $tokens, array $data = array(), array $options = array()) {

  $replacements = array();

  if ($type == 'file' && !empty($data['file'])) {

    foreach (filehash_algos() as $algo) {

    foreach (filehash_algos() as $column => $algo) {

      // Generate the file hash if it is needed but does not yet exist.

      if (empty($data['file']->filehash[$algo])) {

      if (empty($data['file']->filehash[$column])) {

        foreach ($tokens as $name => $original) {

          if (strpos($name, "filehash-$algo") === 0) {

            $data['file']->filehash[$algo] = hash_file($algo, $data['file']->uri);

          if (strpos($name, "filehash-$column") === 0) {

            $data['file']->filehash[$column] = hash_file($algo, $data['file']->uri);

            break;

          }

        }

      }

      if (isset($tokens["filehash-$algo"])) {

        $replacements[$tokens["filehash-$algo"]] = $data['file']->filehash[$algo];

      if (isset($tokens["filehash-$column"])) {

        $replacements[$tokens["filehash-$column"]] = $data['file']->filehash[$column];

      }

      if (isset($tokens["filehash-$algo-pair-1"])) {

        $replacements[$tokens["filehash-$algo-pair-1"]] = substr($data['file']->filehash[$algo], 0, 2);

      if (isset($tokens["filehash-$column-pair-1"])) {

        $replacements[$tokens["filehash-$column-pair-1"]] = substr($data['file']->filehash[$column], 0, 2);

      }

      if (isset($tokens["filehash-$algo-pair-2"])) {

        $replacements[$tokens["filehash-$algo-pair-2"]] = substr($data['file']->filehash[$algo], 2, 2);

      if (isset($tokens["filehash-$column-pair-2"])) {

        $replacements[$tokens["filehash-$column-pair-2"]] = substr($data['file']->filehash[$column], 2, 2);

      }

    }

  }