Issue #1742894 by Sutharsan, webflo, attiks, Gábor Hojtsy: Added Get status of...

Issue #1742894 by Sutharsan, webflo, attiks, Gábor Hojtsy: Added Get status of local and remote translation files.

core/modules/locale/config/locale.settings.yml

 translation:
+  use_source: 'remote_and_local'
   check_disabled_modules: false
+  default_filename: '%project-%version.%language.po'
   default_server_pattern: 'http://ftp.drupal.org/files/translations/%core/%project/%project-%version.%language.po'

core/modules/locale/lib/Drupal/locale/Tests/LocaleCompareTest.php

@@ -14,6 +14,13 @@
  */
 class LocaleCompareTest extends WebTestBase {
 
+  /**
+   * The path of the translations directory where local translations are stored.
+   *
+   * @var string
+   */
+  private $tranlations_directory;
+
   /**
    * Modules to enable.
    *
@@ -29,6 +36,73 @@ public static function getInfo() {
     );
   }
 
+  /**
+   * Setup the test environment.
+   *
+   * We use German as default test language. Due to hardcoded configurations in
+   * the locale_test module, the language can not be chosen randomly.
+   */
+  function setUp() {
+    parent::setUp();
+    module_load_include('compare.inc', 'locale');
+    $admin_user = $this->drupalCreateUser(array('administer site configuration', 'administer languages', 'access administration pages', 'translate interface'));
+    $this->drupalLogin($admin_user);
+    $this->drupalPost('admin/config/regional/language/add', array('predefined_langcode' => 'de'), t('Add language'));
+  }
+
+  /**
+   * Set the value of the default translations directory.
+   *
+   * @param string $path
+   *   Path of the translations directory relative to the drupal installation
+   *   directory.
+   */
+  private function setTranslationsDirectory($path) {
+    $this->tranlations_directory = $path;
+    file_prepare_directory($path, FILE_CREATE_DIRECTORY);
+    variable_set('locale_translate_file_directory', $path);
+  }
+
+  /**
+   * Creates a translation file and tests its timestamp.
+   *
+   * @param string $path
+   *   Path of the file relative to the public file path.
+   * @param string $filename
+   *   Name of the file to create.
+   * @param integer $timestamp
+   *   Timestamp to set the file to. Defaults to current time.
+   * @param string $data
+   *   Translation data to put into the file. Po header data will be added.
+   */
+  private function makePoFile($path, $filename, $timestamp = NULL, $data = '') {
+    $timestamp = $timestamp ? $timestamp : REQUEST_TIME;
+    $path = 'public://' . $path;
+    $po_header = <<<EOF
+msgid ""
+msgstr ""
+"Project-Id-Version: Drupal 8\\n"
+"MIME-Version: 1.0\\n"
+"Content-Type: text/plain; charset=UTF-8\\n"
+"Content-Transfer-Encoding: 8bit\\n"
+"Plural-Forms: nplurals=2; plural=(n > 1);\\n"
+
+EOF;
+
+    file_prepare_directory($path, FILE_CREATE_DIRECTORY);
+    $file = entity_create('file', array(
+      'uid' => 1,
+      'filename' => $filename,
+      'uri' => $path . '/' . $filename,
+      'filemime' => 'text/x-gettext-translation',
+      'timestamp' => $timestamp,
+      'status' => FILE_STATUS_PERMANENT,
+    ));
+    file_put_contents($file->uri, $po_header . $data);
+    touch(drupal_realpath($file->uri), $timestamp);
+    $file->save();
+  }
+
   /**
    * Test for translation status storage and translation status comparison.
    */
@@ -46,7 +120,7 @@ function testLocaleCompare() {
     // Make the test modules look like a normal custom module. i.e. make the
     // modules not hidden. locale_test_system_info_alter() modifies the project
     // info of the locale_test and locale_test_disabled modules.
-    variable_set('locale_translation_test_system_info_alter', TRUE);
+    state()->set('locale_translation_test_system_info_alter', TRUE);
 
     // Reset static system list caches to reflect info changes.
     drupal_static_reset('locale_translation_project_list');
@@ -54,7 +128,7 @@ function testLocaleCompare() {
 
     // Check if interface translation data is collected from hook_info.
     $projects = locale_translation_project_list();
-    $this->assertEqual($projects['locale_test']['info']['interface translation server pattern'], 'core/modules/locale/test/modules/locale_test/%project-%version.%language.po', 'Interface translation parameter found in project info.');
+    $this->assertEqual($projects['locale_test']['info']['interface translation server pattern'], 'core/modules/locale/test/test.%language.po', 'Interface translation parameter found in project info.');
     $this->assertEqual($projects['locale_test']['name'] , 'locale_test', format_string('%key found in project info.', array('%key' => 'interface translation project')));
 
     // Get the locale settings.
@@ -72,12 +146,89 @@ function testLocaleCompare() {
     drupal_static_reset('locale_translation_project_list');
     $projects = locale_translation_get_projects();
     $this->assertEqual($projects['drupal']->name, 'drupal', 'Core project found');
-    $this->assertEqual($projects['locale_test']->server_pattern, 'core/modules/locale/test/modules/locale_test/%project-%version.%language.po', 'Interface translation parameter found in project info.');
+    $this->assertEqual($projects['locale_test']->server_pattern, 'core/modules/locale/test/test.%language.po', 'Interface translation parameter found in project info.');
     $this->assertEqual($projects['locale_test_disabled']->status, '0', 'Disabled module found');
     $config->delete('translation.check_disabled_modules');
 
     // Return the locale test modules back to their hidden state.
-    variable_del('locale_translation_test_system_info_alter');
+    state()->delete('locale_translation_test_system_info_alter');
   }
 
+  /**
+   * Checks if local or remote translation sources are detected.
+   *
+   * This test requires a simulated environment for local and remote files.
+   * Normally remote files are located at a remote server (e.g. ftp.drupal.org).
+   * For testing we can not rely on this. A directory in the file system of the
+   * test site is designated for remote files and is addressed using an absolute
+   * URL. Because Drupal does not allow files with a po extension to be accessed
+   * (denied in .htaccess) the translation files get a txt extension. Another
+   * directory is designated for local translation files.
+   *
+   * The translation status process by default checks the status of the
+   * installed projects. For testing purpose a predefined set of modules with
+   * fixed file names and release versions is used. Using a
+   * hook_locale_translation_projects_alter implementation in the locale_test
+   * module this custom project definition is applied.
+   *
+   * This test generates a set of local and remote translation files in their
+   * respective local and remote translation directory. The test checks whether
+   * the most recent files are selected in the different check scenarios: check
+   * for local files only, check for remote files only, check for both local and
+   * remote files.
+   */
+  function testCompareCheckLocal() {
+    $config = config('locale.settings');
+
+    // A flag is set to let the locale_test module replace the project data with
+    // a set of test projects.
+    state()->set('locale_translation_test_projects', TRUE);
+
+    // Setup timestamps to identify old and new translation sources.
+    $timestamp_old = REQUEST_TIME - 100;
+    $timestamp_new = REQUEST_TIME;
+
+    // Set up the environment.
+    $public_path = variable_get('file_public_path', conf_path() . '/files');
+    $this->setTranslationsDirectory($public_path . '/local');
+    $config->set('translation.default_filename', '%project-%version.%language.txt')->save();
+
+    // Add a number of files to the local file system to serve as remote
+    // translation server and match the project definitions set in
+    // locale_test_locale_translation_projects_alter().
+    $this->makePoFile('remote/8.x/contrib_module_one', 'contrib_module_one-8.x-1.1.de.txt', $timestamp_new);
+    $this->makePoFile('remote/8.x/contrib_module_two', 'contrib_module_two-8.x-2.0-beta4.de.txt', $timestamp_old);
+    $this->makePoFile('remote/8.x/contrib_module_three', 'contrib_module_three-8.x-1.0.de.txt', $timestamp_old);
+
+    // Add a number of files to the local file system to serve as local
+    // translation files and match the project definitions set in
+    // locale_test_locale_translation_projects_alter().
+    $this->makePoFile('local', 'contrib_module_one-8.x-1.1.de.txt', $timestamp_old);
+    $this->makePoFile('local', 'contrib_module_two-8.x-2.0-beta4.de.txt', $timestamp_new);
+    $this->makePoFile('local', 'custom_module_one.de.po', $timestamp_new);
+
+    // Get status of translation sources at local file system.
+    $config->set('translation.use_source', LOCALE_TRANSLATION_USE_SOURCE_LOCAL)->save();
+    $this->drupalGet('admin/reports/translations/check');
+    $result = state()->get('locale_translation_status');
+    $this->assertEqual($result['contrib_module_one']['de']->type, 'local', 'Translation of contrib_module_one found');
+    $this->assertEqual($result['contrib_module_one']['de']->timestamp, $timestamp_old, 'Translation timestamp found');
+    $this->assertEqual($result['contrib_module_two']['de']->type, 'local', 'Translation of contrib_module_two found');
+    $this->assertEqual($result['contrib_module_two']['de']->timestamp, $timestamp_new, 'Translation timestamp found');
+    $this->assertEqual($result['locale_test']['de']->type, 'local', 'Translation of locale_test found');
+    $this->assertEqual($result['custom_module_one']['de']->type, 'local', 'Translation of custom_module_one found');
+
+    // Get status of translation sources at both local and remote the locations.
+    $config->set('translation.use_source', LOCALE_TRANSLATION_USE_SOURCE_REMOTE_AND_LOCAL)->save();
+    $this->drupalGet('admin/reports/translations/check');
+    $result = state()->get('locale_translation_status');
+    $this->assertEqual($result['contrib_module_one']['de']->type, 'remote', 'Translation of contrib_module_one found');
+    $this->assertEqual($result['contrib_module_one']['de']->timestamp, $timestamp_new, 'Translation timestamp found');
+    $this->assertEqual($result['contrib_module_two']['de']->type, 'local', 'Translation of contrib_module_two found');
+    $this->assertEqual($result['contrib_module_two']['de']->timestamp, $timestamp_new, 'Translation timestamp found');
+    $this->assertEqual($result['contrib_module_three']['de']->type, 'remote', 'Translation of contrib_module_three found');
+    $this->assertEqual($result['contrib_module_three']['de']->timestamp, $timestamp_old, 'Translation timestamp found');
+    $this->assertEqual($result['locale_test']['de']->type, 'local', 'Translation of locale_test found');
+    $this->assertEqual($result['custom_module_one']['de']->type, 'local', 'Translation of custom_module_one found');
+  }
 }

core/modules/locale/locale.api.php

@@ -28,7 +28,18 @@
  * the module's folder.
  * @code
  * interface translation project = example_module
- * interface translation server pattern = sites/example.com/modules/custom/example_module/%project-%version.%language.po
+ * interface translation server pattern = modules/custom/example_module/%project-%version.%language.po
+ * @endcode
+ *
+ * Streamwrappers can be used in the server pattern definition. The interface
+ * translations directory (Configuration > Media > File system) can be addressed
+ * using the "translations://" streamwrapper. But also other streamwrappers can
+ * be used.
+ * @code
+ * interface translation server pattern = translations://%project-%version.%language.po
+ * @endcode
+ * @code
+ * interface translation server pattern = public://translations/%project-%version.%language.po
  * @endcode
  *
  * Multiple custom modules or themes sharing the same po file should have
@@ -97,7 +108,7 @@
  * @param array $projects
  *   Project data as returned by update_get_projects().
  *
- * @see locale_project_list().
+ * @see locale_translation_project_list().
  */
 function hook_locale_translation_projects_alter(&$projects) {
   // The translations are located at a custom translation sever.

core/modules/locale/locale.batch.inc

+<?php
+
+/**
+ * @file
+ *   Batch process to check the availability of remote or local po files.
+ */
+
+/**
+ * Build a batch to get the status of remote and local translation files.
+ *
+ * The batch process fetches the state of both remote and (if configured) local
+ * translation files. The data of the most recent translation is stored per
+ * per project and per language. This data is stored in a state variable
+ * 'locale_translation_status'. The timestamp it was last updated is stored
+ * in the state variable 'locale_translation_status_last_update'.
+ *
+ * @param array $sources
+ *   Array of translation source objects for which to check the state of
+ *   translation source files.
+ */
+function locale_translation_batch_status_build($sources) {
+  $operations = array();
+
+  // Set the batch processes for remote sources.
+  foreach ($sources as $source) {
+    $operations[] = array('locale_translation_batch_status_fetch_remote', array($source));
+  }
+
+  // Check for local sources, compare the results of local and remote and store
+  // the most recent.
+  $operations[] = array('locale_translation_batch_status_fetch_local', array($sources));
+  $operations[] = array('locale_translation_batch_status_compare', array());
+
+  $batch = array(
+    'operations' => $operations,
+    'title' => t('Checking available translations'),
+    'finished' => 'locale_translation_batch_status_finished',
+    'error_message' => t('Error checking available interface translation updates.'),
+    'file' => drupal_get_path('module', 'locale') . '/locale.batch.inc',
+  );
+  return $batch;
+}
+
+/**
+ * Batch operation callback: Check the availability of a remote po file.
+ *
+ * Checks the presence and creation time of one po file per batch process. The
+ * file URL and timestamp are stored.
+ *
+ * @param array $source
+ *   A translation source object of the project for which to check the state of
+ *   a remote po file.
+ * @param array $context
+ *   The batch context array. The collected state is stored in the 'results'
+ *   parameter of the context.
+ *
+ * @see locale_translation_batch_status_fetch_local()
+ * @see locale_translation_batch_status_compare()
+*/
+function locale_translation_batch_status_fetch_remote($source, &$context) {
+  // Check the translation file at the remote server and update the source
+  // data with the remote status.
+  if (isset($source->files['remote'])) {
+    $remote_file = $source->files['remote'];
+    $result = locale_translation_http_check($remote_file->url);
+
+    // Update the file object with the result data. In case of a redirect we
+    // store the resulting url.
+    if ($result && !empty($result->updated)) {
+      $remote_file->url = isset($result->redirect_url) ? $result->redirect_url : $remote_file->url;
+      $remote_file->timestamp = $result->updated;
+      $source->files['remote'] = $remote_file;
+    }
+    $context['results'][$source->name][$source->language] = $source;
+  }
+}
+
+/**
+ * Batch operation callback: Check the availability of local po files.
+ *
+ * Checks the presence and creation time of po files in the local file system.
+ * The file path and the timestamp are stored.
+ *
+ * @param array $sources
+ *   Array of translation source objects of projects for which to check the
+ *   state of local po files.
+ * @param array $context
+ *   The batch context array. The collected state is stored in the 'results'
+ *   parameter of the context.
+ *
+ * @see locale_translation_batch_status_fetch_remote()
+ * @see locale_translation_batch_status_compare()
+ */
+function locale_translation_batch_status_fetch_local($sources, &$context) {
+  module_load_include('compare.inc', 'locale');
+
+  // Get the status of local translation files and store the result data in the
+  // batch results for later processing.
+  foreach ($sources as $source) {
+    if (isset($source->files['local'])) {
+      locale_translation_source_check_file($source);
+
+      // If remote data was collected before, we merge it into the newly
+      // collected result.
+      if (isset($context['results'][$source->name][$source->language])) {
+        $source->files['remote'] = $context['results'][$source->name][$source->language]->files['remote'];
+      }
+      $context['results'][$source->name][$source->language] = $source;
+    }
+  }
+}
+
+/**
+ * Batch operation callback: Compare states and store the result.
+ *
+ * In the preceding batch processes data of remote and local translation sources
+ * is collected. Here we compare the collected results and update the source
+ * object with the data of the most recent translation file. The end result is
+ * stored in the 'locale_translation_status' state variable. Other
+ * processes can collect this data after the batch process is completed.
+ *
+ * @param array $context
+ *   The batch context array. The 'results' element contains a structured array
+ *   of project data with languages, local and remote source data.
+ *
+ * @see locale_translation_batch_status_fetch_remote()
+ * @see locale_translation_batch_status_fetch_local()
+ */
+function locale_translation_batch_status_compare(&$context) {
+  module_load_include('compare.inc', 'locale');
+  $results = array();
+
+  foreach ($context['results'] as $project => $langcodes) {
+    foreach ($langcodes as $langcode => $source) {
+      $local = isset($source->files['local']) ? $source->files['local'] : NULL;
+      $remote = isset($source->files['remote']) ? $source->files['remote'] : NULL;
+
+      // The available translation files are compare and data of the most recent
+      // file is used to update the source object.
+      $file = _locale_translation_source_compare($local, $remote) == LOCALE_TRANSLATION_SOURCE_COMPARE_LT ? $remote : $local;
+      if (isset($file->timestamp)) {
+        $source->type = $file->type;
+        $source->timestamp = $file->timestamp;
+        $results[$project][$langcode] = $source;
+      }
+    }
+  }
+
+  state()->set('locale_translation_status', $results);
+  state()->set('locale_translation_status_last_update', REQUEST_TIME);
+}
+
+/**
+ * Batch finished callback: Set result message.
+ *
+ * @param boolean $success
+ *   TRUE if batch succesfully completed.
+ * @param array $results
+ *   Batch results.
+ */
+function locale_translation_batch_status_finished($success, $results) {
+  $t = get_t();
+  if($success) {
+    if ($results) {
+      drupal_set_message(format_plural(
+        count($results),
+        'Checked available interface translation updates for one project.',
+        'Checked available interface translation updates for @count projects.'
+      ));
+    }
+  }
+  else {
+    drupal_set_message($t('An error occurred trying to check available interface translation updates.'), 'error');
+  }
+}
+
+/**
+ * Check if remote file exists and when it was last updated.
+ *
+ * @param string $url
+ *   URL of remote file.
+ * @param array $headers
+ *   HTTP request headers.
+ * @return stdClass
+ *   Result object containing the HTTP request headers, response code, headers,
+ *   data, redirect status and updated timestamp.
+ */
+function locale_translation_http_check($url, $headers = array()) {
+  $result = drupal_http_request($url, array('headers' => $headers, 'method' => 'HEAD'));
+  if ($result && $result->code == '200') {
+    $result->updated = isset($result->headers['last-modified']) ? strtotime($result->headers['last-modified']) : 0;
+  }
+  return $result;
+}

core/modules/locale/locale.compare.inc

@@ -6,11 +6,37 @@
  */
 
 /**
- * Default location of gettext file on the translation server.
+ * Threshold for timestamp comparison.
  *
- * @see locale_translation_default_translation_server().
+ * Eliminates a difference between the download time and the actual .po file
+ * timestamp in seconds. The download time is stored in the database in
+ * {locale_file}.timestamp.
  */
-const LOCALE_TRANSLATION_DEFAULT_SERVER_PATTERN = 'http://ftp.drupal.org/files/translations/%core/%project/%project-%version.%language.po';
+const LOCALE_TRANSLATION_TIMESTAMP_THRESHOLD = 2;
+
+/**
+ * Comparison result of source files timestamps.
+ *
+ * Timestamp of source 1 is less than the timestamp of source 2.
+ * @see _locale_translation_source_compare()
+ */
+const LOCALE_TRANSLATION_SOURCE_COMPARE_LT = -1;
+
+/**
+ * Comparison result of source files timestamps.
+ *
+ * Timestamp of source 1 is equal to the timestamp of source 2.
+ * @see _locale_translation_source_compare()
+ */
+const LOCALE_TRANSLATION_SOURCE_COMPARE_EQ = 0;
+
+/**
+ * Comparison result of source files timestamps.
+ *
+ * Timestamp of source 1 is greater than the timestamp of source 2.
+ * @see _locale_translation_source_compare()
+ */
+const LOCALE_TRANSLATION_SOURCE_COMPARE_GT = 1;
 
 use Drupal\Core\Cache;
 
@@ -254,7 +280,6 @@ function locale_translation_default_translation_server() {
  *   - "%version": Project version.
  *   - "%core": Project core version.
  *   - "%language": Language code.
- *   - "%filename": Project file name.
  *
  * @return string
  *   String with replaced placeholders.
@@ -265,7 +290,275 @@ function locale_translation_build_server_pattern($project, $template) {
     '%version' => $project->version,
     '%core' => $project->core,
     '%language' => isset($project->language) ? $project->language : '%language',
-    '%filename' => isset($project->filename) ? $project->filename : '%filename',
   );
   return strtr($template, $variables);
 }
+
+/**
+ * Check for the latest release of project translations.
+ *
+ * @param array $projects
+ *   Projects to check (objects).
+ * @param string $langcodes
+ *   Array of language codes to check for. Leave empty to check all languages.
+ *
+ * @return array
+ *   Available sources indexed by project and language.
+ */
+function locale_translation_check_projects($projects, $langcodes = NULL) {
+  module_load_include('batch.inc', 'locale');
+
+  if (config('locale.settings')->get('translation.use_source') == LOCALE_TRANSLATION_USE_SOURCE_REMOTE_AND_LOCAL) {
+    // Retrieve the status of both remote and local translation sources by
+    // using a batch process.
+    locale_translation_check_projects_batch($projects, $langcodes);
+  }
+  else {
+    // Retrieve and save the status of local translations only.
+    locale_translation_check_projects_local($projects, $langcodes);
+  }
+}
+
+/**
+ * Gets and stores the status and timestamp of remote po files.
+ *
+ * A batch process is used to check for po files at remote locations and (when
+ * configured) to check for po files in the local file system. The most recent
+ * translation source states are stored in the state variable
+ * 'locale_translation_status'.
+ *
+ * @params array $projects
+ *   Array of translatable projects.
+ * @params array $langcodes
+ *   Array of language codes to check for. Leave empty to check all languages.
+ */
+function locale_translation_check_projects_batch($projects, $langcodes = NULL) {
+  $langcodes = $langcodes ? $langcodes : array_keys(locale_translatable_language_list());
+  $sources = array();
+  foreach ($projects as $name => $project) {
+    foreach ($langcodes as $langcode) {
+      $source = locale_translation_source_build($project, $langcode);
+      $sources[] = $source;
+    }
+  }
+
+  // Build and set the batch process.
+  module_load_include('batch.inc', 'locale');
+  $batch = locale_translation_batch_status_build($sources);
+  batch_set($batch);
+}
+
+/**
+ * Check and store the status and timestamp of local po files.
+ *
+ * Only po files in the local file system are checked. Any remote translation
+ * sources will be ignored. Results are stored in the state variable
+ * 'locale_translation_status'.
+ *
+ * Projects may contain a server_pattern option containing a pattern of the
+ * path to the po source files. If no server_pattern is defined the default
+ * translation directory is checked for the po file. When a server_pattern is
+ * defined the specified location is checked. The server_pattern can be set in
+ * the module's .info file or by using hook_locale_translation_projects_alter().
+ *
+ * @params array $projects
+ *   Array of translatable projects.
+ * @params array $langcodes
+ *   Array of language codes to check for. Leave empty to check all languages.
+ */
+function locale_translation_check_projects_local($projects, $langcodes = NULL) {
+  $langcodes = $langcodes ? $langcodes : array_keys(locale_translatable_language_list());
+  $results = array();
+
+  // For each project and each language we check if a local po file is
+  // available. When found the source object is updated with the appropriate
+  // type and timestamp of the po file.
+  foreach ($projects as $name => $project) {
+    foreach ($langcodes as $langcode) {
+      $source = locale_translation_source_build($project, $langcode);
+      if (locale_translation_source_check_file($source)) {
+        $source->type = 'local';
+        $source->timestamp = $source->files['local']->timestamp;
+        $results[$name][$langcode] = $source;
+      }
+    }
+  }
+
+  state()->set('locale_translation_status', $results);
+  state()->set('locale_translation_status_last_update', REQUEST_TIME);
+}
+
+/**
+ * Check whether a po file exists in the local filesystem.
+ *
+ * It will search in the directory set in the translation source. Which defaults
+ * to the "translations://" stream wrapper path. The directory may contain any
+ * valid stream wrapper.
+ *
+ * The "local" files property of the source object contains the definition of a
+ * po file we are looking for. The file name defaults to
+ * LOCALE_TRANSLATION_DEFAULT_FILENAME. Per project this value
+ * can be overridden using the server_pattern directive in the module's .info
+ * file or by using hook_locale_translation_projects_alter().
+ *
+ * @param stdClass $source
+ *   Translation source object.
+ *   @see locale_translation_source_build()
+ *
+ * @return stdClass
+ *   File object (filename, basename, name) updated with data of the po file.
+ *   On success the files property of the source object is updated.
+ *   files['local']:
+ *   - "uri": File name and path.
+ *   - "timestamp": Last updated time of the po file.
+ *   FALSE if the file is not found.
+ */
+function locale_translation_source_check_file(&$source) {
+  if (isset($source->files['local'])) {
+    $directory = $source->files['local']->directory;
+