Batch translation & Drupal 11 support

README.md

@@ -3,7 +3,7 @@ Automated website translation solution for Drupal websites.
 This module can pre-translate all entities (articles, pages, comments, tags), UI and config strings and translate new entities when they are created or updated. Before loading any translatable entity, this module also translates them, if translation in selected language does not exist. It does that by adding/updating entity, UI and config string translations provided by Drupal core localization modules. 
 
 ## Dependencies
-Module depends on Drupal 10 built-in localization modules:
+Module depends on Drupal built-in localization modules:
 - language - provides website language selection functionality;
 - content_translation - provides API for Drupal entity (page, article, etc.) translation;
 - config_translation - provides API for configuration item translation;

src/Controller/EtranslationCallbackController.php

@@ -15,7 +15,6 @@ use Symfony\Component\HttpFoundation\Response;
  * Controller that handles eTranslation async responses.
  */
 class EtranslationCallbackController extends ControllerBase {
-
   /**
    * The translation manager service.
    *
@@ -36,7 +35,6 @@ class EtranslationCallbackController extends ControllerBase {
    * @var \Drupal\Core\Entity\EntityTypeManagerInterface
    */
   protected $entityTypeManager;
-
   /**
    * Constructs a new class instance.
    *
@@ -86,7 +84,7 @@ class EtranslationCallbackController extends ControllerBase {
         $entityInfoParts = explode(':', $entityInfo);
         $entity_id       = $entityInfoParts[0];
         $entity_type     = $entityInfoParts[1];
-        $entity          = $this->loadEntityByIdAndType($entity_id, $entity_type);
+        $entity          = $this->translationManager->loadEntityByIdAndType($entity_id, $entity_type);
         if ($entity) {
           $this->processEntityTranslation($entity, $translated_content, Html::escape($request->get('target-language')));
         }
@@ -128,7 +126,6 @@ class EtranslationCallbackController extends ControllerBase {
    *   Language of translation received.
    */
   private function processEntityTranslation($entity, $translated_content, $target_language) {
-    $source_langcode     = $entity->language()->getId();
     $target_langcode     = $this->getLangcodeByLanguage($target_language);
     $translation_service = $this->translationManager->translationService;
     if ($translation_service instanceof EtranslationService) {
@@ -138,31 +135,8 @@ class EtranslationCallbackController extends ControllerBase {
       $original_values     = $this->translationManager->extractTranslatableValues($entity, $translatable_fields);
       if (count($original_values) === count($xml_translations)) {
         $translations = $translation_service->decodeXmlTranslations($original_values, $xml_translations);
-        $this->translationManager->translateEntity($entity, $source_langcode, $target_langcode, $translations);
-      }
-    }
+        $this->translationManager->translateEntity($entity->id(), $entity->getEntityTypeId(), $target_langcode, FALSE, $translations);
       }
-
-  /**
-   * Retrieve entity by ID.
-   *
-   * @param [type] $id
-   *   Identifier of entity.
-   * @param [type] $type
-   *   Type of entity.
-   *
-   * @return \Drupal\Core\Entity\Entity
-   *   Entity or null if enitity was not found
-   */
-  private function loadEntityByIdAndType($id, $type) {
-    $storage = $this->entityTypeManager->getStorage($type);
-    $entity  = $storage->load($id);
-    if ($entity) {
-      return $entity;
-    }
-    else {
-      $this->logger->error("Could not load entity by ID (@entity_id). Please make sure such entity exists!", ['@entity_id' => $id]);
-      return NULL;
     }
   }
 

src/TranslationManager.php

@@ -2,6 +2,7 @@
 
 namespace Drupal\webt;
 
+use Drupal\Core\Entity\ContentEntityBase;
 use Drupal\Core\Config\ConfigFactoryInterface;
 use Drupal\Core\Logger\LoggerChannelTrait;
 use Drupal\webt\Model\ProcessingResponse;
@@ -188,43 +189,32 @@ class TranslationManager {
     return $this->getProcessingResponse($translated_versions_created, $translated_versions_to_create);
   }
 
-  /**
-   * Creates or updates all translations of an enitity.
-   *
-   * @param \Drupal\Core\Entity\Entity $entity
-   *   Entity to translate.
-   */
-  public function entityUpdateAllTranslations($entity) {
-    $this->logger->info('Updating all translations for entity');
-    $source_langcode = $entity->language()->getId();
-    $langcodes       = $this->languageManager->getLanguages();
-    $langcodes_list  = array_keys($langcodes);
-
-    foreach ($langcodes_list as $langcode) {
-      if ($langcode === $source_langcode || !$this->translationService->isLanguageSupported($langcode)) {
-        continue;
-      }
-      $this->translateEntity($entity, $source_langcode, $langcode);
-    }
-  }
-
   /**
    * Translate entity from specified source language to target language.
    *
-   * @param \Drupal\Core\Entity\Entity $entity
-   *   Enitity to translate.
-   * @param string $src_lang
-   *   Source language code.
-   * @param string $trg_lang
-   *   Target language code.
+   * @param int $entityId
+   *   Entity ID
+   * @param string $entityTypeId
+   *   Entity type ID
+   * @param string[] $trg_lang
+   *   Target language code
+   * @param boolean $skip_if_translation_exists
+   *   Whether to skip translation if translation already exists
    * @param string[] $translations
    *   If provided, uses given string translations, skips machine translation.
    */
-  public function translateEntity($entity, $src_lang, $trg_lang, $translations = NULL) {
-    $this->logger->info("Translating entity (@src_lang-@trg_lang)", ['@src_lang' => $src_lang, '@trg_lang' => $trg_lang]);
+  public function translateEntity($entityId, $entityTypeId, $trg_lang, $skip_if_translation_exists = FALSE, $translations = NULL) {
+    $entity = $this->loadEntityByIdAndType($entityId, $entityTypeId);
+    if (!$entity) {
+      return;
+    }    
+    $src_lang = $entity->language()->getId();
+
+    if ($entity instanceof ContentEntityBase && $entity->isTranslatable() && (!$skip_if_translation_exists || !$entity->hasTranslation($trg_lang))) {
       $translatable_fields = $this->getTranslatableFields($entity);
+      if (count($translatable_fields) > 0) {
+        $this->logger->info("Translating entity (@src_lang-@trg_lang)", ['@src_lang' => $src_lang, '@trg_lang' => $trg_lang]);
   
-    if (count($translatable_fields) > 0 && $entity->isTranslatable()) {
         $strings_to_translate = $this->extractTranslatableValues($entity, $translatable_fields);
   
         if (!$translations) {
@@ -237,6 +227,7 @@ class TranslationManager {
         $this->entityCreateTranslation($entity, $trg_lang, $translatable_fields, $translations);
       }
     }
+  }
 
   /**
    * Delete translations of entities in specified languages.
@@ -265,6 +256,29 @@ class TranslationManager {
     return ProcessingResponse::$fullyProcessed;
   }
 
+  /**
+   * Retrieves translation languages for entity
+   *
+   * @param \Drupal\Core\Entity\Entity $entity
+   *   Entity to translate.
+   * @return string[]
+   *   List of languages
+   */
+  public function getTranslationLanguagesForEntity($entity) {
+    $source_langcode       = $entity->language()->getId();
+    $langcodes             = $this->languageManager->getLanguages();
+    $langcodes_list        = array_keys($langcodes);
+    $translation_languages = [$source_langcode => []];
+
+    foreach ($langcodes_list as $langcode) {
+      if ($langcode === $source_langcode || !$this->translationService->isLanguageSupported($langcode)) {
+        continue;
+      }
+      $translation_languages[$source_langcode][] = $langcode;
+    }
+    return $translation_languages;
+  }
+
   /**
    * Translate given UI strings and save translations.
    *
@@ -757,4 +771,26 @@ class TranslationManager {
     return $translatable_values;
   }
 
+  /**
+   * Retrieve entity by ID.
+   *
+   * @param [type] $id
+   *   Identifier of entity.
+   * @param [type] $type
+   *   Type of entity.
+   *
+   * @return \Drupal\Core\Entity\Entity
+   *   Entity or null if enitity was not found
+   */
+  public function loadEntityByIdAndType($id, $type) {
+    $storage = $this->entityTypeManager->getStorage($type);
+    $entity  = $storage->load($id);
+    if ($entity) {
+      return $entity;
+    }
+    else {
+      $this->logger->error("Could not load entity by ID (@entity_id). Please make sure such entity exists!", ['@entity_id' => $id]);
+      return NULL;
+    }
+  }
 }

src/translation_engines/AbstractTranslationService.php

@@ -238,26 +238,25 @@ abstract class AbstractTranslationService {
 
     for ($i = 0; $i < $request_count; $i++) {
       $this->checkClientDisconnected($reference_id, $entity);
-      // Replace necessary special characters removed on HTML tree creation;
-      // currently needed only for UI strings containing multiple values
-      // in same string, separated by ETX char.
-      $request_value_lists[$i] = str_replace(array_keys($this->specialCharDict), $this->specialCharDict, $request_value_lists[$i]);
 
-      $contains_only_empty_strings = empty(
-      array_filter(
+      $non_empty_strings = array_filter(
         $request_value_lists[$i],
-        function ($a) {
-          return !empty($a);
+        function( $text ) {
+          return !empty( $text );
         }
-      )
       );
 
-      if ($contains_only_empty_strings) {
+      if (empty($non_empty_strings)) {
         $full_result = array_merge($full_result, $request_value_lists[$i]);
       }
       else {
+        // Replace necessary special characters removed on HTML tree creation;
+        // currently needed only for UI strings containing multiple values
+        // in same string, separated by ETX char.
+        $non_empty_strings = str_replace(array_keys($this->specialCharDict), $this->specialCharDict, $non_empty_strings);
+
         // Convert string[] to XML[].
-        $xml_values = $this->translatableValuesToXml($request_value_lists[$i]);
+        $xml_values = $this->translatableValuesToXml($non_empty_strings);
 
         $translatable_values = $xml_values;
         $encode_response     = NULL;
@@ -304,11 +303,11 @@ abstract class AbstractTranslationService {
           $xml_translations = $this->decodePlaceholders($xml_translations, $encode_response->placeholders);
         }
         // Convert XML[] back to string[].
-        $decoded_translations = $this->decodeXmlTranslations($request_value_lists[$i], $xml_translations);
+        $decoded_translations = $this->decodeXmlTranslations($non_empty_strings, $xml_translations);
         // Restore special characters (for some UI strings).
         $decoded_translations = str_replace($this->specialCharDict, array_keys($this->specialCharDict), $decoded_translations);
         // Merge with previous translations.
-        $full_result = array_merge($full_result, $decoded_translations);
+        $full_result = $this->mergeTranslationResults($request_value_lists[$i], $decoded_translations, $full_result);
       }
       // Update percentage for pre-translation progress bar.
       $percentage_completed += 100 / $request_count;
@@ -690,4 +689,24 @@ abstract class AbstractTranslationService {
     }
   }
 
+  /**
+   * Adds translated string batch to full result list. If source string is empty, creates and adds empty translation string.
+   *
+   * @param string[] $source_strings List of source strings
+   * @param string[] $translated_strings List of translations for non-empty source strings
+   * @param string[] $all_translations All translations, including previously added
+   * @return string[] Merged translatioins
+   */
+  private function mergeTranslationResults($source_strings, $translated_strings, $all_translations) {
+    $translation_offset = 0;
+    foreach ( $source_strings as $original ) {
+      if ( empty( $original ) ) {
+        $all_translations[] = '';
+      } else {
+        $all_translations[] = $translated_strings[ $translation_offset++ ];
+      }
+    }
+    return $all_translations;
+  }
+
 }

webt.info.yml

@@ -2,7 +2,7 @@ name: WEB-T
 description: Automated website content translation with WEB-T module
 package: Multilingual
 type: module
-core_version_requirement: ^8.8 || ^9 || ^10
+core_version_requirement: ^8.8 || ^9 || ^10 || ^11
 
 configure: webt.settings
 dependencies:

webt.module

@@ -4,10 +4,8 @@
  * @file
  * WEB-T module functions.
  */
-
+use Drupal\Core\Batch\BatchBuilder;
 use Drupal\Core\Entity\ContentEntityBase;
-use Drupal\webt\WebtConstInterface;
-
 /***
  * Hooks for updating translations on entity insert/update.
  */
@@ -16,28 +14,14 @@ use Drupal\webt\WebtConstInterface;
  * {@inheritDoc}
  */
 function webt_entity_insert($entity) {
-  webt_mark_entity_for_translation($entity);
+  webt_batch_translate_entity($entity);
 }
 
 /**
  * {@inheritDoc}
  */
 function webt_entity_update($entity) {
-  webt_mark_entity_for_translation($entity);
-}
-
-/**
- * {@inheritDoc}
- */
-function webt_entity_load($entities) {
-  foreach ($entities as $entity) {
-    $marked_entity = \Drupal::state()->get(WebtConstInterface::WEBT_ENTITY_UPDATE);
-    if ($marked_entity && $marked_entity->id() === $entity->id() && $marked_entity->getEntityTypeId() === $entity->getEntityTypeId()) {
-      \Drupal::state()->delete(WebtConstInterface::WEBT_ENTITY_UPDATE);
-      $translation_manager = \Drupal::service('webt.translation_manager');
-      $translation_manager->entityUpdateAllTranslations($entity);
-    }
-  }
+  webt_batch_translate_entity($entity);
 }
 
 /**
@@ -51,9 +35,60 @@ function webt_entity_load($entities) {
  * @param \Drupal\Core\Entity\Entity $entity
  *   Entity to translate.
  */
-function webt_mark_entity_for_translation($entity) {
+function webt_batch_translate_entity($entity) {
   if ($entity instanceof ContentEntityBase && $entity->isTranslatable() && array_key_exists('default_langcode', $entity->toArray()) && $entity->default_langcode && $entity->default_langcode[0]->value && \Drupal::service('webt.translation_manager')->entityTranslatableContentChanged($entity)) {
-    \Drupal::state()->set(WebtConstInterface::WEBT_ENTITY_UPDATE, $entity);
+    $batch_builder = new BatchBuilder();
+    $batch_builder->setTitle(t('Translating entity'))
+      ->setInitMessage(t('Updating all translations for entity'))
+      ->setErrorMessage(t('An error occurred during entity translation.'));
+  
+    $batch_builder->setFinishCallback('webt_batch_finished');
+  
+    $translation_manager = \Drupal::service('webt.translation_manager');
+    $languages = $translation_manager->getTranslationLanguagesForEntity($entity);
+  
+    foreach ($languages as $src_lang => $trg_langs) {
+      foreach($trg_langs as $trg_lang) {
+        $batch_builder->addOperation('webt_batch_update_entity_translations', [$entity->id(), $entity->getEntityTypeId(), $trg_lang]);
+      }
+    }
+    batch_set($batch_builder->toArray());
+  }
+}
+
+/**
+ * Batch operation callback.
+ */
+function webt_batch_update_entity_translations($entityId, $entityTypeId, $target_langcode, array &$context): void {
+  if (!isset($context['results']['translated'])) {
+    $context['results']['translated'] = [];
+  }
+
+  $context['message'] = t('Translating @entity_type to @target_language', ['@entity_type' => $entityTypeId, '@target_language' => strtoupper($target_langcode)]);
+
+  $translation_manager = \Drupal::service('webt.translation_manager');
+  $translation_manager->translateEntity($entityId, $entityTypeId, $target_langcode);
+  
+  $context['results']['translated'][] = $target_langcode;
+}
+
+/**
+ * Batch finished callback.
+ */
+function webt_batch_finished(bool $success, array $results, array $operations, string $elapsed) {
+  $messenger = \Drupal::messenger();
+  if ($success) {
+    $messenger->addMessage(t('Entity translation completed'));
+  }
+  else {
+    $error_operation = reset($operations);
+    if ($error_operation) {
+      $message = t('An error occurred while processing %error_operation with arguments: @arguments', [
+        '%error_operation' => print_r($error_operation[0]),
+        '@arguments' => print_r($error_operation[1], TRUE),
+      ]);
+      $messenger->addError($message);
+    }
   }
 }
 
@@ -68,16 +103,11 @@ function webt_entity_preload(array $ids, $entity_type_id) {
       // when loading entity from storage.
       \Drupal::state()->set($key, TRUE);
 
-      $entity_type_manager = \Drupal::service('entity_type.manager');
       $selected_language = \Drupal::languageManager()->getCurrentLanguage()->getId();
-      $storage = $entity_type_manager->getStorage($entity_type_id);
       foreach ($ids as $id) {
-        $entity = $storage->load($id);
-        if ($entity && $entity instanceof ContentEntityBase && $entity->isTranslatable() && !$entity->hasTranslation($selected_language)) {
         $translation_manager = \Drupal::service('webt.translation_manager');
         if ($translation_manager->translationService->isLanguageSupported($selected_language)) {
-            $translation_manager->translateEntity($entity, $entity->language()->getId(), $selected_language);
-          }
+          $translation_manager->translateEntity($id, $entity_type_id, $selected_language, TRUE);
         }
       }
     } finally {