Fix invalid utf-8 character handling

smp-server-library/src/main/java/eu/europa/ec/edelivery/smp/services/mail/MailTemplateService.java

@@ -43,7 +43,7 @@ import java.util.Properties;
 public class MailTemplateService {
     private static final String DEFAULT_LANGUAGE = "en";
     private static final String MAIL_TEMPLATE = "/mail-messages/mail-template.htm";
-
+    private static final String MAIL_TEMPLATE_CHARSET= "UTF-8";
     private static final String MAIL_HEADER = "MAIL_HEADER";
     private static final String MAIL_FOOTER = "MAIL_FOOTER";
     private static final String MAIL_TITLE = "MAIL_TITLE";
@@ -58,7 +58,7 @@ public class MailTemplateService {
             modelData.put(MAIL_FOOTER, getMailFooter(model));
             modelData.put(MAIL_TITLE, getMailTitle(model));
             modelData.put(MAIL_CONTENT, getMailBody(model));
-            return StringNamedSubstitutor.resolve(templateIS, modelData);
+            return StringNamedSubstitutor.resolve(templateIS, modelData, MAIL_TEMPLATE_CHARSET);
         } catch (IOException e) {
             throw new SMPRuntimeException(ErrorCode.INTERNAL_ERROR, "Error reading mail template", ExceptionUtils.getRootCauseMessage(e));
         }

smp-server-library/src/main/java/eu/europa/ec/edelivery/smp/services/resource/AbstractResourceHandler.java

@@ -50,6 +50,7 @@ import java.util.stream.Collectors;
 
 public class AbstractResourceHandler {
     protected static final SMPLogger LOG = SMPLoggerFactory.getLogger(AbstractResourceHandler.class);
+    private static final String EXPECTED_RESOURCE_CHARSET= "UTF-8";
     // the Spring beans for the resource definitions
     final List<ResourceDefinitionSpi> resourceDefinitionSpiList;
     final ResourceStorage resourceStorage;
@@ -117,7 +118,7 @@ public class AbstractResourceHandler {
         Map<String, Object> docProp = resourceStorage.getResourceProperties(resource);
 
         try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
-            StringNamedSubstitutor.resolve(new ByteArrayInputStream(content), docProp, baos);
+            StringNamedSubstitutor.resolve(new ByteArrayInputStream(content), docProp, baos, EXPECTED_RESOURCE_CHARSET);
             ByteArrayInputStream inputStream = new ByteArrayInputStream(baos.toByteArray());
             return buildRequestDataForResource(domain,
                     resource,
@@ -145,7 +146,7 @@ public class AbstractResourceHandler {
         Map<String, Object> docProp = resourceStorage.getSubresourceProperties(resource, subresource);
 
         try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
-            StringNamedSubstitutor.resolve(new ByteArrayInputStream(content), docProp, baos);
+            StringNamedSubstitutor.resolve(new ByteArrayInputStream(content), docProp, baos, EXPECTED_RESOURCE_CHARSET);
             return new SpiRequestData(domain.getDomainCode(),
                     SPIUtils.toUrlIdentifier(resource),
                     SPIUtils.toUrlIdentifier(subresource),

smp-server-library/src/main/java/eu/europa/ec/edelivery/smp/utils/StringNamedSubstitutor.java

@@ -19,8 +19,10 @@
 package eu.europa.ec.edelivery.smp.utils;
 
 import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
 
 import java.io.*;
+import java.nio.charset.Charset;
 import java.util.Map;
 import java.util.stream.Collectors;
 
@@ -33,6 +35,7 @@ import java.util.stream.Collectors;
  * @since 4.2
  */
 public class StringNamedSubstitutor {
+    private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(StringNamedSubstitutor.class);
     private static final String START_NAME = "${";
     private static final char END_NAME = '}';
 
@@ -44,15 +47,62 @@ public class StringNamedSubstitutor {
      * Substitute named variables in the string with key value pairs from the map.
      * The variables are in the form of ${name} and are case-insensitive and can contain only letters, digits, _ and .
      *
+     * @param string the string to resolve
+     * @param config the config to use
+     * @return the resolved string
+     */
+    public static String resolve(String string, Map<String, Object> config) {
+        String charset = Charset.defaultCharset().name();
+        LOG.debug("Using default charset: [{}]", charset);
+        return resolve(string, config, charset);
+    }
+    /**
+     * Substitute named variables in the string with key value pairs from the map.
+     * The variables are in the form of ${name} and are case-insensitive and can contain only letters, digits, _ and .
+     *
+     * @param string the string to resolve
+     * @param config the config to use
+     * @param charset the character of the input stream
+     * @return the resolved string
+     */
+    public static String resolve(String string, Map<String, Object> config, String charset) {
+        try {
+            return resolve(new ByteArrayInputStream(string.getBytes()), config, charset);
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    /**
+     * Substitute named variables in the string with key value pairs from the map.
+     * The variables are in the form of ${name} and are case-insensitive and can contain only letters, digits, _ and .
+     * The input stream is ready with default charset.
+     *
      * @param templateIS the InputStream to resolve
-     * @param config     the config to use
+     * @param config the map of property names and its values
      * @return the resolved string
+     * @throws IOException if an I/O error occurs
      */
     public static String resolve(InputStream templateIS, Map<String, Object> config) throws IOException {
+        String charset = Charset.defaultCharset().name();
+        LOG.debug("Using default charset: [{}]", charset);
+        return resolve(templateIS, config, charset);
+    }
+
+    /**
+     * Substitute named variables in the string with key value pairs from the map.
+     * The variables are in the form of ${name} and are case-insensitive and can contain only letters, digits, _ and .
+     *
+     * @param templateIS the InputStream to resolve
+     * @param config     the config to use
+     * @param charset    the character of the input stream
+     * @return the resolved string
+     */
+    public static String resolve(InputStream templateIS, Map<String, Object> config, String charset) throws IOException {
         Map<String, Object> lowerCaseMap = config.entrySet().stream()
                 .collect(Collectors.toMap(e -> StringUtils.lowerCase(e.getKey()), Map.Entry::getValue));
         try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
-            resolve(templateIS, lowerCaseMap, byteArrayOutputStream);
+            resolve(templateIS, lowerCaseMap, byteArrayOutputStream, charset);
             return byteArrayOutputStream.toString();
         }
     }
@@ -64,42 +114,45 @@ public class StringNamedSubstitutor {
      * @param templateIS   the InputStream to resolve
      * @param config       the config to use
      * @param outputStream the output stream to write the resolved string
+     * @param charset      the charset to use
      * @throws IOException if an I/O error occurs
      */
-    public static void resolve(InputStream templateIS, Map<String, Object> config, OutputStream outputStream) throws IOException {
+    public static void resolve(InputStream templateIS, Map<String, Object> config,
+                               OutputStream outputStream, String charset) throws IOException {
         Map<String, Object> lowerCaseMap = config.entrySet().stream()
                 .collect(Collectors.toMap(e -> StringUtils.lowerCase(e.getKey()), Map.Entry::getValue));
 
-        try (BufferedReader template = new BufferedReader(new InputStreamReader(templateIS))) {
+        try (BufferedReader template = new BufferedReader(new InputStreamReader(templateIS, charset));
+             Writer writer = new OutputStreamWriter(outputStream, charset)) {
             int read;
             while ((read = template.read()) != -1) {
                 if (read != START_NAME.charAt(0) || !isStartSequence(template)) {
-                    outputStream.write((char) read);
+                    writer.write((char) read);
                     continue;
                 }
 
                 template.skip(1L);
                 String name = readName(template, END_NAME);
                 if (name == null) {
-                    outputStream.write(START_NAME.getBytes());
+                    writer.write(START_NAME);
                 } else {
                     String key = StringUtils.lowerCase(name);
                     Object objValue = lowerCaseMap.get(key);
                     String value = objValue != null ? String.valueOf(lowerCaseMap.get(key)) : null;
 
                     if (value != null) {
-                        outputStream.write(value.getBytes());
+                        writer.write(value);
                     } else {
-                        outputStream.write(START_NAME.getBytes());
-                        outputStream.write(name.getBytes());
-                        outputStream.write(END_NAME);
+                        writer.write(START_NAME);
+                        writer.write(name);
+                        writer.write(END_NAME);
                     }
                 }
             }
         }
     }
 
-    public static boolean isStartSequence(BufferedReader reader) throws IOException {
+    private static boolean isStartSequence(BufferedReader reader) throws IOException {
         reader.mark(START_NAME.length());
         int read = reader.read();
         if (read == -1) {
@@ -144,20 +197,4 @@ public class StringNamedSubstitutor {
         }
         return null;
     }
-
-    /**
-     * Substitute named variables in the string with key value pairs from the map.
-     * The variables are in the form of ${name} and are case-insensitive and can contain only letters, digits, _ and .
-     *
-     * @param string the string to resolve
-     * @param config the config to use
-     * @return the resolved string
-     */
-    public static String resolve(String string, Map<String, Object> config) {
-        try {
-            return resolve(new ByteArrayInputStream(string.getBytes()), config);
-        } catch (IOException e) {
-            throw new IllegalArgumentException(e);
-        }
-    }
 }

smp-server-library/src/test/java/eu/europa/ec/edelivery/smp/utils/StringNamedSubstitutorTest.java

@@ -18,9 +18,14 @@
  */
 package eu.europa.ec.edelivery.smp.utils;
 
+import eu.europa.ec.smp.spi.enums.TransientDocumentPropertyType;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
 
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -29,6 +34,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 
 class StringNamedSubstitutorTest {
 
+    private static final String  TEST_UTF8_STRING = "Test%C4%85%C3%B3%C5%BC%C4%99%C4%85%E1%BA%9E%C3%B6+Greek+%C3%80%C3%86%C3%87%C3%9F%C3%A3%C3%BF%CE%B1%CE%A9%C6%92%CE%91+char";
+    // partially URL encoded service group with UTF8 characters.
+    // test characters are URL encoded to "survive various development local settings  :)" to use this first url decode string!
+    private static final String SERVICE_GROUP_WITH_UTF8 = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n" +
+            "<ServiceGroup xmlns=\"http://docs.oasis-open.org/bdxr/ns/SMP/2016/05\">\n" +
+            "    <ParticipantIdentifier scheme=\"${resource.identifier.scheme}\">${resource.identifier.value}</ParticipantIdentifier>\n" +
+            "    <ServiceMetadataReferenceCollection/>\n" +
+            "    <Extension>\n" +
+            "        <ex:Test xmlns:ex=\"http://test.eu\">"+TEST_UTF8_STRING+"</ex:Test>\n" +
+            "    </Extension>\n" +
+            "</ServiceGroup>";
 
     @ParameterizedTest
     @CsvSource({
@@ -50,4 +66,36 @@ class StringNamedSubstitutorTest {
         String result = StringNamedSubstitutor.resolve(testString, mapVal);
         assertEquals(expected, result);
     }
+
+    @Test
+    void testTransientResolutionForResourceWithUTF8() throws UnsupportedEncodingException {
+        Map<String, Object> mapProperties = new HashMap<>();
+        mapProperties.put(TransientDocumentPropertyType.RESOURCE_IDENTIFIER_VALUE.getPropertyName(), "value");
+        mapProperties.put(TransientDocumentPropertyType.RESOURCE_IDENTIFIER_SCHEME.getPropertyName(), "scheme");
+        String serviceGroupWithUt8 = URLDecoder.decode(SERVICE_GROUP_WITH_UTF8, "UTF-8");
+        String testStringInMessage = URLDecoder.decode(TEST_UTF8_STRING, "UTF-8");
+        // when
+        System.out.println(serviceGroupWithUt8);
+        String resolved = StringNamedSubstitutor.resolve(serviceGroupWithUt8, mapProperties);
+        System.out.println(resolved);
+        //then
+        Assertions.assertThat(resolved)
+                .doesNotContain(TransientDocumentPropertyType.SUBRESOURCE_IDENTIFIER_VALUE.getPropertyPlaceholder())
+                .doesNotContain(TransientDocumentPropertyType.SUBRESOURCE_IDENTIFIER_SCHEME.getPropertyPlaceholder())
+                .contains(testStringInMessage);
+    }
+
+    @Test
+    void testTransientResolutionWithUTF8String() throws UnsupportedEncodingException {
+        Map<String, Object> mapProperties = new HashMap<>();
+        mapProperties.put(TransientDocumentPropertyType.RESOURCE_IDENTIFIER_VALUE.getPropertyName(), "value");
+        mapProperties.put(TransientDocumentPropertyType.RESOURCE_IDENTIFIER_SCHEME.getPropertyName(), "scheme");
+        String serviceGroupWithUt8 = URLDecoder.decode(TEST_UTF8_STRING, "UTF-8");
+        // when
+        System.out.println(serviceGroupWithUt8);
+        String resolved = StringNamedSubstitutor.resolve(serviceGroupWithUt8, mapProperties);
+        System.out.println(resolved);
+        //then
+        assertEquals(serviceGroupWithUt8, resolved);
+    }
 }

smp-webapp/src/test/java/eu/europa/ec/edelivery/smp/controllers/ResourceControllerSingleDomainTest.java

@@ -8,9 +8,9 @@
  * versions of the EUPL (the "Licence");
  * You may not use this work except in compliance with the Licence.
  * You may obtain a copy of the Licence at:
- * 
+ *
  * [PROJECT_HOME]\license\eupl-1.2\license.txt or https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software distributed under the Licence is
  * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the Licence for the specific language governing permissions and limitations under the Licence.
@@ -18,7 +18,9 @@
  */
 package eu.europa.ec.edelivery.smp.controllers;
 
+import eu.europa.ec.edelivery.smp.server.security.SignatureUtil;
 import eu.europa.ec.edelivery.smp.ui.AbstractControllerTest;
+import org.apache.commons.io.IOUtils;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -29,6 +31,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpHeaders;
 
 import java.io.IOException;
+import java.io.InputStream;
 
 import static eu.europa.ec.edelivery.smp.ServiceGroupBodyUtil.getSampleServiceGroupBodyWithScheme;
 import static java.lang.String.format;
@@ -105,6 +108,23 @@ public class ResourceControllerSingleDomainTest extends AbstractControllerTest {
                 .andExpect(status().is(expectedStatus));
     }
 
+    @Test
+    void addServiceGroupWithUTF8() throws Exception {
+        String resourceLocation = "/input/ServiceGroupWithUTF8.xml";
+        LOG.info(resourceLocation);
+
+        InputStream inputStream = SignatureUtil.class.getResourceAsStream(resourceLocation);
+        byte[] data = IOUtils.toByteArray(inputStream);
+
+        mvc.perform(put(URL_PATH)
+                        .with(ADMIN_CREDENTIALS)
+                        .contentType(APPLICATION_XML_VALUE)
+                        .content(data))
+                .andExpect(status().isCreated());
+        // get service group
+        mvc.perform(get(URL_PATH))
+                .andExpect(status().isOk());
+    }
 
     @Test
     void anonymousUserCannotCreateServiceGroup() throws Exception {

smp-webapp/src/test/resources/input/ServiceGroupWithUTF8.xml

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<ServiceGroup xmlns="http://docs.oasis-open.org/bdxr/ns/SMP/2016/05">
+    <ParticipantIdentifier scheme="${resource.identifier.scheme}">${resource.identifier.value}</ParticipantIdentifier>
+    <ServiceMetadataReferenceCollection/>
+    <Extension>
+        <ex:Test xmlns:ex="http://test.eu">Testąóżęąẞö Greek ÀÆÇßãÿαΩƒΑ char</ex:Test>
+    </Extension>
+</ServiceGroup>