1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 package info.magnolia.i18nsystem;
35
36 import info.magnolia.cms.i18n.MessagesManager;
37 import info.magnolia.i18nsystem.util.LocaleUtils;
38 import info.magnolia.objectfactory.Components;
39 import info.magnolia.resourceloader.Resource;
40 import info.magnolia.resourceloader.ResourceChangeHandler;
41 import info.magnolia.resourceloader.ResourceOrigin;
42 import info.magnolia.resourceloader.ResourceOriginChange;
43 import info.magnolia.resourceloader.util.FileResourceCollectorVisitor;
44 import info.magnolia.resourceloader.util.Functions;
45
46 import java.io.IOException;
47 import java.io.InputStream;
48 import java.io.InputStreamReader;
49 import java.io.Reader;
50 import java.util.Collection;
51 import java.util.Collections;
52 import java.util.HashMap;
53 import java.util.Locale;
54 import java.util.Map;
55 import java.util.Properties;
56 import java.util.Set;
57 import java.util.TreeMap;
58 import java.util.TreeSet;
59 import java.util.function.Predicate;
60
61 import javax.inject.Inject;
62 import javax.inject.Singleton;
63
64 import org.apache.commons.io.input.BOMInputStream;
65 import org.jsoup.Jsoup;
66 import org.jsoup.nodes.Document;
67 import org.jsoup.safety.Cleaner;
68 import org.jsoup.safety.Whitelist;
69 import org.slf4j.Logger;
70 import org.slf4j.LoggerFactory;
71
72
73
74
75 @Singleton
76 public class DefaultMessageBundlesLoader {
77
78 public static final String MGNL_I18N_PATH = "^/[^/]+/i18n/";
79 public static final String MGNL_I18N_PROPERTIES = MGNL_I18N_PATH + ".*\\.properties$";
80
81 private static final String OLD_I18N_PATH = "^/mgnl-i18n/";
82 private static final String OLD_I18N_PROPERTIES = OLD_I18N_PATH + ".*\\.properties$";
83
84 protected static final Predicate<Resource> DIRECTORY_PREDICATE = dir -> true;
85 protected static final Predicate<Resource> RESOURCE_PREDICATE = Functions.pathMatches(OLD_I18N_PROPERTIES)
86 .or(Functions.pathMatches(MGNL_I18N_PROPERTIES));
87
88 private static final Logger log = LoggerFactory.getLogger(DefaultMessageBundlesLoader.class);
89 private static final Cleaner CLEANER = new Cleaner(Whitelist.basic());
90
91 private Map<Locale, Properties> messages = Collections.unmodifiableMap(new HashMap<>());
92
93
94 private final Map<String, DuplicateEntry> duplicateEntriesMap = new HashMap<>();
95
96 @Inject
97 public DefaultMessageBundlesLoader(final ResourceOrigin resourceOrigin) {
98 loadMessages(resourceOrigin, newVisitor());
99
100
101
102
103
104
105
106 log.info("Starting monitoring of {} to load translation files", resourceOrigin);
107 resourceOrigin.registerResourceChangeHandler(new ResourceChangeHandler() {
108 @Override
109 public void onResourceChanged(ResourceOriginChange change) {
110
111
112
113
114
115
116 if (change.getRelatedResourcePath().matches(MGNL_I18N_PROPERTIES)) {
117 loadMessages(resourceOrigin, newVisitor());
118 }
119 }
120 });
121 }
122
123
124
125
126 @Deprecated
127 public DefaultMessageBundlesLoader() {
128 this(Components.getComponent(ResourceOrigin.class));
129 }
130
131 Map<Locale, Properties> getMessages() {
132 return messages;
133 }
134
135 private FileResourceCollectorVisitor newVisitor() {
136 return FileResourceCollectorVisitor.on(DIRECTORY_PREDICATE, RESOURCE_PREDICATE);
137 }
138
139 private void loadMessages(ResourceOrigin resourceOrigin, FileResourceCollectorVisitor visitor) {
140 final Map<Locale, Properties> newMessages = new HashMap<>();
141
142 resourceOrigin.traverseWith(visitor);
143 final Collection<Resource> collected = visitor.getCollectedResources();
144 for (Resource langFile : collected) {
145 final String fileName = langFile.getName();
146 final Locale locale = resolveLocale(fileName);
147 loadResourcesInPropertiesMap(newMessages, langFile, locale);
148 }
149
150 this.messages = Collections.unmodifiableMap(newMessages);
151
152
153 logDuplicates();
154 }
155
156
157
158
159
160
161
162
163
164
165
166
167
168 private void loadResourcesInPropertiesMap(final Map<Locale, Properties> newMessages, Resource propertiesFile, Locale locale) {
169 try (InputStream in = propertiesFile.openStream()) {
170 log.debug("Loading properties file at [{}] with locale [{}]...", propertiesFile, locale);
171
172
173 final Reader inStream = new InputStreamReader(new BOMInputStream(in), "UTF-8");
174
175 final Properties properties = new Properties();
176 properties.load(inStream);
177 Properties existingProperties = newMessages.get(locale);
178
179 if (existingProperties != null) {
180 checkForDuplicates(existingProperties, properties, locale, propertiesFile);
181
182 log.debug("Adding properties to already existing ones under {} locale", locale);
183 properties.putAll(existingProperties);
184 }
185
186 for (Map.Entry<Object, Object> property : properties.entrySet()) {
187 String propertyValue = property.getValue().toString();
188 Document document = Jsoup.parseBodyFragment(propertyValue, "");
189 if (!CLEANER.isValid(document)) {
190 properties.replace(property.getKey(), CLEANER.clean(document).body().html());
191 }
192 }
193
194 newMessages.put(locale, properties);
195 } catch (IOException e) {
196 log.warn("An IO error occurred while trying to read properties file at [{}]", propertiesFile, e);
197 }
198 }
199
200 private void checkForDuplicates(Properties existingProperties, Properties newProperties, Locale locale, Resource resource) {
201 for (String key : newProperties.stringPropertyNames()) {
202 if (existingProperties.containsKey(key)) {
203 DuplicateEntry duplicateEntry = new DuplicateEntry(resource, locale, key);
204 duplicateEntriesMap.put(duplicateEntry.getKeyWithLocaleAndUrl(), duplicateEntry);
205 }
206 }
207 }
208
209 private void logDuplicates() {
210
211 Map<String, DuplicateEntry> perLocaleDuplicates = new TreeMap<>();
212
213 Map<String, DuplicateEntry> perKeyDuplicates = new TreeMap<>();
214 Set<String> urlsSet = new TreeSet<>();
215
216 for (DuplicateEntry duplicateEntry : duplicateEntriesMap.values()) {
217 perLocaleDuplicates.put(duplicateEntry.getKeyWithLocale(), duplicateEntry);
218 perKeyDuplicates.put(duplicateEntry.getKey(), duplicateEntry);
219 urlsSet.add(duplicateEntry.getResource().toString());
220 }
221
222 if (duplicateEntriesMap.size() > 0) {
223 StringBuilder logMsg = new StringBuilder("\n");
224 logMsg.append("------------------------------------\n");
225 logMsg.append("Duplicated keys found while loading message bundles from ./mgnl-i18n :\n");
226 logMsg.append("------------------------------------\n");
227 logMsg.append("Number of duplicates based on key pattern <key>_<locale>_<bundle-url>: ").append(duplicateEntriesMap.size()).append("\n");
228 logMsg.append("Number of duplicates based on key pattern <key>_<locale>: ").append(perLocaleDuplicates.size()).append("\n");
229 logMsg.append("Number of duplicates based on key pattern <key>: ").append(perKeyDuplicates.size()).append("\n");
230 logMsg.append("To get more details concerning the keys, raise the log level to 'DEBUG' for ").append(this.getClass().getName()).append(".\n");
231 logMsg.append("If you encounter a large number of duplicates, it's possible that you are running in a development environment where you have multiple copies of the web-apps in the overlays folder of your web-app.\n");
232 logMsg.append("URLs of the affected files creating duplicate entries:\n");
233 for (String url : urlsSet) {
234 logMsg.append(url).append("\n");
235 }
236 logMsg.append("------------------------------------");
237 log.info(logMsg.toString());
238
239
240 for (DuplicateEntry duplicateEntry : perLocaleDuplicates.values()) {
241 log.debug("Duplicate key found: [{}]; for locale [{}]; in resource [{}]", duplicateEntry.getKey(), duplicateEntry.getLocale(), duplicateEntry.getResource());
242 }
243 }
244
245 duplicateEntriesMap.clear();
246 }
247
248
249
250
251 protected Locale resolveLocale(final String fileName) {
252 return LocaleUtils.parseFromFilename(fileName, getFallbackLocale());
253 }
254
255
256 private Locale getFallbackLocale() {
257 return MessagesManager.getInstance().getDefaultLocale();
258 }
259
260
261 private class DuplicateEntry {
262 private Resource resource;
263 private Locale locale;
264 private String key;
265 private String keyWithLocale;
266 private String keyWithLocaleAndUrl;
267
268 private DuplicateEntry(Resource resource, Locale locale, String key) {
269 this.resource = resource;
270 this.locale = locale;
271 this.key = key;
272 this.keyWithLocale = this.key + "_" + locale.toString();
273 this.keyWithLocaleAndUrl = this.keyWithLocale + "_" + resource.getPath();
274 }
275
276 public Resource getResource() {
277 return resource;
278 }
279
280 private String getKeyWithLocale() {
281 return keyWithLocale;
282 }
283
284 private String getKeyWithLocaleAndUrl() {
285 return keyWithLocaleAndUrl;
286 }
287
288 private Locale getLocale() {
289 return locale;
290 }
291
292 private String getKey() {
293 return key;
294 }
295
296 }
297
298 }