1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 package info.magnolia.importexport;
35
36 import static javax.xml.transform.OutputKeys.*;
37
38 import info.magnolia.cms.beans.runtime.Document;
39 import info.magnolia.cms.core.SystemProperty;
40 import info.magnolia.cms.core.version.VersionManager;
41 import info.magnolia.context.MgnlContext;
42 import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
43 import info.magnolia.importexport.filters.ImportXmlRootFilter;
44 import info.magnolia.importexport.filters.MagnoliaV2Filter;
45 import info.magnolia.importexport.filters.MetadataUuidFilter;
46 import info.magnolia.importexport.filters.RemoveMixversionableFilter;
47 import info.magnolia.importexport.filters.VersionFilter;
48 import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
49 import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
50 import info.magnolia.importexport.postprocessors.UpdateVersionMixinPostProcessor;
51 import info.magnolia.jcr.RuntimeRepositoryException;
52 import info.magnolia.jcr.util.NodeTypes;
53 import info.magnolia.jcr.util.NodeUtil;
54 import info.magnolia.jcr.util.SessionUtil;
55 import info.magnolia.objectfactory.Components;
56
57 import java.io.File;
58 import java.io.FileInputStream;
59 import java.io.FileNotFoundException;
60 import java.io.FileOutputStream;
61 import java.io.IOException;
62 import java.io.InputStream;
63 import java.io.OutputStream;
64 import java.io.UnsupportedEncodingException;
65 import java.net.URLDecoder;
66 import java.net.URLEncoder;
67 import java.util.Iterator;
68 import java.util.List;
69 import java.util.Properties;
70 import java.util.regex.Matcher;
71 import java.util.regex.Pattern;
72 import java.util.zip.DeflaterOutputStream;
73 import java.util.zip.GZIPInputStream;
74 import java.util.zip.GZIPOutputStream;
75 import java.util.zip.ZipInputStream;
76 import java.util.zip.ZipOutputStream;
77
78 import javax.jcr.ImportUUIDBehavior;
79 import javax.jcr.Node;
80 import javax.jcr.NodeIterator;
81 import javax.jcr.PathNotFoundException;
82 import javax.jcr.RepositoryException;
83 import javax.jcr.Session;
84 import javax.xml.parsers.ParserConfigurationException;
85 import javax.xml.parsers.SAXParser;
86 import javax.xml.parsers.SAXParserFactory;
87 import javax.xml.transform.Source;
88 import javax.xml.transform.Transformer;
89 import javax.xml.transform.TransformerConfigurationException;
90 import javax.xml.transform.sax.SAXTransformerFactory;
91 import javax.xml.transform.sax.TransformerHandler;
92 import javax.xml.transform.stream.StreamResult;
93 import javax.xml.transform.stream.StreamSource;
94
95 import org.apache.commons.io.IOUtils;
96 import org.apache.commons.lang3.StringUtils;
97 import org.slf4j.Logger;
98 import org.slf4j.LoggerFactory;
99 import org.xml.sax.ContentHandler;
100 import org.xml.sax.InputSource;
101 import org.xml.sax.SAXException;
102 import org.xml.sax.XMLFilter;
103 import org.xml.sax.XMLReader;
104
105
106
107
108 public class DataTransporter {
109
110 private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
111
112 private static final int INDENT_VALUE = 2;
113
114 private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
115
116 final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
117
118 public static final String ZIP = ".zip";
119
120 public static final String GZ = ".gz";
121
122 public static final String XML = ".xml";
123
124 public static final String PROPERTIES = ".properties";
125
126 public static final String DOT = ".";
127
128 public static final String SLASH = "/";
129
130 public static final String UTF8 = "UTF-8";
131
132 public static final String JCR_ROOT = "jcr:root";
133
134
135
136
137
138
139
140
141
142
143
144 public static synchronized void importDocument(Document xmlDocument, String repositoryName, String basepath,
145 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
146 boolean createBasepathIfNotExist)
147 throws IOException {
148 File xmlFile = xmlDocument.getFile();
149 importFile(xmlFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
150 createBasepathIfNotExist);
151 }
152
153
154
155
156
157
158
159
160
161
162
163 public static synchronized void importFile(File xmlFile, String repositoryName, String basepath,
164 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
165 boolean createBasepathIfNotExist)
166 throws IOException {
167 String name = xmlFile.getAbsolutePath();
168
169 InputStream xmlStream = getInputStreamForFile(xmlFile);
170 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport,
171 createBasepathIfNotExist);
172 }
173
174 public static void executeBootstrapImport(File xmlFile, String repositoryName) throws IOException {
175 String filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
176 if (filenameWithoutExt.endsWith(XML)) {
177
178
179 filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
180 }
181 String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
182
183 pathName = decodePath(pathName, UTF8);
184
185 String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
186
187 if (xmlFile.getName().endsWith(PROPERTIES)) {
188 Properties properties = new Properties();
189 FileInputStream stream = new FileInputStream(xmlFile);
190 try {
191 properties.load(stream);
192 } finally {
193 IOUtils.closeQuietly(stream);
194 }
195 importProperties(properties, repositoryName);
196 } else {
197 DataTransporter.importFile(xmlFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
198 }
199 }
200
201
202
203
204 @Deprecated
205 public static void importProperties(Properties properties, String repositoryName) {
206 for (Iterator iter = properties.keySet().iterator(); iter.hasNext(); ) {
207 String key = (String) iter.next();
208 String value = (String) properties.get(key);
209
210 String name = StringUtils.substringAfterLast(key, ".");
211 String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
212 Node node = SessionUtil.getNode(repositoryName, path);
213 if (node != null) {
214 try {
215 node.setProperty(name, value);
216 node.getSession().save();
217 } catch (RepositoryException e) {
218 log.error("can't set property {}", key, e);
219 }
220 }
221 }
222
223 }
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
242 String name, boolean keepVersionHistory, int importMode,
243 boolean saveAfterImport, boolean createBasepathIfNotExist)
244 throws IOException {
245 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
246
247 }
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
266 String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
267 boolean saveAfterImport, boolean createBasepathIfNotExist)
268 throws IOException {
269
270
271 if (xmlStream == null) {
272 throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
273 }
274 try {
275 Session session = MgnlContext.getJCRSession(repositoryName);
276
277 log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
278
279 if (!session.nodeExists(basepath) && createBasepathIfNotExist) {
280 try {
281 NodeUtil.createPath(session.getRootNode(), basepath, NodeTypes.Content.NAME);
282 } catch (RepositoryException e) {
283 log.error("can't create path [{}]", basepath);
284 }
285 }
286
287
288 List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
289
290 if (keepVersionHistory) {
291
292 session.importXML(basepath, xmlStream, importMode);
293 } else {
294
295 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
296 saxParserFactory.setNamespaceAware(true);
297 SAXParser saxParser = saxParserFactory.newSAXParser();
298 XMLReader initialReader = saxParser.getXMLReader();
299 try {
300 initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
301 } catch (SAXException e) {
302 log.error("could not set parser feature");
303 }
304
305
306 XMLFilter magnoliaV2Filter = null;
307
308
309 if (new File(name).isFile()) {
310 InputStream xslStream = getXslStreamForXmlFile(new File(name));
311 if (xslStream != null) {
312 Source xslSource = new StreamSource(xslStream);
313 SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
314 XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
315 magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
316 }
317 }
318
319 if (magnoliaV2Filter == null) {
320 magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
321 }
322
323 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
324
325
326
327
328
329 versionFilter = new RemoveMixversionableFilter(versionFilter);
330
331
332 versionFilter = new AccesscontrolNodeFilter(versionFilter);
333
334 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
335
336 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
337 finalReader.setContentHandler(handler);
338
339
340 try {
341 finalReader.parse(new InputSource(xmlStream));
342 } finally {
343 IOUtils.closeQuietly(xmlStream);
344 }
345
346 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
347 String path = basepath;
348 if (!path.endsWith(SLASH)) {
349 path += SLASH;
350 }
351
352 Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
353 for (Iterator iter = dummyRoot.getNodes(); iter.hasNext(); ) {
354 Node child = (Node) iter.next();
355
356
357 if (session.itemExists(path + child.getName())) {
358 session.getItem(path + child.getName()).remove();
359 }
360
361 session.move(child.getPath(), path + child.getName());
362 }
363
364 dummyRoot.remove();
365 }
366
367
368 VersionManager versionManager = Components.getComponent(VersionManager.class);
369 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
370 while (nodesAfterImport.hasNext()) {
371 Node nodeAfterImport = nodesAfterImport.nextNode();
372 boolean existedBeforeImport = false;
373 for (Node nodeBeforeImport : nodesBeforeImport) {
374 if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
375 existedBeforeImport = true;
376 break;
377 }
378 }
379 if (!existedBeforeImport) {
380 postProcessAfterImport(nodeAfterImport, forceUnpublishState, importMode, versionManager);
381 }
382 }
383 }
384 if (saveAfterImport) {
385 session.save();
386 }
387 } catch (Exception e) {
388 throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
389 } finally {
390 IOUtils.closeQuietly(xmlStream);
391 }
392 }
393
394 private static void postProcessAfterImport(Node node, boolean forceUnpublishState, int importMode, VersionManager versionManager) throws RepositoryException {
395 try {
396 new MetaDataImportPostProcessor().postProcessNode(node);
397 if (forceUnpublishState) {
398 new ActivationStatusImportPostProcessor().postProcessNode(node);
399 }
400 new UpdateVersionMixinPostProcessor(importMode, versionManager).postProcessNode(node);
401 } catch (RepositoryException e) {
402 throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
403 }
404 }
405
406
407
408
409 protected static InputStream getXslStreamForXmlFile(File file) {
410 InputStream xslStream = null;
411 String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
412 File xslFile = new File(xlsFilename);
413 if (xslFile.exists()) {
414 try {
415 xslStream = new FileInputStream(xslFile);
416 log.info("XSL file for [{}] found ({})", file.getName(), xslFile.getName());
417 } catch (FileNotFoundException e) {
418 e.printStackTrace();
419 }
420 }
421 return xslStream;
422 }
423
424
425
426
427
428
429 private static InputStream getInputStreamForFile(File xmlFile) throws IOException {
430 InputStream xmlStream;
431
432 if (xmlFile.getName().endsWith(ZIP)) {
433 xmlStream = new ZipInputStream((new FileInputStream(xmlFile)));
434 } else if (xmlFile.getName().endsWith(GZ)) {
435 xmlStream = new GZIPInputStream((new FileInputStream(xmlFile)));
436 } else {
437 xmlStream = new FileInputStream(xmlFile);
438 }
439 return xmlStream;
440 }
441
442 public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
443 Session session, String basepath, String repository, String ext) throws IOException {
444 OutputStream outputStream = baseOutputStream;
445 if (ext.endsWith(ZIP)) {
446 outputStream = new ZipOutputStream(baseOutputStream);
447 } else if (ext.endsWith(GZ)) {
448 outputStream = new GZIPOutputStream(baseOutputStream);
449 }
450
451 try {
452 if (keepVersionHistory) {
453
454
455 if (!format) {
456 session.exportSystemView(basepath, outputStream, false, false);
457 } else {
458 parseAndFormat(outputStream, null, repository, basepath, session, false);
459 }
460 } else {
461 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
462 saxParserFactory.setNamespaceAware(true);
463 SAXParser saxParser = saxParserFactory.newSAXParser();
464 XMLReader reader = saxParser.getXMLReader();
465
466 reader = new AccesscontrolNodeFilter(reader);
467 parseAndFormat(outputStream, reader, repository, basepath, session, false);
468 }
469 } catch (IOException | SAXException | ParserConfigurationException | TransformerConfigurationException e) {
470 throw new RuntimeException(e);
471 } catch (RepositoryException e) {
472 throw new RuntimeRepositoryException(e);
473 }
474
475
476
477 if (outputStream instanceof DeflaterOutputStream) {
478 ((DeflaterOutputStream) outputStream).finish();
479 }
480
481 baseOutputStream.flush();
482 IOUtils.closeQuietly(baseOutputStream);
483 }
484
485
486
487
488
489
490
491
492
493
494
495 public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
496 Session session, boolean noRecurse)
497 throws IOException, SAXException, ParserConfigurationException, TransformerConfigurationException, PathNotFoundException, RepositoryException {
498
499 if (reader == null) {
500 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
501 saxParserFactory.setNamespaceAware(true);
502 SAXParser saxParser = saxParserFactory.newSAXParser();
503 reader = saxParser.getXMLReader();
504 }
505
506
507 File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
508 OutputStream fileStream = new FileOutputStream(tempFile);
509
510 try {
511 session.exportSystemView(basepath, fileStream, false, noRecurse);
512 } finally {
513 IOUtils.closeQuietly(fileStream);
514 }
515
516 readFormatted(reader, tempFile, stream);
517
518 if (!tempFile.delete()) {
519 log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
520 }
521 }
522
523 protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
524 throws FileNotFoundException, IOException, SAXException, TransformerConfigurationException {
525 InputStream fileInputStream = new FileInputStream(inputFile);
526 readFormatted(reader, fileInputStream, outputStream);
527 IOUtils.closeQuietly(fileInputStream);
528 }
529
530 protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
531 throws FileNotFoundException, IOException, SAXException, TransformerConfigurationException {
532
533 SAXTransformerFactory transformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
534 TransformerHandler transformerHandler = transformerFactory.newTransformerHandler();
535 Transformer transformer = transformerHandler.getTransformer();
536 transformer.setOutputProperty(INDENT, "yes");
537 transformer.setOutputProperty(DOCTYPE_PUBLIC, "yes");
538 transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", String.valueOf(INDENT_VALUE));
539 transformerHandler.setResult(new StreamResult(outputStream));
540
541 final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces");
542 MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces);
543 metadataUuidFilter.setContentHandler(transformerHandler);
544 metadataUuidFilter.parse(new InputSource(inputStream));
545
546 IOUtils.closeQuietly(inputStream);
547 }
548
549
550
551
552
553
554
555 public static String encodePath(String path, String separator, String enc) {
556 StringBuilder pathEncoded = new StringBuilder();
557 try {
558 if (!StringUtils.contains(path, separator)) {
559 return URLEncoder.encode(path, enc);
560 }
561 for (int i = 0; i < path.length(); i++) {
562 String ch = String.valueOf(path.charAt(i));
563 if (separator.equals(ch)) {
564 pathEncoded.append(ch);
565 } else {
566 pathEncoded.append(URLEncoder.encode(ch, enc));
567 }
568 }
569 } catch (UnsupportedEncodingException e) {
570 return path;
571 }
572 return pathEncoded.toString();
573 }
574
575
576
577
578
579
580
581
582 public static String decodePath(String path, String enc) {
583 String pathEncoded = StringUtils.EMPTY;
584 try {
585 pathEncoded = URLDecoder.decode(path, enc);
586 } catch (UnsupportedEncodingException e) {
587 return path;
588 }
589 return pathEncoded;
590 }
591
592
593
594
595
596
597
598 public static String createExportPath(String path) {
599
600 String newPath = path.replace(".", "..");
601 newPath = newPath.replace("/", ".");
602 return newPath;
603 }
604
605
606
607
608
609 public static String revertExportPath(String exportPath) {
610 if (".".equals(exportPath)) {
611 return "/";
612 }
613
614
615 Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
616
617 StringBuilder reversed = new StringBuilder(exportPath.length());
618
619 while (matcher.find()) {
620 String group = matcher.group();
621 int dotsNumber = StringUtils.countMatches(group, ".");
622 if (dotsNumber == 1) {
623 reversed.append(group.replaceFirst("\\.", "/"));
624 } else {
625 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
626 String name = StringUtils.substringAfterLast(group, ".");
627 reversed.append(dots);
628
629 if (dotsNumber % 2 != 0) {
630 reversed.append("/");
631 }
632 reversed.append(name);
633 }
634 }
635 return reversed.toString();
636 }
637
638 }