1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 package info.magnolia.importexport;
35
36 import info.magnolia.cms.beans.runtime.Document;
37 import info.magnolia.cms.core.Content;
38 import info.magnolia.cms.core.HierarchyManager;
39 import info.magnolia.cms.core.ItemType;
40 import info.magnolia.cms.core.SystemProperty;
41 import info.magnolia.cms.util.ContentUtil;
42 import info.magnolia.cms.util.NodeDataUtil;
43 import info.magnolia.context.MgnlContext;
44 import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
45 import info.magnolia.importexport.filters.ImportXmlRootFilter;
46 import info.magnolia.importexport.filters.MagnoliaV2Filter;
47 import info.magnolia.importexport.filters.MetadataUuidFilter;
48 import info.magnolia.importexport.filters.RemoveMixversionableFilter;
49 import info.magnolia.importexport.filters.VersionFilter;
50 import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
51 import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
52 import info.magnolia.jcr.util.NodeUtil;
53
54 import java.io.File;
55 import java.io.FileInputStream;
56 import java.io.FileNotFoundException;
57 import java.io.FileOutputStream;
58 import java.io.IOException;
59 import java.io.InputStream;
60 import java.io.OutputStream;
61 import java.io.UnsupportedEncodingException;
62 import java.net.URLDecoder;
63 import java.net.URLEncoder;
64 import java.text.MessageFormat;
65 import java.util.Iterator;
66 import java.util.List;
67 import java.util.Properties;
68 import java.util.regex.Matcher;
69 import java.util.regex.Pattern;
70 import java.util.zip.DeflaterOutputStream;
71 import java.util.zip.GZIPInputStream;
72 import java.util.zip.GZIPOutputStream;
73 import java.util.zip.ZipInputStream;
74 import java.util.zip.ZipOutputStream;
75
76 import javax.jcr.ImportUUIDBehavior;
77 import javax.jcr.Node;
78 import javax.jcr.NodeIterator;
79 import javax.jcr.PathNotFoundException;
80 import javax.jcr.RepositoryException;
81 import javax.jcr.Session;
82 import javax.jcr.Workspace;
83 import javax.xml.transform.Source;
84 import javax.xml.transform.sax.SAXTransformerFactory;
85 import javax.xml.transform.stream.StreamSource;
86
87 import org.apache.commons.io.IOUtils;
88 import org.apache.commons.lang.StringUtils;
89 import org.apache.xml.serialize.OutputFormat;
90 import org.apache.xml.serialize.XMLSerializer;
91 import org.slf4j.Logger;
92 import org.slf4j.LoggerFactory;
93 import org.xml.sax.ContentHandler;
94 import org.xml.sax.InputSource;
95 import org.xml.sax.SAXException;
96 import org.xml.sax.XMLFilter;
97 import org.xml.sax.XMLReader;
98 import org.xml.sax.helpers.XMLReaderFactory;
99
100
101
102
103
104
105
106 public class DataTransporter {
107
108 private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
109
110 private static final int INDENT_VALUE = 2;
111
112 private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
113
114 final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
115
116 public static final String ZIP = ".zip";
117
118 public static final String GZ = ".gz";
119
120 public static final String XML = ".xml";
121
122 public static final String PROPERTIES = ".properties";
123
124 public static final String DOT = ".";
125
126 public static final String SLASH = "/";
127
128 public static final String UTF8 = "UTF-8";
129
130 public static final String JCR_ROOT = "jcr:root";
131
132
133
134
135
136
137
138
139
140
141
142
143
144 public static synchronized void importDocument(Document xmlDocument, String repositoryName, String basepath,
145 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
146 boolean createBasepathIfNotExist)
147 throws IOException {
148 File xmlFile = xmlDocument.getFile();
149 importFile(xmlFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
150 createBasepathIfNotExist);
151 }
152
153
154
155
156
157
158
159
160
161
162
163
164
165 public static synchronized void importFile(File xmlFile, String repositoryName, String basepath,
166 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
167 boolean createBasepathIfNotExist)
168 throws IOException {
169 String name = xmlFile.getAbsolutePath();
170
171 InputStream xmlStream = getInputStreamForFile(xmlFile);
172 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport,
173 createBasepathIfNotExist);
174 }
175
176
177
178
179
180
181 public static void executeBootstrapImport(File xmlFile, String repositoryName) throws IOException {
182 String filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
183 if (filenameWithoutExt.endsWith(XML)) {
184
185
186 filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
187 }
188 String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
189
190 pathName = decodePath(pathName, UTF8);
191
192 String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
193
194 if (xmlFile.getName().endsWith(PROPERTIES)) {
195 Properties properties = new Properties();
196 FileInputStream stream = new FileInputStream(xmlFile);
197 properties.load(stream);
198 stream.close();
199 importProperties(properties, repositoryName);
200 } else {
201 DataTransporter.importFile(xmlFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
202 }
203 }
204
205
206
207
208
209
210
211 public static void importProperties(Properties properties, String repositoryName) {
212 for (Iterator iter = properties.keySet().iterator(); iter.hasNext();) {
213 String key = (String) iter.next();
214 String value = (String) properties.get(key);
215
216 String name = StringUtils.substringAfterLast(key, ".");
217 String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
218 Content node = ContentUtil.getContent(repositoryName, path);
219 if (node != null) {
220 try {
221 NodeDataUtil.getOrCreate(node, name).setValue(value);
222 node.save();
223 }
224 catch (RepositoryException e) {
225 log.error("can't set property " + key, e);
226 }
227 }
228 }
229
230 }
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
251 String name, boolean keepVersionHistory, int importMode,
252 boolean saveAfterImport, boolean createBasepathIfNotExist)
253 throws IOException {
254 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
255
256 }
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
278 String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
279 boolean saveAfterImport, boolean createBasepathIfNotExist)
280 throws IOException {
281
282
283 if (xmlStream == null) {
284 throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
285 }
286
287 HierarchyManager hm = MgnlContext.getHierarchyManager(repositoryName);
288 if (hm == null) {
289 throw new IllegalStateException("Can't import " + name + " since repository " + repositoryName + " does not exist.");
290 }
291 Workspace ws = hm.getWorkspace();
292
293 if (log.isDebugEnabled()) {
294 log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]",
295 new Object[]{repositoryName, name, basepath});
296 }
297
298 if (!hm.isExist(basepath) && createBasepathIfNotExist) {
299 try {
300 ContentUtil.createPath(hm, basepath, ItemType.CONTENT);
301 }
302 catch (RepositoryException e) {
303 log.error("can't create path [{}]", basepath);
304 }
305 }
306
307 Session session = ws.getSession();
308
309 try {
310
311
312 List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
313
314 if (keepVersionHistory) {
315
316 session.importXML(basepath, xmlStream, importMode);
317 } else {
318
319 XMLReader initialReader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
320 try{
321 initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
322 }catch (SAXException e) {
323 log.error("could not set parser feature");
324 }
325
326
327 XMLFilter magnoliaV2Filter = null;
328
329
330 if (new File(name).isFile()) {
331 InputStream xslStream = getXslStreamForXmlFile(new File(name));
332 if (xslStream != null) {
333 Source xslSource = new StreamSource(xslStream);
334 SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
335 XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
336 magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
337 }
338 }
339
340 if (magnoliaV2Filter == null) {
341 magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
342 }
343
344 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
345
346
347
348
349
350 versionFilter = new RemoveMixversionableFilter(versionFilter);
351
352
353 versionFilter = new AccesscontrolNodeFilter(versionFilter);
354
355 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
356
357 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
358 finalReader.setContentHandler(handler);
359
360
361 try {
362 finalReader.parse(new InputSource(xmlStream));
363 }
364 finally {
365 IOUtils.closeQuietly(xmlStream);
366 }
367
368 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
369 String path = basepath;
370 if (!path.endsWith(SLASH)) {
371 path += SLASH;
372 }
373
374 Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
375 for (Iterator iter = dummyRoot.getNodes(); iter.hasNext();) {
376 Node child = (Node) iter.next();
377
378
379 if (session.itemExists(path + child.getName())) {
380 session.getItem(path + child.getName()).remove();
381 }
382
383 session.move(child.getPath(), path + child.getName());
384 }
385
386 dummyRoot.remove();
387 }
388
389
390 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
391 while (nodesAfterImport.hasNext()) {
392 Node nodeAfterImport = nodesAfterImport.nextNode();
393 boolean existedBeforeImport = false;
394 for (Node nodeBeforeImport : nodesBeforeImport) {
395 if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
396 existedBeforeImport = true;
397 break;
398 }
399 }
400 if (!existedBeforeImport) {
401 postProcessAfterImport(nodeAfterImport, forceUnpublishState);
402 }
403 }
404 }
405 }
406 catch (Exception e) {
407 throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
408 }
409 finally {
410 IOUtils.closeQuietly(xmlStream);
411 }
412
413 try {
414 if (saveAfterImport) {
415 session.save();
416 }
417 }
418 catch (RepositoryException e) {
419 log.error(MessageFormat.format(
420 "Unable to save changes to the [{0}] repository due to a {1} Exception: {2}.",
421 new Object[]{repositoryName, e.getClass().getName(), e.getMessage()}), e);
422 throw new IOException(e.getMessage());
423 }
424 }
425
426 private static void postProcessAfterImport(Node node, boolean forceUnpublishState) throws RepositoryException {
427 try {
428 new MetaDataImportPostProcessor().postProcessNode(node);
429 if (forceUnpublishState) {
430 new ActivationStatusImportPostProcessor().postProcessNode(node);
431 }
432 } catch (RepositoryException e) {
433 throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
434 }
435 }
436
437
438
439
440
441 protected static InputStream getXslStreamForXmlFile(File file) {
442 InputStream xslStream = null;
443 String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
444 File xslFile = new File(xlsFilename);
445 if (xslFile.exists()) {
446 try {
447 xslStream = new FileInputStream(xslFile);
448 log.info("XSL file for [" + file.getName() + "] found (" + xslFile.getName() + ")");
449 } catch (FileNotFoundException e) {
450 e.printStackTrace();
451 }
452 }
453 return xslStream;
454 }
455
456
457
458
459
460
461
462 private static InputStream getInputStreamForFile(File xmlFile) throws IOException {
463 InputStream xmlStream;
464
465 if (xmlFile.getName().endsWith(ZIP)) {
466 xmlStream = new ZipInputStream((new FileInputStream(xmlFile)));
467 } else if (xmlFile.getName().endsWith(GZ)) {
468 xmlStream = new GZIPInputStream((new FileInputStream(xmlFile)));
469 } else {
470 xmlStream = new FileInputStream(xmlFile);
471 }
472 return xmlStream;
473 }
474
475 public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
476 Session session, String basepath, String repository, String ext) throws IOException {
477 OutputStream outputStream = baseOutputStream;
478 if (ext.endsWith(ZIP)) {
479 outputStream = new ZipOutputStream(baseOutputStream);
480 } else if (ext.endsWith(GZ)) {
481 outputStream = new GZIPOutputStream(baseOutputStream);
482 }
483
484 try {
485 if (keepVersionHistory) {
486
487
488 if (!format) {
489 session.exportSystemView(basepath, outputStream, false, false);
490 } else {
491 parseAndFormat(outputStream, null, repository, basepath, session, false);
492 }
493 } else {
494
495
496 XMLReader reader = new VersionFilter(XMLReaderFactory
497 .createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName()));
498 reader = new AccesscontrolNodeFilter(reader);
499 parseAndFormat(outputStream, reader, repository, basepath, session, false);
500 }
501 }
502 catch (IOException e) {
503 throw new RuntimeException(e);
504 }
505 catch (SAXException e) {
506 throw new RuntimeException(e);
507 }
508 catch (RepositoryException e) {
509 throw new RuntimeException(e);
510 }
511
512
513
514 if (outputStream instanceof DeflaterOutputStream) {
515 ((DeflaterOutputStream) outputStream).finish();
516 }
517
518 baseOutputStream.flush();
519 IOUtils.closeQuietly(baseOutputStream);
520 }
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536 public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
537 Session session, boolean noRecurse)
538 throws IOException, SAXException, PathNotFoundException, RepositoryException {
539
540 if (reader == null) {
541 reader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
542 }
543
544
545 File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
546 OutputStream fileStream = new FileOutputStream(tempFile);
547
548 try {
549 session.exportSystemView(basepath, fileStream, false, noRecurse);
550 }
551 finally {
552 IOUtils.closeQuietly(fileStream);
553 }
554
555 readFormatted(reader, tempFile, stream);
556
557 if (!tempFile.delete()) {
558 log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
559 }
560 }
561
562
563
564
565
566
567
568
569
570 protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
571 throws FileNotFoundException, IOException, SAXException {
572 InputStream fileInputStream = new FileInputStream(inputFile);
573 readFormatted(reader, fileInputStream, outputStream);
574 IOUtils.closeQuietly(fileInputStream);
575 }
576
577
578
579
580
581
582
583
584
585 protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
586 throws FileNotFoundException, IOException, SAXException {
587
588 OutputFormat outputFormat = new OutputFormat();
589
590 outputFormat.setPreserveSpace(false);
591 outputFormat.setIndenting(true);
592 outputFormat.setIndent(INDENT_VALUE);
593 outputFormat.setLineWidth(120);
594
595 final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces");
596 MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces);
597 metadataUuidFilter.setContentHandler(new XMLSerializer(outputStream, outputFormat));
598 metadataUuidFilter.parse(new InputSource(inputStream));
599
600 IOUtils.closeQuietly(inputStream);
601 }
602
603
604
605
606
607
608
609
610 public static String encodePath(String path, String separator, String enc)
611 {
612 StringBuilder pathEncoded = new StringBuilder();
613 try
614 {
615 if (!StringUtils.contains(path, separator))
616 {
617 return URLEncoder.encode(path, enc);
618 }
619 for(int i=0; i < path.length(); i++) {
620 String ch = String.valueOf(path.charAt(i));
621 if(separator.equals(ch)) {
622 pathEncoded.append(ch);
623 } else {
624 pathEncoded.append(URLEncoder.encode(ch, enc));
625 }
626 }
627 }
628 catch (UnsupportedEncodingException e)
629 {
630 return path;
631 }
632 return pathEncoded.toString();
633 }
634
635
636
637
638
639
640
641 public static String decodePath(String path, String enc)
642 {
643 String pathEncoded = StringUtils.EMPTY;
644 try
645 {
646 pathEncoded = URLDecoder.decode(path, enc);
647 }
648 catch (UnsupportedEncodingException e)
649 {
650 return path;
651 }
652 return pathEncoded;
653 }
654
655
656
657
658
659
660
661 public static String createExportPath(String path) {
662
663 String newPath = path.replace(".", "..");
664 newPath = newPath.replace("/", ".");
665 return newPath;
666 }
667
668
669
670
671 public static String revertExportPath(String exportPath) {
672 if(".".equals(exportPath)) {
673 return "/";
674 }
675
676
677 Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
678
679 StringBuilder reversed = new StringBuilder(exportPath.length());
680
681 while(matcher.find()){
682 String group = matcher.group();
683 int dotsNumber = StringUtils.countMatches(group, ".");
684 if(dotsNumber == 1) {
685 reversed.append(group.replaceFirst("\\.", "/"));
686 } else {
687 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
688 String name = StringUtils.substringAfterLast(group, ".");
689 reversed.append(dots);
690
691 if(dotsNumber % 2 != 0) {
692 reversed.append("/");
693 }
694 reversed.append(name);
695 }
696 }
697 return reversed.toString();
698 }
699
700 }