1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 package info.magnolia.importexport;
35
36 import info.magnolia.cms.beans.runtime.Document;
37 import info.magnolia.cms.core.Content;
38 import info.magnolia.cms.core.HierarchyManager;
39 import info.magnolia.cms.core.ItemType;
40 import info.magnolia.cms.core.SystemProperty;
41 import info.magnolia.cms.util.ContentUtil;
42 import info.magnolia.cms.util.NodeDataUtil;
43 import info.magnolia.context.MgnlContext;
44 import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
45 import info.magnolia.importexport.filters.ImportXmlRootFilter;
46 import info.magnolia.importexport.filters.MagnoliaV2Filter;
47 import info.magnolia.importexport.filters.MetadataUuidFilter;
48 import info.magnolia.importexport.filters.RemoveMixversionableFilter;
49 import info.magnolia.importexport.filters.VersionFilter;
50 import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
51 import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
52 import info.magnolia.jcr.util.NodeUtil;
53
54 import java.io.File;
55 import java.io.FileInputStream;
56 import java.io.FileNotFoundException;
57 import java.io.FileOutputStream;
58 import java.io.IOException;
59 import java.io.InputStream;
60 import java.io.OutputStream;
61 import java.io.UnsupportedEncodingException;
62 import java.net.URLDecoder;
63 import java.net.URLEncoder;
64 import java.text.MessageFormat;
65 import java.util.Iterator;
66 import java.util.List;
67 import java.util.Properties;
68 import java.util.regex.Matcher;
69 import java.util.regex.Pattern;
70 import java.util.zip.DeflaterOutputStream;
71 import java.util.zip.GZIPInputStream;
72 import java.util.zip.GZIPOutputStream;
73 import java.util.zip.ZipInputStream;
74 import java.util.zip.ZipOutputStream;
75
76 import javax.jcr.ImportUUIDBehavior;
77 import javax.jcr.Node;
78 import javax.jcr.NodeIterator;
79 import javax.jcr.PathNotFoundException;
80 import javax.jcr.RepositoryException;
81 import javax.jcr.Session;
82 import javax.jcr.Workspace;
83 import javax.xml.transform.Source;
84 import javax.xml.transform.sax.SAXTransformerFactory;
85 import javax.xml.transform.stream.StreamSource;
86
87 import org.apache.commons.io.IOUtils;
88 import org.apache.commons.lang3.StringUtils;
89 import org.apache.xml.serialize.OutputFormat;
90 import org.apache.xml.serialize.XMLSerializer;
91 import org.slf4j.Logger;
92 import org.slf4j.LoggerFactory;
93 import org.xml.sax.ContentHandler;
94 import org.xml.sax.InputSource;
95 import org.xml.sax.SAXException;
96 import org.xml.sax.XMLFilter;
97 import org.xml.sax.XMLReader;
98 import org.xml.sax.helpers.XMLReaderFactory;
99
100
101
102
103
104 public class DataTransporter {
105
106 private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
107
108 private static final int INDENT_VALUE = 2;
109
110 private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
111
112 final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
113
114 public static final String ZIP = ".zip";
115
116 public static final String GZ = ".gz";
117
118 public static final String XML = ".xml";
119
120 public static final String PROPERTIES = ".properties";
121
122 public static final String DOT = ".";
123
124 public static final String SLASH = "/";
125
126 public static final String UTF8 = "UTF-8";
127
128 public static final String JCR_ROOT = "jcr:root";
129
130
131
132
133
134
135
136
137
138
139
140 public static synchronized void importDocument(Document xmlDocument, String repositoryName, String basepath,
141 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
142 boolean createBasepathIfNotExist)
143 throws IOException {
144 File xmlFile = xmlDocument.getFile();
145 importFile(xmlFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
146 createBasepathIfNotExist);
147 }
148
149
150
151
152
153
154
155
156
157
158
159 public static synchronized void importFile(File xmlFile, String repositoryName, String basepath,
160 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
161 boolean createBasepathIfNotExist)
162 throws IOException {
163 String name = xmlFile.getAbsolutePath();
164
165 InputStream xmlStream = getInputStreamForFile(xmlFile);
166 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport,
167 createBasepathIfNotExist);
168 }
169
170 public static void executeBootstrapImport(File xmlFile, String repositoryName) throws IOException {
171 String filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
172 if (filenameWithoutExt.endsWith(XML)) {
173
174
175 filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
176 }
177 String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
178
179 pathName = decodePath(pathName, UTF8);
180
181 String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
182
183 if (xmlFile.getName().endsWith(PROPERTIES)) {
184 Properties properties = new Properties();
185 FileInputStream stream = new FileInputStream(xmlFile);
186 try {
187 properties.load(stream);
188 } finally {
189 IOUtils.closeQuietly(stream);
190 }
191 importProperties(properties, repositoryName);
192 } else {
193 DataTransporter.importFile(xmlFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
194 }
195 }
196
197
198
199
200 public static void importProperties(Properties properties, String repositoryName) {
201 for (Iterator iter = properties.keySet().iterator(); iter.hasNext();) {
202 String key = (String) iter.next();
203 String value = (String) properties.get(key);
204
205 String name = StringUtils.substringAfterLast(key, ".");
206 String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
207 Content node = ContentUtil.getContent(repositoryName, path);
208 if (node != null) {
209 try {
210 NodeDataUtil.getOrCreate(node, name).setValue(value);
211 node.save();
212 }
213 catch (RepositoryException e) {
214 log.error("can't set property {}", key, e);
215 }
216 }
217 }
218
219 }
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
238 String name, boolean keepVersionHistory, int importMode,
239 boolean saveAfterImport, boolean createBasepathIfNotExist)
240 throws IOException {
241 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
242
243 }
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
263 String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
264 boolean saveAfterImport, boolean createBasepathIfNotExist)
265 throws IOException {
266
267
268 if (xmlStream == null) {
269 throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
270 }
271
272 HierarchyManager hm = MgnlContext.getHierarchyManager(repositoryName);
273 if (hm == null) {
274 throw new IllegalStateException("Can't import " + name + " since repository " + repositoryName + " does not exist.");
275 }
276 Workspace ws = hm.getWorkspace();
277
278 log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
279
280
281 if (!hm.isExist(basepath) && createBasepathIfNotExist) {
282 try {
283 ContentUtil.createPath(hm, basepath, ItemType.CONTENT);
284 }
285 catch (RepositoryException e) {
286 log.error("can't create path [{}]", basepath);
287 }
288 }
289
290 Session session = ws.getSession();
291
292 try {
293
294
295 List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
296
297 if (keepVersionHistory) {
298
299 session.importXML(basepath, xmlStream, importMode);
300 } else {
301
302 XMLReader initialReader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
303 try{
304 initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
305 }catch (SAXException e) {
306 log.error("could not set parser feature");
307 }
308
309
310 XMLFilter magnoliaV2Filter = null;
311
312
313 if (new File(name).isFile()) {
314 InputStream xslStream = getXslStreamForXmlFile(new File(name));
315 if (xslStream != null) {
316 Source xslSource = new StreamSource(xslStream);
317 SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
318 XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
319 magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
320 }
321 }
322
323 if (magnoliaV2Filter == null) {
324 magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
325 }
326
327 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
328
329
330
331
332
333 versionFilter = new RemoveMixversionableFilter(versionFilter);
334
335
336 versionFilter = new AccesscontrolNodeFilter(versionFilter);
337
338 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
339
340 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
341 finalReader.setContentHandler(handler);
342
343
344 try {
345 finalReader.parse(new InputSource(xmlStream));
346 }
347 finally {
348 IOUtils.closeQuietly(xmlStream);
349 }
350
351 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
352 String path = basepath;
353 if (!path.endsWith(SLASH)) {
354 path += SLASH;
355 }
356
357 Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
358 for (Iterator iter = dummyRoot.getNodes(); iter.hasNext();) {
359 Node child = (Node) iter.next();
360
361
362 if (session.itemExists(path + child.getName())) {
363 session.getItem(path + child.getName()).remove();
364 }
365
366 session.move(child.getPath(), path + child.getName());
367 }
368
369 dummyRoot.remove();
370 }
371
372
373 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
374 while (nodesAfterImport.hasNext()) {
375 Node nodeAfterImport = nodesAfterImport.nextNode();
376 boolean existedBeforeImport = false;
377 for (Node nodeBeforeImport : nodesBeforeImport) {
378 if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
379 existedBeforeImport = true;
380 break;
381 }
382 }
383 if (!existedBeforeImport) {
384 postProcessAfterImport(nodeAfterImport, forceUnpublishState);
385 }
386 }
387 }
388 }
389 catch (Exception e) {
390 throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
391 }
392 finally {
393 IOUtils.closeQuietly(xmlStream);
394 }
395
396 try {
397 if (saveAfterImport) {
398 session.save();
399 }
400 }
401 catch (RepositoryException e) {
402 log.error(MessageFormat.format(
403 "Unable to save changes to the [{0}] repository due to a {1} Exception: {2}.",
404 new Object[]{repositoryName, e.getClass().getName(), e.getMessage()}), e);
405 throw new IOException(e.getMessage());
406 }
407 }
408
409 private static void postProcessAfterImport(Node node, boolean forceUnpublishState) throws RepositoryException {
410 try {
411 new MetaDataImportPostProcessor().postProcessNode(node);
412 if (forceUnpublishState) {
413 new ActivationStatusImportPostProcessor().postProcessNode(node);
414 }
415 } catch (RepositoryException e) {
416 throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
417 }
418 }
419
420
421
422
423 protected static InputStream getXslStreamForXmlFile(File file) {
424 InputStream xslStream = null;
425 String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
426 File xslFile = new File(xlsFilename);
427 if (xslFile.exists()) {
428 try {
429 xslStream = new FileInputStream(xslFile);
430 log.info("XSL file for [{}] found ({})", file.getName(), xslFile.getName());
431 } catch (FileNotFoundException e) {
432 e.printStackTrace();
433 }
434 }
435 return xslStream;
436 }
437
438
439
440
441
442
443 private static InputStream getInputStreamForFile(File xmlFile) throws IOException {
444 InputStream xmlStream;
445
446 if (xmlFile.getName().endsWith(ZIP)) {
447 xmlStream = new ZipInputStream((new FileInputStream(xmlFile)));
448 } else if (xmlFile.getName().endsWith(GZ)) {
449 xmlStream = new GZIPInputStream((new FileInputStream(xmlFile)));
450 } else {
451 xmlStream = new FileInputStream(xmlFile);
452 }
453 return xmlStream;
454 }
455
456 public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
457 Session session, String basepath, String repository, String ext) throws IOException {
458 OutputStream outputStream = baseOutputStream;
459 if (ext.endsWith(ZIP)) {
460 outputStream = new ZipOutputStream(baseOutputStream);
461 } else if (ext.endsWith(GZ)) {
462 outputStream = new GZIPOutputStream(baseOutputStream);
463 }
464
465 try {
466 if (keepVersionHistory) {
467
468
469 if (!format) {
470 session.exportSystemView(basepath, outputStream, false, false);
471 } else {
472 parseAndFormat(outputStream, null, repository, basepath, session, false);
473 }
474 } else {
475
476
477 XMLReader reader = new VersionFilter(XMLReaderFactory
478 .createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName()));
479 reader = new AccesscontrolNodeFilter(reader);
480 parseAndFormat(outputStream, reader, repository, basepath, session, false);
481 }
482 }
483 catch (IOException e) {
484 throw new RuntimeException(e);
485 }
486 catch (SAXException e) {
487 throw new RuntimeException(e);
488 }
489 catch (RepositoryException e) {
490 throw new RuntimeException(e);
491 }
492
493
494
495 if (outputStream instanceof DeflaterOutputStream) {
496 ((DeflaterOutputStream) outputStream).finish();
497 }
498
499 baseOutputStream.flush();
500 IOUtils.closeQuietly(baseOutputStream);
501 }
502
503
504
505
506
507
508
509
510
511
512 public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
513 Session session, boolean noRecurse)
514 throws IOException, SAXException, PathNotFoundException, RepositoryException {
515
516 if (reader == null) {
517 reader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
518 }
519
520
521 File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
522 OutputStream fileStream = new FileOutputStream(tempFile);
523
524 try {
525 session.exportSystemView(basepath, fileStream, false, noRecurse);
526 }
527 finally {
528 IOUtils.closeQuietly(fileStream);
529 }
530
531 readFormatted(reader, tempFile, stream);
532
533 if (!tempFile.delete()) {
534 log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
535 }
536 }
537
538 protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
539 throws FileNotFoundException, IOException, SAXException {
540 InputStream fileInputStream = new FileInputStream(inputFile);
541 readFormatted(reader, fileInputStream, outputStream);
542 IOUtils.closeQuietly(fileInputStream);
543 }
544
545 protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
546 throws FileNotFoundException, IOException, SAXException {
547
548 OutputFormat outputFormat = new OutputFormat();
549
550 outputFormat.setPreserveSpace(false);
551 outputFormat.setIndenting(true);
552 outputFormat.setIndent(INDENT_VALUE);
553 outputFormat.setLineWidth(120);
554
555 final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces");
556 MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces);
557 metadataUuidFilter.setContentHandler(new XMLSerializer(outputStream, outputFormat));
558 metadataUuidFilter.parse(new InputSource(inputStream));
559
560 IOUtils.closeQuietly(inputStream);
561 }
562
563
564
565
566
567
568
569
570 public static String encodePath(String path, String separator, String enc)
571 {
572 StringBuilder pathEncoded = new StringBuilder();
573 try
574 {
575 if (!StringUtils.contains(path, separator))
576 {
577 return URLEncoder.encode(path, enc);
578 }
579 for(int i=0; i < path.length(); i++) {
580 String ch = String.valueOf(path.charAt(i));
581 if(separator.equals(ch)) {
582 pathEncoded.append(ch);
583 } else {
584 pathEncoded.append(URLEncoder.encode(ch, enc));
585 }
586 }
587 }
588 catch (UnsupportedEncodingException e)
589 {
590 return path;
591 }
592 return pathEncoded.toString();
593 }
594
595
596
597
598
599
600
601 public static String decodePath(String path, String enc)
602 {
603 String pathEncoded = StringUtils.EMPTY;
604 try
605 {
606 pathEncoded = URLDecoder.decode(path, enc);
607 }
608 catch (UnsupportedEncodingException e)
609 {
610 return path;
611 }
612 return pathEncoded;
613 }
614
615
616
617
618
619
620
621 public static String createExportPath(String path) {
622
623 String newPath = path.replace(".", "..");
624 newPath = newPath.replace("/", ".");
625 return newPath;
626 }
627
628
629
630
631 public static String revertExportPath(String exportPath) {
632 if(".".equals(exportPath)) {
633 return "/";
634 }
635
636
637 Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
638
639 StringBuilder reversed = new StringBuilder(exportPath.length());
640
641 while(matcher.find()){
642 String group = matcher.group();
643 int dotsNumber = StringUtils.countMatches(group, ".");
644 if(dotsNumber == 1) {
645 reversed.append(group.replaceFirst("\\.", "/"));
646 } else {
647 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
648 String name = StringUtils.substringAfterLast(group, ".");
649 reversed.append(dots);
650
651 if(dotsNumber % 2 != 0) {
652 reversed.append("/");
653 }
654 reversed.append(name);
655 }
656 }
657 return reversed.toString();
658 }
659
660 }