1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 package info.magnolia.importexport;
35
36 import info.magnolia.cms.beans.runtime.Document;
37 import info.magnolia.cms.core.Content;
38 import info.magnolia.cms.core.HierarchyManager;
39 import info.magnolia.cms.core.ItemType;
40 import info.magnolia.cms.core.SystemProperty;
41 import info.magnolia.cms.util.ContentUtil;
42 import info.magnolia.cms.util.NodeDataUtil;
43 import info.magnolia.context.MgnlContext;
44 import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
45 import info.magnolia.importexport.filters.ImportXmlRootFilter;
46 import info.magnolia.importexport.filters.MagnoliaV2Filter;
47 import info.magnolia.importexport.filters.MetadataUuidFilter;
48 import info.magnolia.importexport.filters.RemoveMixversionableFilter;
49 import info.magnolia.importexport.filters.VersionFilter;
50 import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
51 import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
52 import info.magnolia.jcr.util.NodeUtil;
53
54 import java.io.File;
55 import java.io.FileInputStream;
56 import java.io.FileNotFoundException;
57 import java.io.FileOutputStream;
58 import java.io.IOException;
59 import java.io.InputStream;
60 import java.io.OutputStream;
61 import java.io.UnsupportedEncodingException;
62 import java.net.URLDecoder;
63 import java.net.URLEncoder;
64 import java.text.MessageFormat;
65 import java.util.Iterator;
66 import java.util.List;
67 import java.util.Properties;
68 import java.util.regex.Matcher;
69 import java.util.regex.Pattern;
70 import java.util.zip.DeflaterOutputStream;
71 import java.util.zip.GZIPInputStream;
72 import java.util.zip.GZIPOutputStream;
73 import java.util.zip.ZipInputStream;
74 import java.util.zip.ZipOutputStream;
75
76 import javax.jcr.ImportUUIDBehavior;
77 import javax.jcr.Node;
78 import javax.jcr.NodeIterator;
79 import javax.jcr.PathNotFoundException;
80 import javax.jcr.RepositoryException;
81 import javax.jcr.Session;
82 import javax.jcr.Workspace;
83 import javax.xml.transform.Source;
84 import javax.xml.transform.sax.SAXTransformerFactory;
85 import javax.xml.transform.stream.StreamSource;
86
87 import org.apache.commons.io.IOUtils;
88 import org.apache.commons.lang.StringUtils;
89 import org.apache.xml.serialize.OutputFormat;
90 import org.apache.xml.serialize.XMLSerializer;
91 import org.slf4j.Logger;
92 import org.slf4j.LoggerFactory;
93 import org.xml.sax.ContentHandler;
94 import org.xml.sax.InputSource;
95 import org.xml.sax.SAXException;
96 import org.xml.sax.XMLFilter;
97 import org.xml.sax.XMLReader;
98 import org.xml.sax.helpers.XMLReaderFactory;
99
100
101
102
103
104 public class DataTransporter {
105
106 private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
107
108 private static final int INDENT_VALUE = 2;
109
110 private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
111
112 final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
113
114 public static final String ZIP = ".zip";
115
116 public static final String GZ = ".gz";
117
118 public static final String XML = ".xml";
119
120 public static final String PROPERTIES = ".properties";
121
122 public static final String DOT = ".";
123
124 public static final String SLASH = "/";
125
126 public static final String UTF8 = "UTF-8";
127
128 public static final String JCR_ROOT = "jcr:root";
129
130
131
132
133
134
135
136
137
138
139
140 public static synchronized void importDocument(Document xmlDocument, String repositoryName, String basepath,
141 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
142 boolean createBasepathIfNotExist)
143 throws IOException {
144 File xmlFile = xmlDocument.getFile();
145 importFile(xmlFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
146 createBasepathIfNotExist);
147 }
148
149
150
151
152
153
154
155
156
157
158
159 public static synchronized void importFile(File xmlFile, String repositoryName, String basepath,
160 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
161 boolean createBasepathIfNotExist)
162 throws IOException {
163 String name = xmlFile.getAbsolutePath();
164
165 InputStream xmlStream = getInputStreamForFile(xmlFile);
166 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport,
167 createBasepathIfNotExist);
168 }
169
170 public static void executeBootstrapImport(File xmlFile, String repositoryName) throws IOException {
171 String filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
172 if (filenameWithoutExt.endsWith(XML)) {
173
174
175 filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
176 }
177 String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
178
179 pathName = decodePath(pathName, UTF8);
180
181 String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
182
183 if (xmlFile.getName().endsWith(PROPERTIES)) {
184 Properties properties = new Properties();
185 FileInputStream stream = new FileInputStream(xmlFile);
186 try {
187 properties.load(stream);
188 } finally {
189 IOUtils.closeQuietly(stream);
190 }
191 importProperties(properties, repositoryName);
192 } else {
193 DataTransporter.importFile(xmlFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
194 }
195 }
196
197
198
199
200 public static void importProperties(Properties properties, String repositoryName) {
201 for (Iterator iter = properties.keySet().iterator(); iter.hasNext();) {
202 String key = (String) iter.next();
203 String value = (String) properties.get(key);
204
205 String name = StringUtils.substringAfterLast(key, ".");
206 String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
207 Content node = ContentUtil.getContent(repositoryName, path);
208 if (node != null) {
209 try {
210 NodeDataUtil.getOrCreate(node, name).setValue(value);
211 node.save();
212 }
213 catch (RepositoryException e) {
214 log.error("can't set property " + key, e);
215 }
216 }
217 }
218
219 }
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
238 String name, boolean keepVersionHistory, int importMode,
239 boolean saveAfterImport, boolean createBasepathIfNotExist)
240 throws IOException {
241 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
242
243 }
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
263 String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
264 boolean saveAfterImport, boolean createBasepathIfNotExist)
265 throws IOException {
266
267
268 if (xmlStream == null) {
269 throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
270 }
271
272 HierarchyManager hm = MgnlContext.getHierarchyManager(repositoryName);
273 if (hm == null) {
274 throw new IllegalStateException("Can't import " + name + " since repository " + repositoryName + " does not exist.");
275 }
276 Workspace ws = hm.getWorkspace();
277
278 if (log.isDebugEnabled()) {
279 log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]",
280 new Object[]{repositoryName, name, basepath});
281 }
282
283 if (!hm.isExist(basepath) && createBasepathIfNotExist) {
284 try {
285 ContentUtil.createPath(hm, basepath, ItemType.CONTENT);
286 }
287 catch (RepositoryException e) {
288 log.error("can't create path [{}]", basepath);
289 }
290 }
291
292 Session session = ws.getSession();
293
294 try {
295
296
297 List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
298
299 if (keepVersionHistory) {
300
301 session.importXML(basepath, xmlStream, importMode);
302 } else {
303
304 XMLReader initialReader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
305 try{
306 initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
307 }catch (SAXException e) {
308 log.error("could not set parser feature");
309 }
310
311
312 XMLFilter magnoliaV2Filter = null;
313
314
315 if (new File(name).isFile()) {
316 InputStream xslStream = getXslStreamForXmlFile(new File(name));
317 if (xslStream != null) {
318 Source xslSource = new StreamSource(xslStream);
319 SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
320 XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
321 magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
322 }
323 }
324
325 if (magnoliaV2Filter == null) {
326 magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
327 }
328
329 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
330
331
332
333
334
335 versionFilter = new RemoveMixversionableFilter(versionFilter);
336
337
338 versionFilter = new AccesscontrolNodeFilter(versionFilter);
339
340 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
341
342 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
343 finalReader.setContentHandler(handler);
344
345
346 try {
347 finalReader.parse(new InputSource(xmlStream));
348 }
349 finally {
350 IOUtils.closeQuietly(xmlStream);
351 }
352
353 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
354 String path = basepath;
355 if (!path.endsWith(SLASH)) {
356 path += SLASH;
357 }
358
359 Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
360 for (Iterator iter = dummyRoot.getNodes(); iter.hasNext();) {
361 Node child = (Node) iter.next();
362
363
364 if (session.itemExists(path + child.getName())) {
365 session.getItem(path + child.getName()).remove();
366 }
367
368 session.move(child.getPath(), path + child.getName());
369 }
370
371 dummyRoot.remove();
372 }
373
374
375 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
376 while (nodesAfterImport.hasNext()) {
377 Node nodeAfterImport = nodesAfterImport.nextNode();
378 boolean existedBeforeImport = false;
379 for (Node nodeBeforeImport : nodesBeforeImport) {
380 if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
381 existedBeforeImport = true;
382 break;
383 }
384 }
385 if (!existedBeforeImport) {
386 postProcessAfterImport(nodeAfterImport, forceUnpublishState);
387 }
388 }
389 }
390 }
391 catch (Exception e) {
392 throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
393 }
394 finally {
395 IOUtils.closeQuietly(xmlStream);
396 }
397
398 try {
399 if (saveAfterImport) {
400 session.save();
401 }
402 }
403 catch (RepositoryException e) {
404 log.error(MessageFormat.format(
405 "Unable to save changes to the [{0}] repository due to a {1} Exception: {2}.",
406 new Object[]{repositoryName, e.getClass().getName(), e.getMessage()}), e);
407 throw new IOException(e.getMessage());
408 }
409 }
410
411 private static void postProcessAfterImport(Node node, boolean forceUnpublishState) throws RepositoryException {
412 try {
413 new MetaDataImportPostProcessor().postProcessNode(node);
414 if (forceUnpublishState) {
415 new ActivationStatusImportPostProcessor().postProcessNode(node);
416 }
417 } catch (RepositoryException e) {
418 throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
419 }
420 }
421
422
423
424
425 protected static InputStream getXslStreamForXmlFile(File file) {
426 InputStream xslStream = null;
427 String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
428 File xslFile = new File(xlsFilename);
429 if (xslFile.exists()) {
430 try {
431 xslStream = new FileInputStream(xslFile);
432 log.info("XSL file for [" + file.getName() + "] found (" + xslFile.getName() + ")");
433 } catch (FileNotFoundException e) {
434 e.printStackTrace();
435 }
436 }
437 return xslStream;
438 }
439
440
441
442
443
444
445 private static InputStream getInputStreamForFile(File xmlFile) throws IOException {
446 InputStream xmlStream;
447
448 if (xmlFile.getName().endsWith(ZIP)) {
449 xmlStream = new ZipInputStream((new FileInputStream(xmlFile)));
450 } else if (xmlFile.getName().endsWith(GZ)) {
451 xmlStream = new GZIPInputStream((new FileInputStream(xmlFile)));
452 } else {
453 xmlStream = new FileInputStream(xmlFile);
454 }
455 return xmlStream;
456 }
457
458 public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
459 Session session, String basepath, String repository, String ext) throws IOException {
460 OutputStream outputStream = baseOutputStream;
461 if (ext.endsWith(ZIP)) {
462 outputStream = new ZipOutputStream(baseOutputStream);
463 } else if (ext.endsWith(GZ)) {
464 outputStream = new GZIPOutputStream(baseOutputStream);
465 }
466
467 try {
468 if (keepVersionHistory) {
469
470
471 if (!format) {
472 session.exportSystemView(basepath, outputStream, false, false);
473 } else {
474 parseAndFormat(outputStream, null, repository, basepath, session, false);
475 }
476 } else {
477
478
479 XMLReader reader = new VersionFilter(XMLReaderFactory
480 .createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName()));
481 reader = new AccesscontrolNodeFilter(reader);
482 parseAndFormat(outputStream, reader, repository, basepath, session, false);
483 }
484 }
485 catch (IOException e) {
486 throw new RuntimeException(e);
487 }
488 catch (SAXException e) {
489 throw new RuntimeException(e);
490 }
491 catch (RepositoryException e) {
492 throw new RuntimeException(e);
493 }
494
495
496
497 if (outputStream instanceof DeflaterOutputStream) {
498 ((DeflaterOutputStream) outputStream).finish();
499 }
500
501 baseOutputStream.flush();
502 IOUtils.closeQuietly(baseOutputStream);
503 }
504
505
506
507
508
509
510
511
512
513
514 public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
515 Session session, boolean noRecurse)
516 throws IOException, SAXException, PathNotFoundException, RepositoryException {
517
518 if (reader == null) {
519 reader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
520 }
521
522
523 File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
524 OutputStream fileStream = new FileOutputStream(tempFile);
525
526 try {
527 session.exportSystemView(basepath, fileStream, false, noRecurse);
528 }
529 finally {
530 IOUtils.closeQuietly(fileStream);
531 }
532
533 readFormatted(reader, tempFile, stream);
534
535 if (!tempFile.delete()) {
536 log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
537 }
538 }
539
540 protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
541 throws FileNotFoundException, IOException, SAXException {
542 InputStream fileInputStream = new FileInputStream(inputFile);
543 readFormatted(reader, fileInputStream, outputStream);
544 IOUtils.closeQuietly(fileInputStream);
545 }
546
547 protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
548 throws FileNotFoundException, IOException, SAXException {
549
550 OutputFormat outputFormat = new OutputFormat();
551
552 outputFormat.setPreserveSpace(false);
553 outputFormat.setIndenting(true);
554 outputFormat.setIndent(INDENT_VALUE);
555 outputFormat.setLineWidth(120);
556
557 final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces");
558 MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces);
559 metadataUuidFilter.setContentHandler(new XMLSerializer(outputStream, outputFormat));
560 metadataUuidFilter.parse(new InputSource(inputStream));
561
562 IOUtils.closeQuietly(inputStream);
563 }
564
565
566
567
568
569
570
571
572 public static String encodePath(String path, String separator, String enc)
573 {
574 StringBuilder pathEncoded = new StringBuilder();
575 try
576 {
577 if (!StringUtils.contains(path, separator))
578 {
579 return URLEncoder.encode(path, enc);
580 }
581 for(int i=0; i < path.length(); i++) {
582 String ch = String.valueOf(path.charAt(i));
583 if(separator.equals(ch)) {
584 pathEncoded.append(ch);
585 } else {
586 pathEncoded.append(URLEncoder.encode(ch, enc));
587 }
588 }
589 }
590 catch (UnsupportedEncodingException e)
591 {
592 return path;
593 }
594 return pathEncoded.toString();
595 }
596
597
598
599
600
601
602
603 public static String decodePath(String path, String enc)
604 {
605 String pathEncoded = StringUtils.EMPTY;
606 try
607 {
608 pathEncoded = URLDecoder.decode(path, enc);
609 }
610 catch (UnsupportedEncodingException e)
611 {
612 return path;
613 }
614 return pathEncoded;
615 }
616
617
618
619
620
621
622
623 public static String createExportPath(String path) {
624
625 String newPath = path.replace(".", "..");
626 newPath = newPath.replace("/", ".");
627 return newPath;
628 }
629
630
631
632
633 public static String revertExportPath(String exportPath) {
634 if(".".equals(exportPath)) {
635 return "/";
636 }
637
638
639 Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
640
641 StringBuilder reversed = new StringBuilder(exportPath.length());
642
643 while(matcher.find()){
644 String group = matcher.group();
645 int dotsNumber = StringUtils.countMatches(group, ".");
646 if(dotsNumber == 1) {
647 reversed.append(group.replaceFirst("\\.", "/"));
648 } else {
649 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
650 String name = StringUtils.substringAfterLast(group, ".");
651 reversed.append(dots);
652
653 if(dotsNumber % 2 != 0) {
654 reversed.append("/");
655 }
656 reversed.append(name);
657 }
658 }
659 return reversed.toString();
660 }
661
662 }