1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 package info.magnolia.importexport;
35
36 import info.magnolia.cms.beans.runtime.Document;
37 import info.magnolia.cms.core.Content;
38 import info.magnolia.cms.core.HierarchyManager;
39 import info.magnolia.cms.core.ItemType;
40 import info.magnolia.cms.core.SystemProperty;
41 import info.magnolia.cms.util.ContentUtil;
42 import info.magnolia.cms.util.NodeDataUtil;
43 import info.magnolia.context.MgnlContext;
44 import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
45 import info.magnolia.importexport.filters.ImportXmlRootFilter;
46 import info.magnolia.importexport.filters.MagnoliaV2Filter;
47 import info.magnolia.importexport.filters.MetadataUuidFilter;
48 import info.magnolia.importexport.filters.RemoveMixversionableFilter;
49 import info.magnolia.importexport.filters.VersionFilter;
50 import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
51 import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
52 import info.magnolia.jcr.util.NodeUtil;
53
54 import java.io.File;
55 import java.io.FileInputStream;
56 import java.io.FileNotFoundException;
57 import java.io.FileOutputStream;
58 import java.io.IOException;
59 import java.io.InputStream;
60 import java.io.OutputStream;
61 import java.io.UnsupportedEncodingException;
62 import java.net.URLDecoder;
63 import java.net.URLEncoder;
64 import java.text.MessageFormat;
65 import java.util.Iterator;
66 import java.util.List;
67 import java.util.Properties;
68 import java.util.regex.Matcher;
69 import java.util.regex.Pattern;
70 import java.util.zip.DeflaterOutputStream;
71 import java.util.zip.GZIPInputStream;
72 import java.util.zip.GZIPOutputStream;
73 import java.util.zip.ZipInputStream;
74 import java.util.zip.ZipOutputStream;
75
76 import javax.jcr.ImportUUIDBehavior;
77 import javax.jcr.Node;
78 import javax.jcr.NodeIterator;
79 import javax.jcr.PathNotFoundException;
80 import javax.jcr.RepositoryException;
81 import javax.jcr.Session;
82 import javax.jcr.Workspace;
83 import javax.xml.transform.Source;
84 import javax.xml.transform.sax.SAXTransformerFactory;
85 import javax.xml.transform.stream.StreamSource;
86
87 import org.apache.commons.io.IOUtils;
88 import org.apache.commons.lang3.StringUtils;
89 import org.apache.xml.serialize.OutputFormat;
90 import org.apache.xml.serialize.XMLSerializer;
91 import org.slf4j.Logger;
92 import org.slf4j.LoggerFactory;
93 import org.xml.sax.ContentHandler;
94 import org.xml.sax.InputSource;
95 import org.xml.sax.SAXException;
96 import org.xml.sax.XMLFilter;
97 import org.xml.sax.XMLReader;
98 import org.xml.sax.helpers.XMLReaderFactory;
99
100
101
102
103
104 public class DataTransporter {
105
106 private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
107
108 private static final int INDENT_VALUE = 2;
109
110 private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
111
112 final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
113
114 public static final String ZIP = ".zip";
115
116 public static final String GZ = ".gz";
117
118 public static final String XML = ".xml";
119
120 public static final String PROPERTIES = ".properties";
121
122 public static final String DOT = ".";
123
124 public static final String SLASH = "/";
125
126 public static final String UTF8 = "UTF-8";
127
128 public static final String JCR_ROOT = "jcr:root";
129
130
131
132
133
134
135
136
137
138
139
140 public static synchronized void importDocument(Document xmlDocument, String repositoryName, String basepath,
141 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
142 boolean createBasepathIfNotExist)
143 throws IOException {
144 File xmlFile = xmlDocument.getFile();
145 importFile(xmlFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
146 createBasepathIfNotExist);
147 }
148
149
150
151
152
153
154
155
156
157
158
159 public static synchronized void importFile(File xmlFile, String repositoryName, String basepath,
160 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
161 boolean createBasepathIfNotExist)
162 throws IOException {
163 String name = xmlFile.getAbsolutePath();
164
165 InputStream xmlStream = getInputStreamForFile(xmlFile);
166 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport,
167 createBasepathIfNotExist);
168 }
169
170 public static void executeBootstrapImport(File xmlFile, String repositoryName) throws IOException {
171 String filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
172 if (filenameWithoutExt.endsWith(XML)) {
173
174
175 filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
176 }
177 String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
178
179 pathName = decodePath(pathName, UTF8);
180
181 String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
182
183 if (xmlFile.getName().endsWith(PROPERTIES)) {
184 Properties properties = new Properties();
185 FileInputStream stream = new FileInputStream(xmlFile);
186 try {
187 properties.load(stream);
188 } finally {
189 IOUtils.closeQuietly(stream);
190 }
191 importProperties(properties, repositoryName);
192 } else {
193 DataTransporter.importFile(xmlFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
194 }
195 }
196
197
198
199
200 public static void importProperties(Properties properties, String repositoryName) {
201 for (Iterator iter = properties.keySet().iterator(); iter.hasNext(); ) {
202 String key = (String) iter.next();
203 String value = (String) properties.get(key);
204
205 String name = StringUtils.substringAfterLast(key, ".");
206 String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
207 Content node = ContentUtil.getContent(repositoryName, path);
208 if (node != null) {
209 try {
210 NodeDataUtil.getOrCreate(node, name).setValue(value);
211 node.save();
212 } catch (RepositoryException e) {
213 log.error("can't set property {}", key, e);
214 }
215 }
216 }
217
218 }
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
237 String name, boolean keepVersionHistory, int importMode,
238 boolean saveAfterImport, boolean createBasepathIfNotExist)
239 throws IOException {
240 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
241
242 }
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
261 String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
262 boolean saveAfterImport, boolean createBasepathIfNotExist)
263 throws IOException {
264
265
266 if (xmlStream == null) {
267 throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
268 }
269
270 HierarchyManager hm = MgnlContext.getHierarchyManager(repositoryName);
271 if (hm == null) {
272 throw new IllegalStateException("Can't import " + name + " since repository " + repositoryName + " does not exist.");
273 }
274 Workspace ws = hm.getWorkspace();
275
276 log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
277
278
279 if (!hm.isExist(basepath) && createBasepathIfNotExist) {
280 try {
281 ContentUtil.createPath(hm, basepath, ItemType.CONTENT);
282 } catch (RepositoryException e) {
283 log.error("can't create path [{}]", basepath);
284 }
285 }
286
287 Session session = ws.getSession();
288
289 try {
290
291
292 List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
293
294 if (keepVersionHistory) {
295
296 session.importXML(basepath, xmlStream, importMode);
297 } else {
298
299 XMLReader initialReader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
300 try {
301 initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
302 } catch (SAXException e) {
303 log.error("could not set parser feature");
304 }
305
306
307 XMLFilter magnoliaV2Filter = null;
308
309
310 if (new File(name).isFile()) {
311 InputStream xslStream = getXslStreamForXmlFile(new File(name));
312 if (xslStream != null) {
313 Source xslSource = new StreamSource(xslStream);
314 SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
315 XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
316 magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
317 }
318 }
319
320 if (magnoliaV2Filter == null) {
321 magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
322 }
323
324 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
325
326
327
328
329
330 versionFilter = new RemoveMixversionableFilter(versionFilter);
331
332
333 versionFilter = new AccesscontrolNodeFilter(versionFilter);
334
335 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
336
337 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
338 finalReader.setContentHandler(handler);
339
340
341 try {
342 finalReader.parse(new InputSource(xmlStream));
343 } finally {
344 IOUtils.closeQuietly(xmlStream);
345 }
346
347 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
348 String path = basepath;
349 if (!path.endsWith(SLASH)) {
350 path += SLASH;
351 }
352
353 Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
354 for (Iterator iter = dummyRoot.getNodes(); iter.hasNext(); ) {
355 Node child = (Node) iter.next();
356
357
358 if (session.itemExists(path + child.getName())) {
359 session.getItem(path + child.getName()).remove();
360 }
361
362 session.move(child.getPath(), path + child.getName());
363 }
364
365 dummyRoot.remove();
366 }
367
368
369 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
370 while (nodesAfterImport.hasNext()) {
371 Node nodeAfterImport = nodesAfterImport.nextNode();
372 boolean existedBeforeImport = false;
373 for (Node nodeBeforeImport : nodesBeforeImport) {
374 if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
375 existedBeforeImport = true;
376 break;
377 }
378 }
379 if (!existedBeforeImport) {
380 postProcessAfterImport(nodeAfterImport, forceUnpublishState);
381 }
382 }
383 }
384 } catch (Exception e) {
385 throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
386 } finally {
387 IOUtils.closeQuietly(xmlStream);
388 }
389
390 try {
391 if (saveAfterImport) {
392 session.save();
393 }
394 } catch (RepositoryException e) {
395 log.error(MessageFormat.format(
396 "Unable to save changes to the [{0}] repository due to a {1} Exception: {2}.",
397 new Object[]{repositoryName, e.getClass().getName(), e.getMessage()}), e);
398 throw new IOException(e.getMessage());
399 }
400 }
401
402 private static void postProcessAfterImport(Node node, boolean forceUnpublishState) throws RepositoryException {
403 try {
404 new MetaDataImportPostProcessor().postProcessNode(node);
405 if (forceUnpublishState) {
406 new ActivationStatusImportPostProcessor().postProcessNode(node);
407 }
408 } catch (RepositoryException e) {
409 throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
410 }
411 }
412
413
414
415
416 protected static InputStream getXslStreamForXmlFile(File file) {
417 InputStream xslStream = null;
418 String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
419 File xslFile = new File(xlsFilename);
420 if (xslFile.exists()) {
421 try {
422 xslStream = new FileInputStream(xslFile);
423 log.info("XSL file for [{}] found ({})", file.getName(), xslFile.getName());
424 } catch (FileNotFoundException e) {
425 e.printStackTrace();
426 }
427 }
428 return xslStream;
429 }
430
431
432
433
434
435
436 private static InputStream getInputStreamForFile(File xmlFile) throws IOException {
437 InputStream xmlStream;
438
439 if (xmlFile.getName().endsWith(ZIP)) {
440 xmlStream = new ZipInputStream((new FileInputStream(xmlFile)));
441 } else if (xmlFile.getName().endsWith(GZ)) {
442 xmlStream = new GZIPInputStream((new FileInputStream(xmlFile)));
443 } else {
444 xmlStream = new FileInputStream(xmlFile);
445 }
446 return xmlStream;
447 }
448
449 public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
450 Session session, String basepath, String repository, String ext) throws IOException {
451 OutputStream outputStream = baseOutputStream;
452 if (ext.endsWith(ZIP)) {
453 outputStream = new ZipOutputStream(baseOutputStream);
454 } else if (ext.endsWith(GZ)) {
455 outputStream = new GZIPOutputStream(baseOutputStream);
456 }
457
458 try {
459 if (keepVersionHistory) {
460
461
462 if (!format) {
463 session.exportSystemView(basepath, outputStream, false, false);
464 } else {
465 parseAndFormat(outputStream, null, repository, basepath, session, false);
466 }
467 } else {
468
469
470 XMLReader reader = new VersionFilter(XMLReaderFactory
471 .createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName()));
472 reader = new AccesscontrolNodeFilter(reader);
473 parseAndFormat(outputStream, reader, repository, basepath, session, false);
474 }
475 } catch (IOException e) {
476 throw new RuntimeException(e);
477 } catch (SAXException e) {
478 throw new RuntimeException(e);
479 } catch (RepositoryException e) {
480 throw new RuntimeException(e);
481 }
482
483
484
485 if (outputStream instanceof DeflaterOutputStream) {
486 ((DeflaterOutputStream) outputStream).finish();
487 }
488
489 baseOutputStream.flush();
490 IOUtils.closeQuietly(baseOutputStream);
491 }
492
493
494
495
496
497
498
499
500
501
502
503 public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
504 Session session, boolean noRecurse)
505 throws IOException, SAXException, PathNotFoundException, RepositoryException {
506
507 if (reader == null) {
508 reader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
509 }
510
511
512 File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
513 OutputStream fileStream = new FileOutputStream(tempFile);
514
515 try {
516 session.exportSystemView(basepath, fileStream, false, noRecurse);
517 } finally {
518 IOUtils.closeQuietly(fileStream);
519 }
520
521 readFormatted(reader, tempFile, stream);
522
523 if (!tempFile.delete()) {
524 log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
525 }
526 }
527
528 protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
529 throws FileNotFoundException, IOException, SAXException {
530 InputStream fileInputStream = new FileInputStream(inputFile);
531 readFormatted(reader, fileInputStream, outputStream);
532 IOUtils.closeQuietly(fileInputStream);
533 }
534
535 protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
536 throws FileNotFoundException, IOException, SAXException {
537
538 OutputFormat outputFormat = new OutputFormat();
539
540 outputFormat.setPreserveSpace(false);
541 outputFormat.setIndenting(true);
542 outputFormat.setIndent(INDENT_VALUE);
543 outputFormat.setLineWidth(120);
544
545 final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces");
546 MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces);
547 metadataUuidFilter.setContentHandler(new XMLSerializer(outputStream, outputFormat));
548 metadataUuidFilter.parse(new InputSource(inputStream));
549
550 IOUtils.closeQuietly(inputStream);
551 }
552
553
554
555
556
557
558
559 public static String encodePath(String path, String separator, String enc) {
560 StringBuilder pathEncoded = new StringBuilder();
561 try {
562 if (!StringUtils.contains(path, separator)) {
563 return URLEncoder.encode(path, enc);
564 }
565 for (int i = 0; i < path.length(); i++) {
566 String ch = String.valueOf(path.charAt(i));
567 if (separator.equals(ch)) {
568 pathEncoded.append(ch);
569 } else {
570 pathEncoded.append(URLEncoder.encode(ch, enc));
571 }
572 }
573 } catch (UnsupportedEncodingException e) {
574 return path;
575 }
576 return pathEncoded.toString();
577 }
578
579
580
581
582
583
584
585
586 public static String decodePath(String path, String enc) {
587 String pathEncoded = StringUtils.EMPTY;
588 try {
589 pathEncoded = URLDecoder.decode(path, enc);
590 } catch (UnsupportedEncodingException e) {
591 return path;
592 }
593 return pathEncoded;
594 }
595
596
597
598
599
600
601
602 public static String createExportPath(String path) {
603
604 String newPath = path.replace(".", "..");
605 newPath = newPath.replace("/", ".");
606 return newPath;
607 }
608
609
610
611
612
613 public static String revertExportPath(String exportPath) {
614 if (".".equals(exportPath)) {
615 return "/";
616 }
617
618
619 Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
620
621 StringBuilder reversed = new StringBuilder(exportPath.length());
622
623 while (matcher.find()) {
624 String group = matcher.group();
625 int dotsNumber = StringUtils.countMatches(group, ".");
626 if (dotsNumber == 1) {
627 reversed.append(group.replaceFirst("\\.", "/"));
628 } else {
629 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
630 String name = StringUtils.substringAfterLast(group, ".");
631 reversed.append(dots);
632
633 if (dotsNumber % 2 != 0) {
634 reversed.append("/");
635 }
636 reversed.append(name);
637 }
638 }
639 return reversed.toString();
640 }
641
642 }