1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 package info.magnolia.importexport;
35
36 import info.magnolia.cms.beans.runtime.Document;
37 import info.magnolia.cms.core.Content;
38 import info.magnolia.cms.core.HierarchyManager;
39 import info.magnolia.cms.core.ItemType;
40 import info.magnolia.cms.core.SystemProperty;
41 import info.magnolia.cms.core.version.VersionManager;
42 import info.magnolia.cms.util.ContentUtil;
43 import info.magnolia.cms.util.NodeDataUtil;
44 import info.magnolia.context.MgnlContext;
45 import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
46 import info.magnolia.importexport.filters.ImportXmlRootFilter;
47 import info.magnolia.importexport.filters.MagnoliaV2Filter;
48 import info.magnolia.importexport.filters.MetadataUuidFilter;
49 import info.magnolia.importexport.filters.RemoveMixversionableFilter;
50 import info.magnolia.importexport.filters.VersionFilter;
51 import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
52 import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
53 import info.magnolia.importexport.postprocessors.UpdateVersionMixinPostProcessor;
54 import info.magnolia.jcr.util.NodeUtil;
55 import info.magnolia.objectfactory.Components;
56
57 import java.io.File;
58 import java.io.FileInputStream;
59 import java.io.FileNotFoundException;
60 import java.io.FileOutputStream;
61 import java.io.IOException;
62 import java.io.InputStream;
63 import java.io.OutputStream;
64 import java.io.UnsupportedEncodingException;
65 import java.net.URLDecoder;
66 import java.net.URLEncoder;
67 import java.text.MessageFormat;
68 import java.util.Iterator;
69 import java.util.List;
70 import java.util.Properties;
71 import java.util.regex.Matcher;
72 import java.util.regex.Pattern;
73 import java.util.zip.DeflaterOutputStream;
74 import java.util.zip.GZIPInputStream;
75 import java.util.zip.GZIPOutputStream;
76 import java.util.zip.ZipInputStream;
77 import java.util.zip.ZipOutputStream;
78
79 import javax.jcr.ImportUUIDBehavior;
80 import javax.jcr.Node;
81 import javax.jcr.NodeIterator;
82 import javax.jcr.PathNotFoundException;
83 import javax.jcr.RepositoryException;
84 import javax.jcr.Session;
85 import javax.jcr.Workspace;
86 import javax.xml.transform.Source;
87 import javax.xml.transform.sax.SAXTransformerFactory;
88 import javax.xml.transform.stream.StreamSource;
89
90 import org.apache.commons.io.IOUtils;
91 import org.apache.commons.lang3.StringUtils;
92 import org.apache.xml.serialize.OutputFormat;
93 import org.apache.xml.serialize.XMLSerializer;
94 import org.slf4j.Logger;
95 import org.slf4j.LoggerFactory;
96 import org.xml.sax.ContentHandler;
97 import org.xml.sax.InputSource;
98 import org.xml.sax.SAXException;
99 import org.xml.sax.XMLFilter;
100 import org.xml.sax.XMLReader;
101 import org.xml.sax.helpers.XMLReaderFactory;
102
103
104
105
106
107 public class DataTransporter {
108
109 private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
110
111 private static final int INDENT_VALUE = 2;
112
113 private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
114
115 final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
116
117 public static final String ZIP = ".zip";
118
119 public static final String GZ = ".gz";
120
121 public static final String XML = ".xml";
122
123 public static final String PROPERTIES = ".properties";
124
125 public static final String DOT = ".";
126
127 public static final String SLASH = "/";
128
129 public static final String UTF8 = "UTF-8";
130
131 public static final String JCR_ROOT = "jcr:root";
132
133
134
135
136
137
138
139
140
141
142
143 public static synchronized void importDocument(Document xmlDocument, String repositoryName, String basepath,
144 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
145 boolean createBasepathIfNotExist)
146 throws IOException {
147 File xmlFile = xmlDocument.getFile();
148 importFile(xmlFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
149 createBasepathIfNotExist);
150 }
151
152
153
154
155
156
157
158
159
160
161
162 public static synchronized void importFile(File xmlFile, String repositoryName, String basepath,
163 boolean keepVersionHistory, int importMode, boolean saveAfterImport,
164 boolean createBasepathIfNotExist)
165 throws IOException {
166 String name = xmlFile.getAbsolutePath();
167
168 InputStream xmlStream = getInputStreamForFile(xmlFile);
169 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport,
170 createBasepathIfNotExist);
171 }
172
173 public static void executeBootstrapImport(File xmlFile, String repositoryName) throws IOException {
174 String filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
175 if (filenameWithoutExt.endsWith(XML)) {
176
177
178 filenameWithoutExt = StringUtils.substringBeforeLast(xmlFile.getName(), DOT);
179 }
180 String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
181
182 pathName = decodePath(pathName, UTF8);
183
184 String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
185
186 if (xmlFile.getName().endsWith(PROPERTIES)) {
187 Properties properties = new Properties();
188 FileInputStream stream = new FileInputStream(xmlFile);
189 try {
190 properties.load(stream);
191 } finally {
192 IOUtils.closeQuietly(stream);
193 }
194 importProperties(properties, repositoryName);
195 } else {
196 DataTransporter.importFile(xmlFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
197 }
198 }
199
200
201
202
203 public static void importProperties(Properties properties, String repositoryName) {
204 for (Iterator iter = properties.keySet().iterator(); iter.hasNext(); ) {
205 String key = (String) iter.next();
206 String value = (String) properties.get(key);
207
208 String name = StringUtils.substringAfterLast(key, ".");
209 String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
210 Content node = ContentUtil.getContent(repositoryName, path);
211 if (node != null) {
212 try {
213 NodeDataUtil.getOrCreate(node, name).setValue(value);
214 node.save();
215 } catch (RepositoryException e) {
216 log.error("can't set property {}", key, e);
217 }
218 }
219 }
220
221 }
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
240 String name, boolean keepVersionHistory, int importMode,
241 boolean saveAfterImport, boolean createBasepathIfNotExist)
242 throws IOException {
243 importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
244
245 }
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263 public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
264 String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
265 boolean saveAfterImport, boolean createBasepathIfNotExist)
266 throws IOException {
267
268
269 if (xmlStream == null) {
270 throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
271 }
272
273 HierarchyManager hm = MgnlContext.getHierarchyManager(repositoryName);
274 if (hm == null) {
275 throw new IllegalStateException("Can't import " + name + " since repository " + repositoryName + " does not exist.");
276 }
277 Workspace ws = hm.getWorkspace();
278
279 log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
280
281
282 if (!hm.isExist(basepath) && createBasepathIfNotExist) {
283 try {
284 ContentUtil.createPath(hm, basepath, ItemType.CONTENT);
285 } catch (RepositoryException e) {
286 log.error("can't create path [{}]", basepath);
287 }
288 }
289
290 Session session = ws.getSession();
291
292 try {
293
294
295 List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
296
297 if (keepVersionHistory) {
298
299 session.importXML(basepath, xmlStream, importMode);
300 } else {
301
302 XMLReader initialReader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
303 try {
304 initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
305 } catch (SAXException e) {
306 log.error("could not set parser feature");
307 }
308
309
310 XMLFilter magnoliaV2Filter = null;
311
312
313 if (new File(name).isFile()) {
314 InputStream xslStream = getXslStreamForXmlFile(new File(name));
315 if (xslStream != null) {
316 Source xslSource = new StreamSource(xslStream);
317 SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
318 XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
319 magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
320 }
321 }
322
323 if (magnoliaV2Filter == null) {
324 magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
325 }
326
327 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
328
329
330
331
332
333 versionFilter = new RemoveMixversionableFilter(versionFilter);
334
335
336 versionFilter = new AccesscontrolNodeFilter(versionFilter);
337
338 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
339
340 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
341 finalReader.setContentHandler(handler);
342
343
344 try {
345 finalReader.parse(new InputSource(xmlStream));
346 } finally {
347 IOUtils.closeQuietly(xmlStream);
348 }
349
350 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
351 String path = basepath;
352 if (!path.endsWith(SLASH)) {
353 path += SLASH;
354 }
355
356 Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
357 for (Iterator iter = dummyRoot.getNodes(); iter.hasNext(); ) {
358 Node child = (Node) iter.next();
359
360
361 if (session.itemExists(path + child.getName())) {
362 session.getItem(path + child.getName()).remove();
363 }
364
365 session.move(child.getPath(), path + child.getName());
366 }
367
368 dummyRoot.remove();
369 }
370
371
372 VersionManager versionManager = Components.getComponent(VersionManager.class);
373 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
374 while (nodesAfterImport.hasNext()) {
375 Node nodeAfterImport = nodesAfterImport.nextNode();
376 boolean existedBeforeImport = false;
377 for (Node nodeBeforeImport : nodesBeforeImport) {
378 if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
379 existedBeforeImport = true;
380 break;
381 }
382 }
383 if (!existedBeforeImport) {
384 postProcessAfterImport(nodeAfterImport, forceUnpublishState, importMode, versionManager);
385 }
386 }
387 }
388 } catch (Exception e) {
389 throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
390 } finally {
391 IOUtils.closeQuietly(xmlStream);
392 }
393
394 try {
395 if (saveAfterImport) {
396 session.save();
397 }
398 } catch (RepositoryException e) {
399 log.error(MessageFormat.format(
400 "Unable to save changes to the [{0}] repository due to a {1} Exception: {2}.",
401 new Object[]{repositoryName, e.getClass().getName(), e.getMessage()}), e);
402 throw new IOException(e.getMessage());
403 }
404 }
405
406 private static void postProcessAfterImport(Node node, boolean forceUnpublishState, int importMode, VersionManager versionManager) throws RepositoryException {
407 try {
408 new MetaDataImportPostProcessor().postProcessNode(node);
409 if (forceUnpublishState) {
410 new ActivationStatusImportPostProcessor().postProcessNode(node);
411 }
412 new UpdateVersionMixinPostProcessor(importMode, versionManager).postProcessNode(node);
413 } catch (RepositoryException e) {
414 throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
415 }
416 }
417
418
419
420
421 protected static InputStream getXslStreamForXmlFile(File file) {
422 InputStream xslStream = null;
423 String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
424 File xslFile = new File(xlsFilename);
425 if (xslFile.exists()) {
426 try {
427 xslStream = new FileInputStream(xslFile);
428 log.info("XSL file for [{}] found ({})", file.getName(), xslFile.getName());
429 } catch (FileNotFoundException e) {
430 e.printStackTrace();
431 }
432 }
433 return xslStream;
434 }
435
436
437
438
439
440
441 private static InputStream getInputStreamForFile(File xmlFile) throws IOException {
442 InputStream xmlStream;
443
444 if (xmlFile.getName().endsWith(ZIP)) {
445 xmlStream = new ZipInputStream((new FileInputStream(xmlFile)));
446 } else if (xmlFile.getName().endsWith(GZ)) {
447 xmlStream = new GZIPInputStream((new FileInputStream(xmlFile)));
448 } else {
449 xmlStream = new FileInputStream(xmlFile);
450 }
451 return xmlStream;
452 }
453
454 public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
455 Session session, String basepath, String repository, String ext) throws IOException {
456 OutputStream outputStream = baseOutputStream;
457 if (ext.endsWith(ZIP)) {
458 outputStream = new ZipOutputStream(baseOutputStream);
459 } else if (ext.endsWith(GZ)) {
460 outputStream = new GZIPOutputStream(baseOutputStream);
461 }
462
463 try {
464 if (keepVersionHistory) {
465
466
467 if (!format) {
468 session.exportSystemView(basepath, outputStream, false, false);
469 } else {
470 parseAndFormat(outputStream, null, repository, basepath, session, false);
471 }
472 } else {
473
474
475 XMLReader reader = new VersionFilter(XMLReaderFactory
476 .createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName()));
477 reader = new AccesscontrolNodeFilter(reader);
478 parseAndFormat(outputStream, reader, repository, basepath, session, false);
479 }
480 } catch (IOException e) {
481 throw new RuntimeException(e);
482 } catch (SAXException e) {
483 throw new RuntimeException(e);
484 } catch (RepositoryException e) {
485 throw new RuntimeException(e);
486 }
487
488
489
490 if (outputStream instanceof DeflaterOutputStream) {
491 ((DeflaterOutputStream) outputStream).finish();
492 }
493
494 baseOutputStream.flush();
495 IOUtils.closeQuietly(baseOutputStream);
496 }
497
498
499
500
501
502
503
504
505
506
507
508 public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
509 Session session, boolean noRecurse)
510 throws IOException, SAXException, PathNotFoundException, RepositoryException {
511
512 if (reader == null) {
513 reader = XMLReaderFactory.createXMLReader(org.apache.xerces.parsers.SAXParser.class.getName());
514 }
515
516
517 File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
518 OutputStream fileStream = new FileOutputStream(tempFile);
519
520 try {
521 session.exportSystemView(basepath, fileStream, false, noRecurse);
522 } finally {
523 IOUtils.closeQuietly(fileStream);
524 }
525
526 readFormatted(reader, tempFile, stream);
527
528 if (!tempFile.delete()) {
529 log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
530 }
531 }
532
533 protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
534 throws FileNotFoundException, IOException, SAXException {
535 InputStream fileInputStream = new FileInputStream(inputFile);
536 readFormatted(reader, fileInputStream, outputStream);
537 IOUtils.closeQuietly(fileInputStream);
538 }
539
540 protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
541 throws FileNotFoundException, IOException, SAXException {
542
543 OutputFormat outputFormat = new OutputFormat();
544
545 outputFormat.setPreserveSpace(false);
546 outputFormat.setIndenting(true);
547 outputFormat.setIndent(INDENT_VALUE);
548 outputFormat.setLineWidth(120);
549
550 final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces");
551 MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces);
552 metadataUuidFilter.setContentHandler(new XMLSerializer(outputStream, outputFormat));
553 metadataUuidFilter.parse(new InputSource(inputStream));
554
555 IOUtils.closeQuietly(inputStream);
556 }
557
558
559
560
561
562
563
564 public static String encodePath(String path, String separator, String enc) {
565 StringBuilder pathEncoded = new StringBuilder();
566 try {
567 if (!StringUtils.contains(path, separator)) {
568 return URLEncoder.encode(path, enc);
569 }
570 for (int i = 0; i < path.length(); i++) {
571 String ch = String.valueOf(path.charAt(i));
572 if (separator.equals(ch)) {
573 pathEncoded.append(ch);
574 } else {
575 pathEncoded.append(URLEncoder.encode(ch, enc));
576 }
577 }
578 } catch (UnsupportedEncodingException e) {
579 return path;
580 }
581 return pathEncoded.toString();
582 }
583
584
585
586
587
588
589
590
591 public static String decodePath(String path, String enc) {
592 String pathEncoded = StringUtils.EMPTY;
593 try {
594 pathEncoded = URLDecoder.decode(path, enc);
595 } catch (UnsupportedEncodingException e) {
596 return path;
597 }
598 return pathEncoded;
599 }
600
601
602
603
604
605
606
607 public static String createExportPath(String path) {
608
609 String newPath = path.replace(".", "..");
610 newPath = newPath.replace("/", ".");
611 return newPath;
612 }
613
614
615
616
617
618 public static String revertExportPath(String exportPath) {
619 if (".".equals(exportPath)) {
620 return "/";
621 }
622
623
624 Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
625
626 StringBuilder reversed = new StringBuilder(exportPath.length());
627
628 while (matcher.find()) {
629 String group = matcher.group();
630 int dotsNumber = StringUtils.countMatches(group, ".");
631 if (dotsNumber == 1) {
632 reversed.append(group.replaceFirst("\\.", "/"));
633 } else {
634 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
635 String name = StringUtils.substringAfterLast(group, ".");
636 reversed.append(dots);
637
638 if (dotsNumber % 2 != 0) {
639 reversed.append("/");
640 }
641 reversed.append(name);
642 }
643 }
644 return reversed.toString();
645 }
646
647 }