View Javadoc
1   /**
2    * This file Copyright (c) 2003-2018 Magnolia International
3    * Ltd.  (http://www.magnolia-cms.com). All rights reserved.
4    *
5    *
6    * This file is dual-licensed under both the Magnolia
7    * Network Agreement and the GNU General Public License.
8    * You may elect to use one or the other of these licenses.
9    *
10   * This file is distributed in the hope that it will be
11   * useful, but AS-IS and WITHOUT ANY WARRANTY; without even the
12   * implied warranty of MERCHANTABILITY or FITNESS FOR A
13   * PARTICULAR PURPOSE, TITLE, or NONINFRINGEMENT.
14   * Redistribution, except as permitted by whichever of the GPL
15   * or MNA you select, is prohibited.
16   *
17   * 1. For the GPL license (GPL), you can redistribute and/or
18   * modify this file under the terms of the GNU General
19   * Public License, Version 3, as published by the Free Software
20   * Foundation.  You should have received a copy of the GNU
21   * General Public License, Version 3 along with this program;
22   * if not, write to the Free Software Foundation, Inc., 51
23   * Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
24   *
25   * 2. For the Magnolia Network Agreement (MNA), this file
26   * and the accompanying materials are made available under the
27   * terms of the MNA which accompanies this distribution, and
28   * is available at http://www.magnolia-cms.com/mna.html
29   *
30   * Any modifications to this file must keep this entire header
31   * intact.
32   *
33   */
34  package info.magnolia.importexport;
35  
36  import static javax.xml.transform.OutputKeys.*;
37  
38  import info.magnolia.cms.beans.runtime.Document;
39  import info.magnolia.cms.core.SystemProperty;
40  import info.magnolia.cms.core.version.VersionManager;
41  import info.magnolia.context.MgnlContext;
42  import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
43  import info.magnolia.importexport.filters.ImportXmlRootFilter;
44  import info.magnolia.importexport.filters.MagnoliaV2Filter;
45  import info.magnolia.importexport.filters.MetadataUuidFilter;
46  import info.magnolia.importexport.filters.RemoveMixversionableFilter;
47  import info.magnolia.importexport.filters.VersionFilter;
48  import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
49  import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
50  import info.magnolia.importexport.postprocessors.UpdateVersionMixinPostProcessor;
51  import info.magnolia.jcr.RuntimeRepositoryException;
52  import info.magnolia.jcr.util.NodeTypes;
53  import info.magnolia.jcr.util.NodeUtil;
54  import info.magnolia.jcr.util.PropertyUtil;
55  import info.magnolia.jcr.util.SessionUtil;
56  import info.magnolia.objectfactory.Components;
57  
58  import java.io.File;
59  import java.io.FileInputStream;
60  import java.io.FileNotFoundException;
61  import java.io.FileOutputStream;
62  import java.io.IOException;
63  import java.io.InputStream;
64  import java.io.InputStreamReader;
65  import java.io.OutputStream;
66  import java.io.UnsupportedEncodingException;
67  import java.net.URLDecoder;
68  import java.net.URLEncoder;
69  import java.nio.charset.StandardCharsets;
70  import java.util.Arrays;
71  import java.util.Collection;
72  import java.util.HashMap;
73  import java.util.Iterator;
74  import java.util.List;
75  import java.util.Map;
76  import java.util.Properties;
77  import java.util.regex.Matcher;
78  import java.util.regex.Pattern;
79  import java.util.zip.DeflaterOutputStream;
80  import java.util.zip.GZIPInputStream;
81  import java.util.zip.GZIPOutputStream;
82  import java.util.zip.ZipInputStream;
83  import java.util.zip.ZipOutputStream;
84  
85  import javax.jcr.ImportUUIDBehavior;
86  import javax.jcr.ItemNotFoundException;
87  import javax.jcr.Node;
88  import javax.jcr.NodeIterator;
89  import javax.jcr.PropertyType;
90  import javax.jcr.RepositoryException;
91  import javax.jcr.Session;
92  import javax.jcr.ValueFactory;
93  import javax.xml.parsers.ParserConfigurationException;
94  import javax.xml.parsers.SAXParser;
95  import javax.xml.parsers.SAXParserFactory;
96  import javax.xml.transform.Source;
97  import javax.xml.transform.Transformer;
98  import javax.xml.transform.TransformerConfigurationException;
99  import javax.xml.transform.sax.SAXTransformerFactory;
100 import javax.xml.transform.sax.TransformerHandler;
101 import javax.xml.transform.stream.StreamResult;
102 import javax.xml.transform.stream.StreamSource;
103 
104 import org.apache.commons.io.IOUtils;
105 import org.apache.commons.lang3.StringUtils;
106 import org.apache.jackrabbit.JcrConstants;
107 import org.apache.jackrabbit.core.NodeImpl;
108 import org.apache.jackrabbit.value.ValueHelper;
109 import org.slf4j.Logger;
110 import org.slf4j.LoggerFactory;
111 import org.xml.sax.ContentHandler;
112 import org.xml.sax.InputSource;
113 import org.xml.sax.SAXException;
114 import org.xml.sax.XMLFilter;
115 import org.xml.sax.XMLReader;
116 import org.yaml.snakeyaml.Yaml;
117 import org.yaml.snakeyaml.constructor.AbstractConstruct;
118 import org.yaml.snakeyaml.constructor.Constructor;
119 import org.yaml.snakeyaml.nodes.NodeId;
120 import org.yaml.snakeyaml.nodes.ScalarNode;
121 import org.yaml.snakeyaml.nodes.Tag;
122 
123 /**
124  * Utility class for manipulation of XML/YAML files (mainly JCR XML).
125  */
126 public class DataTransporter {
127 
128     private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
129 
130     private static final int INDENT_VALUE = 2;
131 
132     private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
133 
134     final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
135 
136     public static final String ZIP = ".zip";
137 
138     public static final String GZ = ".gz";
139 
140     public static final String XML = ".xml";
141 
142     public static final String YAML = ".yaml";
143 
144     public static final String PROPERTIES = ".properties";
145 
146     public static final String DOT = ".";
147 
148     public static final String SLASH = "/";
149 
150     public static final String UTF8 = "UTF-8";
151 
152     public static final String JCR_ROOT = "jcr:root";
153 
154     /**
155      * Converts a xml/yaml document into a file.
156      *
157      * @param bsDocument uploaded file
158      * @param repositoryName selected repository
159      * @param basepath base path in repository
160      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
161      * @param importMode a valid value for ImportUUIDBehavior
162      * @see ImportUUIDBehavior
163      */
164     public static synchronized void importDocument(Document bsDocument, String repositoryName, String basepath,
165                                                    boolean keepVersionHistory, int importMode, boolean saveAfterImport,
166                                                    boolean createBasepathIfNotExist)
167             throws IOException {
168         File bsFile = bsDocument.getFile();
169         importFile(bsFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
170                 createBasepathIfNotExist);
171     }
172 
173     /**
174      * Creates an <code>InputStream</code> backed by the specified xml/yaml file.
175      *
176      * @param bsFile (zipped/gzipped) XML/YAML file to import
177      * @param repositoryName selected repository
178      * @param basepath base path in repository
179      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
180      * @param importMode a valid value for ImportUUIDBehavior
181      * @see ImportUUIDBehavior
182      */
183     public static synchronized void importFile(File bsFile, String repositoryName, String basepath,
184                                                boolean keepVersionHistory, int importMode, boolean saveAfterImport,
185                                                boolean createBasepathIfNotExist)
186             throws IOException {
187         String name = bsFile.getAbsolutePath();
188 
189         InputStream bsStream = getInputStreamForFile(bsFile);
190         if (bsFile.getName().endsWith(YAML)) {
191             importYamlStream(bsStream, repositoryName, basepath, null, name, saveAfterImport, importMode, saveAfterImport, createBasepathIfNotExist);
192         } else { // bsFile.getName().endsWith(XML)
193             importXmlStream(bsStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport, createBasepathIfNotExist);
194         }
195     }
196 
197     public static void executeBootstrapImport(File bsFile, String repositoryName) throws IOException {
198         String filenameWithoutExt = StringUtils.substringBeforeLast(bsFile.getName(), DOT);
199         if (filenameWithoutExt.endsWith(XML) || filenameWithoutExt.endsWith(YAML)) {
200             // if file ends in .xml.gz, .xml.zip, .yaml.gz or .yaml.zip
201             // need to keep the .xml/.yaml to be able to view it after decompression
202             filenameWithoutExt = StringUtils.substringBeforeLast(bsFile.getName(), DOT);
203         }
204         String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
205 
206         pathName = decodePath(pathName, UTF8);
207 
208         String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
209 
210         if (bsFile.getName().endsWith(PROPERTIES)) {
211             Properties properties = new Properties();
212             FileInputStream stream = new FileInputStream(bsFile);
213             try {
214                 properties.load(stream);
215             } finally {
216                 IOUtils.closeQuietly(stream);
217             }
218             importProperties(properties, repositoryName);
219         } else {
220             DataTransporter.importFile(bsFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
221         }
222     }
223 
224     /**
225      * @deprecated since 4.0 - use the PropertiesImportExport class instead.
226      */
227     @Deprecated
228     public static void importProperties(Properties properties, String repositoryName) {
229         for (Iterator iter = properties.keySet().iterator(); iter.hasNext(); ) {
230             String key = (String) iter.next();
231             String value = (String) properties.get(key);
232 
233             String name = StringUtils.substringAfterLast(key, ".");
234             String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
235             Node node = SessionUtil.getNode(repositoryName, path);
236             if (node != null) {
237                 try {
238                     node.setProperty(name, value);
239                     node.getSession().save();
240                 } catch (RepositoryException e) {
241                     log.error("can't set property {}", key, e);
242                 }
243             }
244         }
245 
246     }
247 
248     /**
249      * Imports XML stream into repository.
250      * XML is filtered by <code>MagnoliaV2Filter</code>, <code>VersionFilter</code> and <code>ImportXmlRootFilter</code>
251      * if <code>keepVersionHistory</code> is set to <code>false</code>
252      *
253      * @param xmlStream XML stream to import
254      * @param repositoryName selected repository
255      * @param basepath base path in repository
256      * @param name (absolute path of <code>File</code>)
257      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
258      * @param importMode a valid value for ImportUUIDBehavior
259      * @see ImportUUIDBehavior
260      * @see ImportXmlRootFilter
261      * @see VersionFilter
262      * @see MagnoliaV2Filter
263      */
264     public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
265                                                     String name, boolean keepVersionHistory, int importMode,
266                                                     boolean saveAfterImport, boolean createBasepathIfNotExist)
267             throws IOException {
268         importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
269 
270     }
271 
272     /**
273      * Imports XML stream into repository.
274      * XML is filtered by <code>MagnoliaV2Filter</code>, <code>VersionFilter</code> and <code>ImportXmlRootFilter</code> if <code>keepVersionHistory</code> is set to <code>false</code>
275      *
276      * @param xmlStream XML stream to import
277      * @param repositoryName selected repository
278      * @param basepath base path in repository
279      * @param name (absolute path of <code>File</code>)
280      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
281      * @param forceUnpublishState if <code>true</code> then activation state of node will be change to unpublished.
282      * @param importMode a valid value for ImportUUIDBehavior
283      * @see ImportUUIDBehavior
284      * @see ImportXmlRootFilter
285      * @see VersionFilter
286      * @see MagnoliaV2Filter
287      */
288     public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
289                                                     String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
290                                                     boolean saveAfterImport, boolean createBasepathIfNotExist)
291             throws IOException {
292 
293         // TODO hopefully this will be fixed with a more useful message with the Bootstrapper refactoring
294         if (xmlStream == null) {
295             throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
296         }
297         try {
298             Session session = MgnlContext.getJCRSession(repositoryName);
299 
300             log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
301 
302             if (!session.nodeExists(basepath) && createBasepathIfNotExist) {
303                 try {
304                     NodeUtil.createPath(session.getRootNode(), basepath, NodeTypes.Content.NAME);
305                 } catch (RepositoryException e) {
306                     log.error("can't create path [{}]", basepath);
307                 }
308             }
309 
310             // Collects a list with all nodes at the basepath before import so we can see exactly which nodes were imported afterwards
311             List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
312 
313             if (keepVersionHistory) {
314                 // do not manipulate
315                 session.importXML(basepath, xmlStream, importMode);
316             } else {
317                 // create readers/filters and chain
318                 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
319                 saxParserFactory.setNamespaceAware(true);
320                 SAXParser saxParser = saxParserFactory.newSAXParser();
321                 XMLReader initialReader = saxParser.getXMLReader();
322                 try {
323                     initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
324                 } catch (SAXException e) {
325                     log.error("could not set parser feature");
326                 }
327 
328 
329                 XMLFilter magnoliaV2Filter = null;
330 
331                 // if stream is from regular file, test for belonging XSL file to apply XSL transformation to XML
332                 if (new File(name).isFile()) {
333                     InputStream xslStream = getXslStreamForXmlFile(new File(name));
334                     if (xslStream != null) {
335                         Source xslSource = new StreamSource(xslStream);
336                         SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
337                         XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
338                         magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
339                     }
340                 }
341 
342                 if (magnoliaV2Filter == null) {
343                     magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
344                 }
345 
346                 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
347 
348                 // enable this to strip useless "name" properties from dialogs
349                 // versionFilter = new UselessNameFilter(versionFilter);
350 
351                 // enable this to strip mix:versionable from pre 3.6 xml files
352                 versionFilter = new RemoveMixversionableFilter(versionFilter);
353 
354                 // strip rep:accesscontrol node
355                 versionFilter = new AccesscontrolNodeFilter(versionFilter);
356 
357                 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
358 
359                 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
360                 finalReader.setContentHandler(handler);
361 
362                 // parse XML, import is done by handler from session
363                 try {
364                     finalReader.parse(new InputSource(xmlStream));
365                 } finally {
366                     IOUtils.closeQuietly(xmlStream);
367                 }
368 
369                 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
370                     String path = basepath;
371                     if (!path.endsWith(SLASH)) {
372                         path += SLASH;
373                     }
374 
375                     Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
376                     for (Iterator iter = dummyRoot.getNodes(); iter.hasNext(); ) {
377                         Node child = (Node) iter.next();
378                         // move childs to real root
379 
380                         if (session.itemExists(path + child.getName())) {
381                             session.getItem(path + child.getName()).remove();
382                         }
383 
384                         session.move(child.getPath(), path + child.getName());
385                     }
386                     // delete the dummy node
387                     dummyRoot.remove();
388                 }
389 
390                 // Post process all nodes that were imported
391                 VersionManager versionManager = Components.getComponent(VersionManager.class);
392                 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
393                 while (nodesAfterImport.hasNext()) {
394                     Node nodeAfterImport = nodesAfterImport.nextNode();
395                     boolean existedBeforeImport = false;
396                     for (Node nodeBeforeImport : nodesBeforeImport) {
397                         if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
398                             existedBeforeImport = true;
399                             break;
400                         }
401                     }
402                     if (!existedBeforeImport) {
403                         postProcessAfterImport(nodeAfterImport, forceUnpublishState, importMode, versionManager);
404                     }
405                 }
406             }
407             if (saveAfterImport) {
408                 session.save();
409             }
410         } catch (Exception e) {
411             throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
412         } finally {
413             IOUtils.closeQuietly(xmlStream);
414         }
415     }
416 
417     /**
418      * Imports YAML stream into repository.
419      *
420      * @param yamlStream YAML stream to import
421      * @param importPath sub path in YAML file
422      * @param repositoryName selected repository
423      * @param basepath base path in repository
424      * @param name (absolute path of <code>File</code>)
425      * @param forceUnpublishState if <code>true</code> then activation state of node will be change to unpublished.
426      * @param saveAfterImport saves session after import
427      *
428      * @deprecated since 6.2.5 - use the {@link #importYamlStream(InputStream, String, String, String, String, boolean, int, boolean, boolean)} class instead.
429      */
430     @Deprecated
431     public static synchronized void importYamlStream(InputStream yamlStream, Node node, String repositoryName, String basepath, String importPath,
432                                                      String name, boolean forceUnpublishState, boolean saveAfterImport, boolean createBasepathIfNotExist) {
433         importYamlStream(yamlStream, repositoryName, basepath, importPath, name, forceUnpublishState, ImportUUIDBehavior.IMPORT_UUID_COLLISION_THROW, saveAfterImport, createBasepathIfNotExist);
434     }
435 
436     /**
437      * Imports YAML stream into repository.
438      *
439      * @param yamlStream YAML stream to import
440      * @param repositoryName selected repository
441      * @param basepath base path in repository
442      * @param importPath sub path in YAML file
443      * @param name (absolute path of <code>InputStream</code>)
444      * @param forceUnpublishState if <code>true</code> then activation state of node will be change to unpublished.
445      * @param importMode a valid value for ImportUUIDBehavior
446      * @param saveAfterImport saves session after import
447      * @param createBasepathIfNotExist if <code>true</code> the basepath is created if not existing
448      *
449      * @see ImportUUIDBehavior
450      */
451     public static synchronized void importYamlStream(InputStream yamlStream, String repositoryName, String basepath, String importPath,
452                                                      String name, boolean forceUnpublishState, int importMode, boolean saveAfterImport, boolean createBasepathIfNotExist) {
453 
454         try {
455             log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
456 
457             final Node node = getNodeToImportTo(repositoryName, basepath, createBasepathIfNotExist);
458             final ValueFactory valueFactory = node.getSession().getValueFactory();
459 
460             final InputStreamReader inputStream = new InputStreamReader(yamlStream, StandardCharsets.UTF_8);
461             final Yaml yaml = new Yaml(new JcrPropertySupportingConstructor(valueFactory));
462             Map content = yaml.load(inputStream);
463             if (importPath != null) {
464                 final String[] path = StringUtils.split(importPath, "/");
465                 for (String nodeName : path) {
466                     content = (Map) content.get(nodeName);
467                 }
468                 final Map<String, Object> wrappedTarget = new HashMap<>();
469                 wrappedTarget.put(path[path.length - 1], content);
470                 content = wrappedTarget;
471             }
472             yaml2Jcr(node, content, importMode);
473             if (saveAfterImport) {
474                 node.getSession().save();
475             }
476             if (forceUnpublishState) {
477                 new ActivationStatusImportPostProcessor().postProcessNode(node);
478             }
479         } catch (RepositoryException e) {
480             throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
481         }
482     }
483 
484     private static Node getNodeToImportTo(String repositoryName, String basepath, boolean createBasepathIfNotExist) throws RepositoryException {
485         Session session = MgnlContext.getJCRSession(repositoryName);
486         if (!session.nodeExists(basepath) && createBasepathIfNotExist) {
487             try {
488                 NodeUtil.createPath(session.getRootNode(), basepath, NodeTypes.Content.NAME);
489             } catch (RepositoryException e) {
490                 log.error("can't create path [{}]", basepath);
491             }
492         }
493         return session.getNode(basepath);
494     }
495 
496     private static void postProcessAfterImport(Node node, boolean forceUnpublishState, int importMode, VersionManager versionManager) throws RepositoryException {
497         try {
498             new MetaDataImportPostProcessor().postProcessNode(node);
499             if (forceUnpublishState) {
500                 new ActivationStatusImportPostProcessor().postProcessNode(node);
501             }
502             new UpdateVersionMixinPostProcessor(importMode, versionManager).postProcessNode(node);
503         } catch (RepositoryException e) {
504             throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
505         }
506     }
507 
508     /**
509      * @return XSL stream for Xml file or <code>null</code>
510      */
511     protected static InputStream getXslStreamForXmlFile(File file) {
512         InputStream xslStream = null;
513         String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
514         Filee/File.html#File">File xslFile = new File(xlsFilename);
515         if (xslFile.exists()) {
516             try {
517                 xslStream = new FileInputStream(xslFile);
518                 log.info("XSL file for [{}] found ({})", file.getName(), xslFile.getName());
519             } catch (FileNotFoundException e) { // should never happen (xslFile.exists())
520                 e.printStackTrace();
521             }
522         }
523         return xslStream;
524     }
525 
526     /**
527      * Creates a stream from the (zipped/gzipped) XML/YAML file.
528      *
529      * @return stream of the file
530      */
531     public static InputStream getInputStreamForFile(File bsFile) throws IOException {
532         InputStream bsStream;
533         // looks like the zip one is buggy. It throws exception when trying to use it
534         if (bsFile.getName().endsWith(ZIP)) {
535             bsStream = new ZipInputStream((new FileInputStream(bsFile)));
536         } else if (bsFile.getName().endsWith(GZ)) {
537             bsStream = new GZIPInputStream((new FileInputStream(bsFile)));
538         } else { // if(fileName.endsWith(XML) || fileName.endsWith(YAML))
539             bsStream = new FileInputStream(bsFile);
540         }
541         return bsStream;
542     }
543 
544     public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
545                                      Session session, String basepath, String repository, String ext) throws IOException {
546         OutputStream outputStream = baseOutputStream;
547         if (ext.endsWith(ZIP)) {
548             outputStream = new ZipOutputStream(baseOutputStream);
549         } else if (ext.endsWith(GZ)) {
550             outputStream = new GZIPOutputStream(baseOutputStream);
551         }
552 
553         try {
554             if (keepVersionHistory) {
555                 // use exportSystemView in order to preserve property types
556                 // http://issues.apache.org/jira/browse/JCR-115
557                 if (!format) {
558                     session.exportSystemView(basepath, outputStream, false, false);
559                 } else {
560                     parseAndFormat(outputStream, null, repository, basepath, session, false);
561                 }
562             } else {
563                 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
564                 saxParserFactory.setNamespaceAware(true);
565                 SAXParser saxParser = saxParserFactory.newSAXParser();
566                 XMLReader reader = saxParser.getXMLReader();
567 
568                 reader = new AccesscontrolNodeFilter(reader);
569                 parseAndFormat(outputStream, reader, repository, basepath, session, false);
570             }
571         } catch (IOException | SAXException | ParserConfigurationException | TransformerConfigurationException e) {
572             throw new RuntimeException(e);
573         } catch (RepositoryException e) {
574             throw new RuntimeRepositoryException(e);
575         }
576 
577         // finish the stream properly if zip stream
578         // this is not done by the IOUtils
579         if (outputStream instanceof DeflaterOutputStream) {
580             ((DeflaterOutputStream) outputStream).finish();
581         }
582 
583         baseOutputStream.flush();
584         IOUtils.closeQuietly(baseOutputStream);
585     }
586 
587     /**
588      * Exports the content of the repository, and format it if necessary.
589      *
590      * @param stream the stream to write the content to
591      * @param reader the reader to use to parse the xml content (so that we can perform filtering), if null instanciate
592      * a default one
593      * @param repository the repository to export
594      * @param basepath the basepath in the repository
595      * @param session the session to use to export the data from the repository
596      */
597     public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
598                                       Session session, boolean noRecurse)
599             throws IOException, SAXException, ParserConfigurationException, TransformerConfigurationException, RepositoryException {
600 
601         if (reader == null) {
602             SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
603             saxParserFactory.setNamespaceAware(true);
604             SAXParser saxParser = saxParserFactory.newSAXParser();
605             reader = saxParser.getXMLReader();
606         }
607 
608         // write to a temp file and then re-read it to remove version history
609         File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
610         OutputStream fileStream = new FileOutputStream(tempFile);
611 
612         try {
613             session.exportSystemView(basepath, fileStream, false, noRecurse);
614         } finally {
615             IOUtils.closeQuietly(fileStream);
616         }
617 
618         readFormatted(reader, tempFile, stream);
619 
620         if (!tempFile.delete()) {
621             log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
622         }
623     }
624 
625     protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
626             throws IOException, SAXException, TransformerConfigurationException {
627         InputStream fileInputStream = new FileInputStream(inputFile);
628         readFormatted(reader, fileInputStream, outputStream);
629         IOUtils.closeQuietly(fileInputStream);
630     }
631 
632     protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
633             throws IOException, SAXException, TransformerConfigurationException {
634 
635         SAXTransformerFactory transformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
636         TransformerHandler transformerHandler = transformerFactory.newTransformerHandler();
637         Transformer transformer = transformerHandler.getTransformer();
638         transformer.setOutputProperty(INDENT, "yes");
639         transformer.setOutputProperty(DOCTYPE_PUBLIC, "yes"); // effectively insert a line-break after the xml declaration
640         transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", String.valueOf(INDENT_VALUE));
641         transformerHandler.setResult(new StreamResult(outputStream));
642 
643         final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces"); // MAGNOLIA-2960
644         MetadataUuidFiltertml#MetadataUuidFilter">MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces); // MAGNOLIA-1650
645         metadataUuidFilter.setContentHandler(transformerHandler);
646         metadataUuidFilter.parse(new InputSource(inputStream));
647 
648         IOUtils.closeQuietly(inputStream);
649     }
650 
651     /**
652      * @param path path to encode
653      * @param separator "." (dot) or "/", it will be not encoded if found
654      * @param enc charset
655      * @return the path encoded
656      */
657     public static String encodePath(String path, String separator, String enc) {
658         StringBuilder pathEncoded = new StringBuilder();
659         try {
660             if (!StringUtils.contains(path, separator)) {
661                 return URLEncoder.encode(path, enc);
662             }
663             for (int i = 0; i < path.length(); i++) {
664                 String ch = String.valueOf(path.charAt(i));
665                 if (separator.equals(ch)) {
666                     pathEncoded.append(ch);
667                 } else {
668                     pathEncoded.append(URLEncoder.encode(ch, enc));
669                 }
670             }
671         } catch (UnsupportedEncodingException e) {
672             return path;
673         }
674         return pathEncoded.toString();
675     }
676 
677     /**
678      * decode a path (ex. %D0%9D%D0%B0.%B2%D0%BE%D0%BB%D0%BD)
679      *
680      * @param path path to decode
681      * @param enc charset
682      * @return the path decoded
683      */
684     public static String decodePath(String path, String enc) {
685         try {
686             return URLDecoder.decode(path, enc);
687         } catch (UnsupportedEncodingException e) {
688             return path;
689         }
690     }
691 
692     /**
693      * Prior to 4.5 Magnolia used to produce export xml filenames where the / (slash) separating sub nodes was replaced by a dot.
694      * Since 4.5, Magnolia enables dots in path names, therefore dots which are part of the node name have to be escaped by doubling them.
695      * I.e. given a path like this <code>/foo/bar.baz/test../dir/baz..bar</code>, this method will produce
696      * <code>.foo.bar..baz.test.....dir.baz....bar</code>.
697      */
698     public static String createExportPath(String path) {
699         //TODO if someone is smarter than me (not an impossible thing) and can do this with one single elegant regex, please do it.
700         String newPath = path.replace(".", "..");
701         newPath = newPath.replace("/", ".");
702         return newPath;
703     }
704 
705     /**
706      * The opposite of {@link #createExportPath(String)}.
707      * I.e. given a path like this <code>.foo.bar..baz.test.....dir.baz....bar</code>, this method will produce <code>/foo/bar.baz/test../dir/baz..bar</code>.
708      */
709     public static String revertExportPath(String exportPath) {
710         if (".".equals(exportPath)) {
711             return "/";
712         }
713 
714         //TODO I have a feeling there's a simpler way to achieve our goal.
715         Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
716 
717         StringBuilder reversed = new StringBuilder(exportPath.length());
718 
719         while (matcher.find()) {
720             String group = matcher.group();
721             int dotsNumber = StringUtils.countMatches(group, ".");
722             if (dotsNumber == 1) {
723                 reversed.append(group.replaceFirst("\\.", "/"));
724             } else {
725                 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
726                 String name = StringUtils.substringAfterLast(group, ".");
727                 reversed.append(dots);
728                 //if number is odd, the last dot has to be replaced with a slash
729                 if (dotsNumber % 2 != 0) {
730                     reversed.append("/");
731                 }
732                 reversed.append(name);
733             }
734         }
735         return reversed.toString();
736     }
737 
738     private static void yaml2Jcr(Node root, Map<?, ?> map, int importMode) throws RepositoryException {
739         for (Map.Entry entry : map.entrySet()) {
740             String key = String.valueOf(entry.getKey());
741             String nodeType = NodeTypes.ContentNode.NAME;
742             if (entry.getValue() == null || entry.getValue() instanceof Map) {
743                 Map subMap = (Map) entry.getValue();
744                 if (subMap == null) { //an empty node
745                     root.addNode(key, nodeType);
746                 } else {
747                     if (subMap.get(JcrConstants.JCR_PRIMARYTYPE) != null) {
748                         nodeType = (String) subMap.get(JcrConstants.JCR_PRIMARYTYPE);
749                     }
750                     Node node;
751                     if (subMap.get(JcrConstants.JCR_UUID) != null && NodeUtil.unwrap(root) instanceof NodeImpl) {
752                         final NodeImpl unwrapped = (NodeImpl) NodeUtil.unwrap(root);
753                         final String uuid = String.valueOf(subMap.get(JcrConstants.JCR_UUID));
754                         try {
755                             Node existingNode = unwrapped.getSession().getNodeByIdentifier(uuid);
756                             switch (importMode) {
757                             case ImportUUIDBehavior.IMPORT_UUID_CREATE_NEW:
758                                 node = unwrapped.addNode(key, nodeType);
759                                 break;
760                             case ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING:
761                                 final Node existingParent = NodeUtil.unwrap(existingNode.getParent());
762                                 existingNode.remove();
763                                 node = ((NodeImpl) existingParent).addNodeWithUuid(key, nodeType, uuid);
764                                 break;
765                             case ImportUUIDBehavior.IMPORT_UUID_COLLISION_REMOVE_EXISTING:
766                                 existingNode.remove();
767                                 node = unwrapped.addNodeWithUuid(key, nodeType, uuid);
768                                 break;
769                             default:
770                                 // will throw ItemExistsException
771                                 node = unwrapped.addNodeWithUuid(key, nodeType, uuid);
772                                 break;
773                             }
774                         } catch (ItemNotFoundException e) {
775                             node = unwrapped.addNodeWithUuid(key, nodeType, uuid);
776                         }
777                     } else {
778                         node = root.addNode(key, nodeType);
779                     }
780                     final Object mixins = subMap.get(JcrConstants.JCR_MIXINTYPES);
781                     if (mixins instanceof Collection) {
782                         for (Object mixin : (Collection) mixins) {
783                             node.addMixin(String.valueOf(mixin));
784                         }
785                     }
786                     yaml2Jcr(node, (Map) entry.getValue(), importMode);
787                 }
788             } else if (!Arrays.asList(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.JCR_UUID, JcrConstants.JCR_MIXINTYPES).contains(key)) {
789                 PropertyUtil.setProperty(root, key, entry.getValue());
790             }
791         }
792     }
793 
794     private static class JcrPropertySupportingConstructor extends Constructor {
795 
796         private JcrPropertySupportingConstructor(ValueFactory valueFactory) {
797             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_PATH)), new JcrConstruct(PropertyType.PATH, valueFactory));
798             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_URI)), new JcrConstruct(PropertyType.URI, valueFactory));
799             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_REFERENCE)), new JcrConstruct(PropertyType.REFERENCE, valueFactory));
800             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_WEAKREFERENCE)), new JcrConstruct(PropertyType.WEAKREFERENCE, valueFactory));
801             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_BINARY)), new JcrConstruct(PropertyType.BINARY, valueFactory));
802 
803             //Prevents sequence node being loaded, see more in MAGNOLIA-7914.
804             yamlClassConstructors.remove(NodeId.sequence);
805         }
806     }
807 
808     private static class JcrConstruct extends AbstractConstruct {
809         private final int propertyType;
810         private final ValueFactory valueFactory;
811 
812         private JcrConstruct(int propertyType, ValueFactory valueFactory) {
813             this.propertyType = propertyType;
814             this.valueFactory = valueFactory;
815         }
816 
817         @Override
818         public Object construct(org.yaml.snakeyaml.nodes.Node node) {
819             try {
820                 return ValueHelper.deserialize(((ScalarNode) node).getValue(), propertyType, false, valueFactory);
821             } catch (RepositoryException e) {
822                 log.error(e.getMessage(), e);
823                 return null;
824             }
825         }
826     }
827 
828 }