View Javadoc
1   /**
2    * This file Copyright (c) 2003-2018 Magnolia International
3    * Ltd.  (http://www.magnolia-cms.com). All rights reserved.
4    *
5    *
6    * This file is dual-licensed under both the Magnolia
7    * Network Agreement and the GNU General Public License.
8    * You may elect to use one or the other of these licenses.
9    *
10   * This file is distributed in the hope that it will be
11   * useful, but AS-IS and WITHOUT ANY WARRANTY; without even the
12   * implied warranty of MERCHANTABILITY or FITNESS FOR A
13   * PARTICULAR PURPOSE, TITLE, or NONINFRINGEMENT.
14   * Redistribution, except as permitted by whichever of the GPL
15   * or MNA you select, is prohibited.
16   *
17   * 1. For the GPL license (GPL), you can redistribute and/or
18   * modify this file under the terms of the GNU General
19   * Public License, Version 3, as published by the Free Software
20   * Foundation.  You should have received a copy of the GNU
21   * General Public License, Version 3 along with this program;
22   * if not, write to the Free Software Foundation, Inc., 51
23   * Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
24   *
25   * 2. For the Magnolia Network Agreement (MNA), this file
26   * and the accompanying materials are made available under the
27   * terms of the MNA which accompanies this distribution, and
28   * is available at http://www.magnolia-cms.com/mna.html
29   *
30   * Any modifications to this file must keep this entire header
31   * intact.
32   *
33   */
34  package info.magnolia.importexport;
35  
36  import static javax.xml.transform.OutputKeys.*;
37  
38  import info.magnolia.cms.beans.runtime.Document;
39  import info.magnolia.cms.core.SystemProperty;
40  import info.magnolia.cms.core.version.VersionManager;
41  import info.magnolia.context.MgnlContext;
42  import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
43  import info.magnolia.importexport.filters.ImportXmlRootFilter;
44  import info.magnolia.importexport.filters.MagnoliaV2Filter;
45  import info.magnolia.importexport.filters.MetadataUuidFilter;
46  import info.magnolia.importexport.filters.RemoveMixversionableFilter;
47  import info.magnolia.importexport.filters.VersionFilter;
48  import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
49  import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
50  import info.magnolia.importexport.postprocessors.UpdateVersionMixinPostProcessor;
51  import info.magnolia.jcr.RuntimeRepositoryException;
52  import info.magnolia.jcr.util.NodeTypes;
53  import info.magnolia.jcr.util.NodeUtil;
54  import info.magnolia.jcr.util.PropertyUtil;
55  import info.magnolia.jcr.util.SessionUtil;
56  import info.magnolia.objectfactory.Components;
57  
58  import java.io.File;
59  import java.io.FileInputStream;
60  import java.io.FileNotFoundException;
61  import java.io.FileOutputStream;
62  import java.io.IOException;
63  import java.io.InputStream;
64  import java.io.InputStreamReader;
65  import java.io.OutputStream;
66  import java.io.UnsupportedEncodingException;
67  import java.net.URLDecoder;
68  import java.net.URLEncoder;
69  import java.nio.charset.StandardCharsets;
70  import java.util.Arrays;
71  import java.util.Collection;
72  import java.util.HashMap;
73  import java.util.Iterator;
74  import java.util.List;
75  import java.util.Map;
76  import java.util.Properties;
77  import java.util.regex.Matcher;
78  import java.util.regex.Pattern;
79  import java.util.zip.DeflaterOutputStream;
80  import java.util.zip.GZIPInputStream;
81  import java.util.zip.GZIPOutputStream;
82  import java.util.zip.ZipInputStream;
83  import java.util.zip.ZipOutputStream;
84  
85  import javax.jcr.ImportUUIDBehavior;
86  import javax.jcr.Node;
87  import javax.jcr.NodeIterator;
88  import javax.jcr.PropertyType;
89  import javax.jcr.RepositoryException;
90  import javax.jcr.Session;
91  import javax.jcr.ValueFactory;
92  import javax.xml.parsers.ParserConfigurationException;
93  import javax.xml.parsers.SAXParser;
94  import javax.xml.parsers.SAXParserFactory;
95  import javax.xml.transform.Source;
96  import javax.xml.transform.Transformer;
97  import javax.xml.transform.TransformerConfigurationException;
98  import javax.xml.transform.sax.SAXTransformerFactory;
99  import javax.xml.transform.sax.TransformerHandler;
100 import javax.xml.transform.stream.StreamResult;
101 import javax.xml.transform.stream.StreamSource;
102 
103 import org.apache.commons.io.IOUtils;
104 import org.apache.commons.lang3.StringUtils;
105 import org.apache.jackrabbit.JcrConstants;
106 import org.apache.jackrabbit.core.NodeImpl;
107 import org.apache.jackrabbit.value.ValueHelper;
108 import org.slf4j.Logger;
109 import org.slf4j.LoggerFactory;
110 import org.xml.sax.ContentHandler;
111 import org.xml.sax.InputSource;
112 import org.xml.sax.SAXException;
113 import org.xml.sax.XMLFilter;
114 import org.xml.sax.XMLReader;
115 import org.yaml.snakeyaml.Yaml;
116 import org.yaml.snakeyaml.constructor.AbstractConstruct;
117 import org.yaml.snakeyaml.constructor.Constructor;
118 import org.yaml.snakeyaml.nodes.ScalarNode;
119 import org.yaml.snakeyaml.nodes.Tag;
120 
121 /**
122  * Utility class for manipulation of XML/YAML files (mainly JCR XML).
123  */
124 public class DataTransporter {
125 
126     private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
127 
128     private static final int INDENT_VALUE = 2;
129 
130     private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
131 
132     final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
133 
134     public static final String ZIP = ".zip";
135 
136     public static final String GZ = ".gz";
137 
138     public static final String XML = ".xml";
139 
140     public static final String YAML = ".yaml";
141 
142     public static final String PROPERTIES = ".properties";
143 
144     public static final String DOT = ".";
145 
146     public static final String SLASH = "/";
147 
148     public static final String UTF8 = "UTF-8";
149 
150     public static final String JCR_ROOT = "jcr:root";
151 
152     /**
153      * Converts a xml/yaml document into a file.
154      *
155      * @param bsDocument uploaded file
156      * @param repositoryName selected repository
157      * @param basepath base path in repository
158      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
159      * @param importMode a valid value for ImportUUIDBehavior
160      * @see ImportUUIDBehavior
161      */
162     public static synchronized void importDocument(Document bsDocument, String repositoryName, String basepath,
163                                                    boolean keepVersionHistory, int importMode, boolean saveAfterImport,
164                                                    boolean createBasepathIfNotExist)
165             throws IOException {
166         File bsFile = bsDocument.getFile();
167         importFile(bsFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
168                 createBasepathIfNotExist);
169     }
170 
171     /**
172      * Creates an <code>InputStream</code> backed by the specified xml/yaml file.
173      *
174      * @param bsFile (zipped/gzipped) XML/YAML file to import
175      * @param repositoryName selected repository
176      * @param basepath base path in repository
177      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
178      * @param importMode a valid value for ImportUUIDBehavior
179      * @see ImportUUIDBehavior
180      */
181     public static synchronized void importFile(File bsFile, String repositoryName, String basepath,
182                                                boolean keepVersionHistory, int importMode, boolean saveAfterImport,
183                                                boolean createBasepathIfNotExist)
184             throws IOException {
185         String name = bsFile.getAbsolutePath();
186 
187         InputStream bsStream = getInputStreamForFile(bsFile);
188         if (bsFile.getName().endsWith(YAML)) {
189             importYamlStream(bsStream, null, repositoryName, basepath, null, name, saveAfterImport, saveAfterImport, createBasepathIfNotExist);
190         } else { // bsFile.getName().endsWith(XML)
191             importXmlStream(bsStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport, createBasepathIfNotExist);
192         }
193     }
194 
195     public static void executeBootstrapImport(File bsFile, String repositoryName) throws IOException {
196         String filenameWithoutExt = StringUtils.substringBeforeLast(bsFile.getName(), DOT);
197         if (filenameWithoutExt.endsWith(XML) || filenameWithoutExt.endsWith(YAML)) {
198             // if file ends in .xml.gz, .xml.zip, .yaml.gz or .yaml.zip
199             // need to keep the .xml/.yaml to be able to view it after decompression
200             filenameWithoutExt = StringUtils.substringBeforeLast(bsFile.getName(), DOT);
201         }
202         String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
203 
204         pathName = decodePath(pathName, UTF8);
205 
206         String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
207 
208         if (bsFile.getName().endsWith(PROPERTIES)) {
209             Properties properties = new Properties();
210             FileInputStream stream = new FileInputStream(bsFile);
211             try {
212                 properties.load(stream);
213             } finally {
214                 IOUtils.closeQuietly(stream);
215             }
216             importProperties(properties, repositoryName);
217         } else {
218             DataTransporter.importFile(bsFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
219         }
220     }
221 
222     /**
223      * @deprecated since 4.0 - use the PropertiesImportExport class instead.
224      */
225     @Deprecated
226     public static void importProperties(Properties properties, String repositoryName) {
227         for (Iterator iter = properties.keySet().iterator(); iter.hasNext(); ) {
228             String key = (String) iter.next();
229             String value = (String) properties.get(key);
230 
231             String name = StringUtils.substringAfterLast(key, ".");
232             String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
233             Node node = SessionUtil.getNode(repositoryName, path);
234             if (node != null) {
235                 try {
236                     node.setProperty(name, value);
237                     node.getSession().save();
238                 } catch (RepositoryException e) {
239                     log.error("can't set property {}", key, e);
240                 }
241             }
242         }
243 
244     }
245 
246     /**
247      * Imports XML stream into repository.
248      * XML is filtered by <code>MagnoliaV2Filter</code>, <code>VersionFilter</code> and <code>ImportXmlRootFilter</code>
249      * if <code>keepVersionHistory</code> is set to <code>false</code>
250      *
251      * @param xmlStream XML stream to import
252      * @param repositoryName selected repository
253      * @param basepath base path in repository
254      * @param name (absolute path of <code>File</code>)
255      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
256      * @param importMode a valid value for ImportUUIDBehavior
257      * @see ImportUUIDBehavior
258      * @see ImportXmlRootFilter
259      * @see VersionFilter
260      * @see MagnoliaV2Filter
261      */
262     public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
263                                                     String name, boolean keepVersionHistory, int importMode,
264                                                     boolean saveAfterImport, boolean createBasepathIfNotExist)
265             throws IOException {
266         importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
267 
268     }
269 
270     /**
271      * Imports XML stream into repository.
272      * XML is filtered by <code>MagnoliaV2Filter</code>, <code>VersionFilter</code> and <code>ImportXmlRootFilter</code> if <code>keepVersionHistory</code> is set to <code>false</code>
273      *
274      * @param xmlStream XML stream to import
275      * @param repositoryName selected repository
276      * @param basepath base path in repository
277      * @param name (absolute path of <code>File</code>)
278      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
279      * @param forceUnpublishState if <code>true</code> then activation state of node will be change to unpublished.
280      * @param importMode a valid value for ImportUUIDBehavior
281      * @see ImportUUIDBehavior
282      * @see ImportXmlRootFilter
283      * @see VersionFilter
284      * @see MagnoliaV2Filter
285      */
286     public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
287                                                     String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
288                                                     boolean saveAfterImport, boolean createBasepathIfNotExist)
289             throws IOException {
290 
291         // TODO hopefully this will be fixed with a more useful message with the Bootstrapper refactoring
292         if (xmlStream == null) {
293             throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
294         }
295         try {
296             Session session = MgnlContext.getJCRSession(repositoryName);
297 
298             log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
299 
300             if (!session.nodeExists(basepath) && createBasepathIfNotExist) {
301                 try {
302                     NodeUtil.createPath(session.getRootNode(), basepath, NodeTypes.Content.NAME);
303                 } catch (RepositoryException e) {
304                     log.error("can't create path [{}]", basepath);
305                 }
306             }
307 
308             // Collects a list with all nodes at the basepath before import so we can see exactly which nodes were imported afterwards
309             List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
310 
311             if (keepVersionHistory) {
312                 // do not manipulate
313                 session.importXML(basepath, xmlStream, importMode);
314             } else {
315                 // create readers/filters and chain
316                 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
317                 saxParserFactory.setNamespaceAware(true);
318                 SAXParser saxParser = saxParserFactory.newSAXParser();
319                 XMLReader initialReader = saxParser.getXMLReader();
320                 try {
321                     initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
322                 } catch (SAXException e) {
323                     log.error("could not set parser feature");
324                 }
325 
326 
327                 XMLFilter magnoliaV2Filter = null;
328 
329                 // if stream is from regular file, test for belonging XSL file to apply XSL transformation to XML
330                 if (new File(name).isFile()) {
331                     InputStream xslStream = getXslStreamForXmlFile(new File(name));
332                     if (xslStream != null) {
333                         Source xslSource = new StreamSource(xslStream);
334                         SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
335                         XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
336                         magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
337                     }
338                 }
339 
340                 if (magnoliaV2Filter == null) {
341                     magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
342                 }
343 
344                 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
345 
346                 // enable this to strip useless "name" properties from dialogs
347                 // versionFilter = new UselessNameFilter(versionFilter);
348 
349                 // enable this to strip mix:versionable from pre 3.6 xml files
350                 versionFilter = new RemoveMixversionableFilter(versionFilter);
351 
352                 // strip rep:accesscontrol node
353                 versionFilter = new AccesscontrolNodeFilter(versionFilter);
354 
355                 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
356 
357                 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
358                 finalReader.setContentHandler(handler);
359 
360                 // parse XML, import is done by handler from session
361                 try {
362                     finalReader.parse(new InputSource(xmlStream));
363                 } finally {
364                     IOUtils.closeQuietly(xmlStream);
365                 }
366 
367                 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
368                     String path = basepath;
369                     if (!path.endsWith(SLASH)) {
370                         path += SLASH;
371                     }
372 
373                     Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
374                     for (Iterator iter = dummyRoot.getNodes(); iter.hasNext(); ) {
375                         Node child = (Node) iter.next();
376                         // move childs to real root
377 
378                         if (session.itemExists(path + child.getName())) {
379                             session.getItem(path + child.getName()).remove();
380                         }
381 
382                         session.move(child.getPath(), path + child.getName());
383                     }
384                     // delete the dummy node
385                     dummyRoot.remove();
386                 }
387 
388                 // Post process all nodes that were imported
389                 VersionManager versionManager = Components.getComponent(VersionManager.class);
390                 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
391                 while (nodesAfterImport.hasNext()) {
392                     Node nodeAfterImport = nodesAfterImport.nextNode();
393                     boolean existedBeforeImport = false;
394                     for (Node nodeBeforeImport : nodesBeforeImport) {
395                         if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
396                             existedBeforeImport = true;
397                             break;
398                         }
399                     }
400                     if (!existedBeforeImport) {
401                         postProcessAfterImport(nodeAfterImport, forceUnpublishState, importMode, versionManager);
402                     }
403                 }
404             }
405             if (saveAfterImport) {
406                 session.save();
407             }
408         } catch (Exception e) {
409             throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
410         } finally {
411             IOUtils.closeQuietly(xmlStream);
412         }
413     }
414 
415     /**
416      * Imports YAML stream into repository.
417      *
418      * @param yamlStream YAML stream to import
419      * @param importPath sub path in YAML file
420      * @param repositoryName selected repository
421      * @param basepath base path in repository
422      * @param name (absolute path of <code>File</code>)
423      * @param forceUnpublishState if <code>true</code> then activation state of node will be change to unpublished.
424      * @param saveAfterImport saves session after import
425      *
426      */
427     public static synchronized void importYamlStream(InputStream yamlStream, Node node, String repositoryName, String basepath, String importPath,
428                                                     String name, boolean forceUnpublishState, boolean saveAfterImport, boolean createBasepathIfNotExist) {
429 
430         try {
431             log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
432 
433             if (node == null) {
434                 Session session = MgnlContext.getJCRSession(repositoryName);
435                 if (!session.nodeExists(basepath) && createBasepathIfNotExist) {
436                     try {
437                         NodeUtil.createPath(session.getRootNode(), basepath, NodeTypes.Content.NAME);
438                     } catch (RepositoryException e) {
439                         log.error("can't create path [{}]", basepath);
440                     }}
441                 node = session.getNode(basepath);
442             }
443             final ValueFactory valueFactory = node.getSession().getValueFactory();
444 
445 
446             final InputStreamReader inputStream = new InputStreamReader(yamlStream, StandardCharsets.UTF_8);
447             final Yaml yaml = new Yaml(new JcrPropertySupportingConstructor(valueFactory));
448             Map content = yaml.load(inputStream);
449             if (importPath != null) {
450                 final String[] path = StringUtils.split(importPath, "/");
451                 for (String nodeName : path) {
452                     content = (Map) content.get(nodeName);
453                 }
454                 final Map<String, Object> wrappedTarget = new HashMap<>();
455                 wrappedTarget.put(path[path.length - 1], content);
456                 content = wrappedTarget;
457             }
458             yaml2Jcr(node, content);
459             if (saveAfterImport) {
460                 node.getSession().save();
461             }
462             if (forceUnpublishState) {
463                 new ActivationStatusImportPostProcessor().postProcessNode(node);
464             }
465         } catch (RepositoryException e) {
466             throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
467         }
468     }
469 
470     private static void postProcessAfterImport(Node node, boolean forceUnpublishState, int importMode, VersionManager versionManager) throws RepositoryException {
471         try {
472             new MetaDataImportPostProcessor().postProcessNode(node);
473             if (forceUnpublishState) {
474                 new ActivationStatusImportPostProcessor().postProcessNode(node);
475             }
476             new UpdateVersionMixinPostProcessor(importMode, versionManager).postProcessNode(node);
477         } catch (RepositoryException e) {
478             throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
479         }
480     }
481 
482     /**
483      * @return XSL stream for Xml file or <code>null</code>
484      */
485     protected static InputStream getXslStreamForXmlFile(File file) {
486         InputStream xslStream = null;
487         String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
488         Filee/File.html#File">File xslFile = new File(xlsFilename);
489         if (xslFile.exists()) {
490             try {
491                 xslStream = new FileInputStream(xslFile);
492                 log.info("XSL file for [{}] found ({})", file.getName(), xslFile.getName());
493             } catch (FileNotFoundException e) { // should never happen (xslFile.exists())
494                 e.printStackTrace();
495             }
496         }
497         return xslStream;
498     }
499 
500     /**
501      * Creates a stream from the (zipped/gzipped) XML/YAML file.
502      *
503      * @return stream of the file
504      */
505     private static InputStream getInputStreamForFile(File bsFile) throws IOException {
506         InputStream bsStream;
507         // looks like the zip one is buggy. It throws exception when trying to use it
508         if (bsFile.getName().endsWith(ZIP)) {
509             bsStream = new ZipInputStream((new FileInputStream(bsFile)));
510         } else if (bsFile.getName().endsWith(GZ)) {
511             bsStream = new GZIPInputStream((new FileInputStream(bsFile)));
512         } else { // if(fileName.endsWith(XML) || fileName.endsWith(YAML))
513             bsStream = new FileInputStream(bsFile);
514         }
515         return bsStream;
516     }
517 
518     public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
519                                      Session session, String basepath, String repository, String ext) throws IOException {
520         OutputStream outputStream = baseOutputStream;
521         if (ext.endsWith(ZIP)) {
522             outputStream = new ZipOutputStream(baseOutputStream);
523         } else if (ext.endsWith(GZ)) {
524             outputStream = new GZIPOutputStream(baseOutputStream);
525         }
526 
527         try {
528             if (keepVersionHistory) {
529                 // use exportSystemView in order to preserve property types
530                 // http://issues.apache.org/jira/browse/JCR-115
531                 if (!format) {
532                     session.exportSystemView(basepath, outputStream, false, false);
533                 } else {
534                     parseAndFormat(outputStream, null, repository, basepath, session, false);
535                 }
536             } else {
537                 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
538                 saxParserFactory.setNamespaceAware(true);
539                 SAXParser saxParser = saxParserFactory.newSAXParser();
540                 XMLReader reader = saxParser.getXMLReader();
541 
542                 reader = new AccesscontrolNodeFilter(reader);
543                 parseAndFormat(outputStream, reader, repository, basepath, session, false);
544             }
545         } catch (IOException | SAXException | ParserConfigurationException | TransformerConfigurationException e) {
546             throw new RuntimeException(e);
547         } catch (RepositoryException e) {
548             throw new RuntimeRepositoryException(e);
549         }
550 
551         // finish the stream properly if zip stream
552         // this is not done by the IOUtils
553         if (outputStream instanceof DeflaterOutputStream) {
554             ((DeflaterOutputStream) outputStream).finish();
555         }
556 
557         baseOutputStream.flush();
558         IOUtils.closeQuietly(baseOutputStream);
559     }
560 
561     /**
562      * Exports the content of the repository, and format it if necessary.
563      *
564      * @param stream the stream to write the content to
565      * @param reader the reader to use to parse the xml content (so that we can perform filtering), if null instanciate
566      * a default one
567      * @param repository the repository to export
568      * @param basepath the basepath in the repository
569      * @param session the session to use to export the data from the repository
570      */
571     public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
572                                       Session session, boolean noRecurse)
573             throws IOException, SAXException, ParserConfigurationException, TransformerConfigurationException, RepositoryException {
574 
575         if (reader == null) {
576             SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
577             saxParserFactory.setNamespaceAware(true);
578             SAXParser saxParser = saxParserFactory.newSAXParser();
579             reader = saxParser.getXMLReader();
580         }
581 
582         // write to a temp file and then re-read it to remove version history
583         File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
584         OutputStream fileStream = new FileOutputStream(tempFile);
585 
586         try {
587             session.exportSystemView(basepath, fileStream, false, noRecurse);
588         } finally {
589             IOUtils.closeQuietly(fileStream);
590         }
591 
592         readFormatted(reader, tempFile, stream);
593 
594         if (!tempFile.delete()) {
595             log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
596         }
597     }
598 
599     protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
600             throws IOException, SAXException, TransformerConfigurationException {
601         InputStream fileInputStream = new FileInputStream(inputFile);
602         readFormatted(reader, fileInputStream, outputStream);
603         IOUtils.closeQuietly(fileInputStream);
604     }
605 
606     protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
607             throws IOException, SAXException, TransformerConfigurationException {
608 
609         SAXTransformerFactory transformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
610         TransformerHandler transformerHandler = transformerFactory.newTransformerHandler();
611         Transformer transformer = transformerHandler.getTransformer();
612         transformer.setOutputProperty(INDENT, "yes");
613         transformer.setOutputProperty(DOCTYPE_PUBLIC, "yes"); // effectively insert a line-break after the xml declaration
614         transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", String.valueOf(INDENT_VALUE));
615         transformerHandler.setResult(new StreamResult(outputStream));
616 
617         final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces"); // MAGNOLIA-2960
618         MetadataUuidFiltertml#MetadataUuidFilter">MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces); // MAGNOLIA-1650
619         metadataUuidFilter.setContentHandler(transformerHandler);
620         metadataUuidFilter.parse(new InputSource(inputStream));
621 
622         IOUtils.closeQuietly(inputStream);
623     }
624 
625     /**
626      * @param path path to encode
627      * @param separator "." (dot) or "/", it will be not encoded if found
628      * @param enc charset
629      * @return the path encoded
630      */
631     public static String encodePath(String path, String separator, String enc) {
632         StringBuilder pathEncoded = new StringBuilder();
633         try {
634             if (!StringUtils.contains(path, separator)) {
635                 return URLEncoder.encode(path, enc);
636             }
637             for (int i = 0; i < path.length(); i++) {
638                 String ch = String.valueOf(path.charAt(i));
639                 if (separator.equals(ch)) {
640                     pathEncoded.append(ch);
641                 } else {
642                     pathEncoded.append(URLEncoder.encode(ch, enc));
643                 }
644             }
645         } catch (UnsupportedEncodingException e) {
646             return path;
647         }
648         return pathEncoded.toString();
649     }
650 
651     /**
652      * decode a path (ex. %D0%9D%D0%B0.%B2%D0%BE%D0%BB%D0%BD)
653      *
654      * @param path path to decode
655      * @param enc charset
656      * @return the path decoded
657      */
658     public static String decodePath(String path, String enc) {
659         try {
660             return URLDecoder.decode(path, enc);
661         } catch (UnsupportedEncodingException e) {
662             return path;
663         }
664     }
665 
666     /**
667      * Prior to 4.5 Magnolia used to produce export xml filenames where the / (slash) separating sub nodes was replaced by a dot.
668      * Since 4.5, Magnolia enables dots in path names, therefore dots which are part of the node name have to be escaped by doubling them.
669      * I.e. given a path like this <code>/foo/bar.baz/test../dir/baz..bar</code>, this method will produce
670      * <code>.foo.bar..baz.test.....dir.baz....bar</code>.
671      */
672     public static String createExportPath(String path) {
673         //TODO if someone is smarter than me (not an impossible thing) and can do this with one single elegant regex, please do it.
674         String newPath = path.replace(".", "..");
675         newPath = newPath.replace("/", ".");
676         return newPath;
677     }
678 
679     /**
680      * The opposite of {@link #createExportPath(String)}.
681      * I.e. given a path like this <code>.foo.bar..baz.test.....dir.baz....bar</code>, this method will produce <code>/foo/bar.baz/test../dir/baz..bar</code>.
682      */
683     public static String revertExportPath(String exportPath) {
684         if (".".equals(exportPath)) {
685             return "/";
686         }
687 
688         //TODO I have a feeling there's a simpler way to achieve our goal.
689         Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
690 
691         StringBuilder reversed = new StringBuilder(exportPath.length());
692 
693         while (matcher.find()) {
694             String group = matcher.group();
695             int dotsNumber = StringUtils.countMatches(group, ".");
696             if (dotsNumber == 1) {
697                 reversed.append(group.replaceFirst("\\.", "/"));
698             } else {
699                 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
700                 String name = StringUtils.substringAfterLast(group, ".");
701                 reversed.append(dots);
702                 //if number is odd, the last dot has to be replaced with a slash
703                 if (dotsNumber % 2 != 0) {
704                     reversed.append("/");
705                 }
706                 reversed.append(name);
707             }
708         }
709         return reversed.toString();
710     }
711 
712     private static void yaml2Jcr(Node root, Map<?, ?> map) throws RepositoryException {
713         for (Map.Entry entry : map.entrySet()) {
714             String key = String.valueOf(entry.getKey());
715             String nodeType = NodeTypes.ContentNode.NAME;
716             if (entry.getValue() == null || entry.getValue() instanceof Map) {
717                 Map subMap = (Map) entry.getValue();
718                 if (subMap == null) { //an empty node
719                     root.addNode(key, nodeType);
720                 } else {
721                     if (subMap.get(JcrConstants.JCR_PRIMARYTYPE) != null) {
722                         nodeType = (String) subMap.get(JcrConstants.JCR_PRIMARYTYPE);
723                     }
724                     Node node;
725                     if (subMap.get(JcrConstants.JCR_UUID) != null && NodeUtil.unwrap(root) instanceof NodeImpl) {
726                         final Node unwrapped = NodeUtil.unwrap(root);
727                         node = ((NodeImpl) unwrapped).addNodeWithUuid(key, nodeType, String.valueOf(subMap.get(JcrConstants.JCR_UUID)));
728                     } else {
729                         node = root.addNode(key, nodeType);
730                     }
731                     final Object mixins = subMap.get(JcrConstants.JCR_MIXINTYPES);
732                     if (mixins instanceof Collection) {
733                         for (Object mixin : (Collection) mixins) {
734                             node.addMixin(String.valueOf(mixin));
735                         }
736                     }
737                     yaml2Jcr(node, (Map) entry.getValue());
738                 }
739             } else if (!Arrays.asList(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.JCR_UUID, JcrConstants.JCR_MIXINTYPES).contains(key)) {
740                 PropertyUtil.setProperty(root, key, entry.getValue());
741             }
742         }
743     }
744 
745     private static class JcrPropertySupportingConstructor extends Constructor {
746 
747         private JcrPropertySupportingConstructor(ValueFactory valueFactory) {
748             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_PATH)), new JcrConstruct(PropertyType.PATH, valueFactory));
749             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_URI)), new JcrConstruct(PropertyType.URI, valueFactory));
750             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_REFERENCE)), new JcrConstruct(PropertyType.REFERENCE, valueFactory));
751             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_WEAKREFERENCE)), new JcrConstruct(PropertyType.WEAKREFERENCE, valueFactory));
752             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_BINARY)), new JcrConstruct(PropertyType.BINARY, valueFactory));
753         }
754     }
755 
756     private static class JcrConstruct extends AbstractConstruct {
757         private final int propertyType;
758         private final ValueFactory valueFactory;
759 
760         private JcrConstruct(int propertyType, ValueFactory valueFactory) {
761             this.propertyType = propertyType;
762             this.valueFactory = valueFactory;
763         }
764 
765         @Override
766         public Object construct(org.yaml.snakeyaml.nodes.Node node) {
767             try {
768                 return ValueHelper.deserialize(((ScalarNode) node).getValue(), propertyType, false, valueFactory);
769             } catch (RepositoryException e) {
770                 log.error(e.getMessage(), e);
771                 return null;
772             }
773         }
774     }
775 
776 }