View Javadoc
1   /**
2    * This file Copyright (c) 2003-2018 Magnolia International
3    * Ltd.  (http://www.magnolia-cms.com). All rights reserved.
4    *
5    *
6    * This file is dual-licensed under both the Magnolia
7    * Network Agreement and the GNU General Public License.
8    * You may elect to use one or the other of these licenses.
9    *
10   * This file is distributed in the hope that it will be
11   * useful, but AS-IS and WITHOUT ANY WARRANTY; without even the
12   * implied warranty of MERCHANTABILITY or FITNESS FOR A
13   * PARTICULAR PURPOSE, TITLE, or NONINFRINGEMENT.
14   * Redistribution, except as permitted by whichever of the GPL
15   * or MNA you select, is prohibited.
16   *
17   * 1. For the GPL license (GPL), you can redistribute and/or
18   * modify this file under the terms of the GNU General
19   * Public License, Version 3, as published by the Free Software
20   * Foundation.  You should have received a copy of the GNU
21   * General Public License, Version 3 along with this program;
22   * if not, write to the Free Software Foundation, Inc., 51
23   * Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
24   *
25   * 2. For the Magnolia Network Agreement (MNA), this file
26   * and the accompanying materials are made available under the
27   * terms of the MNA which accompanies this distribution, and
28   * is available at http://www.magnolia-cms.com/mna.html
29   *
30   * Any modifications to this file must keep this entire header
31   * intact.
32   *
33   */
34  package info.magnolia.importexport;
35  
36  import static javax.xml.transform.OutputKeys.*;
37  
38  import info.magnolia.cms.beans.runtime.Document;
39  import info.magnolia.cms.core.SystemProperty;
40  import info.magnolia.cms.core.version.VersionManager;
41  import info.magnolia.context.MgnlContext;
42  import info.magnolia.importexport.filters.AccesscontrolNodeFilter;
43  import info.magnolia.importexport.filters.ImportXmlRootFilter;
44  import info.magnolia.importexport.filters.MagnoliaV2Filter;
45  import info.magnolia.importexport.filters.MetadataUuidFilter;
46  import info.magnolia.importexport.filters.RemoveMixversionableFilter;
47  import info.magnolia.importexport.filters.VersionFilter;
48  import info.magnolia.importexport.postprocessors.ActivationStatusImportPostProcessor;
49  import info.magnolia.importexport.postprocessors.MetaDataImportPostProcessor;
50  import info.magnolia.importexport.postprocessors.UpdateVersionMixinPostProcessor;
51  import info.magnolia.jcr.RuntimeRepositoryException;
52  import info.magnolia.jcr.util.NodeTypes;
53  import info.magnolia.jcr.util.NodeUtil;
54  import info.magnolia.jcr.util.PropertyUtil;
55  import info.magnolia.jcr.util.SessionUtil;
56  import info.magnolia.objectfactory.Components;
57  
58  import java.io.File;
59  import java.io.FileInputStream;
60  import java.io.FileNotFoundException;
61  import java.io.FileOutputStream;
62  import java.io.IOException;
63  import java.io.InputStream;
64  import java.io.InputStreamReader;
65  import java.io.OutputStream;
66  import java.io.UnsupportedEncodingException;
67  import java.net.URLDecoder;
68  import java.net.URLEncoder;
69  import java.nio.charset.StandardCharsets;
70  import java.util.Arrays;
71  import java.util.Collection;
72  import java.util.HashMap;
73  import java.util.Iterator;
74  import java.util.List;
75  import java.util.Map;
76  import java.util.Properties;
77  import java.util.regex.Matcher;
78  import java.util.regex.Pattern;
79  import java.util.zip.DeflaterOutputStream;
80  import java.util.zip.GZIPInputStream;
81  import java.util.zip.GZIPOutputStream;
82  import java.util.zip.ZipInputStream;
83  import java.util.zip.ZipOutputStream;
84  
85  import javax.jcr.ImportUUIDBehavior;
86  import javax.jcr.Node;
87  import javax.jcr.NodeIterator;
88  import javax.jcr.PropertyType;
89  import javax.jcr.RepositoryException;
90  import javax.jcr.Session;
91  import javax.jcr.ValueFactory;
92  import javax.xml.parsers.ParserConfigurationException;
93  import javax.xml.parsers.SAXParser;
94  import javax.xml.parsers.SAXParserFactory;
95  import javax.xml.transform.Source;
96  import javax.xml.transform.Transformer;
97  import javax.xml.transform.TransformerConfigurationException;
98  import javax.xml.transform.sax.SAXTransformerFactory;
99  import javax.xml.transform.sax.TransformerHandler;
100 import javax.xml.transform.stream.StreamResult;
101 import javax.xml.transform.stream.StreamSource;
102 
103 import org.apache.commons.io.IOUtils;
104 import org.apache.commons.lang3.StringUtils;
105 import org.apache.jackrabbit.JcrConstants;
106 import org.apache.jackrabbit.core.NodeImpl;
107 import org.apache.jackrabbit.value.ValueHelper;
108 import org.slf4j.Logger;
109 import org.slf4j.LoggerFactory;
110 import org.xml.sax.ContentHandler;
111 import org.xml.sax.InputSource;
112 import org.xml.sax.SAXException;
113 import org.xml.sax.XMLFilter;
114 import org.xml.sax.XMLReader;
115 import org.yaml.snakeyaml.Yaml;
116 import org.yaml.snakeyaml.constructor.AbstractConstruct;
117 import org.yaml.snakeyaml.constructor.Constructor;
118 import org.yaml.snakeyaml.nodes.NodeId;
119 import org.yaml.snakeyaml.nodes.ScalarNode;
120 import org.yaml.snakeyaml.nodes.Tag;
121 
122 /**
123  * Utility class for manipulation of XML/YAML files (mainly JCR XML).
124  */
125 public class DataTransporter {
126 
127     private static final Pattern DOT_NAME_PATTERN = Pattern.compile("[\\w\\-]*\\.*[\\w\\-]*");
128 
129     private static final int INDENT_VALUE = 2;
130 
131     private static Logger log = LoggerFactory.getLogger(DataTransporter.class.getName());
132 
133     final static int BOOTSTRAP_IMPORT_MODE = ImportUUIDBehavior.IMPORT_UUID_COLLISION_REPLACE_EXISTING;
134 
135     public static final String ZIP = ".zip";
136 
137     public static final String GZ = ".gz";
138 
139     public static final String XML = ".xml";
140 
141     public static final String YAML = ".yaml";
142 
143     public static final String PROPERTIES = ".properties";
144 
145     public static final String DOT = ".";
146 
147     public static final String SLASH = "/";
148 
149     public static final String UTF8 = "UTF-8";
150 
151     public static final String JCR_ROOT = "jcr:root";
152 
153     /**
154      * Converts a xml/yaml document into a file.
155      *
156      * @param bsDocument uploaded file
157      * @param repositoryName selected repository
158      * @param basepath base path in repository
159      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
160      * @param importMode a valid value for ImportUUIDBehavior
161      * @see ImportUUIDBehavior
162      */
163     public static synchronized void importDocument(Document bsDocument, String repositoryName, String basepath,
164                                                    boolean keepVersionHistory, int importMode, boolean saveAfterImport,
165                                                    boolean createBasepathIfNotExist)
166             throws IOException {
167         File bsFile = bsDocument.getFile();
168         importFile(bsFile, repositoryName, basepath, keepVersionHistory, importMode, saveAfterImport,
169                 createBasepathIfNotExist);
170     }
171 
172     /**
173      * Creates an <code>InputStream</code> backed by the specified xml/yaml file.
174      *
175      * @param bsFile (zipped/gzipped) XML/YAML file to import
176      * @param repositoryName selected repository
177      * @param basepath base path in repository
178      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
179      * @param importMode a valid value for ImportUUIDBehavior
180      * @see ImportUUIDBehavior
181      */
182     public static synchronized void importFile(File bsFile, String repositoryName, String basepath,
183                                                boolean keepVersionHistory, int importMode, boolean saveAfterImport,
184                                                boolean createBasepathIfNotExist)
185             throws IOException {
186         String name = bsFile.getAbsolutePath();
187 
188         InputStream bsStream = getInputStreamForFile(bsFile);
189         if (bsFile.getName().endsWith(YAML)) {
190             importYamlStream(bsStream, null, repositoryName, basepath, null, name, saveAfterImport, saveAfterImport, createBasepathIfNotExist);
191         } else { // bsFile.getName().endsWith(XML)
192             importXmlStream(bsStream, repositoryName, basepath, name, keepVersionHistory, importMode, saveAfterImport, createBasepathIfNotExist);
193         }
194     }
195 
196     public static void executeBootstrapImport(File bsFile, String repositoryName) throws IOException {
197         String filenameWithoutExt = StringUtils.substringBeforeLast(bsFile.getName(), DOT);
198         if (filenameWithoutExt.endsWith(XML) || filenameWithoutExt.endsWith(YAML)) {
199             // if file ends in .xml.gz, .xml.zip, .yaml.gz or .yaml.zip
200             // need to keep the .xml/.yaml to be able to view it after decompression
201             filenameWithoutExt = StringUtils.substringBeforeLast(bsFile.getName(), DOT);
202         }
203         String pathName = StringUtils.substringAfter(StringUtils.substringBeforeLast(filenameWithoutExt, DOT), DOT);
204 
205         pathName = decodePath(pathName, UTF8);
206 
207         String basepath = SLASH + StringUtils.replace(pathName, DOT, SLASH);
208 
209         if (bsFile.getName().endsWith(PROPERTIES)) {
210             Properties properties = new Properties();
211             FileInputStream stream = new FileInputStream(bsFile);
212             try {
213                 properties.load(stream);
214             } finally {
215                 IOUtils.closeQuietly(stream);
216             }
217             importProperties(properties, repositoryName);
218         } else {
219             DataTransporter.importFile(bsFile, repositoryName, basepath, false, BOOTSTRAP_IMPORT_MODE, true, true);
220         }
221     }
222 
223     /**
224      * @deprecated since 4.0 - use the PropertiesImportExport class instead.
225      */
226     @Deprecated
227     public static void importProperties(Properties properties, String repositoryName) {
228         for (Iterator iter = properties.keySet().iterator(); iter.hasNext(); ) {
229             String key = (String) iter.next();
230             String value = (String) properties.get(key);
231 
232             String name = StringUtils.substringAfterLast(key, ".");
233             String path = StringUtils.substringBeforeLast(key, ".").replace('.', '/');
234             Node node = SessionUtil.getNode(repositoryName, path);
235             if (node != null) {
236                 try {
237                     node.setProperty(name, value);
238                     node.getSession().save();
239                 } catch (RepositoryException e) {
240                     log.error("can't set property {}", key, e);
241                 }
242             }
243         }
244 
245     }
246 
247     /**
248      * Imports XML stream into repository.
249      * XML is filtered by <code>MagnoliaV2Filter</code>, <code>VersionFilter</code> and <code>ImportXmlRootFilter</code>
250      * if <code>keepVersionHistory</code> is set to <code>false</code>
251      *
252      * @param xmlStream XML stream to import
253      * @param repositoryName selected repository
254      * @param basepath base path in repository
255      * @param name (absolute path of <code>File</code>)
256      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
257      * @param importMode a valid value for ImportUUIDBehavior
258      * @see ImportUUIDBehavior
259      * @see ImportXmlRootFilter
260      * @see VersionFilter
261      * @see MagnoliaV2Filter
262      */
263     public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
264                                                     String name, boolean keepVersionHistory, int importMode,
265                                                     boolean saveAfterImport, boolean createBasepathIfNotExist)
266             throws IOException {
267         importXmlStream(xmlStream, repositoryName, basepath, name, keepVersionHistory, false, importMode, saveAfterImport, createBasepathIfNotExist);
268 
269     }
270 
271     /**
272      * Imports XML stream into repository.
273      * XML is filtered by <code>MagnoliaV2Filter</code>, <code>VersionFilter</code> and <code>ImportXmlRootFilter</code> if <code>keepVersionHistory</code> is set to <code>false</code>
274      *
275      * @param xmlStream XML stream to import
276      * @param repositoryName selected repository
277      * @param basepath base path in repository
278      * @param name (absolute path of <code>File</code>)
279      * @param keepVersionHistory if <code>false</code> version info will be stripped before importing the document
280      * @param forceUnpublishState if <code>true</code> then activation state of node will be change to unpublished.
281      * @param importMode a valid value for ImportUUIDBehavior
282      * @see ImportUUIDBehavior
283      * @see ImportXmlRootFilter
284      * @see VersionFilter
285      * @see MagnoliaV2Filter
286      */
287     public static synchronized void importXmlStream(InputStream xmlStream, String repositoryName, String basepath,
288                                                     String name, boolean keepVersionHistory, boolean forceUnpublishState, int importMode,
289                                                     boolean saveAfterImport, boolean createBasepathIfNotExist)
290             throws IOException {
291 
292         // TODO hopefully this will be fixed with a more useful message with the Bootstrapper refactoring
293         if (xmlStream == null) {
294             throw new IOException("Can't import a null stream into repository: " + repositoryName + ", basepath: " + basepath + ", name: " + name);
295         }
296         try {
297             Session session = MgnlContext.getJCRSession(repositoryName);
298 
299             log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
300 
301             if (!session.nodeExists(basepath) && createBasepathIfNotExist) {
302                 try {
303                     NodeUtil.createPath(session.getRootNode(), basepath, NodeTypes.Content.NAME);
304                 } catch (RepositoryException e) {
305                     log.error("can't create path [{}]", basepath);
306                 }
307             }
308 
309             // Collects a list with all nodes at the basepath before import so we can see exactly which nodes were imported afterwards
310             List<Node> nodesBeforeImport = NodeUtil.asList(NodeUtil.asIterable(session.getNode(basepath).getNodes()));
311 
312             if (keepVersionHistory) {
313                 // do not manipulate
314                 session.importXML(basepath, xmlStream, importMode);
315             } else {
316                 // create readers/filters and chain
317                 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
318                 saxParserFactory.setNamespaceAware(true);
319                 SAXParser saxParser = saxParserFactory.newSAXParser();
320                 XMLReader initialReader = saxParser.getXMLReader();
321                 try {
322                     initialReader.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
323                 } catch (SAXException e) {
324                     log.error("could not set parser feature");
325                 }
326 
327 
328                 XMLFilter magnoliaV2Filter = null;
329 
330                 // if stream is from regular file, test for belonging XSL file to apply XSL transformation to XML
331                 if (new File(name).isFile()) {
332                     InputStream xslStream = getXslStreamForXmlFile(new File(name));
333                     if (xslStream != null) {
334                         Source xslSource = new StreamSource(xslStream);
335                         SAXTransformerFactory saxTransformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
336                         XMLFilter xslFilter = saxTransformerFactory.newXMLFilter(xslSource);
337                         magnoliaV2Filter = new MagnoliaV2Filter(xslFilter);
338                     }
339                 }
340 
341                 if (magnoliaV2Filter == null) {
342                     magnoliaV2Filter = new MagnoliaV2Filter(initialReader);
343                 }
344 
345                 XMLFilter versionFilter = new VersionFilter(magnoliaV2Filter);
346 
347                 // enable this to strip useless "name" properties from dialogs
348                 // versionFilter = new UselessNameFilter(versionFilter);
349 
350                 // enable this to strip mix:versionable from pre 3.6 xml files
351                 versionFilter = new RemoveMixversionableFilter(versionFilter);
352 
353                 // strip rep:accesscontrol node
354                 versionFilter = new AccesscontrolNodeFilter(versionFilter);
355 
356                 XMLReader finalReader = new ImportXmlRootFilter(versionFilter);
357 
358                 ContentHandler handler = session.getImportContentHandler(basepath, importMode);
359                 finalReader.setContentHandler(handler);
360 
361                 // parse XML, import is done by handler from session
362                 try {
363                     finalReader.parse(new InputSource(xmlStream));
364                 } finally {
365                     IOUtils.closeQuietly(xmlStream);
366                 }
367 
368                 if (((ImportXmlRootFilter) finalReader).rootNodeFound) {
369                     String path = basepath;
370                     if (!path.endsWith(SLASH)) {
371                         path += SLASH;
372                     }
373 
374                     Node dummyRoot = (Node) session.getItem(path + JCR_ROOT);
375                     for (Iterator iter = dummyRoot.getNodes(); iter.hasNext(); ) {
376                         Node child = (Node) iter.next();
377                         // move childs to real root
378 
379                         if (session.itemExists(path + child.getName())) {
380                             session.getItem(path + child.getName()).remove();
381                         }
382 
383                         session.move(child.getPath(), path + child.getName());
384                     }
385                     // delete the dummy node
386                     dummyRoot.remove();
387                 }
388 
389                 // Post process all nodes that were imported
390                 VersionManager versionManager = Components.getComponent(VersionManager.class);
391                 NodeIterator nodesAfterImport = session.getNode(basepath).getNodes();
392                 while (nodesAfterImport.hasNext()) {
393                     Node nodeAfterImport = nodesAfterImport.nextNode();
394                     boolean existedBeforeImport = false;
395                     for (Node nodeBeforeImport : nodesBeforeImport) {
396                         if (NodeUtil.isSame(nodeAfterImport, nodeBeforeImport)) {
397                             existedBeforeImport = true;
398                             break;
399                         }
400                     }
401                     if (!existedBeforeImport) {
402                         postProcessAfterImport(nodeAfterImport, forceUnpublishState, importMode, versionManager);
403                     }
404                 }
405             }
406             if (saveAfterImport) {
407                 session.save();
408             }
409         } catch (Exception e) {
410             throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
411         } finally {
412             IOUtils.closeQuietly(xmlStream);
413         }
414     }
415 
416     /**
417      * Imports YAML stream into repository.
418      *
419      * @param yamlStream YAML stream to import
420      * @param importPath sub path in YAML file
421      * @param repositoryName selected repository
422      * @param basepath base path in repository
423      * @param name (absolute path of <code>File</code>)
424      * @param forceUnpublishState if <code>true</code> then activation state of node will be change to unpublished.
425      * @param saveAfterImport saves session after import
426      *
427      */
428     public static synchronized void importYamlStream(InputStream yamlStream, Node node, String repositoryName, String basepath, String importPath,
429                                                     String name, boolean forceUnpublishState, boolean saveAfterImport, boolean createBasepathIfNotExist) {
430 
431         try {
432             log.debug("Importing content into repository: [{}] from: [{}] into path: [{}]", repositoryName, name, basepath);
433 
434             node = getNodeToImportTo(node, repositoryName, basepath, createBasepathIfNotExist);
435             final ValueFactory valueFactory = node.getSession().getValueFactory();
436 
437 
438             final InputStreamReader inputStream = new InputStreamReader(yamlStream, StandardCharsets.UTF_8);
439             final Yaml yaml = new Yaml(new JcrPropertySupportingConstructor(valueFactory));
440             Map content = yaml.load(inputStream);
441             if (importPath != null) {
442                 final String[] path = StringUtils.split(importPath, "/");
443                 for (String nodeName : path) {
444                     content = (Map) content.get(nodeName);
445                 }
446                 final Map<String, Object> wrappedTarget = new HashMap<>();
447                 wrappedTarget.put(path[path.length - 1], content);
448                 content = wrappedTarget;
449             }
450             yaml2Jcr(node, content);
451             if (saveAfterImport) {
452                 node.getSession().save();
453             }
454             if (forceUnpublishState) {
455                 new ActivationStatusImportPostProcessor().postProcessNode(node);
456             }
457         } catch (RepositoryException e) {
458             throw new RuntimeException("Error importing " + name + ": " + e.getMessage(), e);
459         }
460     }
461 
462     private static Node getNodeToImportTo(Node node, String repositoryName, String basepath, boolean createBasepathIfNotExist) throws RepositoryException {
463         if (node == null) {
464             Session session = MgnlContext.getJCRSession(repositoryName);
465             if (!session.nodeExists(basepath) && createBasepathIfNotExist) {
466                 try {
467                     NodeUtil.createPath(session.getRootNode(), basepath, NodeTypes.Content.NAME);
468                 } catch (RepositoryException e) {
469                     log.error("can't create path [{}]", basepath);
470                 }
471             }
472             node = session.getNode(basepath);
473         }
474         return node;
475     }
476 
477     private static void postProcessAfterImport(Node node, boolean forceUnpublishState, int importMode, VersionManager versionManager) throws RepositoryException {
478         try {
479             new MetaDataImportPostProcessor().postProcessNode(node);
480             if (forceUnpublishState) {
481                 new ActivationStatusImportPostProcessor().postProcessNode(node);
482             }
483             new UpdateVersionMixinPostProcessor(importMode, versionManager).postProcessNode(node);
484         } catch (RepositoryException e) {
485             throw new RepositoryException("Failed to post process imported nodes at path " + NodeUtil.getNodePathIfPossible(node) + ": " + e.getMessage(), e);
486         }
487     }
488 
489     /**
490      * @return XSL stream for Xml file or <code>null</code>
491      */
492     protected static InputStream getXslStreamForXmlFile(File file) {
493         InputStream xslStream = null;
494         String xlsFilename = StringUtils.substringBeforeLast(file.getAbsolutePath(), ".") + ".xsl";
495         Filee/File.html#File">File xslFile = new File(xlsFilename);
496         if (xslFile.exists()) {
497             try {
498                 xslStream = new FileInputStream(xslFile);
499                 log.info("XSL file for [{}] found ({})", file.getName(), xslFile.getName());
500             } catch (FileNotFoundException e) { // should never happen (xslFile.exists())
501                 e.printStackTrace();
502             }
503         }
504         return xslStream;
505     }
506 
507     /**
508      * Creates a stream from the (zipped/gzipped) XML/YAML file.
509      *
510      * @return stream of the file
511      */
512     public static InputStream getInputStreamForFile(File bsFile) throws IOException {
513         InputStream bsStream;
514         // looks like the zip one is buggy. It throws exception when trying to use it
515         if (bsFile.getName().endsWith(ZIP)) {
516             bsStream = new ZipInputStream((new FileInputStream(bsFile)));
517         } else if (bsFile.getName().endsWith(GZ)) {
518             bsStream = new GZIPInputStream((new FileInputStream(bsFile)));
519         } else { // if(fileName.endsWith(XML) || fileName.endsWith(YAML))
520             bsStream = new FileInputStream(bsFile);
521         }
522         return bsStream;
523     }
524 
525     public static void executeExport(OutputStream baseOutputStream, boolean keepVersionHistory, boolean format,
526                                      Session session, String basepath, String repository, String ext) throws IOException {
527         OutputStream outputStream = baseOutputStream;
528         if (ext.endsWith(ZIP)) {
529             outputStream = new ZipOutputStream(baseOutputStream);
530         } else if (ext.endsWith(GZ)) {
531             outputStream = new GZIPOutputStream(baseOutputStream);
532         }
533 
534         try {
535             if (keepVersionHistory) {
536                 // use exportSystemView in order to preserve property types
537                 // http://issues.apache.org/jira/browse/JCR-115
538                 if (!format) {
539                     session.exportSystemView(basepath, outputStream, false, false);
540                 } else {
541                     parseAndFormat(outputStream, null, repository, basepath, session, false);
542                 }
543             } else {
544                 SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
545                 saxParserFactory.setNamespaceAware(true);
546                 SAXParser saxParser = saxParserFactory.newSAXParser();
547                 XMLReader reader = saxParser.getXMLReader();
548 
549                 reader = new AccesscontrolNodeFilter(reader);
550                 parseAndFormat(outputStream, reader, repository, basepath, session, false);
551             }
552         } catch (IOException | SAXException | ParserConfigurationException | TransformerConfigurationException e) {
553             throw new RuntimeException(e);
554         } catch (RepositoryException e) {
555             throw new RuntimeRepositoryException(e);
556         }
557 
558         // finish the stream properly if zip stream
559         // this is not done by the IOUtils
560         if (outputStream instanceof DeflaterOutputStream) {
561             ((DeflaterOutputStream) outputStream).finish();
562         }
563 
564         baseOutputStream.flush();
565         IOUtils.closeQuietly(baseOutputStream);
566     }
567 
568     /**
569      * Exports the content of the repository, and format it if necessary.
570      *
571      * @param stream the stream to write the content to
572      * @param reader the reader to use to parse the xml content (so that we can perform filtering), if null instanciate
573      * a default one
574      * @param repository the repository to export
575      * @param basepath the basepath in the repository
576      * @param session the session to use to export the data from the repository
577      */
578     public static void parseAndFormat(OutputStream stream, XMLReader reader, String repository, String basepath,
579                                       Session session, boolean noRecurse)
580             throws IOException, SAXException, ParserConfigurationException, TransformerConfigurationException, RepositoryException {
581 
582         if (reader == null) {
583             SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
584             saxParserFactory.setNamespaceAware(true);
585             SAXParser saxParser = saxParserFactory.newSAXParser();
586             reader = saxParser.getXMLReader();
587         }
588 
589         // write to a temp file and then re-read it to remove version history
590         File tempFile = File.createTempFile("export-" + repository + session.getUserID(), ".xml");
591         OutputStream fileStream = new FileOutputStream(tempFile);
592 
593         try {
594             session.exportSystemView(basepath, fileStream, false, noRecurse);
595         } finally {
596             IOUtils.closeQuietly(fileStream);
597         }
598 
599         readFormatted(reader, tempFile, stream);
600 
601         if (!tempFile.delete()) {
602             log.warn("Could not delete temporary export file {}", tempFile.getAbsolutePath());
603         }
604     }
605 
606     protected static void readFormatted(XMLReader reader, File inputFile, OutputStream outputStream)
607             throws IOException, SAXException, TransformerConfigurationException {
608         InputStream fileInputStream = new FileInputStream(inputFile);
609         readFormatted(reader, fileInputStream, outputStream);
610         IOUtils.closeQuietly(fileInputStream);
611     }
612 
613     protected static void readFormatted(XMLReader reader, InputStream inputStream, OutputStream outputStream)
614             throws IOException, SAXException, TransformerConfigurationException {
615 
616         SAXTransformerFactory transformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
617         TransformerHandler transformerHandler = transformerFactory.newTransformerHandler();
618         Transformer transformer = transformerHandler.getTransformer();
619         transformer.setOutputProperty(INDENT, "yes");
620         transformer.setOutputProperty(DOCTYPE_PUBLIC, "yes"); // effectively insert a line-break after the xml declaration
621         transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", String.valueOf(INDENT_VALUE));
622         transformerHandler.setResult(new StreamResult(outputStream));
623 
624         final boolean removeUnwantedNamespaces = !SystemProperty.getBooleanProperty("magnolia.export.keep_extra_namespaces"); // MAGNOLIA-2960
625         MetadataUuidFiltertml#MetadataUuidFilter">MetadataUuidFilter metadataUuidFilter = new MetadataUuidFilter(reader, removeUnwantedNamespaces); // MAGNOLIA-1650
626         metadataUuidFilter.setContentHandler(transformerHandler);
627         metadataUuidFilter.parse(new InputSource(inputStream));
628 
629         IOUtils.closeQuietly(inputStream);
630     }
631 
632     /**
633      * @param path path to encode
634      * @param separator "." (dot) or "/", it will be not encoded if found
635      * @param enc charset
636      * @return the path encoded
637      */
638     public static String encodePath(String path, String separator, String enc) {
639         StringBuilder pathEncoded = new StringBuilder();
640         try {
641             if (!StringUtils.contains(path, separator)) {
642                 return URLEncoder.encode(path, enc);
643             }
644             for (int i = 0; i < path.length(); i++) {
645                 String ch = String.valueOf(path.charAt(i));
646                 if (separator.equals(ch)) {
647                     pathEncoded.append(ch);
648                 } else {
649                     pathEncoded.append(URLEncoder.encode(ch, enc));
650                 }
651             }
652         } catch (UnsupportedEncodingException e) {
653             return path;
654         }
655         return pathEncoded.toString();
656     }
657 
658     /**
659      * decode a path (ex. %D0%9D%D0%B0.%B2%D0%BE%D0%BB%D0%BD)
660      *
661      * @param path path to decode
662      * @param enc charset
663      * @return the path decoded
664      */
665     public static String decodePath(String path, String enc) {
666         try {
667             return URLDecoder.decode(path, enc);
668         } catch (UnsupportedEncodingException e) {
669             return path;
670         }
671     }
672 
673     /**
674      * Prior to 4.5 Magnolia used to produce export xml filenames where the / (slash) separating sub nodes was replaced by a dot.
675      * Since 4.5, Magnolia enables dots in path names, therefore dots which are part of the node name have to be escaped by doubling them.
676      * I.e. given a path like this <code>/foo/bar.baz/test../dir/baz..bar</code>, this method will produce
677      * <code>.foo.bar..baz.test.....dir.baz....bar</code>.
678      */
679     public static String createExportPath(String path) {
680         //TODO if someone is smarter than me (not an impossible thing) and can do this with one single elegant regex, please do it.
681         String newPath = path.replace(".", "..");
682         newPath = newPath.replace("/", ".");
683         return newPath;
684     }
685 
686     /**
687      * The opposite of {@link #createExportPath(String)}.
688      * I.e. given a path like this <code>.foo.bar..baz.test.....dir.baz....bar</code>, this method will produce <code>/foo/bar.baz/test../dir/baz..bar</code>.
689      */
690     public static String revertExportPath(String exportPath) {
691         if (".".equals(exportPath)) {
692             return "/";
693         }
694 
695         //TODO I have a feeling there's a simpler way to achieve our goal.
696         Matcher matcher = DOT_NAME_PATTERN.matcher(exportPath);
697 
698         StringBuilder reversed = new StringBuilder(exportPath.length());
699 
700         while (matcher.find()) {
701             String group = matcher.group();
702             int dotsNumber = StringUtils.countMatches(group, ".");
703             if (dotsNumber == 1) {
704                 reversed.append(group.replaceFirst("\\.", "/"));
705             } else {
706                 String dots = StringUtils.substringBeforeLast(group, ".").replace("..", ".");
707                 String name = StringUtils.substringAfterLast(group, ".");
708                 reversed.append(dots);
709                 //if number is odd, the last dot has to be replaced with a slash
710                 if (dotsNumber % 2 != 0) {
711                     reversed.append("/");
712                 }
713                 reversed.append(name);
714             }
715         }
716         return reversed.toString();
717     }
718 
719     private static void yaml2Jcr(Node root, Map<?, ?> map) throws RepositoryException {
720         for (Map.Entry entry : map.entrySet()) {
721             String key = String.valueOf(entry.getKey());
722             String nodeType = NodeTypes.ContentNode.NAME;
723             if (entry.getValue() == null || entry.getValue() instanceof Map) {
724                 Map subMap = (Map) entry.getValue();
725                 if (subMap == null) { //an empty node
726                     root.addNode(key, nodeType);
727                 } else {
728                     if (subMap.get(JcrConstants.JCR_PRIMARYTYPE) != null) {
729                         nodeType = (String) subMap.get(JcrConstants.JCR_PRIMARYTYPE);
730                     }
731                     Node node;
732                     if (subMap.get(JcrConstants.JCR_UUID) != null && NodeUtil.unwrap(root) instanceof NodeImpl) {
733                         final Node unwrapped = NodeUtil.unwrap(root);
734                         node = ((NodeImpl) unwrapped).addNodeWithUuid(key, nodeType, String.valueOf(subMap.get(JcrConstants.JCR_UUID)));
735                     } else {
736                         node = root.addNode(key, nodeType);
737                     }
738                     final Object mixins = subMap.get(JcrConstants.JCR_MIXINTYPES);
739                     if (mixins instanceof Collection) {
740                         for (Object mixin : (Collection) mixins) {
741                             node.addMixin(String.valueOf(mixin));
742                         }
743                     }
744                     yaml2Jcr(node, (Map) entry.getValue());
745                 }
746             } else if (!Arrays.asList(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.JCR_UUID, JcrConstants.JCR_MIXINTYPES).contains(key)) {
747                 PropertyUtil.setProperty(root, key, entry.getValue());
748             }
749         }
750     }
751 
752     private static class JcrPropertySupportingConstructor extends Constructor {
753 
754         private JcrPropertySupportingConstructor(ValueFactory valueFactory) {
755             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_PATH)), new JcrConstruct(PropertyType.PATH, valueFactory));
756             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_URI)), new JcrConstruct(PropertyType.URI, valueFactory));
757             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_REFERENCE)), new JcrConstruct(PropertyType.REFERENCE, valueFactory));
758             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_WEAKREFERENCE)), new JcrConstruct(PropertyType.WEAKREFERENCE, valueFactory));
759             yamlConstructors.put(new Tag("!" + StringUtils.lowerCase(PropertyType.TYPENAME_BINARY)), new JcrConstruct(PropertyType.BINARY, valueFactory));
760 
761             //Prevents sequence node being loaded, see more in MAGNOLIA-7914.
762             yamlClassConstructors.remove(NodeId.sequence);
763         }
764     }
765 
766     private static class JcrConstruct extends AbstractConstruct {
767         private final int propertyType;
768         private final ValueFactory valueFactory;
769 
770         private JcrConstruct(int propertyType, ValueFactory valueFactory) {
771             this.propertyType = propertyType;
772             this.valueFactory = valueFactory;
773         }
774 
775         @Override
776         public Object construct(org.yaml.snakeyaml.nodes.Node node) {
777             try {
778                 return ValueHelper.deserialize(((ScalarNode) node).getValue(), propertyType, false, valueFactory);
779             } catch (RepositoryException e) {
780                 log.error(e.getMessage(), e);
781                 return null;
782             }
783         }
784     }
785 
786 }