View Javadoc
1   /**
2    * This file Copyright (c) 2008-2015 Magnolia International
3    * Ltd.  (http://www.magnolia-cms.com). All rights reserved.
4    *
5    *
6    * This file is dual-licensed under both the Magnolia
7    * Network Agreement and the GNU General Public License.
8    * You may elect to use one or the other of these licenses.
9    *
10   * This file is distributed in the hope that it will be
11   * useful, but AS-IS and WITHOUT ANY WARRANTY; without even the
12   * implied warranty of MERCHANTABILITY or FITNESS FOR A
13   * PARTICULAR PURPOSE, TITLE, or NONINFRINGEMENT.
14   * Redistribution, except as permitted by whichever of the GPL
15   * or MNA you select, is prohibited.
16   *
17   * 1. For the GPL license (GPL), you can redistribute and/or
18   * modify this file under the terms of the GNU General
19   * Public License, Version 3, as published by the Free Software
20   * Foundation.  You should have received a copy of the GNU
21   * General Public License, Version 3 along with this program;
22   * if not, write to the Free Software Foundation, Inc., 51
23   * Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
24   *
25   * 2. For the Magnolia Network Agreement (MNA), this file
26   * and the accompanying materials are made available under the
27   * terms of the MNA which accompanies this distribution, and
28   * is available at http://www.magnolia-cms.com/mna.html
29   *
30   * Any modifications to this file must keep this entire header
31   * intact.
32   *
33   */
34  package info.magnolia.module.rssaggregator.importhandler;
35  
36  import info.magnolia.objectfactory.ComponentProvider;
37  
38  import java.util.ArrayList;
39  import java.util.List;
40  import java.util.Set;
41  import java.util.concurrent.CancellationException;
42  import java.util.concurrent.ExecutionException;
43  import java.util.concurrent.ExecutorService;
44  import java.util.concurrent.Executors;
45  import java.util.concurrent.Future;
46  import java.util.concurrent.TimeUnit;
47  
48  import javax.inject.Inject;
49  
50  import org.slf4j.Logger;
51  import org.slf4j.LoggerFactory;
52  
53  import com.sun.syndication.fetcher.impl.HashMapFeedInfoCache;
54  
55  /**
56   * A multithreaded {@link RSSFeedFetcher} for feed retrieval over <code>http</code> that follows redirects and supports
57   * conditional gets. <p/> Will attempt to fetch all {@link FeedChannel feed channels} defined in the provided {@link
58   * AggregateFeed aggregate feeds}, processing {@value #NUMBER_OF_THREADS} feeds simultaneously. Failures to fetch a feed
59   * result for a given feed channel are logged at ERROR level.
60   *
61   * @author Rob van der Linden Vooren
62   * @see RSSFeedFetcher
63   */
64  public class FastRSSFeedFetcher implements RSSFeedFetcher {
65  
66      private static final Logger log = LoggerFactory.getLogger(FastRSSFeedFetcher.class);
67  
68      private static final int NUMBER_OF_THREADS = 3;
69  
70      private Class<? extends Runnable> feedChannelFetchTask;
71  
72      private int shutdownTimeout = 300000; // in milliseconds
73  
74      private final ExecutorService executorService;
75  
76      private ComponentProvider componentProvider;
77  
78      private ArrayList<Future<?>> futureList = new ArrayList<Future<?>>();
79  
80      @Inject
81      public FastRSSFeedFetcher(ComponentProvider componentProvider) {
82          this.componentProvider = componentProvider;
83          setFeedChannelFetchTask(DefaultFeedChannelFetchTask.class);
84          executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS);
85      }
86  
87      @Override
88      public Set<AggregateFeed> fetchAggregateFeeds(Set<AggregateFeed> aggregateFeeds) {
89          for (AggregateFeed aggregate : aggregateFeeds) {
90              for (FeedChannel channel : aggregate.getChannels()) {
91                  Runnable task = componentProvider.newInstance(getFeedChannelFetchTaskClass(), new MgnlHttpURLFeedFetcher(HashMapFeedInfoCache.getInstance()), channel, aggregate.getName());
92                  Future<?> future = executorService.submit(task);
93                  futureList.add(future);
94              }
95          }
96          // Each Future will block on get() when not done, thus ensuring this method will return only when all
97          // attempts to fetch feed aggregates have completed.
98          for (Future<?> future : futureList) {
99              waitForFutureResult(future);
100         }
101         return aggregateFeeds;
102     }
103 
104     protected void waitForFutureResult(Future<?> future) {
105         try {
106             future.get();
107         } catch (InterruptedException ie) {
108             // Restore the interrupted status
109             Thread.currentThread().interrupt();
110         } catch (ExecutionException ee) {
111             log.error(ee.getMessage(), ee);
112         } catch (CancellationException e) {
113             //set as debug to not flood logs on shutdown
114             log.debug("RSS feed update has been cancelled.");
115         }
116     }
117 
118     /**
119      * All feeds that are being fetched will be shut down.
120      */
121     public void shutdown() {
122         List<Runnable> runnables = executorService.shutdownNow();
123         log.info("Canceling {} RSS Update Tasks because of shutdown...", runnables.size());
124 
125         // stop all running updates (if this is not done scheduler will hang on unfinished job(s) and instance won't shut down)
126         for (Future<?> f : futureList) {
127             f.cancel(true);
128         }
129 
130         try {
131             if (executorService.awaitTermination(getShutdownTimeout(), TimeUnit.MILLISECONDS)) {
132                 log.info("Termination successful.");
133             } else {
134                 log.error("Termination timed out.");
135             }
136         } catch (InterruptedException e) {
137             log.error("Termination error.", e);
138         }
139     }
140 
141     public Class<? extends Runnable> getFeedChannelFetchTaskClass() {
142         return feedChannelFetchTask;
143     }
144 
145     public void setFeedChannelFetchTask(Class<? extends Runnable> feedChannelFetchTask) {
146         this.feedChannelFetchTask = feedChannelFetchTask;
147     }
148 
149     public int getShutdownTimeout() {
150         return shutdownTimeout;
151     }
152 
153     public void setShutdownTimeout(int shutdownTimeout) {
154         this.shutdownTimeout = shutdownTimeout;
155     }
156 
157     protected ExecutorService getExecutorService() {
158         return executorService;
159     }
160 }