View Javadoc

1   /*
2    * Copyright 2012 The Netty Project
3    *
4    * The Netty Project licenses this file to you under the Apache License,
5    * version 2.0 (the "License"); you may not use this file except in compliance
6    * with the License. You may obtain a copy of the License at:
7    *
8    *   http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13   * License for the specific language governing permissions and limitations
14   * under the License.
15   */
16  package org.jboss.netty.channel.socket.nio;
17  
18  import org.jboss.netty.channel.Channel;
19  import org.jboss.netty.channel.ChannelException;
20  import org.jboss.netty.channel.ChannelFuture;
21  import org.jboss.netty.logging.InternalLogger;
22  import org.jboss.netty.logging.InternalLoggerFactory;
23  import org.jboss.netty.util.ThreadNameDeterminer;
24  import org.jboss.netty.util.ThreadRenamingRunnable;
25  import org.jboss.netty.util.internal.DeadLockProofWorker;
26  
27  import java.io.IOException;
28  import java.nio.channels.CancelledKeyException;
29  import java.nio.channels.DatagramChannel;
30  import java.nio.channels.SelectableChannel;
31  import java.nio.channels.SelectionKey;
32  import java.nio.channels.Selector;
33  import java.nio.channels.SocketChannel;
34  import java.util.ConcurrentModificationException;
35  import java.util.Queue;
36  import java.util.concurrent.ConcurrentLinkedQueue;
37  import java.util.concurrent.CountDownLatch;
38  import java.util.concurrent.Executor;
39  import java.util.concurrent.RejectedExecutionException;
40  import java.util.concurrent.atomic.AtomicBoolean;
41  import java.util.concurrent.atomic.AtomicInteger;
42  
43  abstract class AbstractNioSelector implements NioSelector {
44  
45      private static final AtomicInteger nextId = new AtomicInteger();
46  
47      private final int id = nextId.incrementAndGet();
48  
49      /**
50       * Internal Netty logger.
51       */
52      protected static final InternalLogger logger = InternalLoggerFactory
53              .getInstance(AbstractNioSelector.class);
54  
55      private static final int CLEANUP_INTERVAL = 256; // XXX Hard-coded value, but won't need customization.
56  
57      /**
58       * Executor used to execute {@link Runnable}s such as channel registration
59       * task.
60       */
61      private final Executor executor;
62  
63      /**
64       * If this worker has been started thread will be a reference to the thread
65       * used when starting. i.e. the current thread when the run method is executed.
66       */
67      protected volatile Thread thread;
68  
69      /**
70       * The NIO {@link Selector}.
71       */
72      protected volatile Selector selector;
73  
74      /**
75       * Boolean that controls determines if a blocked Selector.select should
76       * break out of its selection process. In our case we use a timeone for
77       * the select method and the select method will block for that time unless
78       * waken up.
79       */
80      protected final AtomicBoolean wakenUp = new AtomicBoolean();
81  
82      private final Queue<Runnable> taskQueue = new ConcurrentLinkedQueue<Runnable>();
83  
84      private volatile int cancelledKeys; // should use AtomicInteger but we just need approximation
85  
86      private final CountDownLatch shutdownLatch = new CountDownLatch(1);
87      private volatile boolean shutdown;
88  
89      AbstractNioSelector(Executor executor) {
90          this(executor, null);
91      }
92  
93      AbstractNioSelector(Executor executor, ThreadNameDeterminer determiner) {
94          this.executor = executor;
95          openSelector(determiner);
96      }
97  
98      public void register(Channel channel, ChannelFuture future) {
99          Runnable task = createRegisterTask(channel, future);
100         registerTask(task);
101     }
102 
103     protected final void registerTask(Runnable task) {
104         taskQueue.add(task);
105 
106         Selector selector = this.selector;
107 
108         if (selector != null) {
109             if (wakenUp.compareAndSet(false, true)) {
110                 selector.wakeup();
111             }
112         } else {
113             if (taskQueue.remove(task)) {
114                 // the selector was null this means the Worker has already been shutdown.
115                 throw new RejectedExecutionException("Worker has already been shutdown");
116             }
117         }
118     }
119 
120     protected final boolean isIoThread() {
121         return Thread.currentThread() == thread;
122     }
123 
124     public void rebuildSelector() {
125         if (!isIoThread()) {
126             taskQueue.add(new Runnable() {
127                 public void run() {
128                     rebuildSelector();
129                 }
130             });
131             return;
132         }
133 
134         final Selector oldSelector = selector;
135         final Selector newSelector;
136 
137         if (oldSelector == null) {
138             return;
139         }
140 
141         try {
142             newSelector = Selector.open();
143         } catch (Exception e) {
144             logger.warn("Failed to create a new Selector.", e);
145             return;
146         }
147 
148         // Register all channels to the new Selector.
149         int nChannels = 0;
150         for (;;) {
151             try {
152                 for (SelectionKey key: oldSelector.keys()) {
153                     try {
154                         if (key.channel().keyFor(newSelector) != null) {
155                             continue;
156                         }
157 
158                         int interestOps = key.interestOps();
159                         key.cancel();
160                         key.channel().register(newSelector, interestOps, key.attachment());
161                         nChannels ++;
162                     } catch (Exception e) {
163                         logger.warn("Failed to re-register a Channel to the new Selector,", e);
164                         close(key);
165                     }
166                 }
167             } catch (ConcurrentModificationException e) {
168                 // Probably due to concurrent modification of the key set.
169                 continue;
170             }
171 
172             break;
173         }
174 
175         selector = newSelector;
176 
177         try {
178             // time to close the old selector as everything else is registered to the new one
179             oldSelector.close();
180         } catch (Throwable t) {
181             if (logger.isWarnEnabled()) {
182                 logger.warn("Failed to close the old Selector.", t);
183             }
184         }
185 
186         logger.info("Migrated " + nChannels + " channel(s) to the new Selector,");
187     }
188 
189     public void run() {
190         thread = Thread.currentThread();
191 
192         int selectReturnsImmediately = 0;
193         Selector selector = this.selector;
194 
195         if (selector == null) {
196             return;
197         }
198         // use 80% of the timeout for measure
199         final long minSelectTimeout = SelectorUtil.SELECT_TIMEOUT_NANOS * 80 / 100;
200         boolean wakenupFromLoop = false;
201         for (;;) {
202             wakenUp.set(false);
203 
204             try {
205                 long beforeSelect = System.nanoTime();
206                 int selected = select(selector);
207                 if (SelectorUtil.EPOLL_BUG_WORKAROUND && selected == 0 && !wakenupFromLoop && !wakenUp.get()) {
208                     long timeBlocked = System.nanoTime() - beforeSelect;
209 
210                     if (timeBlocked < minSelectTimeout) {
211                         boolean notConnected = false;
212                         // loop over all keys as the selector may was unblocked because of a closed channel
213                         for (SelectionKey key: selector.keys()) {
214                             SelectableChannel ch = key.channel();
215                             try {
216                                 if (ch instanceof DatagramChannel && !((DatagramChannel) ch).isConnected() ||
217                                         ch instanceof SocketChannel && !((SocketChannel) ch).isConnected()) {
218                                     notConnected = true;
219                                     // cancel the key just to be on the safe side
220                                     key.cancel();
221                                 }
222                             } catch (CancelledKeyException e) {
223                                 // ignore
224                             }
225                         }
226                         if (notConnected) {
227                             selectReturnsImmediately = 0;
228                         } else {
229                             // returned before the minSelectTimeout elapsed with nothing select.
230                             // this may be the cause of the jdk epoll(..) bug, so increment the counter
231                             // which we use later to see if its really the jdk bug.
232                             selectReturnsImmediately ++;
233                         }
234                     } else {
235                         selectReturnsImmediately = 0;
236                     }
237 
238                     if (selectReturnsImmediately == 1024) {
239                         // The selector returned immediately for 10 times in a row,
240                         // so recreate one selector as it seems like we hit the
241                         // famous epoll(..) jdk bug.
242                         rebuildSelector();
243                         selector = this.selector;
244                         selectReturnsImmediately = 0;
245                         wakenupFromLoop = false;
246                         // try to select again
247                         continue;
248                     }
249                 } else {
250                     // reset counter
251                     selectReturnsImmediately = 0;
252                 }
253 
254                 // 'wakenUp.compareAndSet(false, true)' is always evaluated
255                 // before calling 'selector.wakeup()' to reduce the wake-up
256                 // overhead. (Selector.wakeup() is an expensive operation.)
257                 //
258                 // However, there is a race condition in this approach.
259                 // The race condition is triggered when 'wakenUp' is set to
260                 // true too early.
261                 //
262                 // 'wakenUp' is set to true too early if:
263                 // 1) Selector is waken up between 'wakenUp.set(false)' and
264                 //    'selector.select(...)'. (BAD)
265                 // 2) Selector is waken up between 'selector.select(...)' and
266                 //    'if (wakenUp.get()) { ... }'. (OK)
267                 //
268                 // In the first case, 'wakenUp' is set to true and the
269                 // following 'selector.select(...)' will wake up immediately.
270                 // Until 'wakenUp' is set to false again in the next round,
271                 // 'wakenUp.compareAndSet(false, true)' will fail, and therefore
272                 // any attempt to wake up the Selector will fail, too, causing
273                 // the following 'selector.select(...)' call to block
274                 // unnecessarily.
275                 //
276                 // To fix this problem, we wake up the selector again if wakenUp
277                 // is true immediately after selector.select(...).
278                 // It is inefficient in that it wakes up the selector for both
279                 // the first case (BAD - wake-up required) and the second case
280                 // (OK - no wake-up required).
281 
282                 if (wakenUp.get()) {
283                     wakenupFromLoop = true;
284                     selector.wakeup();
285                 } else {
286                     wakenupFromLoop = false;
287                 }
288 
289                 cancelledKeys = 0;
290                 processTaskQueue();
291                 selector = this.selector; // processTaskQueue() can call rebuildSelector()
292 
293                 if (shutdown) {
294                     this.selector = null;
295 
296                     // process one time again
297                     processTaskQueue();
298 
299                     for (SelectionKey k: selector.keys()) {
300                         close(k);
301                     }
302 
303                     try {
304                         selector.close();
305                     } catch (IOException e) {
306                         logger.warn(
307                                 "Failed to close a selector.", e);
308                     }
309                     shutdownLatch.countDown();
310                     break;
311                 } else {
312                     process(selector);
313                 }
314             } catch (Throwable t) {
315                 logger.warn(
316                         "Unexpected exception in the selector loop.", t);
317 
318                 // Prevent possible consecutive immediate failures that lead to
319                 // excessive CPU consumption.
320                 try {
321                     Thread.sleep(1000);
322                 } catch (InterruptedException e) {
323                     // Ignore.
324                 }
325             }
326         }
327     }
328 
329     /**
330      * Start the {@link AbstractNioWorker} and return the {@link Selector} that will be used for
331      * the {@link AbstractNioChannel}'s when they get registered
332      */
333     private void openSelector(ThreadNameDeterminer determiner) {
334         try {
335             selector = Selector.open();
336         } catch (Throwable t) {
337             throw new ChannelException("Failed to create a selector.", t);
338         }
339 
340         // Start the worker thread with the new Selector.
341         boolean success = false;
342         try {
343             DeadLockProofWorker.start(executor, newThreadRenamingRunnable(id, determiner));
344             success = true;
345         } finally {
346             if (!success) {
347                 // Release the Selector if the execution fails.
348                 try {
349                     selector.close();
350                 } catch (Throwable t) {
351                     logger.warn("Failed to close a selector.", t);
352                 }
353                 selector = null;
354                 // The method will return to the caller at this point.
355             }
356         }
357         assert selector != null && selector.isOpen();
358     }
359 
360     private void processTaskQueue() {
361         for (;;) {
362             final Runnable task = taskQueue.poll();
363             if (task == null) {
364                 break;
365             }
366             task.run();
367             try {
368                 cleanUpCancelledKeys();
369             } catch (IOException e) {
370                 // Ignore
371             }
372         }
373     }
374 
375     protected final void increaseCancelledKeys() {
376         cancelledKeys ++;
377     }
378 
379     protected final boolean cleanUpCancelledKeys() throws IOException {
380         if (cancelledKeys >= CLEANUP_INTERVAL) {
381             cancelledKeys = 0;
382             selector.selectNow();
383             return true;
384         }
385         return false;
386     }
387 
388     public void shutdown() {
389         if (isIoThread()) {
390             throw new IllegalStateException("Must not be called from a I/O-Thread to prevent deadlocks!");
391         }
392 
393         Selector selector = this.selector;
394         shutdown = true;
395         if (selector != null) {
396             selector.wakeup();
397         }
398         try {
399             shutdownLatch.await();
400         } catch (InterruptedException e) {
401             logger.error("Interrupted while wait for resources to be released #" + id);
402             Thread.currentThread().interrupt();
403         }
404     }
405 
406     protected abstract void process(Selector selector) throws IOException;
407 
408     protected int select(Selector selector) throws IOException {
409         return SelectorUtil.select(selector);
410     }
411 
412     protected abstract void close(SelectionKey k);
413 
414     protected abstract ThreadRenamingRunnable newThreadRenamingRunnable(int id, ThreadNameDeterminer determiner);
415 
416     protected abstract Runnable createRegisterTask(Channel channel, ChannelFuture future);
417 }