背景:

通过shell脚本定时启动的java进程一直都没有退出

#0 */2 * * * /shell/command.sh
echo $@
now=$(date "+%Y-%m-%d_%H-%M-%S")
setsid java -Xms2G -Xmx2G -Djava.ext.dirs=/lib -Dlog4j.configuration=file:/conf/log4j.xml com.ZMain arg1 arg2 > /log/out.$now 2>&1 &

分析:

通过jstack dump出线程列表

2017-01-06 09:43:12
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.71-b15 mixed mode):"elasticsearch[Boomerang][generic][T#189]" #546 daemon prio=5 os_prio=0 tid=0x00007f085c003800 nid=0x68a8 waiting on condition [0x00007f08edee3000]java.lang.Thread.State: TIMED_WAITING (parking)at sun.misc.Unsafe.park(Native Method)- parking to wait for  <0x000000008000c9f8> (a java.util.concurrent.SynchronousQueue$TransferStack)at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- None"elasticsearch[Slick][generic][T#201]" #505 daemon prio=5 os_prio=0 tid=0x00007f08a8003000 nid=0x17a6 waiting on condition [0x00007f08eedec000]java.lang.Thread.State: TIMED_WAITING (parking)at sun.misc.Unsafe.park(Native Method)- parking to wait for  <0x000000008020fb80> (a java.util.concurrent.SynchronousQueue$TransferStack)at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- None"Attach Listener" #504 daemon prio=9 os_prio=0 tid=0x00007f08c8001000 nid=0x1342 waiting on condition [0x0000000000000000]java.lang.Thread.State: RUNNABLELocked ownable synchronizers:- None"DestroyJavaVM" #162 prio=5 os_prio=0 tid=0x00007f09240bf800 nid=0x1422 waiting on condition [0x0000000000000000]java.lang.Thread.State: RUNNABLELocked ownable synchronizers:- None"pool-8-thread-10" #159 prio=5 os_prio=0 tid=0x00007f0924ad2800 nid=0x21ac waiting on condition [0x00007f08ee4e9000]java.lang.Thread.State: WAITING (parking)at sun.misc.Unsafe.park(Native Method)- parking to wait for  <0x000000008eae3710> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- None"pool-8-thread-9" #158 prio=5 os_prio=0 tid=0x00007f0924ad2000 nid=0x21a2 waiting on condition [0x00007f08ed8df000]java.lang.Thread.State: WAITING (parking)at sun.misc.Unsafe.park(Native Method)- parking to wait for  <0x000000008eae3710> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- None"pool-8-thread-6" #155 prio=5 os_prio=0 tid=0x00007f09240c5000 nid=0x219b waiting on condition [0x00007f08edde2000]java.lang.Thread.State: WAITING (parking)at sun.misc.Unsafe.park(Native Method)- parking to wait for  <0x000000008eae3710> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- None"pool-8-thread-5" #154 prio=5 os_prio=0 tid=0x00007f09255fb000 nid=0x2198 waiting on condition [0x00007f08ed7de000]java.lang.Thread.State: WAITING (parking)at sun.misc.Unsafe.park(Native Method)- parking to wait for  <0x000000008eae3710> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- None"pool-8-thread-4" #153 prio=5 os_prio=0 tid=0x00007f0924a1b000 nid=0x2197 waiting on condition [0x00007f08edfe4000]java.lang.Thread.State: WAITING (parking)at sun.misc.Unsafe.park(Native Method)- parking to wait for  <0x000000008eae3710> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- None"elasticsearch[Boomerang][transport_client_boss][T#1]{New I/O boss #34}" #51 daemon prio=5 os_prio=0 tid=0x00007f0925428000 nid=0x14d7 runnable [0x00007f08eefee000]java.lang.Thread.State: RUNNABLEat sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)- locked <0x000000008000cf70> (a sun.nio.ch.Util$2)- locked <0x000000008000cf60> (a java.util.Collections$UnmodifiableSet)- locked <0x000000008000cf18> (a sun.nio.ch.EPollSelectorImpl)at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434)at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)at org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42)at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- <0x000000008000d170> (a java.util.concurrent.ThreadPoolExecutor$Worker)"elasticsearch[Boomerang][transport_client_worker][T#16]{New I/O worker #33}" #49 daemon prio=5 os_prio=0 tid=0x00007f09253fd000 nid=0x14d6 runnable [0x00007f08ef0ef000]java.lang.Thread.State: RUNNABLEat sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)- locked <0x00000000800149c0> (a sun.nio.ch.Util$2)- locked <0x00000000800149b0> (a java.util.Collections$UnmodifiableSet)- locked <0x0000000080014968> (a sun.nio.ch.EPollSelectorImpl)at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434)at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)at java.lang.Thread.run(Thread.java:745)Locked ownable synchronizers:- <0x0000000080014ab8> (a java.util.concurrent.ThreadPoolExecutor$Worker)

发现其中有线程是waiting状态,而且根据上下文显示,是在ThreadPoolExecutor$Worker.run中产生的.

发现SelectorImpl.lockAndDoSelect,是在NIO中出现的锁,不见的是死锁,如果是死锁的话,在jvisualvm中就会有提示了

结论:

ThreadPoolExecutor这个类是java的线程池.如果最终没有关闭连接池,会导致程序不退出.

增加异常捕获并且保证线程池能够关闭