1 压力测试
1.1环境
Weblogic版本:9.2
数据库版本:informix:11.5
JDK版本: jrmc-3.1.0-1.5.0
1.2 压力测试问题现象以及分析
1.2.1整体监控
从晚上7:30到晚上10点,混发压力测试。压力测试间多次取thread dump并分析,发现大量的thread 被堵塞。从应用的日志文件中分析,发现有执行超时的线程。
1.2.2异常分析
1) 应用中有执行时间过长的线程。该线程与数据库的操作有关。建议开发工程师和数据库工程师参与分析。
Thread-151 "[STUCK] ExecuteThread: '137' for queue: 'weblogic.kernel.Default (self-tuning)'" <alive, in native, suspended, priority=1, DAEMON> {
jrockit.net.SocketNativeIO.readBytesPinned(SocketNativeIO.java:???)
jrockit.net.SocketNativeIO.socketRead(SocketNativeIO.java:31)
java.net.SocketInputStream.socketRead0(SocketInputStream.java:???)
java.net.SocketInputStream.read(SocketInputStream.java:107)
java.io.BufferedInputStream.fill(BufferedInputStream.java:189)
java.io.BufferedInputStream.read1(BufferedInputStream.java:247)
java.io.BufferedInputStream.read(BufferedInputStream.java:304)
^-- Holding lock: java.io.BufferedInputStream@ffffffffae3e8368[thin lock]
com.informix.asf.IfxDataInputStream.readFully(IfxDataInputStream.java:141)
com.informix.asf.IfxDataInputStream.readSmallInt(IfxDataInputStream.java:453)
com.informix.jdbc.IfxSqli.receiveMessage(IfxSqli.java:2481)
com.informix.jdbc.IfxSqli.a(IfxSqli.java:1733)
com.informix.jdbc.IfxSqli.executeStatementQuery(IfxSqli.java:1645)
com.informix.jdbc.IfxSqli.executeStatementQuery(IfxSqli.java:1635)
com.informix.jdbc.IfxResultSet.a(IfxResultSet.java:201)
com.informix.jdbc.IfxStatement.executeQueryImpl(IfxStatement.java:1198)
com.informix.jdbc.IfxPreparedStatement.executeQuery(IfxPreparedStatement.java:369)
weblogic.jdbc.wrapper.PreparedStatement.executeQuery(PreparedStatement.java:91)
org.hibernate.jdbc.AbstractBatcher.getResultSet(AbstractBatcher.java:208)
org.hibernate.loader.Loader.getResultSet(Loader.java:1805)
org.hibernate.loader.Loader.doQuery(Loader.java:688)
org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:255)
org.hibernate.loader.Loader.doList(Loader.java:2226)
org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2129)
org.hibernate.loader.Loader.list(Loader.java:2117)
org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:400)
org.hibernate.hql.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:346)
org.hibernate.engine.query.HQLQueryPlan.performList(HQLQueryPlan.java:173)
org.hibernate.impl.SessionImpl.list(SessionImpl.java:1138)
org.hibernate.impl.QueryImpl.list(QueryImpl.java:98)
ins.framework.dao.EntityDaoHibernate$5.doInHibernate(EntityDaoHibernate.java:494)
org.springframework.orm.hibernate3.HibernateTemplate.doExecute(HibernateTemplate.java:403)
org.springframework.orm.hibernate3.HibernateTemplate.executeFind(HibernateTemplate.java:343)
ins.framework.dao.EntityDaoHibernate.findByHql(EntityDaoHibernate.java:393)
com.sinosoft.prpall.service.spring.PrpWorkbenchMainServiceSpringImpl.getPrpCFlowInfos(PrpWorkbenchMainServiceSpringImpl.java:165)
com.sinosoft.prpall.service.spring.PrpWorkbenchMainServiceSpringImpl$$FastClassByCGLIB$$6bb9d24b.invoke(<generated>:???)
net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:149)
org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:696)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)
ins.framework.log.RunTimeLogAdvice.invoke(RunTimeLogAdvice.java:30)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)
org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:92)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:86)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)
org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:602)
com.sinosoft.prpall.service.spring.PrpWorkbenchMainServiceSpringImpl$$EnhancerByCGLIB$$b7d0b209.getPrpCFlowInfos(<generated>:???)
com.sinosoft.prpall.web.PrpWorkbenchMainAction.findTodoList(PrpWorkbenchMainAction.java:115)
com.sinosoft.prpall.web.PrpWorkbenchMainAction$$FastClassByCGLIB$$27bc1495.invoke(<generated>:???)
net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:149)
org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:696)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)
ins.framework.log.RunTimeLogAdvice.invoke(RunTimeLogAdvice.java:30)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:86)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)
org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:602)
com.sinosoft.prpall.web.PrpWorkbenchMainAction$$EnhancerByCGLIB$$d8e9d0fa.findTodoList(<generated>:???)
sun.reflect.GeneratedMethodAccessor5830.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
java.lang.reflect.Method.invoke(Method.java:570)
com.opensymphony.xwork2.DefaultActionInvocation.invokeAction(DefaultActionInvocation.java:403)
com.opensymphony.xwork2.DefaultActionInvocation.invokeActionOnly(DefaultActionInvocation.java:279)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.DefaultWorkflowInterceptor.doIntercept(DefaultWorkflowInterceptor.java:132)
com.opensymphony.xwork2.interceptor.MethodFilterInterceptor.intercept(MethodFilterInterceptor.java:86)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.ConversionErrorInterceptor.intercept(ConversionErrorInterceptor.java:82)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.ParametersInterceptor.doIntercept(ParametersInterceptor.java:170)
com.opensymphony.xwork2.interceptor.MethodFilterInterceptor.intercept(MethodFilterInterceptor.java:86)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.ParametersInterceptor.doIntercept(ParametersInterceptor.java:170)
com.opensymphony.xwork2.interceptor.MethodFilterInterceptor.intercept(MethodFilterInterceptor.java:86)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.StaticParametersInterceptor.intercept(StaticParametersInterceptor.java:101)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
org.apache.struts2.interceptor.CheckboxInterceptor.intercept(CheckboxInterceptor.java:67)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
org.apache.struts2.interceptor.FileUploadInterceptor.intercept(FileUploadInterceptor.java:225)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.ModelDrivenInterceptor.intercept(ModelDrivenInterceptor.java:76)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.ScopedModelDrivenInterceptor.intercept(ScopedModelDrivenInterceptor.java:101)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
org.apache.struts2.interceptor.ProfilingActivationInterceptor.intercept(ProfilingActivationInterceptor.java:95)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
org.apache.struts2.interceptor.debugging.DebuggingInterceptor.intercept(DebuggingInterceptor.java:158)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.ChainingInterceptor.intercept(ChainingInterceptor.java:101)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.PrepareInterceptor.doIntercept(PrepareInterceptor.java:111)
com.opensymphony.xwork2.interceptor.MethodFilterInterceptor.intercept(MethodFilterInterceptor.java:86)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.I18nInterceptor.intercept(I18nInterceptor.java:97)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
org.apache.struts2.interceptor.ServletConfigInterceptor.intercept(ServletConfigInterceptor.java:124)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.AliasInterceptor.intercept(AliasInterceptor.java:91)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
com.opensymphony.xwork2.interceptor.ExceptionMappingInterceptor.intercept(ExceptionMappingInterceptor.java:176)
com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:223)
org.apache.struts2.impl.StrutsActionProxy.execute(StrutsActionProxy.java:42)
org.apache.struts2.dispatcher.Dispatcher.serviceAction(Dispatcher.java:435)
org.apache.struts2.dispatcher.FilterDispatcher.doFilter(FilterDispatcher.java:354)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
org.springframework.orm.hibernate3.support.OpenSessionInViewFilter.doFilterInternal(OpenSessionInViewFilter.java:171)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:61)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.sinosoft.prpall.common.util.PowerFilter.doFilter(PowerFilter.java:34)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
com.sinosoft.prpall.common.util.SessionFilter.doFilter(SessionFilter.java:68)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:90)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:61)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
ins.framework.web.CompressFilter.doFilter(CompressFilter.java:51)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
cn.com.sinosoft.cas.filter.CustomCASFilter.doFilter(CustomCASFilter.java:163)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:41)
weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3214)
weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:308)
weblogic.security.service.SecurityManager.runAs(SecurityManager.java:117)
weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:1946)
weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:1868)
weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1331)
weblogic.work.ExecuteThread.execute(ExecuteThread.java:206)
weblogic.work.ExecuteThread.run(ExecuteThread.java:173)
}
2 建议
所有系统中的应用都部署在域的admin server中,建议新建一个被管服务,把应用部署在被管服务中,admin server只做管理用。