Since last friday we experience very strange behaviour of out development portal.
It now happens almost every 30 minutes and we are getting a bit desperate.
The error log is not very descriptive (as usual) so hopefully someone can point us in the right direction.
#1.5 #00096BCE1029006B0000003E0001D08200043DF3E0846AE1#1194017559963#com.sap.portal.prt.runtime.broker#sap.com/irj#com.sap.portal.prt.runtime.broker#Guest#0####d837dfa0895811dcc7c700096bce1029#SAPEngine_Application_Thread[impl:3]_8##0#0#Error#1#/System/Server#Java###[PortalServiceItem.release] exception caught releasing com.sap.netweaver.coll.appl.rtc|RTCClientPollingService
[EXCEPTION]
#1#java.lang.NullPointerException at com.sap.ip.collaboration.core.api.rtmf.core.RTMFMessaging.unsubscribe(RTMFMessaging.java:1040) at com.sap.ip.collaboration.core.api.rtmf.core.cluster.ClusterSystemManager.unsubscribe(ClusterSystemManager.java:133) at com.sap.ip.collaboration.core.api.semantic.services.RTCBaseService.destroy(RTCBaseService.java:146) at com.sapportals.portal.prt.core.broker.PortalServiceItem.release(PortalServiceItem.java:389) at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1159) at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194) at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156) at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194) at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156) at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194) at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156) at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194) at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156) at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194) at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156) at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476) at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194) at com.sapportals.portal.prt.core.broker.PortalAppBroker.releaseAllItems(PortalAppBroker.java:568) at com.sapportals.portal.prt.core.broker.PortalAppBroker.shutdown(PortalAppBroker.java:672) at com.sapportals.portal.prt.runtime.Portal.shutdown(Portal.java:722) at com.sapportals.portal.prt.core.PortalCoreInitializer.shutdown(PortalCoreInitializer.java:111) at com.sapportals.portal.prt.dispatcher.PortalInitializer.shutdown(PortalInitializer.java:164) at com.sapportals.portal.prt.dispatcher.Dispatcher.destroy(Dispatcher.java:581) at com.sap.engine.services.servlets_jsp.server.runtime.context.WebComponents.destroyServlets(WebComponents.java:722) at com.sap.engine.services.servlets_jsp.server.container.ApplicationThreadDestroyer.run(ApplicationThreadDestroyer.java:62) at com.sap.engine.core.thread.impl3.ActionObject.run(ActionObject.java:37) at java.security.AccessController.doPrivileged(AccessController.java:215) at com.sap.engine.core.thread.impl3.SingleThread.execute(SingleThread.java:100) at com.sap.engine.core.thread.impl3.SingleThread.run(SingleThread.java:170) # #1.5 #00096BCE1029006B000000400001D08200043DF3E0847234#1194017559965#com.sap.portal.prt.runtime.broker#sap.com/irj#com.sap.portal.prt.runtime.broker#Guest#0####d837dfa0895811dcc7c700096bce1029#SAPEngine_Application_Thread[impl:3]_8##0#0#Error#1#/System/Server#Java###[PortalServiceItem.release] exception caught releasing com.sap.netweaver.coll.appl.rtc|collaboration_connector_awareness [EXCEPTION] #1#java.lang.NullPointerException
at com.sap.ip.collaboration.core.api.rtmf.core.RTMFMessaging.unsubscribe(RTMFMessaging.java:1040)
at com.sap.ip.collaboration.core.api.rtmf.core.cluster.ClusterSystemManager.unsubscribe(ClusterSystemManager.java:133)
at com.sap.ip.collaboration.core.api.semantic.services.RTCBaseService.destroy(RTCBaseService.java:146)
at com.sapportals.portal.prt.core.broker.PortalServiceItem.release(PortalServiceItem.java:389)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1159)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.PortalAppBroker.releaseAllItems(PortalAppBroker.java:568)
at com.sapportals.portal.prt.core.broker.PortalAppBroker.shutdown(PortalAppBroker.java:672)
at com.sapportals.portal.prt.runtime.Portal.shutdown(Portal.java:722)
at com.sapportals.portal.prt.core.PortalCoreInitializer.shutdown(PortalCoreInitializer.java:111)
at com.sapportals.portal.prt.dispatcher.PortalInitializer.shutdown(PortalInitializer.java:164)
at com.sapportals.portal.prt.dispatcher.Dispatcher.destroy(Dispatcher.java:581)
at com.sap.engine.services.servlets_jsp.server.runtime.context.WebComponents.destroyServlets(WebComponents.java:722)
at com.sap.engine.services.servlets_jsp.server.container.ApplicationThreadDestroyer.run(ApplicationThreadDestroyer.java:62)
at com.sap.engine.core.thread.impl3.ActionObject.run(ActionObject.java:37)
at java.security.AccessController.doPrivileged(AccessController.java:215)
at com.sap.engine.core.thread.impl3.SingleThread.execute(SingleThread.java:100)
at com.sap.engine.core.thread.impl3.SingleThread.run(SingleThread.java:170)
#
#1.5 #00096BCE1029006B000000420001D08200043DF3E08479F0#1194017559967#com.sap.portal.prt.runtime.broker#sap.com/irj#com.sap.portal.prt.runtime.broker#Guest#0####d837dfa0895811dcc7c700096bce1029#SAPEngine_Application_Thread[impl:3]_8##0#0#Error#1#/System/Server#Java###[PortalServiceItem.release] exception caught releasing com.sap.netweaver.coll.appl.rtc|RTCClientManagerService
[EXCEPTION]
#1#java.lang.NullPointerException
at com.sap.ip.collaboration.core.api.rtmf.core.RTMFMessaging.unsubscribe(RTMFMessaging.java:1040)
at com.sap.ip.collaboration.core.api.rtmf.core.cluster.ClusterSystemManager.unsubscribe(ClusterSystemManager.java:133)
at com.sap.ip.collaboration.core.api.semantic.services.RTCBaseService.destroy(RTCBaseService.java:146)
at com.sapportals.portal.prt.core.broker.PortalServiceItem.release(PortalServiceItem.java:389)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1159)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.PortalAppBroker.releaseAllItems(PortalAppBroker.java:568)
at com.sapportals.portal.prt.core.broker.PortalAppBroker.shutdown(PortalAppBroker.java:672)
at com.sapportals.portal.prt.runtime.Portal.shutdown(Portal.java:722)
at com.sapportals.portal.prt.core.PortalCoreInitializer.shutdown(PortalCoreInitializer.java:111)
at com.sapportals.portal.prt.dispatcher.PortalInitializer.shutdown(PortalInitializer.java:164)
at com.sapportals.portal.prt.dispatcher.Dispatcher.destroy(Dispatcher.java:581)
at com.sap.engine.services.servlets_jsp.server.runtime.context.WebComponents.destroyServlets(WebComponents.java:722)
at com.sap.engine.services.servlets_jsp.server.container.ApplicationThreadDestroyer.run(ApplicationThreadDestroyer.java:62)
at com.sap.engine.core.thread.impl3.ActionObject.run(ActionObject.java:37)
at java.security.AccessController.doPrivileged(AccessController.java:215)
at com.sap.engine.core.thread.impl3.SingleThread.execute(SingleThread.java:100)
at com.sap.engine.core.thread.impl3.SingleThread.run(SingleThread.java:170)
#
#1.5 #00096BCE1029006B0000004B0001D08200043DF3E08CACC2#1194017560505#com.sapportals.wcm.repository.runtime.CmSystem#sap.com/irj#com.sapportals.wcm.repository.runtime.CmSystem#Guest#0####d837dfa0895811dcc7c700096bce1029#SAPEngine_Application_Thread[impl:3]_8##0#0#Info#1#/Applications/KMC/RF#Plain###CM system is shutting down...
#
#1.5 #00096BCE1029006B0000004D0001D08200043DF3E08CB866#1194017560508#com.sapportals.wcm.repository.runtime.CmSystem#sap.com/irj#com.sapportals.wcm.repository.runtime.CmSystem#Guest#0####d837dfa0895811dcc7c700096bce1029#SAPEngine_Application_Thread[impl:3]_8##0#0#Info#1#/Applications/KMC/RF#Plain###CM system shutdown triggered by this stacktrace:
java.lang.Exception
at com.sapportals.wcm.repository.runtime.CmSystem.shutDown(CmSystem.java:284)
at com.sapportals.wcm.portal.service.KMServiceImpl.release(KMServiceImpl.java:273)
at com.sapportals.portal.prt.core.broker.PortalServiceItem.deprecatedRelease(PortalServiceItem.java:474)
at com.sapportals.portal.prt.core.broker.PortalServiceItem.release(PortalServiceItem.java:408)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1159)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.ApplicationItem.releaseDependents(ApplicationItem.java:223)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.releaseDependents(PortalApplicationItem.java:1156)
at com.sapportals.portal.prt.core.broker.ApplicationItem.release(ApplicationItem.java:476)
at com.sapportals.portal.prt.core.broker.PortalApplicationItem.release(PortalApplicationItem.java:1194)
at com.sapportals.portal.prt.core.broker.PortalAppBroker.releaseAllItems(PortalAppBroker.java:568)
at com.sapportals.portal.prt.core.broker.PortalAppBroker.shutdown(PortalAppBroker.java:672)
at com.sapportals.portal.prt.runtime.Portal.shutdown(Portal.java:722)
at com.sapportals.portal.prt.core.PortalCoreInitializer.shutdown(PortalCoreInitializer.java:111)
at com.sapportals.portal.prt.dispatcher.PortalInitializer.shutdown(PortalInitializer.java:164)
at com.sapportals.portal.prt.dispatcher.Dispatcher.destroy(Dispatcher.java:581)
at com.sap.engine.services.servlets_jsp.server.runtime.context.WebComponents.destroyServlets(WebComponents.java:722)
at com.sap.engine.services.servlets_jsp.server.container.ApplicationThreadDestroyer.run(ApplicationThreadDestroyer.java:62)
at com.sap.engine.core.thread.impl3.ActionObject.run(ActionObject.java:37)
at java.security.AccessController.doPrivileged(AccessController.java:215)
at com.sap.engine.core.thread.impl3.SingleThread.execute(SingleThread.java:100)
at com.sap.engine.core.thread.impl3.SingleThread.run(SingleThread.java:170)