In our process, recently we have observed when database gives persistence error due to “Socket closed” from database, camunda fails to persist the process variables, process instance id and task id into the database. Due to this the triggered process instance cannot be found, even in the history.
We can understand this, since it was persistence error, but why camunda delays the persistence so much that, it occurs only later (highlighted in the diagram below) in the BPMN and not at the beginning or the start event itself ?
Please note, we have configured our start event to be Asynchronous and our database is not hosted on the same machine as Camunda and its MS SQL Server
Java = Java 15
Camunda = 7.17
Database = MS SQL Server
Sample database error =
14-Mar-2024 09:41:16.060 SEVERE [https-jsse-nio-443-exec-46] org.camunda.commons.logging.BaseLogger.logError ENGINE-16004 Exception while closing command context: An exception occurred in the persistence layer. Please check the server logs for a detailed message and the entire exception stack trace.
org.camunda.bpm.engine.ProcessEngineException: An exception occurred in the persistence layer. Please check the server logs for a detailed message and the entire exception stack trace.
at org.camunda.bpm.engine.impl.util.ExceptionUtil.wrapPersistenceException(ExceptionUtil.java:263)
at org.camunda.bpm.engine.impl.db.EnginePersistenceLogger.flushDbOperationUnexpectedException(EnginePersistenceLogger.java:680)
at org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.flushDbOperations(DbEntityManager.java:344)
at org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.flushDbOperationManager(DbEntityManager.java:323)
at org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.flush(DbEntityManager.java:295)
at org.camunda.bpm.engine.impl.interceptor.CommandContext.flushSessions(CommandContext.java:272)
at org.camunda.bpm.engine.impl.interceptor.CommandContext.close(CommandContext.java:188)
at org.camunda.bpm.engine.impl.interceptor.CommandContextInterceptor.execute(CommandContextInterceptor.java:119)
at org.camunda.bpm.engine.impl.interceptor.ProcessApplicationContextInterceptor.execute(ProcessApplicationContextInterceptor.java:70)
at org.camunda.bpm.engine.impl.interceptor.CommandCounterInterceptor.execute(CommandCounterInterceptor.java:35)
at org.camunda.bpm.engine.impl.interceptor.LogInterceptor.execute(LogInterceptor.java:33)
at org.camunda.bpm.engine.impl.FormServiceImpl.submitTaskForm(FormServiceImpl.java:91)
at org.camunda.bpm.engine.rest.sub.task.impl.TaskResourceImpl.submit(TaskResourceImpl.java:153)
at jdk.internal.reflect.GeneratedMethodAccessor833.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:138)
at org.jboss.resteasy.core.ResourceMethodInvoker.internalInvokeOnTarget(ResourceMethodInvoker.java:546)
at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTargetAfterFilter(ResourceMethodInvoker.java:435)
at org.jboss.resteasy.core.ResourceMethodInvoker.lambda$invokeOnTarget$0(ResourceMethodInvoker.java:396)
at org.jboss.resteasy.core.interception.PreMatchContainerRequestContext.filter(PreMatchContainerRequestContext.java:358)
at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:398)
at org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:365)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:150)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:110)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:141)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:104)
at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:440)
at org.jboss.resteasy.core.SynchronousDispatcher.lambda$invoke$4(SynchronousDispatcher.java:229)
at org.jboss.resteasy.core.SynchronousDispatcher.lambda$preprocess$0(SynchronousDispatcher.java:135)
at org.jboss.resteasy.core.interception.PreMatchContainerRequestContext.filter(PreMatchContainerRequestContext.java:358)
at org.jboss.resteasy.core.SynchronousDispatcher.preprocess(SynchronousDispatcher.java:138)
at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:215)
at org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:245)
at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:61)
at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:56)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:764)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:227)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.apache.catalina.filters.CorsFilter.handleNonCORS(CorsFilter.java:357)
at org.apache.catalina.filters.CorsFilter.doFilter(CorsFilter.java:176)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.camunda.bpm.engine.rest.filter.CacheControlFilter.doFilter(CacheControlFilter.java:45)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.camunda.bpm.engine.rest.filter.EmptyBodyFilter.doFilter(EmptyBodyFilter.java:99)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:197)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:540)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:687)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:359)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:399)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:889)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1735)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)
at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:832)
Caused by: org.camunda.bpm.engine.ProcessEngineException: ENGINE-03083 Unexpected exception while executing database operations with message '
### Error flushing statements. Cause: com.microsoft.sqlserver.jdbc.SQLServerException: Socket closed
### Cause: com.microsoft.sqlserver.jdbc.SQLServerException: Socket closed'. Flush summary:
[
INSERT HistoricVariableInstanceEntity[62f8eb11-e208-11ee-ac41-005056857921]
INSERT HistoricVariableInstanceEntity[62f8eb13-e208-11ee-ac41-005056857921]
INSERT HistoricVariableInstanceEntity[62f8eb15-e208-11ee-ac41-005056857921]
INSERT HistoricVariableInstanceEntity[62f8eb17-e208-11ee-ac41-005056857921]
INSERT HistoricVariableInstanceEntity[62f8eb19-e208-11ee-ac41-005056857921]
INSERT HistoricVariableInstanceEntity[62f8eb1b-e208-11ee-ac41-005056857921]
INSERT HistoricVariableInstanceEntity[62f8eb1d-e208-11ee-ac41-005056857921]
INSERT HistoricVariableInstanceEntity[62f8eb20-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f8eb12-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f8eb14-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f8eb16-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f8eb18-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f8eb1a-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f8eb1c-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f8eb1e-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f8eb22-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f9ae79-e208-11ee-ac41-005056857921]
INSERT HistoricVariableUpdateEventEntity[62f9d58a-e208-11ee-ac41-005056857921]
INSERT HistoricFormPropertyEventEntity[62f8eb24-e208-11ee-ac41-005056857921]
INSERT HistoricFormPropertyEventEntity[62f8eb25-e208-11ee-ac41-005056857921]
INSERT HistoricFormPropertyEventEntity[62f8eb26-e208-11ee-ac41-005056857921]
INSERT HistoricFormPropertyEventEntity[62f8eb27-e208-11ee-ac41-005056857921]
INSERT HistoricFormPropertyEventEntity[62f8eb28-e208-11ee-ac41-005056857921]
INSERT ByteArrayEntity[62f8eb1f-e208-11ee-ac41-005056857921]
INSERT ByteArrayEntity[62f8eb21-e208-11ee-ac41-005056857921]
INSERT ByteArrayEntity[62f8eb23-e208-11ee-ac41-005056857921]
INSERT VariableInstanceEntity[62f8eb11-e208-11ee-ac41-005056857921]
INSERT VariableInstanceEntity[62f8eb13-e208-11ee-ac41-005056857921]
INSERT VariableInstanceEntity[62f8eb15-e208-11ee-ac41-005056857921]
INSERT VariableInstanceEntity[62f8eb17-e208-11ee-ac41-005056857921]
INSERT VariableInstanceEntity[62f8eb19-e208-11ee-ac41-005056857921]
INSERT VariableInstanceEntity[62f8eb1b-e208-11ee-ac41-005056857921]
INSERT VariableInstanceEntity[62f8eb1d-e208-11ee-ac41-005056857921]
INSERT VariableInstanceEntity[62f8eb20-e208-11ee-ac41-005056857921]
DELETE IdentityLinkEntity[d1757122-e206-11ee-ac41-005056857921]
UPDATE VariableInstanceEntity[d0a3f12b-e206-11ee-ac41-005056857921]
UPDATE VariableInstanceEntity[d0a3f12d-e206-11ee-ac41-005056857921]
DELETE TaskEntity[d1752301-e206-11ee-ac41-005056857921]
UPDATE ExecutionEntity[d0a3f130-e206-11ee-ac41-005056857921]
UPDATE ExecutionEntity[d0a3f127-e206-11ee-ac41-005056857921]
UPDATE HistoricActivityInstanceEventEntity[d174fbec-e206-11ee-ac41-005056857921]
UPDATE HistoricTaskInstanceEventEntity[d1752301-e206-11ee-ac41-005056857921]
DELETE_BULK deleteAuthorizationsForResourceId {resourceId=d1752301-e206-11ee-ac41-005056857921, resourceType=7}
UPDATE HistoricVariableInstanceEntity[d0a3f12b-e206-11ee-ac41-005056857921]
UPDATE HistoricVariableInstanceEntity[d0a3f12d-e206-11ee-ac41-005056857921]
]
at org.camunda.bpm.engine.impl.db.EnginePersistenceLogger.flushDbOperationUnexpectedException(EnginePersistenceLogger.java:679)
... 67 more
Caused by: org.apache.ibatis.exceptions.PersistenceException:
### Error flushing statements. Cause: com.microsoft.sqlserver.jdbc.SQLServerException: Socket closed
### Cause: com.microsoft.sqlserver.jdbc.SQLServerException: Socket closed
at org.apache.ibatis.exceptions.ExceptionFactory.wrapException(ExceptionFactory.java:30)
at org.apache.ibatis.session.defaults.DefaultSqlSession.flushStatements(DefaultSqlSession.java:254)
at org.camunda.bpm.engine.impl.db.sql.DbSqlSession.flushBatchOperations(DbSqlSession.java:444)
at org.camunda.bpm.engine.impl.db.sql.BatchDbSqlSession.executeDbOperations(BatchDbSqlSession.java:74)
at org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.flushDbOperations(DbEntityManager.java:341)
... 66 more
Caused by: com.microsoft.sqlserver.jdbc.SQLServerException: Socket closed
at com.microsoft.sqlserver.jdbc.SQLServerConnection.terminate(SQLServerConnection.java:3422)
at com.microsoft.sqlserver.jdbc.TDSChannel.write(IOBuffer.java:2080)
at com.microsoft.sqlserver.jdbc.TDSWriter.flush(IOBuffer.java:4322)
at com.microsoft.sqlserver.jdbc.TDSWriter.writePacket(IOBuffer.java:4224)
at com.microsoft.sqlserver.jdbc.TDSWriter.writeBytes(IOBuffer.java:3886)
at com.microsoft.sqlserver.jdbc.TDSWriter.writeStream(IOBuffer.java:4001)
at com.microsoft.sqlserver.jdbc.TDSWriter.writeRPCInputStream(IOBuffer.java:6247)
at com.microsoft.sqlserver.jdbc.DTV$SendByRPCOp.execute(dtv.java:1406)
at com.microsoft.sqlserver.jdbc.DTV.executeOp(dtv.java:1873)
at com.microsoft.sqlserver.jdbc.DTV.sendByRPC(dtv.java:1915)
at com.microsoft.sqlserver.jdbc.Parameter.sendByRPC(Parameter.java:1158)
at com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement.sendParamsByRPC(SQLServerPreparedStatement.java:710)
at com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement.doPrepExec(SQLServerPreparedStatement.java:1063)
at com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement.doExecutePreparedStatementBatch(SQLServerPreparedStatement.java:2814)
at com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement$PrepStmtBatchExecCmd.doExecute(SQLServerPreparedStatement.java:2694)
at com.microsoft.sqlserver.jdbc.TDSCommand.execute(IOBuffer.java:7417)
at com.microsoft.sqlserver.jdbc.SQLServerConnection.executeCommand(SQLServerConnection.java:3488)
at com.microsoft.sqlserver.jdbc.SQLServerStatement.executeCommand(SQLServerStatement.java:262)
at com.microsoft.sqlserver.jdbc.SQLServerStatement.executeStatement(SQLServerStatement.java:237)
at com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement.executeBatch(SQLServerPreparedStatement.java:2092)
at jdk.internal.reflect.GeneratedMethodAccessor45.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at org.apache.tomcat.jdbc.pool.StatementFacade$StatementProxy.invoke(StatementFacade.java:118)
at com.sun.proxy.$Proxy7.executeBatch(Unknown Source)
at org.apache.ibatis.executor.BatchExecutor.doFlushStatements(BatchExecutor.java:123)
at org.apache.ibatis.executor.BaseExecutor.flushStatements(BaseExecutor.java:129)
at org.apache.ibatis.executor.BaseExecutor.flushStatements(BaseExecutor.java:122)
at org.apache.ibatis.executor.CachingExecutor.flushStatements(CachingExecutor.java:114)
at org.apache.ibatis.session.defaults.DefaultSqlSession.flushStatements(DefaultSqlSession.java:252)
... 69 more
Caused by: java.net.SocketException: Socket closed
at java.base/sun.nio.ch.NioSocketImpl.ensureOpenAndConnected(NioSocketImpl.java:165)
at java.base/sun.nio.ch.NioSocketImpl.beginWrite(NioSocketImpl.java:366)
at java.base/sun.nio.ch.NioSocketImpl.implWrite(NioSocketImpl.java:411)
at java.base/sun.nio.ch.NioSocketImpl.write(NioSocketImpl.java:440)
at java.base/sun.nio.ch.NioSocketImpl$2.write(NioSocketImpl.java:826)
at java.base/java.net.Socket$SocketOutputStream.write(Socket.java:1051)
at com.microsoft.sqlserver.jdbc.TDSChannel.write(IOBuffer.java:2075)
... 97 more