Best practice for sending data outside parallel multi-instance call activities

Scenario: I have a process that should do the following:

  1. Retrieve data
  2. Split data into a list
  3. Process list items in parallel (inside a multi-instance subprocess)
  4. Whenever an item in the list is processed, send an output to the superprocess that should continue to the next steps
  5. When all items are processed, perform a different flow

What I thought would be the proper solution for this was to use escalation, but from what I understand, escalations are not dynamic and don’t seem to be able to pass variables.
Below I have the test processes I used to test this scenario with the escalation, that worked as I intended except for sending the data part.

First process:
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" xmlns:modeler="http://camunda.org/schema/modeler/1.0" id="Definitions_0xi0034" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="5.0.0" modeler:executionPlatform="Camunda Platform" modeler:executionPlatformVersion="7.17.0">
  <bpmn:process id="test_escalationContinue" name="Test escalation Continue" isExecutable="true">
    <bpmn:extensionElements />
    <bpmn:startEvent id="StartEvent_1">
      <bpmn:outgoing>Flow_0k22u10</bpmn:outgoing>
    </bpmn:startEvent>
    <bpmn:sequenceFlow id="Flow_0k22u10" sourceRef="StartEvent_1" targetRef="Activity_02wcqyg" />
    <bpmn:userTask id="Activity_02wcqyg" name="Wait for input">
      <bpmn:incoming>Flow_0k22u10</bpmn:incoming>
      <bpmn:outgoing>Flow_1qow6p8</bpmn:outgoing>
    </bpmn:userTask>
    <bpmn:sequenceFlow id="Flow_1qow6p8" sourceRef="Activity_02wcqyg" targetRef="Activity_1c50ue8" />
    <bpmn:callActivity id="Activity_1c50ue8" name="Test multi instance" calledElement="test_escalationProcess">
      <bpmn:extensionElements />
      <bpmn:incoming>Flow_1qow6p8</bpmn:incoming>
      <bpmn:outgoing>Flow_0nynj89</bpmn:outgoing>
      <bpmn:multiInstanceLoopCharacteristics>
        <bpmn:loopCardinality xsi:type="bpmn:tFormalExpression">${input}</bpmn:loopCardinality>
      </bpmn:multiInstanceLoopCharacteristics>
    </bpmn:callActivity>
    <bpmn:boundaryEvent id="Event_0dvlpqq" cancelActivity="false" attachedToRef="Activity_1c50ue8">
      <bpmn:outgoing>Flow_0ex7pl0</bpmn:outgoing>
      <bpmn:escalationEventDefinition id="EscalationEventDefinition_0iam8qa" escalationRef="Escalation_3bjg3ud" camunda:escalationCodeVariable="testEscVar" />
    </bpmn:boundaryEvent>
    <bpmn:sequenceFlow id="Flow_0ex7pl0" sourceRef="Event_0dvlpqq" targetRef="Activity_0p0oaod" />
    <bpmn:userTask id="Activity_0p0oaod" name="Process output data">
      <bpmn:incoming>Flow_0ex7pl0</bpmn:incoming>
      <bpmn:outgoing>Flow_1sjyli3</bpmn:outgoing>
    </bpmn:userTask>
    <bpmn:endEvent id="Event_0tnuq42">
      <bpmn:incoming>Flow_1sjyli3</bpmn:incoming>
    </bpmn:endEvent>
    <bpmn:sequenceFlow id="Flow_1sjyli3" sourceRef="Activity_0p0oaod" targetRef="Event_0tnuq42" />
    <bpmn:sequenceFlow id="Flow_0nynj89" sourceRef="Activity_1c50ue8" targetRef="Activity_09yv1xf" />
    <bpmn:endEvent id="Event_1m4jkg8">
      <bpmn:incoming>Flow_14jl8qz</bpmn:incoming>
    </bpmn:endEvent>
    <bpmn:sequenceFlow id="Flow_14jl8qz" sourceRef="Activity_09yv1xf" targetRef="Event_1m4jkg8" />
    <bpmn:userTask id="Activity_09yv1xf" name="Regular end">
      <bpmn:incoming>Flow_0nynj89</bpmn:incoming>
      <bpmn:outgoing>Flow_14jl8qz</bpmn:outgoing>
    </bpmn:userTask>
  </bpmn:process>
  <bpmn:escalation id="Escalation_3bjg3ud" name="test_escalation" escalationCode="test_escalation" />
  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="test_escalationContinue">
      <bpmndi:BPMNEdge id="Flow_0k22u10_di" bpmnElement="Flow_0k22u10">
        <di:waypoint x="215" y="117" />
        <di:waypoint x="270" y="117" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_1qow6p8_di" bpmnElement="Flow_1qow6p8">
        <di:waypoint x="370" y="117" />
        <di:waypoint x="430" y="117" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_0ex7pl0_di" bpmnElement="Flow_0ex7pl0">
        <di:waypoint x="548" y="120" />
        <di:waypoint x="600" y="120" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_1sjyli3_di" bpmnElement="Flow_1sjyli3">
        <di:waypoint x="700" y="120" />
        <di:waypoint x="752" y="120" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_0nynj89_di" bpmnElement="Flow_0nynj89">
        <di:waypoint x="480" y="157" />
        <di:waypoint x="480" y="230" />
        <di:waypoint x="600" y="230" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_14jl8qz_di" bpmnElement="Flow_14jl8qz">
        <di:waypoint x="700" y="230" />
        <di:waypoint x="752" y="230" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
        <dc:Bounds x="179" y="99" width="36" height="36" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_16sjfgu_di" bpmnElement="Activity_02wcqyg">
        <dc:Bounds x="270" y="77" width="100" height="80" />
        <bpmndi:BPMNLabel />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_0fudaoi_di" bpmnElement="Activity_1c50ue8">
        <dc:Bounds x="430" y="77" width="100" height="80" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_1dvocxk_di" bpmnElement="Activity_0p0oaod">
        <dc:Bounds x="600" y="80" width="100" height="80" />
        <bpmndi:BPMNLabel />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Event_0tnuq42_di" bpmnElement="Event_0tnuq42">
        <dc:Bounds x="752" y="102" width="36" height="36" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Event_1m4jkg8_di" bpmnElement="Event_1m4jkg8">
        <dc:Bounds x="752" y="212" width="36" height="36" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_014mg7f_di" bpmnElement="Activity_09yv1xf">
        <dc:Bounds x="600" y="190" width="100" height="80" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Event_1y1rc6z_di" bpmnElement="Event_0dvlpqq">
        <dc:Bounds x="512" y="102" width="36" height="36" />
      </bpmndi:BPMNShape>
    </bpmndi:BPMNPlane>
  </bpmndi:BPMNDiagram>
</bpmn:definitions>
Second process:
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" xmlns:modeler="http://camunda.org/schema/modeler/1.0" id="Definitions_1btksc3" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="5.0.0" modeler:executionPlatform="Camunda Platform" modeler:executionPlatformVersion="7.17.0">
  <bpmn:process id="test_escalationProcess" name="Test escalation process" isExecutable="true">
    <bpmn:startEvent id="StartEvent_1">
      <bpmn:outgoing>Flow_06q3kve</bpmn:outgoing>
    </bpmn:startEvent>
    <bpmn:sequenceFlow id="Flow_06q3kve" sourceRef="StartEvent_1" targetRef="Activity_1fef452" />
    <bpmn:userTask id="Activity_1fef452" name="Do something">
      <bpmn:incoming>Flow_06q3kve</bpmn:incoming>
      <bpmn:outgoing>Flow_0ox5kgq</bpmn:outgoing>
    </bpmn:userTask>
    <bpmn:sequenceFlow id="Flow_0ox5kgq" sourceRef="Activity_1fef452" targetRef="Event_0mabqd8" />
    <bpmn:endEvent id="Event_0mabqd8">
      <bpmn:extensionElements />
      <bpmn:incoming>Flow_0ox5kgq</bpmn:incoming>
      <bpmn:escalationEventDefinition id="EscalationEventDefinition_15ohl0m" escalationRef="Escalation_1kuuj8v">
        <bpmn:extensionElements>
          <camunda:out source="a" target="a" />
        </bpmn:extensionElements>
      </bpmn:escalationEventDefinition>
    </bpmn:endEvent>
  </bpmn:process>
  <bpmn:escalation id="Escalation_1kuuj8v" name="test_escalation" escalationCode="test_escalation" />
  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="test_escalationProcess">
      <bpmndi:BPMNEdge id="Flow_0ox5kgq_di" bpmnElement="Flow_0ox5kgq">
        <di:waypoint x="370" y="117" />
        <di:waypoint x="442" y="117" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_06q3kve_di" bpmnElement="Flow_06q3kve">
        <di:waypoint x="215" y="117" />
        <di:waypoint x="270" y="117" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
        <dc:Bounds x="179" y="99" width="36" height="36" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_1lvnal1_di" bpmnElement="Activity_1fef452">
        <dc:Bounds x="270" y="77" width="100" height="80" />
        <bpmndi:BPMNLabel />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Event_0oe34sk_di" bpmnElement="Event_0mabqd8">
        <dc:Bounds x="442" y="99" width="36" height="36" />
        <bpmndi:BPMNLabel>
          <dc:Bounds x="525" y="142" width="11" height="14" />
        </bpmndi:BPMNLabel>
      </bpmndi:BPMNShape>
    </bpmndi:BPMNPlane>
  </bpmndi:BPMNDiagram>
</bpmn:definitions>

image
image

Other solutions that I thought about:
A solution that I think will work is using signal or message events instead of escalation, but this doesn’t seem like the proper way to achieve this goal (or am I mistaken?). I know that signals and messages carry variables, so replacing escalation with signal or message should also solve the issue that I had with not knowing what item was processed.
Another possible solution I thought about was to use end listeners. They have access to the variables (if propagation is set), but I’m not sure how to model properly the continuation from a listener (but I know they can be used to start a new process that can continue execution. This, however doesn’t seem to be a good solution, since it will not work with Camunda 8, which is a very good reason not to use it.
The final solution I thought of was to include the next steps inside the subprocess as other subprocesses. This will definitely work, but breaks separation of concerns principle, which I would rather not do.

So the question would be: what is the proper way to solve this scenario?

It seems I made a mistake within the test process. Escalations do pass variables, but I did not put them inside the output for the Call Activity.
Here is a working BPMN for those who would like to test this scenario out:

<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" xmlns:modeler="http://camunda.org/schema/modeler/1.0" id="Definitions_0xi0034" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="5.0.0" modeler:executionPlatform="Camunda Platform" modeler:executionPlatformVersion="7.17.0">
  <bpmn:process id="test_escalationContinue" name="Test escalation Continue" isExecutable="true">
    <bpmn:extensionElements />
    <bpmn:startEvent id="StartEvent_1">
      <bpmn:outgoing>Flow_0k22u10</bpmn:outgoing>
    </bpmn:startEvent>
    <bpmn:sequenceFlow id="Flow_0k22u10" sourceRef="StartEvent_1" targetRef="Activity_02wcqyg" />
    <bpmn:userTask id="Activity_02wcqyg" name="Wait for input">
      <bpmn:incoming>Flow_0k22u10</bpmn:incoming>
      <bpmn:outgoing>Flow_1qow6p8</bpmn:outgoing>
    </bpmn:userTask>
    <bpmn:sequenceFlow id="Flow_1qow6p8" sourceRef="Activity_02wcqyg" targetRef="Activity_1c50ue8" />
    <bpmn:callActivity id="Activity_1c50ue8" name="Test multi instance" calledElement="test_escalationProcess">
      <bpmn:extensionElements>
        <camunda:out variables="all" />
      </bpmn:extensionElements>
      <bpmn:incoming>Flow_1qow6p8</bpmn:incoming>
      <bpmn:outgoing>Flow_0nynj89</bpmn:outgoing>
      <bpmn:multiInstanceLoopCharacteristics>
        <bpmn:loopCardinality xsi:type="bpmn:tFormalExpression">${input}</bpmn:loopCardinality>
      </bpmn:multiInstanceLoopCharacteristics>
    </bpmn:callActivity>
    <bpmn:sequenceFlow id="Flow_0ex7pl0" sourceRef="Event_0dvlpqq" targetRef="Activity_0p0oaod" />
    <bpmn:userTask id="Activity_0p0oaod" name="Process output data">
      <bpmn:incoming>Flow_0ex7pl0</bpmn:incoming>
      <bpmn:outgoing>Flow_1sjyli3</bpmn:outgoing>
    </bpmn:userTask>
    <bpmn:endEvent id="Event_0tnuq42">
      <bpmn:incoming>Flow_1sjyli3</bpmn:incoming>
    </bpmn:endEvent>
    <bpmn:sequenceFlow id="Flow_1sjyli3" sourceRef="Activity_0p0oaod" targetRef="Event_0tnuq42" />
    <bpmn:sequenceFlow id="Flow_0nynj89" sourceRef="Activity_1c50ue8" targetRef="Activity_09yv1xf" />
    <bpmn:endEvent id="Event_1m4jkg8">
      <bpmn:incoming>Flow_14jl8qz</bpmn:incoming>
    </bpmn:endEvent>
    <bpmn:sequenceFlow id="Flow_14jl8qz" sourceRef="Activity_09yv1xf" targetRef="Event_1m4jkg8" />
    <bpmn:userTask id="Activity_09yv1xf" name="Regular end">
      <bpmn:incoming>Flow_0nynj89</bpmn:incoming>
      <bpmn:outgoing>Flow_14jl8qz</bpmn:outgoing>
    </bpmn:userTask>
    <bpmn:boundaryEvent id="Event_0dvlpqq" cancelActivity="false" attachedToRef="Activity_1c50ue8">
      <bpmn:outgoing>Flow_0ex7pl0</bpmn:outgoing>
      <bpmn:escalationEventDefinition id="EscalationEventDefinition_0zic6gr" escalationRef="Escalation_3bjg3ud" camunda:escalationCodeVariable="testEscVar" />
    </bpmn:boundaryEvent>
  </bpmn:process>
  <bpmn:escalation id="Escalation_3bjg3ud" name="test_escalation" escalationCode="test_escalation" />
  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="test_escalationContinue">
      <bpmndi:BPMNEdge id="Flow_14jl8qz_di" bpmnElement="Flow_14jl8qz">
        <di:waypoint x="700" y="230" />
        <di:waypoint x="752" y="230" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_0nynj89_di" bpmnElement="Flow_0nynj89">
        <di:waypoint x="480" y="157" />
        <di:waypoint x="480" y="230" />
        <di:waypoint x="600" y="230" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_1sjyli3_di" bpmnElement="Flow_1sjyli3">
        <di:waypoint x="700" y="120" />
        <di:waypoint x="752" y="120" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_0ex7pl0_di" bpmnElement="Flow_0ex7pl0">
        <di:waypoint x="548" y="120" />
        <di:waypoint x="600" y="120" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_1qow6p8_di" bpmnElement="Flow_1qow6p8">
        <di:waypoint x="370" y="117" />
        <di:waypoint x="430" y="117" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNEdge id="Flow_0k22u10_di" bpmnElement="Flow_0k22u10">
        <di:waypoint x="215" y="117" />
        <di:waypoint x="270" y="117" />
      </bpmndi:BPMNEdge>
      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
        <dc:Bounds x="179" y="99" width="36" height="36" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_16sjfgu_di" bpmnElement="Activity_02wcqyg">
        <dc:Bounds x="270" y="77" width="100" height="80" />
        <bpmndi:BPMNLabel />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_0fudaoi_di" bpmnElement="Activity_1c50ue8">
        <dc:Bounds x="430" y="77" width="100" height="80" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_1dvocxk_di" bpmnElement="Activity_0p0oaod">
        <dc:Bounds x="600" y="80" width="100" height="80" />
        <bpmndi:BPMNLabel />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Event_0tnuq42_di" bpmnElement="Event_0tnuq42">
        <dc:Bounds x="752" y="102" width="36" height="36" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Event_1m4jkg8_di" bpmnElement="Event_1m4jkg8">
        <dc:Bounds x="752" y="212" width="36" height="36" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Activity_014mg7f_di" bpmnElement="Activity_09yv1xf">
        <dc:Bounds x="600" y="190" width="100" height="80" />
      </bpmndi:BPMNShape>
      <bpmndi:BPMNShape id="Event_18d3lay_di" bpmnElement="Event_0dvlpqq">
        <dc:Bounds x="512" y="102" width="36" height="36" />
      </bpmndi:BPMNShape>
    </bpmndi:BPMNPlane>
  </bpmndi:BPMNDiagram>
</bpmn:definitions>

I used another method. Data from paralell processing passed to rpush redis and collecting result.
Redis is using as temp-table.
After all data in multyinstance activity processed, i just read data from redis by using lrange.

1 Like