Implemented rpc sending sequence

@@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry;
 import org.thingsboard.server.common.data.kv.KvEntry;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.common.data.page.SortOrder;
 import org.thingsboard.server.common.data.relation.EntityRelation;
 import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 import org.thingsboard.server.common.data.rpc.Rpc;
@@ -98,6 +99,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -132,7 +134,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         this.deviceId = deviceId;
         this.attributeSubscriptions = new HashMap<>();
         this.rpcSubscriptions = new HashMap<>();
-        this.toDeviceRpcPendingMap = new HashMap<>();
+        this.toDeviceRpcPendingMap = new LinkedHashMap<>();
         this.sessions = new LinkedHashMapRemoveEldest<>(systemContext.getMaxConcurrentSessionsPerDevice(), this::notifyTransportAboutClosedSessionMaxSessionsLimit);
         if (initAttributes()) {
             restoreSessions();
@@ -294,10 +296,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
             }
             systemContext.getTbCoreDeviceRpcService().processRpcResponseFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
                     null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION));
+            sendNextPendingRequest(context);
         }
     }
 
-    private void sendPendingRequests(TbActorCtx context, UUID sessionId, SessionInfoProto sessionInfo) {
+    private void sendPendingRequest(TbActorCtx context, UUID sessionId, String nodeId) {
         SessionType sessionType = getSessionType(sessionId);
         if (!toDeviceRpcPendingMap.isEmpty()) {
             log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId);
@@ -309,13 +312,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
             log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId);
         }
         Set<Integer> sentOneWayIds = new HashSet<>();
-        if (sessionType == SessionType.ASYNC) {
-            toDeviceRpcPendingMap.entrySet().forEach(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds));
-        } else {
-            toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, sessionInfo.getNodeId(), sentOneWayIds));
-        }
+        toDeviceRpcPendingMap.entrySet().stream().findFirst().ifPresent(processPendingRpc(context, sessionId, nodeId, sentOneWayIds));
+    }
 
-        sentOneWayIds.stream().filter(id -> !toDeviceRpcPendingMap.get(id).getMsg().getMsg().isPersisted()).forEach(toDeviceRpcPendingMap::remove);
+    private void sendNextPendingRequest(TbActorCtx context) {
+        rpcSubscriptions.forEach((id, s) -> sendPendingRequest(context, id, s.getNodeId()));
     }
 
     private Consumer<Map.Entry<Integer, ToDeviceRpcRequestMetadata>> processPendingRpc(TbActorCtx context, UUID sessionId, String nodeId, Set<Integer> sentOneWayIds) {
@@ -337,6 +338,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
                     .setPersisted(request.isPersisted())
                     .build();
             sendToTransport(rpcRequest, sessionId, nodeId);
+
+            if (SessionType.ASYNC.equals(getSessionType(sessionId)) && request.isOneway() && !request.isPersisted()) {
+                toDeviceRpcPendingMap.remove(entry.getKey());
+                sendPendingRequest(context, sessionId, nodeId);
+            }
         };
     }
 
@@ -355,7 +361,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
             processSubscriptionCommands(context, sessionInfo, msg.getSubscribeToRPC());
         }
         if (msg.hasSendPendingRPC()) {
-            sendPendingRequests(context, getSessionId(sessionInfo), sessionInfo);
+            sendPendingRequest(context, getSessionId(sessionInfo), sessionInfo.getNodeId());
         }
         if (msg.hasGetAttributes()) {
             handleGetAttributesRequest(context, sessionInfo, msg.getGetAttributes());
@@ -544,6 +550,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
                 }
                 systemContext.getTbRpcService().save(tenantId, new RpcId(requestMd.getMsg().getMsg().getId()), status, response);
             }
+            sendNextPendingRequest(context);
         } else {
             log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId());
         }
@@ -601,7 +608,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
             sessionMD.setSubscribedToRPC(true);
             log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId);
             rpcSubscriptions.put(sessionId, sessionMD.getSessionInfo());
-            sendPendingRequests(context, sessionId, sessionInfo);
+            sendPendingRequest(context, sessionId, sessionInfo.getNodeId());
             dumpSessions();
         }
     }
@@ -869,7 +876,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
 
     void init(TbActorCtx ctx) {
         schedulePeriodicMsgWithDelay(ctx, SessionTimeoutCheckMsg.instance(), systemContext.getSessionReportTimeout(), systemContext.getSessionReportTimeout());
-        PageLink pageLink = new PageLink(1024);
+        PageLink pageLink = new PageLink(1024, 0, null, new SortOrder("createdTime"));
         PageData<Rpc> pageData;
         do {
             pageData = systemContext.getTbRpcService().findAllByDeviceIdAndStatus(tenantId, deviceId, RpcStatus.QUEUED, pageLink);

@@ -90,6 +90,11 @@ public abstract class AbstractMqttServerSideRpcDefaultIntegrationTest extends Ab
     }
 
     @Test
+    public void testSequenceServerMqttTwoWayRpc() throws Exception {
+        processSequenceTwoWayRpcTest();
+    }
+
+    @Test
     public void testGatewayServerMqttOneWayRpc() throws Exception {
         processOneWayRpcTestGateway("Gateway Device OneWay RPC");
     }

@@ -16,6 +16,7 @@
 package org.thingsboard.server.transport.mqtt.rpc;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.nimbusds.jose.util.StandardCharset;
 import io.netty.handler.codec.mqtt.MqttQoS;
@@ -33,7 +34,9 @@ import org.thingsboard.server.common.data.device.profile.MqttTopics;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -101,6 +104,31 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM
         Assert.assertEquals(expected, result);
     }
 
+    protected void processSequenceTwoWayRpcTest() throws Exception {
+        List<String> expected = new ArrayList<>();
+        List<String> result = new ArrayList<>();
+
+        String deviceId = savedDevice.getId().getId().toString();
+
+        for (int i = 0; i < 10; i++) {
+            ObjectNode request = JacksonUtil.newObjectNode();
+            request.put("method", "test");
+            request.put("params", i);
+            expected.add(JacksonUtil.toString(request));
+            request.put("persistent", true);
+            doPostAsync("/api/rpc/twoway/" + deviceId, JacksonUtil.toString(request), String.class, status().isOk());
+        }
+
+        MqttAsyncClient client = getMqttAsyncClient(accessToken);
+        CountDownLatch latch = new CountDownLatch(10);
+        TestSequenceMqttCallback callback = new TestSequenceMqttCallback(client, latch, result);
+        client.setCallback(callback);
+        client.subscribe(MqttTopics.DEVICE_RPC_REQUESTS_SUB_TOPIC, 1);
+
+        latch.await(30, TimeUnit.SECONDS);
+        Assert.assertEquals(expected, result);
+    }
+
     protected void processTwoWayRpcTestGateway(String deviceName) throws Exception {
         MqttAsyncClient client = getMqttAsyncClient(gatewayAccessToken);
 
@@ -213,4 +241,41 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM
 
         }
     }
+
+    protected class TestSequenceMqttCallback implements MqttCallback {
+
+        private final MqttAsyncClient client;
+        private final CountDownLatch latch;
+        private final  List<String> expected;
+        private Integer qoS;
+
+        TestSequenceMqttCallback(MqttAsyncClient client, CountDownLatch latch, List<String> expected) {
+            this.client = client;
+            this.latch = latch;
+            this.expected = expected;
+        }
+
+        int getQoS() {
+            return qoS;
+        }
+
+        @Override
+        public void connectionLost(Throwable throwable) {
+        }
+
+        @Override
+        public void messageArrived(String requestTopic, MqttMessage mqttMessage) throws Exception {
+            log.info("Message Arrived: " + Arrays.toString(mqttMessage.getPayload()));
+            expected.add(new String(mqttMessage.getPayload()));
+            String responseTopic = requestTopic.replace("request", "response");
+            qoS = mqttMessage.getQos();
+            client.publish(responseTopic, processMessageArrived(requestTopic, mqttMessage));
+            latch.countDown();
+        }
+
+        @Override
+        public void deliveryComplete(IMqttDeliveryToken iMqttDeliveryToken) {
+
+        }
+    }
 }

@@ -723,7 +723,7 @@ public class DefaultCoapClientContext implements CoapClientContext {
     private void cancelRpcSubscription(TbCoapClientState state) {
         if (state.getRpc() != null) {
             clientsByToken.remove(state.getRpc().getToken());
-            CoapExchange exchange = state.getAttrs().getExchange();
+            CoapExchange exchange = state.getRpc().getExchange();
             state.setRpc(null);
             transportService.process(state.getSession(),
                     TransportProtos.SubscribeToRPCMsg.newBuilder().setUnsubscribe(true).build(),