简柏林 / thingskit · Commits

Commit 8b5793f203202feb3b49b34226601851c5c6553f

Authored by Igor Kulikov
2 parents 4e153a93 30a2d19d

Merge branch 'master' of github.com:thingsboard/thingsboard into develop/3.0

Inline Side-by-side
Showing 24 changed files with 1369 additions and 151 deletions
@@ -18,6 +18,7 @@ package org.thingsboard.server.queue.azure.servicebus; @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.azure.servicebus;
18 import com.microsoft.azure.servicebus.management.ManagementClient; 18 import com.microsoft.azure.servicebus.management.ManagementClient;
19 import com.microsoft.azure.servicebus.management.QueueDescription; 19 import com.microsoft.azure.servicebus.management.QueueDescription;
20 import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; 20 import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder;
  21 +import com.microsoft.azure.servicebus.primitives.MessagingEntityAlreadyExistsException;
21 import com.microsoft.azure.servicebus.primitives.ServiceBusException; 22 import com.microsoft.azure.servicebus.primitives.ServiceBusException;
22 import lombok.extern.slf4j.Slf4j; 23 import lombok.extern.slf4j.Slf4j;
23 import org.thingsboard.server.queue.TbQueueAdmin; 24 import org.thingsboard.server.queue.TbQueueAdmin;
@@ -71,7 +72,12 @@ public class TbServiceBusAdmin implements TbQueueAdmin { @@ -71,7 +72,12 @@ public class TbServiceBusAdmin implements TbQueueAdmin {
71 client.createQueue(queueDescription); 72 client.createQueue(queueDescription);
72 queues.add(topic); 73 queues.add(topic);
73 } catch (ServiceBusException | InterruptedException e) { 74 } catch (ServiceBusException | InterruptedException e) {
74 - log.error("Failed to create queue: [{}]", topic, e); 75 + if (e instanceof MessagingEntityAlreadyExistsException) {
  76 + queues.add(topic);
  77 + log.info("[{}] queue already exists.", topic);
  78 + } else {
  79 + log.error("Failed to create queue: [{}]", topic, e);
  80 + }
75 } 81 }
76 } 82 }
77 83
@@ -15,20 +15,25 @@ @@ -15,20 +15,25 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 -import org.thingsboard.server.gen.js.JsInvokeProtos; 23 +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
  24 +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
22 import org.thingsboard.server.gen.transport.TransportProtos; 25 import org.thingsboard.server.gen.transport.TransportProtos;
23 import org.thingsboard.server.queue.TbQueueAdmin; 26 import org.thingsboard.server.queue.TbQueueAdmin;
24 import org.thingsboard.server.queue.TbQueueConsumer; 27 import org.thingsboard.server.queue.TbQueueConsumer;
25 import org.thingsboard.server.queue.TbQueueProducer; 28 import org.thingsboard.server.queue.TbQueueProducer;
26 import org.thingsboard.server.queue.TbQueueRequestTemplate; 29 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  30 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
27 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 31 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
28 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 32 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
29 import org.thingsboard.server.queue.discovery.PartitionService; 33 import org.thingsboard.server.queue.discovery.PartitionService;
30 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; 34 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
31 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 35 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  36 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
32 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 37 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
33 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; 38 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
34 import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; 39 import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
@@ -40,6 +45,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; @@ -40,6 +45,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes;
40 import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; 45 import org.thingsboard.server.queue.sqs.TbAwsSqsSettings;
41 46
42 import javax.annotation.PreDestroy; 47 import javax.annotation.PreDestroy;
  48 +import java.nio.charset.StandardCharsets;
43 49
44 @Component 50 @Component
45 @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") 51 @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'")
@@ -52,6 +58,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -52,6 +58,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
52 private final TbQueueTransportApiSettings transportApiSettings; 58 private final TbQueueTransportApiSettings transportApiSettings;
53 private final TbQueueTransportNotificationSettings transportNotificationSettings; 59 private final TbQueueTransportNotificationSettings transportNotificationSettings;
54 private final TbAwsSqsSettings sqsSettings; 60 private final TbAwsSqsSettings sqsSettings;
  61 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
55 62
56 private final TbQueueAdmin coreAdmin; 63 private final TbQueueAdmin coreAdmin;
57 private final TbQueueAdmin ruleEngineAdmin; 64 private final TbQueueAdmin ruleEngineAdmin;
@@ -65,7 +72,8 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -65,7 +72,8 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
65 TbQueueTransportApiSettings transportApiSettings, 72 TbQueueTransportApiSettings transportApiSettings,
66 TbQueueTransportNotificationSettings transportNotificationSettings, 73 TbQueueTransportNotificationSettings transportNotificationSettings,
67 TbAwsSqsSettings sqsSettings, 74 TbAwsSqsSettings sqsSettings,
68 - TbAwsSqsQueueAttributes sqsQueueAttributes) { 75 + TbAwsSqsQueueAttributes sqsQueueAttributes,
  76 + TbQueueRemoteJsInvokeSettings jsInvokeSettings) {
69 this.partitionService = partitionService; 77 this.partitionService = partitionService;
70 this.coreSettings = coreSettings; 78 this.coreSettings = coreSettings;
71 this.serviceInfoProvider = serviceInfoProvider; 79 this.serviceInfoProvider = serviceInfoProvider;
@@ -73,6 +81,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -73,6 +81,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
73 this.transportApiSettings = transportApiSettings; 81 this.transportApiSettings = transportApiSettings;
74 this.transportNotificationSettings = transportNotificationSettings; 82 this.transportNotificationSettings = transportNotificationSettings;
75 this.sqsSettings = sqsSettings; 83 this.sqsSettings = sqsSettings;
  84 + this.jsInvokeSettings = jsInvokeSettings;
76 85
77 this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); 86 this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes());
78 this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); 87 this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes());
@@ -144,8 +153,26 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -144,8 +153,26 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
144 } 153 }
145 154
146 @Override 155 @Override
147 - public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {  
148 - return null; 156 + @Bean
  157 + public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() {
  158 + TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic());
  159 + TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings,
  160 + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(),
  161 + msg -> {
  162 + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder();
  163 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  164 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  165 + });
  166 +
  167 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  168 + <TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  169 + builder.queueAdmin(jsExecutorAdmin);
  170 + builder.requestTemplate(producer);
  171 + builder.responseTemplate(consumer);
  172 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  173 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  174 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  175 + return builder.build();
149 } 176 }
150 177
151 @PreDestroy 178 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -30,11 +32,13 @@ import org.thingsboard.server.queue.TbQueueAdmin; @@ -30,11 +32,13 @@ import org.thingsboard.server.queue.TbQueueAdmin;
30 import org.thingsboard.server.queue.TbQueueConsumer; 32 import org.thingsboard.server.queue.TbQueueConsumer;
31 import org.thingsboard.server.queue.TbQueueProducer; 33 import org.thingsboard.server.queue.TbQueueProducer;
32 import org.thingsboard.server.queue.TbQueueRequestTemplate; 34 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  35 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
33 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 36 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
34 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 37 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
35 import org.thingsboard.server.queue.discovery.PartitionService; 38 import org.thingsboard.server.queue.discovery.PartitionService;
36 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; 39 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
37 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 40 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  41 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
38 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 42 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
39 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; 43 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
40 import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; 44 import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin;
@@ -44,6 +48,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; @@ -44,6 +48,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes;
44 import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; 48 import org.thingsboard.server.queue.sqs.TbAwsSqsSettings;
45 49
46 import javax.annotation.PreDestroy; 50 import javax.annotation.PreDestroy;
  51 +import java.nio.charset.StandardCharsets;
47 52
48 @Component 53 @Component
49 @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") 54 @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'")
@@ -55,6 +60,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @@ -55,6 +60,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
55 private final TbQueueTransportApiSettings transportApiSettings; 60 private final TbQueueTransportApiSettings transportApiSettings;
56 private final PartitionService partitionService; 61 private final PartitionService partitionService;
57 private final TbServiceInfoProvider serviceInfoProvider; 62 private final TbServiceInfoProvider serviceInfoProvider;
  63 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
58 64
59 private final TbQueueAdmin coreAdmin; 65 private final TbQueueAdmin coreAdmin;
60 private final TbQueueAdmin ruleEngineAdmin; 66 private final TbQueueAdmin ruleEngineAdmin;
@@ -68,6 +74,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @@ -68,6 +74,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
68 TbQueueRuleEngineSettings ruleEngineSettings, 74 TbQueueRuleEngineSettings ruleEngineSettings,
69 PartitionService partitionService, 75 PartitionService partitionService,
70 TbServiceInfoProvider serviceInfoProvider, 76 TbServiceInfoProvider serviceInfoProvider,
  77 + TbQueueRemoteJsInvokeSettings jsInvokeSettings,
71 TbAwsSqsQueueAttributes sqsQueueAttributes) { 78 TbAwsSqsQueueAttributes sqsQueueAttributes) {
72 this.sqsSettings = sqsSettings; 79 this.sqsSettings = sqsSettings;
73 this.coreSettings = coreSettings; 80 this.coreSettings = coreSettings;
@@ -75,6 +82,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @@ -75,6 +82,7 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
75 this.ruleEngineSettings = ruleEngineSettings; 82 this.ruleEngineSettings = ruleEngineSettings;
76 this.partitionService = partitionService; 83 this.partitionService = partitionService;
77 this.serviceInfoProvider = serviceInfoProvider; 84 this.serviceInfoProvider = serviceInfoProvider;
  85 + this.jsInvokeSettings = jsInvokeSettings;
78 86
79 this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); 87 this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes());
80 this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); 88 this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes());
@@ -133,8 +141,26 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @@ -133,8 +141,26 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
133 } 141 }
134 142
135 @Override 143 @Override
  144 + @Bean
136 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 145 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
137 - return null; 146 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic());
  147 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings,
  148 + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(),
  149 + msg -> {
  150 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  151 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  152 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  153 + });
  154 +
  155 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  156 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  157 + builder.queueAdmin(jsExecutorAdmin);
  158 + builder.requestTemplate(producer);
  159 + builder.responseTemplate(consumer);
  160 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  161 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  162 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  163 + return builder.build();
138 } 164 }
139 165
140 @PreDestroy 166 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -27,11 +29,13 @@ import org.thingsboard.server.queue.TbQueueAdmin; @@ -27,11 +29,13 @@ import org.thingsboard.server.queue.TbQueueAdmin;
27 import org.thingsboard.server.queue.TbQueueConsumer; 29 import org.thingsboard.server.queue.TbQueueConsumer;
28 import org.thingsboard.server.queue.TbQueueProducer; 30 import org.thingsboard.server.queue.TbQueueProducer;
29 import org.thingsboard.server.queue.TbQueueRequestTemplate; 31 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  32 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
30 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 33 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
31 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 34 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
32 import org.thingsboard.server.queue.discovery.PartitionService; 35 import org.thingsboard.server.queue.discovery.PartitionService;
33 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; 36 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
34 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 37 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  38 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
35 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 39 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
36 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; 40 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
37 import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; 41 import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin;
@@ -41,6 +45,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; @@ -41,6 +45,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes;
41 import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; 45 import org.thingsboard.server.queue.sqs.TbAwsSqsSettings;
42 46
43 import javax.annotation.PreDestroy; 47 import javax.annotation.PreDestroy;
  48 +import java.nio.charset.StandardCharsets;
44 49
45 @Component 50 @Component
46 @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") 51 @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'")
@@ -51,6 +56,7 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @@ -51,6 +56,7 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
51 private final TbServiceInfoProvider serviceInfoProvider; 56 private final TbServiceInfoProvider serviceInfoProvider;
52 private final TbQueueRuleEngineSettings ruleEngineSettings; 57 private final TbQueueRuleEngineSettings ruleEngineSettings;
53 private final TbAwsSqsSettings sqsSettings; 58 private final TbAwsSqsSettings sqsSettings;
  59 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
54 60
55 private final TbQueueAdmin coreAdmin; 61 private final TbQueueAdmin coreAdmin;
56 private final TbQueueAdmin ruleEngineAdmin; 62 private final TbQueueAdmin ruleEngineAdmin;
@@ -61,12 +67,14 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @@ -61,12 +67,14 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
61 TbQueueRuleEngineSettings ruleEngineSettings, 67 TbQueueRuleEngineSettings ruleEngineSettings,
62 TbServiceInfoProvider serviceInfoProvider, 68 TbServiceInfoProvider serviceInfoProvider,
63 TbAwsSqsSettings sqsSettings, 69 TbAwsSqsSettings sqsSettings,
64 - TbAwsSqsQueueAttributes sqsQueueAttributes) { 70 + TbAwsSqsQueueAttributes sqsQueueAttributes,
  71 + TbQueueRemoteJsInvokeSettings jsInvokeSettings) {
65 this.partitionService = partitionService; 72 this.partitionService = partitionService;
66 this.coreSettings = coreSettings; 73 this.coreSettings = coreSettings;
67 this.serviceInfoProvider = serviceInfoProvider; 74 this.serviceInfoProvider = serviceInfoProvider;
68 this.ruleEngineSettings = ruleEngineSettings; 75 this.ruleEngineSettings = ruleEngineSettings;
69 this.sqsSettings = sqsSettings; 76 this.sqsSettings = sqsSettings;
  77 + this.jsInvokeSettings = jsInvokeSettings;
70 78
71 this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); 79 this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes());
72 this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); 80 this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes());
@@ -113,8 +121,26 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @@ -113,8 +121,26 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
113 } 121 }
114 122
115 @Override 123 @Override
  124 + @Bean
116 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 125 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
117 - return null; 126 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic());
  127 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings,
  128 + jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(),
  129 + msg -> {
  130 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  131 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  132 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  133 + });
  134 +
  135 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  136 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  137 + builder.queueAdmin(jsExecutorAdmin);
  138 + builder.requestTemplate(producer);
  139 + builder.responseTemplate(consumer);
  140 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  141 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  142 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  143 + return builder.build();
118 } 144 }
119 145
120 @PreDestroy 146 @PreDestroy
@@ -15,10 +15,13 @@ @@ -15,10 +15,13 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 -import org.thingsboard.server.gen.js.JsInvokeProtos; 23 +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
  24 +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
22 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
23 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; 26 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
24 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
@@ -30,6 +33,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; @@ -30,6 +33,7 @@ import org.thingsboard.server.queue.TbQueueAdmin;
30 import org.thingsboard.server.queue.TbQueueConsumer; 33 import org.thingsboard.server.queue.TbQueueConsumer;
31 import org.thingsboard.server.queue.TbQueueProducer; 34 import org.thingsboard.server.queue.TbQueueProducer;
32 import org.thingsboard.server.queue.TbQueueRequestTemplate; 35 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  36 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
33 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 37 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
34 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 38 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
35 import org.thingsboard.server.queue.discovery.PartitionService; 39 import org.thingsboard.server.queue.discovery.PartitionService;
@@ -40,12 +44,14 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; @@ -40,12 +44,14 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
40 import org.thingsboard.server.queue.pubsub.TbPubSubSettings; 44 import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
41 import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; 45 import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
42 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 46 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  47 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
43 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 48 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
44 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; 49 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
45 import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; 50 import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
46 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; 51 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
47 52
48 import javax.annotation.PreDestroy; 53 import javax.annotation.PreDestroy;
  54 +import java.nio.charset.StandardCharsets;
49 55
50 @Component 56 @Component
51 @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") 57 @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'")
@@ -58,6 +64,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -58,6 +64,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
58 private final TbQueueTransportNotificationSettings transportNotificationSettings; 64 private final TbQueueTransportNotificationSettings transportNotificationSettings;
59 private final PartitionService partitionService; 65 private final PartitionService partitionService;
60 private final TbServiceInfoProvider serviceInfoProvider; 66 private final TbServiceInfoProvider serviceInfoProvider;
  67 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
61 68
62 private final TbQueueAdmin coreAdmin; 69 private final TbQueueAdmin coreAdmin;
63 private final TbQueueAdmin ruleEngineAdmin; 70 private final TbQueueAdmin ruleEngineAdmin;
@@ -72,7 +79,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -72,7 +79,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
72 TbQueueTransportNotificationSettings transportNotificationSettings, 79 TbQueueTransportNotificationSettings transportNotificationSettings,
73 PartitionService partitionService, 80 PartitionService partitionService,
74 TbServiceInfoProvider serviceInfoProvider, 81 TbServiceInfoProvider serviceInfoProvider,
75 - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { 82 + TbPubSubSubscriptionSettings pubSubSubscriptionSettings,
  83 + TbQueueRemoteJsInvokeSettings jsInvokeSettings) {
76 this.pubSubSettings = pubSubSettings; 84 this.pubSubSettings = pubSubSettings;
77 this.coreSettings = coreSettings; 85 this.coreSettings = coreSettings;
78 this.ruleEngineSettings = ruleEngineSettings; 86 this.ruleEngineSettings = ruleEngineSettings;
@@ -86,6 +94,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -86,6 +94,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
86 this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); 94 this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings());
87 this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); 95 this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings());
88 this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); 96 this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings());
  97 + this.jsInvokeSettings = jsInvokeSettings;
89 } 98 }
90 99
91 @Override 100 @Override
@@ -152,8 +161,26 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -152,8 +161,26 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
152 } 161 }
153 162
154 @Override 163 @Override
155 - public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {  
156 - return null; 164 + @Bean
  165 + public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() {
  166 + TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic());
  167 + TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings,
  168 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  169 + msg -> {
  170 + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder();
  171 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  172 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  173 + });
  174 +
  175 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  176 + <TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  177 + builder.queueAdmin(jsExecutorAdmin);
  178 + builder.requestTemplate(producer);
  179 + builder.responseTemplate(consumer);
  180 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  181 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  182 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  183 + return builder.build();
157 } 184 }
158 185
159 @PreDestroy 186 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -28,21 +30,24 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM @@ -28,21 +30,24 @@ import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestM
28 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; 30 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
29 import org.thingsboard.server.queue.TbQueueAdmin; 31 import org.thingsboard.server.queue.TbQueueAdmin;
30 import org.thingsboard.server.queue.TbQueueConsumer; 32 import org.thingsboard.server.queue.TbQueueConsumer;
  33 +import org.thingsboard.server.queue.TbQueueProducer;
31 import org.thingsboard.server.queue.TbQueueRequestTemplate; 34 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  35 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
32 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 36 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
33 -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;  
34 -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;  
35 -import org.thingsboard.server.queue.settings.TbQueueCoreSettings;  
36 -import org.thingsboard.server.queue.TbQueueProducer;  
37 -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;  
38 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 37 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
39 import org.thingsboard.server.queue.discovery.PartitionService; 38 import org.thingsboard.server.queue.discovery.PartitionService;
40 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; 39 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
  40 +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
41 import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; 41 import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate;
42 import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; 42 import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
43 import org.thingsboard.server.queue.pubsub.TbPubSubSettings; 43 import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
  44 +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
  45 +import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  46 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
  47 +import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
44 48
45 import javax.annotation.PreDestroy; 49 import javax.annotation.PreDestroy;
  50 +import java.nio.charset.StandardCharsets;
46 51
47 @Component 52 @Component
48 @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") 53 @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'")
@@ -53,6 +58,7 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @@ -53,6 +58,7 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
53 private final TbQueueTransportApiSettings transportApiSettings; 58 private final TbQueueTransportApiSettings transportApiSettings;
54 private final PartitionService partitionService; 59 private final PartitionService partitionService;
55 private final TbServiceInfoProvider serviceInfoProvider; 60 private final TbServiceInfoProvider serviceInfoProvider;
  61 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
56 62
57 private final TbQueueAdmin coreAdmin; 63 private final TbQueueAdmin coreAdmin;
58 private final TbQueueAdmin jsExecutorAdmin; 64 private final TbQueueAdmin jsExecutorAdmin;
@@ -64,12 +70,14 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @@ -64,12 +70,14 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
64 TbQueueTransportApiSettings transportApiSettings, 70 TbQueueTransportApiSettings transportApiSettings,
65 PartitionService partitionService, 71 PartitionService partitionService,
66 TbServiceInfoProvider serviceInfoProvider, 72 TbServiceInfoProvider serviceInfoProvider,
  73 + TbQueueRemoteJsInvokeSettings jsInvokeSettings,
67 TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { 74 TbPubSubSubscriptionSettings pubSubSubscriptionSettings) {
68 this.pubSubSettings = pubSubSettings; 75 this.pubSubSettings = pubSubSettings;
69 this.coreSettings = coreSettings; 76 this.coreSettings = coreSettings;
70 this.transportApiSettings = transportApiSettings; 77 this.transportApiSettings = transportApiSettings;
71 this.partitionService = partitionService; 78 this.partitionService = partitionService;
72 this.serviceInfoProvider = serviceInfoProvider; 79 this.serviceInfoProvider = serviceInfoProvider;
  80 + this.jsInvokeSettings = jsInvokeSettings;
73 81
74 this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); 82 this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
75 this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); 83 this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings());
@@ -127,8 +135,26 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @@ -127,8 +135,26 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
127 } 135 }
128 136
129 @Override 137 @Override
  138 + @Bean
130 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 139 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
131 - return null; 140 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic());
  141 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings,
  142 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  143 + msg -> {
  144 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  145 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  146 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  147 + });
  148 +
  149 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  150 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  151 + builder.queueAdmin(jsExecutorAdmin);
  152 + builder.requestTemplate(producer);
  153 + builder.responseTemplate(consumer);
  154 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  155 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  156 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  157 + return builder.build();
132 } 158 }
133 159
134 @PreDestroy 160 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -28,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; @@ -28,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueAdmin;
28 import org.thingsboard.server.queue.TbQueueConsumer; 30 import org.thingsboard.server.queue.TbQueueConsumer;
29 import org.thingsboard.server.queue.TbQueueProducer; 31 import org.thingsboard.server.queue.TbQueueProducer;
30 import org.thingsboard.server.queue.TbQueueRequestTemplate; 32 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  33 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
31 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 34 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
32 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 35 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
33 import org.thingsboard.server.queue.discovery.PartitionService; 36 import org.thingsboard.server.queue.discovery.PartitionService;
@@ -38,10 +41,12 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; @@ -38,10 +41,12 @@ import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate;
38 import org.thingsboard.server.queue.pubsub.TbPubSubSettings; 41 import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
39 import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; 42 import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
40 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 43 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  44 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
41 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 45 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
42 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; 46 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
43 47
44 import javax.annotation.PreDestroy; 48 import javax.annotation.PreDestroy;
  49 +import java.nio.charset.StandardCharsets;
45 50
46 @Component 51 @Component
47 @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") 52 @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'")
@@ -52,6 +57,7 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @@ -52,6 +57,7 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
52 private final TbQueueRuleEngineSettings ruleEngineSettings; 57 private final TbQueueRuleEngineSettings ruleEngineSettings;
53 private final PartitionService partitionService; 58 private final PartitionService partitionService;
54 private final TbServiceInfoProvider serviceInfoProvider; 59 private final TbServiceInfoProvider serviceInfoProvider;
  60 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
55 61
56 private final TbQueueAdmin coreAdmin; 62 private final TbQueueAdmin coreAdmin;
57 private final TbQueueAdmin ruleEngineAdmin; 63 private final TbQueueAdmin ruleEngineAdmin;
@@ -63,12 +69,14 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @@ -63,12 +69,14 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
63 TbQueueRuleEngineSettings ruleEngineSettings, 69 TbQueueRuleEngineSettings ruleEngineSettings,
64 PartitionService partitionService, 70 PartitionService partitionService,
65 TbServiceInfoProvider serviceInfoProvider, 71 TbServiceInfoProvider serviceInfoProvider,
  72 + TbQueueRemoteJsInvokeSettings jsInvokeSettings,
66 TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { 73 TbPubSubSubscriptionSettings pubSubSubscriptionSettings) {
67 this.pubSubSettings = pubSubSettings; 74 this.pubSubSettings = pubSubSettings;
68 this.coreSettings = coreSettings; 75 this.coreSettings = coreSettings;
69 this.ruleEngineSettings = ruleEngineSettings; 76 this.ruleEngineSettings = ruleEngineSettings;
70 this.partitionService = partitionService; 77 this.partitionService = partitionService;
71 this.serviceInfoProvider = serviceInfoProvider; 78 this.serviceInfoProvider = serviceInfoProvider;
  79 + this.jsInvokeSettings = jsInvokeSettings;
72 80
73 this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); 81 this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings());
74 this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); 82 this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings());
@@ -116,8 +124,26 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @@ -116,8 +124,26 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
116 } 124 }
117 125
118 @Override 126 @Override
  127 + @Bean
119 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 128 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
120 - return null; 129 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic());
  130 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings,
  131 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  132 + msg -> {
  133 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  134 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  135 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  136 + });
  137 +
  138 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  139 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  140 + builder.queueAdmin(jsExecutorAdmin);
  141 + builder.requestTemplate(producer);
  142 + builder.responseTemplate(consumer);
  143 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  144 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  145 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  146 + return builder.build();
121 } 147 }
122 148
123 @PreDestroy 149 @PreDestroy
@@ -15,15 +15,19 @@ @@ -15,15 +15,19 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 -import org.thingsboard.server.gen.js.JsInvokeProtos; 23 +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
  24 +import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
22 import org.thingsboard.server.gen.transport.TransportProtos; 25 import org.thingsboard.server.gen.transport.TransportProtos;
23 import org.thingsboard.server.queue.TbQueueAdmin; 26 import org.thingsboard.server.queue.TbQueueAdmin;
24 import org.thingsboard.server.queue.TbQueueConsumer; 27 import org.thingsboard.server.queue.TbQueueConsumer;
25 import org.thingsboard.server.queue.TbQueueProducer; 28 import org.thingsboard.server.queue.TbQueueProducer;
26 import org.thingsboard.server.queue.TbQueueRequestTemplate; 29 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  30 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
27 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 31 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
28 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 32 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
29 import org.thingsboard.server.queue.discovery.PartitionService; 33 import org.thingsboard.server.queue.discovery.PartitionService;
@@ -34,12 +38,14 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; @@ -34,12 +38,14 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
34 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; 38 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
35 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; 39 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
36 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 40 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  41 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
37 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 42 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
38 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; 43 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
39 import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; 44 import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
40 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; 45 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
41 46
42 import javax.annotation.PreDestroy; 47 import javax.annotation.PreDestroy;
  48 +import java.nio.charset.StandardCharsets;
43 49
44 @Component 50 @Component
45 @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") 51 @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'")
@@ -58,6 +64,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE @@ -58,6 +64,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
58 private final TbQueueAdmin jsExecutorAdmin; 64 private final TbQueueAdmin jsExecutorAdmin;
59 private final TbQueueAdmin transportApiAdmin; 65 private final TbQueueAdmin transportApiAdmin;
60 private final TbQueueAdmin notificationAdmin; 66 private final TbQueueAdmin notificationAdmin;
  67 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
61 68
62 public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, 69 public RabbitMqMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings,
63 TbQueueRuleEngineSettings ruleEngineSettings, 70 TbQueueRuleEngineSettings ruleEngineSettings,
@@ -65,7 +72,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE @@ -65,7 +72,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
65 TbQueueTransportApiSettings transportApiSettings, 72 TbQueueTransportApiSettings transportApiSettings,
66 TbQueueTransportNotificationSettings transportNotificationSettings, 73 TbQueueTransportNotificationSettings transportNotificationSettings,
67 TbRabbitMqSettings rabbitMqSettings, 74 TbRabbitMqSettings rabbitMqSettings,
68 - TbRabbitMqQueueArguments queueArguments) { 75 + TbRabbitMqQueueArguments queueArguments,
  76 + TbQueueRemoteJsInvokeSettings jsInvokeSettings) {
69 this.partitionService = partitionService; 77 this.partitionService = partitionService;
70 this.coreSettings = coreSettings; 78 this.coreSettings = coreSettings;
71 this.serviceInfoProvider = serviceInfoProvider; 79 this.serviceInfoProvider = serviceInfoProvider;
@@ -79,6 +87,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE @@ -79,6 +87,7 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
79 this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); 87 this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs());
80 this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); 88 this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs());
81 this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); 89 this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs());
  90 + this.jsInvokeSettings = jsInvokeSettings;
82 } 91 }
83 92
84 @Override 93 @Override
@@ -144,8 +153,26 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE @@ -144,8 +153,26 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
144 } 153 }
145 154
146 @Override 155 @Override
147 - public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {  
148 - return null; 156 + @Bean
  157 + public TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> createRemoteJsRequestTemplate() {
  158 + TbQueueProducer<TbProtoJsQueueMsg<RemoteJsRequest>> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic());
  159 + TbQueueConsumer<TbProtoQueueMsg<RemoteJsResponse>> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings,
  160 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  161 + msg -> {
  162 + RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder();
  163 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  164 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  165 + });
  166 +
  167 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  168 + <TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  169 + builder.queueAdmin(jsExecutorAdmin);
  170 + builder.requestTemplate(producer);
  171 + builder.responseTemplate(consumer);
  172 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  173 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  174 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  175 + return builder.build();
149 } 176 }
150 177
151 @PreDestroy 178 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -30,6 +32,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; @@ -30,6 +32,7 @@ import org.thingsboard.server.queue.TbQueueAdmin;
30 import org.thingsboard.server.queue.TbQueueConsumer; 32 import org.thingsboard.server.queue.TbQueueConsumer;
31 import org.thingsboard.server.queue.TbQueueProducer; 33 import org.thingsboard.server.queue.TbQueueProducer;
32 import org.thingsboard.server.queue.TbQueueRequestTemplate; 34 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  35 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
33 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 36 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
34 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 37 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
35 import org.thingsboard.server.queue.discovery.PartitionService; 38 import org.thingsboard.server.queue.discovery.PartitionService;
@@ -40,10 +43,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; @@ -40,10 +43,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
40 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; 43 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
41 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; 44 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
42 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 45 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  46 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
43 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 47 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
44 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; 48 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
45 49
46 import javax.annotation.PreDestroy; 50 import javax.annotation.PreDestroy;
  51 +import java.nio.charset.StandardCharsets;
47 52
48 @Component 53 @Component
49 @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") 54 @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'")
@@ -55,6 +60,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { @@ -55,6 +60,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
55 private final TbQueueTransportApiSettings transportApiSettings; 60 private final TbQueueTransportApiSettings transportApiSettings;
56 private final PartitionService partitionService; 61 private final PartitionService partitionService;
57 private final TbServiceInfoProvider serviceInfoProvider; 62 private final TbServiceInfoProvider serviceInfoProvider;
  63 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
58 64
59 private final TbQueueAdmin coreAdmin; 65 private final TbQueueAdmin coreAdmin;
60 private final TbQueueAdmin ruleEngineAdmin; 66 private final TbQueueAdmin ruleEngineAdmin;
@@ -68,6 +74,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { @@ -68,6 +74,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
68 TbQueueRuleEngineSettings ruleEngineSettings, 74 TbQueueRuleEngineSettings ruleEngineSettings,
69 PartitionService partitionService, 75 PartitionService partitionService,
70 TbServiceInfoProvider serviceInfoProvider, 76 TbServiceInfoProvider serviceInfoProvider,
  77 + TbQueueRemoteJsInvokeSettings jsInvokeSettings,
71 TbRabbitMqQueueArguments queueArguments) { 78 TbRabbitMqQueueArguments queueArguments) {
72 this.rabbitMqSettings = rabbitMqSettings; 79 this.rabbitMqSettings = rabbitMqSettings;
73 this.coreSettings = coreSettings; 80 this.coreSettings = coreSettings;
@@ -75,6 +82,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { @@ -75,6 +82,7 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
75 this.ruleEngineSettings = ruleEngineSettings; 82 this.ruleEngineSettings = ruleEngineSettings;
76 this.partitionService = partitionService; 83 this.partitionService = partitionService;
77 this.serviceInfoProvider = serviceInfoProvider; 84 this.serviceInfoProvider = serviceInfoProvider;
  85 + this.jsInvokeSettings = jsInvokeSettings;
78 86
79 this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); 87 this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
80 this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); 88 this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
@@ -133,8 +141,26 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { @@ -133,8 +141,26 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
133 } 141 }
134 142
135 @Override 143 @Override
  144 + @Bean
136 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 145 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
137 - return null; 146 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic());
  147 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings,
  148 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  149 + msg -> {
  150 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  151 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  152 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  153 + });
  154 +
  155 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  156 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  157 + builder.queueAdmin(jsExecutorAdmin);
  158 + builder.requestTemplate(producer);
  159 + builder.responseTemplate(consumer);
  160 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  161 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  162 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  163 + return builder.build();
138 } 164 }
139 165
140 @PreDestroy 166 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -28,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueAdmin; @@ -28,6 +30,7 @@ import org.thingsboard.server.queue.TbQueueAdmin;
28 import org.thingsboard.server.queue.TbQueueConsumer; 30 import org.thingsboard.server.queue.TbQueueConsumer;
29 import org.thingsboard.server.queue.TbQueueProducer; 31 import org.thingsboard.server.queue.TbQueueProducer;
30 import org.thingsboard.server.queue.TbQueueRequestTemplate; 32 import org.thingsboard.server.queue.TbQueueRequestTemplate;
  33 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
31 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 34 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
32 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 35 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
33 import org.thingsboard.server.queue.discovery.PartitionService; 36 import org.thingsboard.server.queue.discovery.PartitionService;
@@ -38,10 +41,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; @@ -38,10 +41,12 @@ import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate;
38 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; 41 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
39 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; 42 import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
40 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 43 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  44 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
41 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 45 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
42 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; 46 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
43 47
44 import javax.annotation.PreDestroy; 48 import javax.annotation.PreDestroy;
  49 +import java.nio.charset.StandardCharsets;
45 50
46 @Component 51 @Component
47 @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") 52 @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'")
@@ -52,6 +57,7 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor @@ -52,6 +57,7 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor
52 private final TbServiceInfoProvider serviceInfoProvider; 57 private final TbServiceInfoProvider serviceInfoProvider;
53 private final TbQueueRuleEngineSettings ruleEngineSettings; 58 private final TbQueueRuleEngineSettings ruleEngineSettings;
54 private final TbRabbitMqSettings rabbitMqSettings; 59 private final TbRabbitMqSettings rabbitMqSettings;
  60 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
55 61
56 private final TbQueueAdmin coreAdmin; 62 private final TbQueueAdmin coreAdmin;
57 private final TbQueueAdmin ruleEngineAdmin; 63 private final TbQueueAdmin ruleEngineAdmin;
@@ -62,12 +68,14 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor @@ -62,12 +68,14 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor
62 TbQueueRuleEngineSettings ruleEngineSettings, 68 TbQueueRuleEngineSettings ruleEngineSettings,
63 TbServiceInfoProvider serviceInfoProvider, 69 TbServiceInfoProvider serviceInfoProvider,
64 TbRabbitMqSettings rabbitMqSettings, 70 TbRabbitMqSettings rabbitMqSettings,
  71 + TbQueueRemoteJsInvokeSettings jsInvokeSettings,
65 TbRabbitMqQueueArguments queueArguments) { 72 TbRabbitMqQueueArguments queueArguments) {
66 this.partitionService = partitionService; 73 this.partitionService = partitionService;
67 this.coreSettings = coreSettings; 74 this.coreSettings = coreSettings;
68 this.serviceInfoProvider = serviceInfoProvider; 75 this.serviceInfoProvider = serviceInfoProvider;
69 this.ruleEngineSettings = ruleEngineSettings; 76 this.ruleEngineSettings = ruleEngineSettings;
70 this.rabbitMqSettings = rabbitMqSettings; 77 this.rabbitMqSettings = rabbitMqSettings;
  78 + this.jsInvokeSettings = jsInvokeSettings;
71 79
72 this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); 80 this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs());
73 this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); 81 this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs());
@@ -114,8 +122,26 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor @@ -114,8 +122,26 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor
114 } 122 }
115 123
116 @Override 124 @Override
  125 + @Bean
117 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 126 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
118 - return null; 127 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic());
  128 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings,
  129 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  130 + msg -> {
  131 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  132 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  133 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  134 + });
  135 +
  136 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  137 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  138 + builder.queueAdmin(jsExecutorAdmin);
  139 + builder.requestTemplate(producer);
  140 + builder.responseTemplate(consumer);
  141 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  142 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  143 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  144 + return builder.build();
119 } 145 }
120 146
121 @PreDestroy 147 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -35,19 +37,20 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat @@ -35,19 +37,20 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat
35 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; 37 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
36 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; 38 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
37 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; 39 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
  40 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
38 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 41 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
39 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 42 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
40 import org.thingsboard.server.queue.discovery.PartitionService; 43 import org.thingsboard.server.queue.discovery.PartitionService;
41 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; 44 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
42 -import org.thingsboard.server.queue.kafka.TbKafkaAdmin;  
43 -import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs;  
44 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 45 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  46 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
45 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 47 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
46 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; 48 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
47 import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; 49 import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
48 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; 50 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
49 51
50 import javax.annotation.PreDestroy; 52 import javax.annotation.PreDestroy;
  53 +import java.nio.charset.StandardCharsets;
51 54
52 @Component 55 @Component
53 @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") 56 @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'")
@@ -60,6 +63,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul @@ -60,6 +63,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
60 private final TbQueueTransportApiSettings transportApiSettings; 63 private final TbQueueTransportApiSettings transportApiSettings;
61 private final TbQueueTransportNotificationSettings transportNotificationSettings; 64 private final TbQueueTransportNotificationSettings transportNotificationSettings;
62 private final TbServiceBusSettings serviceBusSettings; 65 private final TbServiceBusSettings serviceBusSettings;
  66 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
63 67
64 private final TbQueueAdmin coreAdmin; 68 private final TbQueueAdmin coreAdmin;
65 private final TbQueueAdmin ruleEngineAdmin; 69 private final TbQueueAdmin ruleEngineAdmin;
@@ -73,6 +77,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul @@ -73,6 +77,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
73 TbQueueTransportApiSettings transportApiSettings, 77 TbQueueTransportApiSettings transportApiSettings,
74 TbQueueTransportNotificationSettings transportNotificationSettings, 78 TbQueueTransportNotificationSettings transportNotificationSettings,
75 TbServiceBusSettings serviceBusSettings, 79 TbServiceBusSettings serviceBusSettings,
  80 + TbQueueRemoteJsInvokeSettings jsInvokeSettings,
76 TbServiceBusQueueConfigs serviceBusQueueConfigs) { 81 TbServiceBusQueueConfigs serviceBusQueueConfigs) {
77 this.partitionService = partitionService; 82 this.partitionService = partitionService;
78 this.coreSettings = coreSettings; 83 this.coreSettings = coreSettings;
@@ -81,6 +86,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul @@ -81,6 +86,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
81 this.transportApiSettings = transportApiSettings; 86 this.transportApiSettings = transportApiSettings;
82 this.transportNotificationSettings = transportNotificationSettings; 87 this.transportNotificationSettings = transportNotificationSettings;
83 this.serviceBusSettings = serviceBusSettings; 88 this.serviceBusSettings = serviceBusSettings;
  89 + this.jsInvokeSettings = jsInvokeSettings;
84 90
85 this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); 91 this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
86 this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); 92 this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
@@ -152,8 +158,26 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul @@ -152,8 +158,26 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
152 } 158 }
153 159
154 @Override 160 @Override
  161 + @Bean
155 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 162 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
156 - return null; 163 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic());
  164 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings,
  165 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  166 + msg -> {
  167 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  168 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  169 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  170 + });
  171 +
  172 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  173 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  174 + builder.queueAdmin(jsExecutorAdmin);
  175 + builder.requestTemplate(producer);
  176 + builder.responseTemplate(consumer);
  177 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  178 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  179 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  180 + return builder.build();
157 } 181 }
158 182
159 @PreDestroy 183 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -34,15 +36,18 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat @@ -34,15 +36,18 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat
34 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; 36 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
35 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; 37 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
36 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; 38 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
  39 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
37 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 40 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
38 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 41 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
39 import org.thingsboard.server.queue.discovery.PartitionService; 42 import org.thingsboard.server.queue.discovery.PartitionService;
40 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; 43 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
41 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 44 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  45 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
42 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 46 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
43 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; 47 import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
44 48
45 import javax.annotation.PreDestroy; 49 import javax.annotation.PreDestroy;
  50 +import java.nio.charset.StandardCharsets;
46 51
47 @Component 52 @Component
48 @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") 53 @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'")
@@ -54,6 +59,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { @@ -54,6 +59,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
54 private final TbQueueTransportApiSettings transportApiSettings; 59 private final TbQueueTransportApiSettings transportApiSettings;
55 private final PartitionService partitionService; 60 private final PartitionService partitionService;
56 private final TbServiceInfoProvider serviceInfoProvider; 61 private final TbServiceInfoProvider serviceInfoProvider;
  62 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
57 63
58 private final TbQueueAdmin coreAdmin; 64 private final TbQueueAdmin coreAdmin;
59 private final TbQueueAdmin ruleEngineAdmin; 65 private final TbQueueAdmin ruleEngineAdmin;
@@ -67,6 +73,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { @@ -67,6 +73,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
67 TbQueueRuleEngineSettings ruleEngineSettings, 73 TbQueueRuleEngineSettings ruleEngineSettings,
68 PartitionService partitionService, 74 PartitionService partitionService,
69 TbServiceInfoProvider serviceInfoProvider, 75 TbServiceInfoProvider serviceInfoProvider,
  76 + TbQueueRemoteJsInvokeSettings jsInvokeSettings,
70 TbServiceBusQueueConfigs serviceBusQueueConfigs) { 77 TbServiceBusQueueConfigs serviceBusQueueConfigs) {
71 this.serviceBusSettings = serviceBusSettings; 78 this.serviceBusSettings = serviceBusSettings;
72 this.coreSettings = coreSettings; 79 this.coreSettings = coreSettings;
@@ -74,6 +81,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { @@ -74,6 +81,7 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
74 this.ruleEngineSettings = ruleEngineSettings; 81 this.ruleEngineSettings = ruleEngineSettings;
75 this.partitionService = partitionService; 82 this.partitionService = partitionService;
76 this.serviceInfoProvider = serviceInfoProvider; 83 this.serviceInfoProvider = serviceInfoProvider;
  84 + this.jsInvokeSettings = jsInvokeSettings;
77 85
78 this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); 86 this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
79 this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); 87 this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
@@ -132,8 +140,26 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { @@ -132,8 +140,26 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
132 } 140 }
133 141
134 @Override 142 @Override
  143 + @Bean
135 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 144 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
136 - return null; 145 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic());
  146 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings,
  147 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  148 + msg -> {
  149 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  150 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  151 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  152 + });
  153 +
  154 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  155 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  156 + builder.queueAdmin(jsExecutorAdmin);
  157 + builder.requestTemplate(producer);
  158 + builder.responseTemplate(consumer);
  159 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  160 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  161 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  162 + return builder.build();
137 } 163 }
138 164
139 @PreDestroy 165 @PreDestroy
@@ -15,7 +15,9 @@ @@ -15,7 +15,9 @@
15 */ 15 */
16 package org.thingsboard.server.queue.provider; 16 package org.thingsboard.server.queue.provider;
17 17
  18 +import com.google.protobuf.util.JsonFormat;
18 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  20 +import org.springframework.context.annotation.Bean;
19 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
20 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
21 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
@@ -32,15 +34,18 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat @@ -32,15 +34,18 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplat
32 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; 34 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate;
33 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; 35 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
34 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; 36 import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
  37 +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
35 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 38 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
36 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 39 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
37 import org.thingsboard.server.queue.discovery.PartitionService; 40 import org.thingsboard.server.queue.discovery.PartitionService;
38 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; 41 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
39 import org.thingsboard.server.queue.settings.TbQueueCoreSettings; 42 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
  43 +import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
40 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 44 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
41 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; 45 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
42 46
43 import javax.annotation.PreDestroy; 47 import javax.annotation.PreDestroy;
  48 +import java.nio.charset.StandardCharsets;
44 49
45 @Component 50 @Component
46 @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") 51 @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'")
@@ -51,6 +56,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact @@ -51,6 +56,7 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
51 private final TbServiceInfoProvider serviceInfoProvider; 56 private final TbServiceInfoProvider serviceInfoProvider;
52 private final TbQueueRuleEngineSettings ruleEngineSettings; 57 private final TbQueueRuleEngineSettings ruleEngineSettings;
53 private final TbServiceBusSettings serviceBusSettings; 58 private final TbServiceBusSettings serviceBusSettings;
  59 + private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
54 60
55 private final TbQueueAdmin coreAdmin; 61 private final TbQueueAdmin coreAdmin;
56 private final TbQueueAdmin ruleEngineAdmin; 62 private final TbQueueAdmin ruleEngineAdmin;
@@ -61,12 +67,14 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact @@ -61,12 +67,14 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
61 TbQueueRuleEngineSettings ruleEngineSettings, 67 TbQueueRuleEngineSettings ruleEngineSettings,
62 TbServiceInfoProvider serviceInfoProvider, 68 TbServiceInfoProvider serviceInfoProvider,
63 TbServiceBusSettings serviceBusSettings, 69 TbServiceBusSettings serviceBusSettings,
  70 + TbQueueRemoteJsInvokeSettings jsInvokeSettings,
64 TbServiceBusQueueConfigs serviceBusQueueConfigs) { 71 TbServiceBusQueueConfigs serviceBusQueueConfigs) {
65 this.partitionService = partitionService; 72 this.partitionService = partitionService;
66 this.coreSettings = coreSettings; 73 this.coreSettings = coreSettings;
67 this.serviceInfoProvider = serviceInfoProvider; 74 this.serviceInfoProvider = serviceInfoProvider;
68 this.ruleEngineSettings = ruleEngineSettings; 75 this.ruleEngineSettings = ruleEngineSettings;
69 this.serviceBusSettings = serviceBusSettings; 76 this.serviceBusSettings = serviceBusSettings;
  77 + this.jsInvokeSettings = jsInvokeSettings;
70 78
71 this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); 79 this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs());
72 this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); 80 this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
@@ -113,8 +121,26 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact @@ -113,8 +121,26 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact
113 } 121 }
114 122
115 @Override 123 @Override
  124 + @Bean
116 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() { 125 public TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> createRemoteJsRequestTemplate() {
117 - return null; 126 + TbQueueProducer<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic());
  127 + TbQueueConsumer<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings,
  128 + jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(),
  129 + msg -> {
  130 + JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
  131 + JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder);
  132 + return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
  133 + });
  134 +
  135 + DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
  136 + <TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
  137 + builder.queueAdmin(jsExecutorAdmin);
  138 + builder.requestTemplate(producer);
  139 + builder.responseTemplate(consumer);
  140 + builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests());
  141 + builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout());
  142 + builder.pollInterval(jsInvokeSettings.getResponsePollInterval());
  143 + return builder.build();
118 } 144 }
119 145
120 @PreDestroy 146 @PreDestroy
@@ -19,7 +19,6 @@ const COMPILATION_ERROR = 0; @@ -19,7 +19,6 @@ const COMPILATION_ERROR = 0;
19 const RUNTIME_ERROR = 1; 19 const RUNTIME_ERROR = 1;
20 const TIMEOUT_ERROR = 2; 20 const TIMEOUT_ERROR = 2;
21 const UNRECOGNIZED = -1; 21 const UNRECOGNIZED = -1;
22 -let headers;  
23 22
24 const config = require('config'), 23 const config = require('config'),
25 logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), 24 logger = require('../config/logger')._logger('JsInvokeMessageProcessor'),
@@ -31,6 +30,7 @@ const useSandbox = config.get('script.use_sandbox') === 'true'; @@ -31,6 +30,7 @@ const useSandbox = config.get('script.use_sandbox') === 'true';
31 const maxActiveScripts = Number(config.get('script.max_active_scripts')); 30 const maxActiveScripts = Number(config.get('script.max_active_scripts'));
32 31
33 function JsInvokeMessageProcessor(producer) { 32 function JsInvokeMessageProcessor(producer) {
  33 + console.log("Producer:", producer);
34 this.producer = producer; 34 this.producer = producer;
35 this.executor = new JsExecutor(useSandbox); 35 this.executor = new JsExecutor(useSandbox);
36 this.scriptMap = {}; 36 this.scriptMap = {};
@@ -40,26 +40,26 @@ function JsInvokeMessageProcessor(producer) { @@ -40,26 +40,26 @@ function JsInvokeMessageProcessor(producer) {
40 40
41 JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { 41 JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) {
42 42
43 - var requestId;  
44 - var responseTopic; 43 + let requestId;
  44 + let responseTopic;
45 try { 45 try {
46 - var request = JSON.parse(message.value.toString('utf8'));  
47 - headers = message.headers;  
48 - var buf = message.headers['requestId']; 46 + let request = JSON.parse(Buffer.from(message.data).toString('utf8'));
  47 + let headers = message.headers;
  48 + let buf = Buffer.from(headers.data['requestId']);
49 requestId = Utils.UUIDFromBuffer(buf); 49 requestId = Utils.UUIDFromBuffer(buf);
50 - buf = message.headers['responseTopic']; 50 + buf = Buffer.from(headers.data['responseTopic']);
51 responseTopic = buf.toString('utf8'); 51 responseTopic = buf.toString('utf8');
52 52
53 logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic); 53 logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic);
54 54
55 if (request.compileRequest) { 55 if (request.compileRequest) {
56 - this.processCompileRequest(requestId, responseTopic, request.compileRequest); 56 + this.processCompileRequest(requestId, responseTopic, headers, request.compileRequest);
57 } else if (request.invokeRequest) { 57 } else if (request.invokeRequest) {
58 - this.processInvokeRequest(requestId, responseTopic, request.invokeRequest); 58 + this.processInvokeRequest(requestId, responseTopic, headers, request.invokeRequest);
59 } else if (request.releaseRequest) { 59 } else if (request.releaseRequest) {
60 - this.processReleaseRequest(requestId, responseTopic, request.releaseRequest); 60 + this.processReleaseRequest(requestId, responseTopic, headers, request.releaseRequest);
61 } else { 61 } else {
62 - logger.error('[%s] Unknown request recevied!', requestId); 62 + logger.error('[%s] Unknown request received!', requestId);
63 } 63 }
64 64
65 } catch (err) { 65 } catch (err) {
@@ -68,7 +68,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { @@ -68,7 +68,7 @@ JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) {
68 } 68 }
69 } 69 }
70 70
71 -JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, compileRequest) { 71 +JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, responseTopic, headers, compileRequest) {
72 var scriptId = getScriptId(compileRequest); 72 var scriptId = getScriptId(compileRequest);
73 logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId); 73 logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId);
74 74
@@ -77,17 +77,17 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r @@ -77,17 +77,17 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r
77 this.cacheScript(scriptId, script); 77 this.cacheScript(scriptId, script);
78 var compileResponse = createCompileResponse(scriptId, true); 78 var compileResponse = createCompileResponse(scriptId, true);
79 logger.debug('[%s] Sending success compile response, scriptId: [%s]', requestId, scriptId); 79 logger.debug('[%s] Sending success compile response, scriptId: [%s]', requestId, scriptId);
80 - this.sendResponse(requestId, responseTopic, scriptId, compileResponse); 80 + this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse);
81 }, 81 },
82 (err) => { 82 (err) => {
83 var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err); 83 var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err);
84 logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId); 84 logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId);
85 - this.sendResponse(requestId, responseTopic, scriptId, compileResponse); 85 + this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse);
86 } 86 }
87 ); 87 );
88 } 88 }
89 89
90 -JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, invokeRequest) { 90 +JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, responseTopic, headers, invokeRequest) {
91 var scriptId = getScriptId(invokeRequest); 91 var scriptId = getScriptId(invokeRequest);
92 logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId); 92 logger.debug('[%s] Processing invoke request, scriptId: [%s]', requestId, scriptId);
93 this.executedScriptsCounter++; 93 this.executedScriptsCounter++;
@@ -103,7 +103,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re @@ -103,7 +103,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re
103 (result) => { 103 (result) => {
104 var invokeResponse = createInvokeResponse(result, true); 104 var invokeResponse = createInvokeResponse(result, true);
105 logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId); 105 logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId);
106 - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); 106 + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse);
107 }, 107 },
108 (err) => { 108 (err) => {
109 var errorCode; 109 var errorCode;
@@ -114,19 +114,19 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re @@ -114,19 +114,19 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re
114 } 114 }
115 var invokeResponse = createInvokeResponse("", false, errorCode, err); 115 var invokeResponse = createInvokeResponse("", false, errorCode, err);
116 logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode); 116 logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode);
117 - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); 117 + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse);
118 } 118 }
119 ) 119 )
120 }, 120 },
121 (err) => { 121 (err) => {
122 var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err); 122 var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err);
123 logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR); 123 logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR);
124 - this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); 124 + this.sendResponse(requestId, responseTopic, headers, scriptId, null, invokeResponse);
125 } 125 }
126 ); 126 );
127 } 127 }
128 128
129 -JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, releaseRequest) { 129 +JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, responseTopic, headers, releaseRequest) {
130 var scriptId = getScriptId(releaseRequest); 130 var scriptId = getScriptId(releaseRequest);
131 logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId); 131 logger.debug('[%s] Processing release request, scriptId: [%s]', requestId, scriptId);
132 if (this.scriptMap[scriptId]) { 132 if (this.scriptMap[scriptId]) {
@@ -138,28 +138,17 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r @@ -138,28 +138,17 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r
138 } 138 }
139 var releaseResponse = createReleaseResponse(scriptId, true); 139 var releaseResponse = createReleaseResponse(scriptId, true);
140 logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId); 140 logger.debug('[%s] Sending success release response, scriptId: [%s]', requestId, scriptId);
141 - this.sendResponse(requestId, responseTopic, scriptId, null, null, releaseResponse); 141 + this.sendResponse(requestId, responseTopic, headers, scriptId, null, null, releaseResponse);
142 } 142 }
143 143
144 -JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) { 144 +JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, headers, scriptId, compileResponse, invokeResponse, releaseResponse) {
145 var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); 145 var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse);
146 var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); 146 var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8');
147 - this.producer.send(  
148 - {  
149 - topic: responseTopic,  
150 - messages: [  
151 - {  
152 - key: scriptId,  
153 - value: rawResponse,  
154 - headers: headers  
155 - }  
156 - ]  
157 - }  
158 - ).then( 147 + this.producer.send(responseTopic, scriptId, rawResponse, headers).then(
159 () => {}, 148 () => {},
160 (err) => { 149 (err) => {
161 if (err) { 150 if (err) {
162 - logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message); 151 + logger.error('[%s] Failed to send response to queue: %s', requestId, err.message);
163 logger.error(err.stack); 152 logger.error(err.stack);
164 } 153 }
165 } 154 }
@@ -14,11 +14,45 @@ @@ -14,11 +14,45 @@
14 # limitations under the License. 14 # limitations under the License.
15 # 15 #
16 16
  17 +service-type: "TB_SERVICE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ)
  18 +request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC"
  19 +
  20 +js:
  21 + response_poll_interval: "REMOTE_JS_RESPONSE_POLL_INTERVAL_MS"
  22 +
17 kafka: 23 kafka:
18 - request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC"  
19 bootstrap: 24 bootstrap:
20 # Kafka Bootstrap Servers 25 # Kafka Bootstrap Servers
21 servers: "TB_KAFKA_SERVERS" 26 servers: "TB_KAFKA_SERVERS"
  27 + replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR"
  28 + topic-properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES"
  29 +
  30 +pubsub:
  31 + project_id: "TB_QUEUE_PUBSUB_PROJECT_ID"
  32 + service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT"
  33 + queue-properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES"
  34 +
  35 +aws_sqs:
  36 + access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID"
  37 + secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY"
  38 + region: "TB_QUEUE_AWS_SQS_REGION"
  39 + queue-properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES"
  40 +
  41 +rabbitmq:
  42 + host: "TB_QUEUE_RABBIT_MQ_HOST"
  43 + port: "TB_QUEUE_RABBIT_MQ_PORT"
  44 + virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST"
  45 + username: "TB_QUEUE_RABBIT_MQ_USERNAME"
  46 + password: "TB_QUEUE_RABBIT_MQ_PASSWORD"
  47 + queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES"
  48 +
  49 +service_bus:
  50 + namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME"
  51 + sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME"
  52 + sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY"
  53 + max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES"
  54 + queue-properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES"
  55 +
22 logger: 56 logger:
23 level: "LOGGER_LEVEL" 57 level: "LOGGER_LEVEL"
24 path: "LOG_FOLDER" 58 path: "LOG_FOLDER"
@@ -14,11 +14,35 @@ @@ -14,11 +14,35 @@
14 # limitations under the License. 14 # limitations under the License.
15 # 15 #
16 16
  17 +service-type: "kafka"
  18 +request_topic: "js_eval.requests"
  19 +
  20 +js:
  21 + response_poll_interval: "25"
  22 +
17 kafka: 23 kafka:
18 - request_topic: "js.eval.requests"  
19 bootstrap: 24 bootstrap:
20 # Kafka Bootstrap Servers 25 # Kafka Bootstrap Servers
21 servers: "localhost:9092" 26 servers: "localhost:9092"
  27 + replication_factor: "1"
  28 + topic-properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600"
  29 +
  30 +pubsub:
  31 + queue-properties: "ackDeadlineInSec:30;messageRetentionInSec:604800"
  32 +
  33 +aws_sqs:
  34 + queue-properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800"
  35 +
  36 +rabbitmq:
  37 + host: "localhost"
  38 + port: "5672"
  39 + virtual_host: "/"
  40 + username: "admin"
  41 + password: "password"
  42 + queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000"
  43 +
  44 +service_bus:
  45 + queue-properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800"
22 46
23 logger: 47 logger:
24 level: "info" 48 level: "info"
@@ -14,7 +14,12 @@ @@ -14,7 +14,12 @@
14 "dependencies": { 14 "dependencies": {
15 "config": "^3.2.2", 15 "config": "^3.2.2",
16 "js-yaml": "^3.12.0", 16 "js-yaml": "^3.12.0",
17 - "kafkajs": "^1.11.0", 17 + "kafkajs": "^1.12.0",
  18 + "@google-cloud/pubsub": "^1.7.1",
  19 + "aws-sdk": "^2.663.0",
  20 + "amqplib": "^0.5.5",
  21 + "@azure/service-bus": "^1.1.6",
  22 + "azure-sb": "^0.11.1",
18 "long": "^4.0.0", 23 "long": "^4.0.0",
19 "uuid-parse": "^1.0.0", 24 "uuid-parse": "^1.0.0",
20 "winston": "^3.0.0", 25 "winston": "^3.0.0",
  1 +/*
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +'use strict';
  18 +
  19 +const config = require('config'),
  20 + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
  21 + logger = require('../config/logger')._logger('awsSqsTemplate');
  22 +
  23 +const requestTopic = config.get('request_topic');
  24 +
  25 +const accessKeyId = config.get('aws_sqs.access_key_id');
  26 +const secretAccessKey = config.get('aws_sqs.secret_access_key');
  27 +const region = config.get('aws_sqs.region');
  28 +const AWS = require('aws-sdk');
  29 +const queueProperties = config.get('aws_sqs.queue-properties');
  30 +const poolInterval = config.get('js.response_poll_interval');
  31 +
  32 +let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'};
  33 +let sqsClient;
  34 +let requestQueueURL;
  35 +const queueUrls = new Map();
  36 +let stopped = false;
  37 +
  38 +function AwsSqsProducer() {
  39 + this.send = async (responseTopic, scriptId, rawResponse, headers) => {
  40 + let msgBody = JSON.stringify(
  41 + {
  42 + key: scriptId,
  43 + data: [...rawResponse],
  44 + headers: headers
  45 + });
  46 +
  47 + let responseQueueUrl = queueUrls.get(topicToSqsQueueName(responseTopic));
  48 +
  49 + if (!responseQueueUrl) {
  50 + responseQueueUrl = await createQueue(responseTopic);
  51 + queueUrls.set(responseTopic, responseQueueUrl);
  52 + }
  53 +
  54 + let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId};
  55 +
  56 + return new Promise((resolve, reject) => {
  57 + sqsClient.sendMessage(params, function (err, data) {
  58 + if (err) {
  59 + reject(err);
  60 + } else {
  61 + resolve(data);
  62 + }
  63 + });
  64 + });
  65 + }
  66 +}
  67 +
  68 +(async () => {
  69 + try {
  70 + logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
  71 + AWS.config.update({accessKeyId: accessKeyId, secretAccessKey: secretAccessKey, region: region});
  72 +
  73 + sqsClient = new AWS.SQS({apiVersion: '2012-11-05'});
  74 +
  75 + const queues = await getQueues();
  76 +
  77 + queues.forEach(queueUrl => {
  78 + const delimiterPosition = queueUrl.lastIndexOf('/');
  79 + const queueName = queueUrl.substring(delimiterPosition + 1);
  80 + queueUrls.set(queueName, queueUrl);
  81 + })
  82 +
  83 + parseQueueProperties();
  84 +
  85 + requestQueueURL = queueUrls.get(topicToSqsQueueName(requestTopic));
  86 + if (!requestQueueURL) {
  87 + requestQueueURL = await createQueue(requestTopic);
  88 + }
  89 +
  90 + const messageProcessor = new JsInvokeMessageProcessor(new AwsSqsProducer());
  91 +
  92 + const params = {
  93 + MaxNumberOfMessages: 10,
  94 + QueueUrl: requestQueueURL,
  95 + WaitTimeSeconds: poolInterval / 1000
  96 + };
  97 + while (!stopped) {
  98 + const messages = await new Promise((resolve, reject) => {
  99 + sqsClient.receiveMessage(params, function (err, data) {
  100 + if (err) {
  101 + reject(err);
  102 + } else {
  103 + resolve(data.Messages);
  104 + }
  105 + });
  106 + });
  107 +
  108 + if (messages && messages.length > 0) {
  109 + const entries = [];
  110 +
  111 + messages.forEach(message => {
  112 + entries.push({
  113 + Id: message.MessageId,
  114 + ReceiptHandle: message.ReceiptHandle
  115 + });
  116 + messageProcessor.onJsInvokeMessage(JSON.parse(message.Body));
  117 + });
  118 +
  119 + const deleteBatch = {
  120 + QueueUrl: requestQueueURL,
  121 + Entries: entries
  122 + };
  123 + sqsClient.deleteMessageBatch(deleteBatch, function (err, data) {
  124 + if (err) {
  125 + logger.error("Failed to delete messages from queue.", err.message);
  126 + } else {
  127 + //do nothing
  128 + }
  129 + });
  130 + }
  131 + }
  132 + } catch (e) {
  133 + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
  134 + logger.error(e.stack);
  135 + exit(-1);
  136 + }
  137 +})();
  138 +
  139 +function createQueue(topic) {
  140 + let queueName = topicToSqsQueueName(topic);
  141 + let queueParams = {QueueName: queueName, Attributes: queueAttributes};
  142 +
  143 + return new Promise((resolve, reject) => {
  144 + sqsClient.createQueue(queueParams, function (err, data) {
  145 + if (err) {
  146 + reject(err);
  147 + } else {
  148 + resolve(data.QueueUrl);
  149 + }
  150 + });
  151 + });
  152 +}
  153 +
  154 +function getQueues() {
  155 + return new Promise((resolve, reject) => {
  156 + sqsClient.listQueues(function (err, data) {
  157 + if (err) {
  158 + reject(err);
  159 + } else {
  160 + resolve(data.QueueUrls);
  161 + }
  162 + });
  163 + });
  164 +}
  165 +
  166 +function topicToSqsQueueName(topic) {
  167 + return topic.replace(/\./g, '_') + '.fifo';
  168 +}
  169 +
  170 +function parseQueueProperties() {
  171 + const props = queueProperties.split(';');
  172 + props.forEach(p => {
  173 + const delimiterPosition = p.indexOf(':');
  174 + queueAttributes[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
  175 + });
  176 +}
  177 +
  178 +process.on('exit', () => {
  179 + stopped = true;
  180 + logger.info('Aws Sqs client stopped.');
  181 + exit(0);
  182 +});
  183 +
  184 +async function exit(status) {
  185 + logger.info('Exiting with status: %d ...', status);
  186 + if (sqsClient) {
  187 + logger.info('Stopping Aws Sqs client.')
  188 + try {
  189 + await sqsClient.close();
  190 + logger.info('Aws Sqs client stopped.')
  191 + process.exit(status);
  192 + } catch (e) {
  193 + logger.info('Aws Sqs client stop error.');
  194 + process.exit(status);
  195 + }
  196 + } else {
  197 + process.exit(status);
  198 + }
  199 +}
  1 +/*
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +const {logLevel, Kafka} = require('kafkajs');
  17 +
  18 +const config = require('config'),
  19 + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
  20 + logger = require('../config/logger')._logger('kafkaTemplate'),
  21 + KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator;
  22 +const replicationFactor = config.get('kafka.replication_factor');
  23 +const topicProperties = config.get('kafka.topic-properties');
  24 +
  25 +let kafkaClient;
  26 +let kafkaAdmin;
  27 +let consumer;
  28 +let producer;
  29 +
  30 +const topics = [];
  31 +const configEntries = [];
  32 +
  33 +function KafkaProducer() {
  34 + this.send = async (responseTopic, scriptId, rawResponse, headers) => {
  35 +
  36 + if (!topics.includes(responseTopic)) {
  37 + let createResponseTopicResult = await createTopic(responseTopic);
  38 + topics.push(responseTopic);
  39 + if (createResponseTopicResult) {
  40 + logger.info('Created new topic: %s', requestTopic);
  41 + }
  42 + }
  43 +
  44 + return producer.send(
  45 + {
  46 + topic: responseTopic,
  47 + messages: [
  48 + {
  49 + key: scriptId,
  50 + value: rawResponse,
  51 + headers: headers.data
  52 + }
  53 + ]
  54 + });
  55 + }
  56 +}
  57 +
  58 +(async () => {
  59 + try {
  60 + logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
  61 +
  62 + const kafkaBootstrapServers = config.get('kafka.bootstrap.servers');
  63 + const requestTopic = config.get('request_topic');
  64 +
  65 + logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
  66 + logger.info('Kafka Requests Topic: %s', requestTopic);
  67 +
  68 + kafkaClient = new Kafka({
  69 + brokers: kafkaBootstrapServers.split(','),
  70 + logLevel: logLevel.INFO,
  71 + logCreator: KafkaJsWinstonLogCreator
  72 + });
  73 +
  74 + parseTopicProperties();
  75 +
  76 + kafkaAdmin = kafkaClient.admin();
  77 + await kafkaAdmin.connect();
  78 +
  79 + let createRequestTopicResult = await createTopic(requestTopic);
  80 +
  81 + if (createRequestTopicResult) {
  82 + logger.info('Created new topic: %s', requestTopic);
  83 + }
  84 +
  85 + consumer = kafkaClient.consumer({groupId: 'js-executor-group'});
  86 + producer = kafkaClient.producer();
  87 + const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer());
  88 + await consumer.connect();
  89 + await producer.connect();
  90 + await consumer.subscribe({topic: requestTopic});
  91 +
  92 + logger.info('Started ThingsBoard JavaScript Executor Microservice.');
  93 + await consumer.run({
  94 + eachMessage: async ({topic, partition, message}) => {
  95 + let headers = message.headers;
  96 + let key = message.key;
  97 + let msg = {};
  98 + msg.key = key.toString('utf8');
  99 + msg.data = message.value;
  100 + msg.headers = {data: headers};
  101 + messageProcessor.onJsInvokeMessage(msg);
  102 + },
  103 + });
  104 +
  105 + } catch (e) {
  106 + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
  107 + logger.error(e.stack);
  108 + exit(-1);
  109 + }
  110 +})();
  111 +
  112 +function createTopic(topic) {
  113 + return kafkaAdmin.createTopics({
  114 + topics: [{
  115 + topic: topic,
  116 + replicationFactor: replicationFactor,
  117 + configEntries: configEntries
  118 + }]
  119 + });
  120 +}
  121 +
  122 +function parseTopicProperties() {
  123 + const props = topicProperties.split(';');
  124 + props.forEach(p => {
  125 + const delimiterPosition = p.indexOf(':');
  126 + configEntries.push({name: p.substring(0, delimiterPosition), value: p.substring(delimiterPosition + 1)});
  127 + });
  128 +}
  129 +
  130 +process.on('exit', () => {
  131 + exit(0);
  132 +});
  133 +
  134 +async function exit(status) {
  135 + logger.info('Exiting with status: %d ...', status);
  136 +
  137 + if (kafkaAdmin) {
  138 + logger.info('Stopping Kafka Admin...');
  139 + await kafkaAdmin.disconnect();
  140 + logger.info('Kafka Admin stopped.');
  141 + }
  142 +
  143 + if (consumer) {
  144 + logger.info('Stopping Kafka Consumer...');
  145 + let _consumer = consumer;
  146 + consumer = null;
  147 + try {
  148 + await _consumer.disconnect();
  149 + logger.info('Kafka Consumer stopped.');
  150 + await disconnectProducer();
  151 + process.exit(status);
  152 + } catch (e) {
  153 + logger.info('Kafka Consumer stop error.');
  154 + await disconnectProducer();
  155 + process.exit(status);
  156 + }
  157 + } else {
  158 + process.exit(status);
  159 + }
  160 +}
  161 +
  162 +async function disconnectProducer() {
  163 + if (producer) {
  164 + logger.info('Stopping Kafka Producer...');
  165 + var _producer = producer;
  166 + producer = null;
  167 + try {
  168 + await _producer.disconnect();
  169 + logger.info('Kafka Producer stopped.');
  170 + } catch (e) {
  171 + logger.info('Kafka Producer stop error.');
  172 + }
  173 + }
  174 +}
  1 +/*
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +'use strict';
  18 +
  19 +const config = require('config'),
  20 + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
  21 + logger = require('../config/logger')._logger('pubSubTemplate');
  22 +const {PubSub} = require('@google-cloud/pubsub');
  23 +
  24 +const projectId = config.get('pubsub.project_id');
  25 +const credentials = JSON.parse(config.get('pubsub.service_account'));
  26 +const requestTopic = config.get('request_topic');
  27 +const queueProperties = config.get('pubsub.queue-properties');
  28 +
  29 +let pubSubClient;
  30 +
  31 +const topics = [];
  32 +const subscriptions = [];
  33 +const queueProps = [];
  34 +
  35 +function PubSubProducer() {
  36 + this.send = async (responseTopic, scriptId, rawResponse, headers) => {
  37 +
  38 + if (!(subscriptions.includes(responseTopic) && topics.includes(requestTopic))) {
  39 + await createTopic(requestTopic);
  40 + }
  41 +
  42 + let data = JSON.stringify(
  43 + {
  44 + key: scriptId,
  45 + data: [...rawResponse],
  46 + headers: headers
  47 + });
  48 + let dataBuffer = Buffer.from(data);
  49 + return pubSubClient.topic(responseTopic).publish(dataBuffer);
  50 + }
  51 +}
  52 +
  53 +(async () => {
  54 + try {
  55 + logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
  56 + pubSubClient = new PubSub({projectId: projectId, credentials: credentials});
  57 +
  58 + parseQueueProperties();
  59 +
  60 + const topicList = await pubSubClient.getTopics();
  61 +
  62 + if (topicList) {
  63 + topicList[0].forEach(topic => {
  64 + topics.push(getName(topic.name));
  65 + });
  66 + }
  67 +
  68 + const subscriptionList = await pubSubClient.getSubscriptions();
  69 +
  70 + if (subscriptionList) {
  71 + topicList[0].forEach(sub => {
  72 + subscriptions.push(getName(sub.name));
  73 + });
  74 + }
  75 +
  76 + if (!(subscriptions.includes(requestTopic) && topics.includes(requestTopic))) {
  77 + await createTopic(requestTopic);
  78 + }
  79 +
  80 + const subscription = pubSubClient.subscription(requestTopic);
  81 +
  82 + const messageProcessor = new JsInvokeMessageProcessor(new PubSubProducer());
  83 +
  84 + const messageHandler = message => {
  85 +
  86 + messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8')));
  87 + message.ack();
  88 + };
  89 +
  90 + subscription.on('message', messageHandler);
  91 +
  92 + } catch (e) {
  93 + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
  94 + logger.error(e.stack);
  95 + exit(-1);
  96 + }
  97 +})();
  98 +
  99 +async function createTopic(topic) {
  100 + if (!topics.includes(topic)) {
  101 + await pubSubClient.createTopic(topic);
  102 + topics.push(topic);
  103 + logger.info('Created new Pub/Sub topic: %s', topic);
  104 + }
  105 + await createSubscription(topic)
  106 +}
  107 +
  108 +async function createSubscription(topic) {
  109 + if (!subscriptions.includes(topic)) {
  110 + await pubSubClient.createSubscription(topic, topic, {
  111 + topic: topic,
  112 + subscription: topic,
  113 + ackDeadlineSeconds: queueProps['ackDeadlineInSec'],
  114 + messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']}
  115 + });
  116 + subscriptions.push(topic);
  117 + logger.info('Created new Pub/Sub subscription: %s', topic);
  118 + }
  119 +}
  120 +
  121 +function parseQueueProperties() {
  122 + const props = queueProperties.split(';');
  123 + props.forEach(p => {
  124 + const delimiterPosition = p.indexOf(':');
  125 + queueProps[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
  126 + });
  127 +}
  128 +
  129 +function getName(fullName) {
  130 + const delimiterPosition = fullName.lastIndexOf('/');
  131 + return fullName.substring(delimiterPosition + 1);
  132 +}
  133 +
  134 +process.on('exit', () => {
  135 + exit(0);
  136 +});
  137 +
  138 +async function exit(status) {
  139 + logger.info('Exiting with status: %d ...', status);
  140 + if (pubSubClient) {
  141 + logger.info('Stopping Pub/Sub client.')
  142 + try {
  143 + await pubSubClient.close();
  144 + logger.info('Pub/Sub client stopped.')
  145 + process.exit(status);
  146 + } catch (e) {
  147 + logger.info('Pub/Sub client stop error.');
  148 + process.exit(status);
  149 + }
  150 + } else {
  151 + process.exit(status);
  152 + }
  153 +}
  154 +
  1 +/*
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +'use strict';
  18 +
  19 +const config = require('config'),
  20 + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
  21 + logger = require('../config/logger')._logger('rabbitmqTemplate');
  22 +
  23 +const requestTopic = config.get('request_topic');
  24 +const host = config.get('rabbitmq.host');
  25 +const port = config.get('rabbitmq.port');
  26 +const vhost = config.get('rabbitmq.virtual_host');
  27 +const username = config.get('rabbitmq.username');
  28 +const password = config.get('rabbitmq.password');
  29 +const queueProperties = config.get('rabbitmq.queue-properties');
  30 +const poolInterval = config.get('js.response_poll_interval');
  31 +
  32 +const amqp = require('amqplib/callback_api');
  33 +
  34 +let queueParams = {durable: false, exclusive: false, autoDelete: false};
  35 +let connection;
  36 +let channel;
  37 +let stopped = false;
  38 +const responseTopics = [];
  39 +
  40 +function RabbitMqProducer() {
  41 + this.send = async (responseTopic, scriptId, rawResponse, headers) => {
  42 +
  43 + if (!responseTopics.includes(responseTopic)) {
  44 + await createQueue(responseTopic);
  45 + responseTopics.push(responseTopic);
  46 + }
  47 +
  48 + let data = JSON.stringify(
  49 + {
  50 + key: scriptId,
  51 + data: [...rawResponse],
  52 + headers: headers
  53 + });
  54 + let dataBuffer = Buffer.from(data);
  55 + channel.sendToQueue(responseTopic, dataBuffer);
  56 + return new Promise((resolve, reject) => {
  57 + channel.waitForConfirms((err) => {
  58 + if (err) {
  59 + reject(err);
  60 + } else {
  61 + resolve();
  62 + }
  63 + });
  64 + });
  65 + }
  66 +}
  67 +
  68 +(async () => {
  69 + try {
  70 + logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
  71 + const url = `amqp://${host}:${port}${vhost}`;
  72 +
  73 + amqp.credentials.amqplain(username, password);
  74 + connection = await new Promise((resolve, reject) => {
  75 + amqp.connect(url, function (err, connection) {
  76 + if (err) {
  77 + reject(err);
  78 + } else {
  79 + resolve(connection);
  80 + }
  81 + });
  82 + });
  83 +
  84 + channel = await new Promise((resolve, reject) => {
  85 + connection.createConfirmChannel(function (err, channel) {
  86 + if (err) {
  87 + reject(err);
  88 + } else {
  89 + resolve(channel);
  90 + }
  91 + });
  92 + });
  93 +
  94 + parseQueueProperties();
  95 +
  96 + await createQueue(requestTopic);
  97 +
  98 + const messageProcessor = new JsInvokeMessageProcessor(new RabbitMqProducer());
  99 +
  100 + while (!stopped) {
  101 + let message = await new Promise((resolve, reject) => {
  102 + channel.get(requestTopic, {}, function (err, msg) {
  103 + if (err) {
  104 + reject(err);
  105 + } else {
  106 + resolve(msg);
  107 + }
  108 + });
  109 + });
  110 +
  111 + if (message) {
  112 + messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8')));
  113 + channel.ack(message);
  114 + } else {
  115 + await sleep(poolInterval);
  116 + }
  117 + }
  118 + } catch (e) {
  119 + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
  120 + logger.error(e.stack);
  121 + exit(-1);
  122 + }
  123 +})();
  124 +
  125 +function parseQueueProperties() {
  126 + const props = queueProperties.split(';');
  127 + props.forEach(p => {
  128 + const delimiterPosition = p.indexOf(':');
  129 + queueParams[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
  130 + });
  131 +}
  132 +
  133 +function createQueue(topic) {
  134 + return new Promise((resolve, reject) => {
  135 + channel.assertQueue(topic, queueParams, function (err) {
  136 + if (err) {
  137 + reject(err);
  138 + } else {
  139 + resolve();
  140 + }
  141 + });
  142 + });
  143 +}
  144 +
  145 +function sleep(ms) {
  146 + return new Promise((resolve) => {
  147 + setTimeout(resolve, ms);
  148 + });
  149 +}
  150 +
  151 +process.on('exit', () => {
  152 + exit(0);
  153 +});
  154 +
  155 +async function exit(status) {
  156 + logger.info('Exiting with status: %d ...', status);
  157 +
  158 + if (channel) {
  159 + logger.info('Stopping RabbitMq chanel.')
  160 + await channel.close();
  161 + logger.info('RabbitMq chanel stopped');
  162 + }
  163 +
  164 + if (connection) {
  165 + logger.info('Stopping RabbitMq connection.')
  166 + try {
  167 + await connection.close();
  168 + logger.info('RabbitMq client connection.')
  169 + process.exit(status);
  170 + } catch (e) {
  171 + logger.info('RabbitMq connection stop error.');
  172 + process.exit(status);
  173 + }
  174 + } else {
  175 + process.exit(status);
  176 + }
  177 +}
  1 +/*
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +'use strict';
  18 +
  19 +const config = require('config'),
  20 + JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
  21 + logger = require('../config/logger')._logger('serviceBusTemplate');
  22 +const {ServiceBusClient, ReceiveMode} = require("@azure/service-bus");
  23 +const azure = require('azure-sb');
  24 +
  25 +const requestTopic = config.get('request_topic');
  26 +const namespaceName = config.get('service_bus.namespace_name');
  27 +const sasKeyName = config.get('service_bus.sas_key_name');
  28 +const sasKey = config.get('service_bus.sas_key');
  29 +const queueProperties = config.get('service_bus.queue-properties');
  30 +
  31 +let sbClient;
  32 +let receiverClient;
  33 +let receiver;
  34 +let serviceBusService;
  35 +
  36 +let queueOptions = {};
  37 +const queues = [];
  38 +const senderMap = new Map();
  39 +
  40 +function ServiceBusProducer() {
  41 + this.send = async (responseTopic, scriptId, rawResponse, headers) => {
  42 + if (!queues.includes(requestTopic)) {
  43 + await createQueueIfNotExist(requestTopic);
  44 + queues.push(requestTopic);
  45 + }
  46 +
  47 + let customSender = senderMap.get(responseTopic);
  48 +
  49 + if (!customSender) {
  50 + customSender = new CustomSender(responseTopic);
  51 + senderMap.set(responseTopic, customSender);
  52 + }
  53 +
  54 + let data = {
  55 + key: scriptId,
  56 + data: [...rawResponse],
  57 + headers: headers
  58 + };
  59 +
  60 + return customSender.send({body: data});
  61 + }
  62 +}
  63 +
  64 +function CustomSender(topic) {
  65 + this.queueClient = sbClient.createQueueClient(topic);
  66 + this.sender = this.queueClient.createSender();
  67 +
  68 + this.send = async (message) => {
  69 + return this.sender.send(message);
  70 + }
  71 +}
  72 +
  73 +(async () => {
  74 + try {
  75 + logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
  76 +
  77 + const connectionString = `Endpoint=sb://${namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${sasKeyName};SharedAccessKey=${sasKey}`;
  78 + sbClient = ServiceBusClient.createFromConnectionString(connectionString);
  79 + serviceBusService = azure.createServiceBusService(connectionString);
  80 +
  81 + parseQueueProperties();
  82 +
  83 + await new Promise((resolve, reject) => {
  84 + serviceBusService.listQueues((err, data) => {
  85 + if (err) {
  86 + reject(err);
  87 + } else {
  88 + data.forEach(queue => {
  89 + queues.push(queue.QueueName);
  90 + });
  91 + resolve();
  92 + }
  93 + });
  94 + });
  95 +
  96 + if (!queues.includes(requestTopic)) {
  97 + await createQueueIfNotExist(requestTopic);
  98 + queues.push(requestTopic);
  99 + }
  100 +
  101 + receiverClient = sbClient.createQueueClient(requestTopic);
  102 + receiver = receiverClient.createReceiver(ReceiveMode.peekLock);
  103 +
  104 + const messageProcessor = new JsInvokeMessageProcessor(new ServiceBusProducer());
  105 +
  106 + const messageHandler = async (message) => {
  107 + if (message) {
  108 + messageProcessor.onJsInvokeMessage(message.body);
  109 + await message.complete();
  110 + }
  111 + };
  112 + const errorHandler = (error) => {
  113 + logger.error('Failed to receive message from queue.', error);
  114 + };
  115 + receiver.registerMessageHandler(messageHandler, errorHandler);
  116 + } catch (e) {
  117 + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
  118 + logger.error(e.stack);
  119 + exit(-1);
  120 + }
  121 +})();
  122 +
  123 +async function createQueueIfNotExist(topic) {
  124 + return new Promise((resolve, reject) => {
  125 + serviceBusService.createQueueIfNotExists(topic, queueOptions, (err) => {
  126 + if (err) {
  127 + reject(err);
  128 + } else {
  129 + resolve();
  130 + }
  131 + });
  132 + });
  133 +}
  134 +
  135 +function parseQueueProperties() {
  136 + let properties = {};
  137 + const props = queueProperties.split(';');
  138 + props.forEach(p => {
  139 + const delimiterPosition = p.indexOf(':');
  140 + properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
  141 + });
  142 + queueOptions = {
  143 + MaxSizeInMegabytes: properties['maxSizeInMb'],
  144 + DefaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`,
  145 + LockDuration: `PT${properties['lockDurationInSec']}S`
  146 + };
  147 +}
  148 +
  149 +process.on('exit', () => {
  150 + exit(0);
  151 +});
  152 +
  153 +async function exit(status) {
  154 + logger.info('Exiting with status: %d ...', status);
  155 + logger.info('Stopping Azure Service Bus resources...')
  156 + if (receiver) {
  157 + try {
  158 + await receiver.close();
  159 + } catch (e) {
  160 +
  161 + }
  162 + }
  163 +
  164 + if (receiverClient) {
  165 + try {
  166 + await receiverClient.close();
  167 + } catch (e) {
  168 +
  169 + }
  170 + }
  171 +
  172 + senderMap.forEach((k, v) => {
  173 + try {
  174 + v.sender.close();
  175 + } catch (e) {
  176 +
  177 + }
  178 + try {
  179 + v.queueClient.close();
  180 + } catch (e) {
  181 +
  182 + }
  183 + });
  184 +
  185 + if (sbClient) {
  186 + try {
  187 + sbClient.close();
  188 + } catch (e) {
  189 +
  190 + }
  191 + }
  192 + logger.info('Azure Service Bus resources stopped.')
  193 + process.exit(status);
  194 +}
@@ -13,89 +13,38 @@ @@ -13,89 +13,38 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -const { logLevel, Kafka } = require('kafkajs');  
17 16
18 -const config = require('config'),  
19 - JsInvokeMessageProcessor = require('./api/jsInvokeMessageProcessor'),  
20 - logger = require('./config/logger')._logger('main'),  
21 - KafkaJsWinstonLogCreator = require('./config/logger').KafkaJsWinstonLogCreator;  
22 -  
23 -var kafkaClient;  
24 -var consumer;  
25 -var producer;  
26 -  
27 -(async() => {  
28 - try {  
29 - logger.info('Starting ThingsBoard JavaScript Executor Microservice...');  
30 -  
31 - const kafkaBootstrapServers = config.get('kafka.bootstrap.servers');  
32 - const kafkaRequestTopic = config.get('kafka.request_topic');  
33 -  
34 - logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);  
35 - logger.info('Kafka Requests Topic: %s', kafkaRequestTopic);  
36 -  
37 - kafkaClient = new Kafka({  
38 - brokers: kafkaBootstrapServers.split(','),  
39 - logLevel: logLevel.INFO,  
40 - logCreator: KafkaJsWinstonLogCreator  
41 - });  
42 -  
43 - consumer = kafkaClient.consumer({ groupId: 'js-executor-group' });  
44 - producer = kafkaClient.producer();  
45 - const messageProcessor = new JsInvokeMessageProcessor(producer);  
46 - await consumer.connect();  
47 - await producer.connect();  
48 - await consumer.subscribe({ topic: kafkaRequestTopic});  
49 -  
50 - logger.info('Started ThingsBoard JavaScript Executor Microservice.');  
51 - await consumer.run({  
52 - eachMessage: async ({ topic, partition, message }) => {  
53 - messageProcessor.onJsInvokeMessage(message);  
54 - },  
55 - });  
56 -  
57 - } catch (e) {  
58 - logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);  
59 - logger.error(e.stack);  
60 - exit(-1);  
61 - }  
62 -})();  
63 -  
64 -process.on('exit', () => {  
65 - exit(0);  
66 -});  
67 -  
68 -async function exit(status) {  
69 - logger.info('Exiting with status: %d ...', status);  
70 - if (consumer) {  
71 - logger.info('Stopping Kafka Consumer...');  
72 - var _consumer = consumer;  
73 - consumer = null;  
74 - try {  
75 - await _consumer.disconnect();  
76 - logger.info('Kafka Consumer stopped.');  
77 - await disconnectProducer();  
78 - process.exit(status);  
79 - } catch (e) {  
80 - logger.info('Kafka Consumer stop error.');  
81 - await disconnectProducer();  
82 - process.exit(status);  
83 - }  
84 - } else {  
85 - process.exit(status);  
86 - } 17 +const config = require('config'), logger = require('./config/logger')._logger('main');
  18 +
  19 +const serviceType = config.get('service-type');
  20 +switch (serviceType) {
  21 + case 'kafka':
  22 + logger.info('Starting kafka template.');
  23 + require('./queue/kafkaTemplate');
  24 + logger.info('kafka template started.');
  25 + break;
  26 + case 'pubsub':
  27 + logger.info('Starting Pub/Sub template.')
  28 + require('./queue/pubSubTemplate');
  29 + logger.info('Pub/Sub template started.')
  30 + break;
  31 + case 'aws-sqs':
  32 + logger.info('Starting Aws Sqs template.')
  33 + require('./queue/awsSqsTemplate');
  34 + logger.info('Aws Sqs template started.')
  35 + break;
  36 + case 'rabbitmq':
  37 + logger.info('Starting RabbitMq template.')
  38 + require('./queue/rabbitmqTemplate');
  39 + logger.info('RabbitMq template started.')
  40 + break;
  41 + case 'service-bus':
  42 + logger.info('Starting Azure Service Bus template.')
  43 + require('./queue/serviceBusTemplate');
  44 + logger.info('Azure Service Bus template started.')
  45 + break;
  46 + default:
  47 + logger.error('Unknown service type: ', serviceType);
  48 + process.exit(-1);
87 } 49 }
88 50
89 -async function disconnectProducer() {  
90 - if (producer) {  
91 - logger.info('Stopping Kafka Producer...');  
92 - var _producer = producer;  
93 - producer = null;  
94 - try {  
95 - await _producer.disconnect();  
96 - logger.info('Kafka Producer stopped.');  
97 - } catch (e) {  
98 - logger.info('Kafka Producer stop error.');  
99 - }  
100 - }  
101 -}  
@@ -41,7 +41,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -41,7 +41,7 @@ import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS;
41 name = "delay", 41 name = "delay",
42 configClazz = TbMsgDelayNodeConfiguration.class, 42 configClazz = TbMsgDelayNodeConfiguration.class,
43 nodeDescription = "Delays incoming message", 43 nodeDescription = "Delays incoming message",
44 - nodeDetails = "Delays messages for configurable period.", 44 + nodeDetails = "Delays messages for configurable period. Please note, this node acknowledges the message from the current queue (message will be removed from queue)",
45 icon = "pause", 45 icon = "pause",
46 uiResources = {"static/rulenode/rulenode-core-config.js"}, 46 uiResources = {"static/rulenode/rulenode-core-config.js"},
47 configDirective = "tbActionNodeMsgDelayConfig" 47 configDirective = "tbActionNodeMsgDelayConfig"