Commit eedb38384536215555c97cb4fcff53ae7ee1bf72

Authored by YevhenBondarenko
1 parent 9a66fc02

AWS improvements

... ... @@ -127,6 +127,11 @@ public class TbAwsSqsConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
127 127 if (!subscribed) {
128 128 List<String> topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList());
129 129 queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet());
  130 +
  131 + if (consumerExecutor != null) {
  132 + consumerExecutor.shutdown();
  133 + }
  134 +
130 135 consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1));
131 136 subscribed = true;
132 137 }
... ... @@ -172,7 +177,6 @@ public class TbAwsSqsConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
172 177 ReceiveMessageRequest request = new ReceiveMessageRequest();
173 178 request
174 179 .withWaitTimeSeconds(waitTimeSeconds)
175   - .withMessageAttributeNames("headers")
176 180 .withQueueUrl(url)
177 181 .withMaxNumberOfMessages(MAX_NUM_MSGS);
178 182 return sqsClient.receiveMessage(request).getMessages();
... ...
... ... @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.TbQueueProducer;
37 37 import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
38 38
39 39 import java.util.Map;
  40 +import java.util.UUID;
40 41 import java.util.concurrent.ConcurrentHashMap;
41 42 import java.util.concurrent.Executors;
42 43
... ... @@ -80,7 +81,9 @@ public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
80 81 sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName()));
81 82 sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg)));
82 83
83   - sendMsgRequest.withMessageGroupId(msg.getKey().toString());
  84 + sendMsgRequest.withMessageGroupId(tpi.getTopic());
  85 + sendMsgRequest.withMessageDeduplicationId(UUID.randomUUID().toString());
  86 +
84 87 ListenableFuture<SendMessageResult> future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest));
85 88
86 89 Futures.addCallback(future, new FutureCallback<SendMessageResult>() {
... ...
... ... @@ -55,7 +55,6 @@ public class TbAwsSqsQueueAttributes {
55 55 @PostConstruct
56 56 private void init() {
57 57 defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true");
58   - defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true");
59 58
60 59 coreAttributes = getConfigs(coreProperties);
61 60 ruleEngineAttributes = getConfigs(ruleEngineProperties);
... ...
... ... @@ -22,6 +22,7 @@
22 22 "azure-sb": "^0.11.1",
23 23 "long": "^4.0.0",
24 24 "uuid-parse": "^1.0.0",
  25 + "uuid-random": "^1.3.0",
25 26 "winston": "^3.0.0",
26 27 "winston-daily-rotate-file": "^3.2.1"
27 28 },
... ...
... ... @@ -19,6 +19,7 @@
19 19 const config = require('config'),
20 20 JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
21 21 logger = require('../config/logger')._logger('awsSqsTemplate');
  22 +const uuid = require('uuid-random');
22 23
23 24 const requestTopic = config.get('request_topic');
24 25
... ... @@ -29,7 +30,7 @@ const AWS = require('aws-sdk');
29 30 const queueProperties = config.get('aws_sqs.queue_properties');
30 31 const poolInterval = config.get('js.response_poll_interval');
31 32
32   -let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'};
  33 +let queueAttributes = {FifoQueue: 'true'};
33 34 let sqsClient;
34 35 let requestQueueURL;
35 36 const queueUrls = new Map();
... ... @@ -51,7 +52,7 @@ function AwsSqsProducer() {
51 52 queueUrls.set(responseTopic, responseQueueUrl);
52 53 }
53 54
54   - let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId};
  55 + let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: 'js_eval', MessageDeduplicationId: uuid()};
55 56
56 57 return new Promise((resolve, reject) => {
57 58 sqsClient.sendMessage(params, function (err, data) {
... ...
... ... @@ -60,9 +60,11 @@ function PubSubProducer() {
60 60 const topicList = await pubSubClient.getTopics();
61 61
62 62 if (topicList) {
63   - topicList[0].forEach(topic => {
64   - topics.push(getName(topic.name));
65   - });
  63 + if (topicList) {
  64 + topicList[0].forEach(topic => {
  65 + topics.push(getName(topic.name));
  66 + });
  67 + }
66 68 }
67 69
68 70 const subscriptionList = await pubSubClient.getSubscriptions();
... ...