简柏林 / thingskit · Commits

Commit 746e63ab632b96772346d55f8ef7e57b1270d6bf

Authored by Adsumus
2 parents 22210e68 9953853f

Merge branch 'develop/3.0' of https://github.com/thingsboard/thingsboard into map/3.0

Inline Side-by-side
Showing 56 changed files with 1942 additions and 663 deletions

Too many changes to show.
Reload with full diff Plain diff Email patch

To preserve performance only 56 of 91 files are displayed.

... ... @@ -193,8 +193,8 @@
193 193 <artifactId>logback-classic</artifactId>
194 194 </dependency>
195 195 <dependency>
196   - <groupId>javax.mail</groupId>
197   - <artifactId>mail</artifactId>
  196 + <groupId>com.sun.mail</groupId>
  197 + <artifactId>javax.mail</artifactId>
198 198 </dependency>
199 199 <dependency>
200 200 <groupId>org.apache.curator</groupId>
... ...
... ... @@ -17,7 +17,7 @@
17 17 },
18 18 "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode",
19 19 "name": "Is Thermostat?",
20   - "debugMode": true,
  20 + "debugMode": false,
21 21 "configuration": {
22 22 "jsScript": "return msg.id.entityType === \"DEVICE\" && msg.type === \"thermostat\";"
23 23 }
... ... @@ -113,7 +113,7 @@
113 113 },
114 114 "type": "org.thingsboard.rule.engine.action.TbCreateRelationNode",
115 115 "name": "Relate to Asset",
116   - "debugMode": true,
  116 + "debugMode": false,
117 117 "configuration": {
118 118 "direction": "FROM",
119 119 "relationType": "ToAlarmPropagationAsset",
... ...
... ... @@ -81,7 +81,7 @@
81 81 },
82 82 "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode",
83 83 "name": "Check Alarms",
84   - "debugMode": true,
  84 + "debugMode": false,
85 85 "configuration": {
86 86 "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;"
87 87 }
... ... @@ -93,7 +93,7 @@
93 93 },
94 94 "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode",
95 95 "name": "Fetch Configuration",
96   - "debugMode": true,
  96 + "debugMode": false,
97 97 "configuration": {
98 98 "clientAttributeNames": [],
99 99 "sharedAttributeNames": [],
... ...
... ... @@ -163,7 +163,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
163 163 String dispatcherName = tenantId.getId().equals(EntityId.NULL_UUID) ?
164 164 DefaultActorService.SYSTEM_RULE_DISPATCHER_NAME : DefaultActorService.TENANT_RULE_DISPATCHER_NAME;
165 165 return context.actorOf(
166   - Props.create(new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleNode.getId()))
  166 + Props.create(new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleNode.getName(), ruleNode.getId()))
167 167 .withDispatcher(dispatcherName), ruleNode.getId().toString());
168 168 }
169 169
... ... @@ -200,7 +200,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
200 200 log.trace("[{}][{}] Processing message [{}]: {}", entityId, firstId, msg.getId(), msg);
201 201 if (envelope.getRelationTypes() == null || envelope.getRelationTypes().isEmpty()) {
202 202 try {
203   - checkActive();
  203 + checkActive(envelope.getTbMsg());
204 204 RuleNodeId targetId = msg.getRuleNodeId();
205 205 RuleNodeCtx targetCtx;
206 206 if (targetId == null) {
... ... @@ -216,6 +216,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
216 216 log.trace("[{}][{}] Rule node does not exist. Probably old message", entityId, targetId);
217 217 msg.getCallback().onSuccess();
218 218 }
  219 + } catch (RuleNodeException rne) {
  220 + envelope.getTbMsg().getCallback().onFailure(rne);
219 221 } catch (Exception e) {
220 222 envelope.getTbMsg().getCallback().onFailure(new RuleEngineException(e.getMessage()));
221 223 }
... ... @@ -225,11 +227,15 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
225 227 }
226 228
227 229 void onRuleChainToRuleChainMsg(RuleChainToRuleChainMsg envelope) {
228   - checkActive();
229   - if (firstNode != null) {
230   - pushMsgToNode(firstNode, envelope.getMsg(), envelope.getFromRelationType());
231   - } else {
232   - envelope.getMsg().getCallback().onSuccess();
  230 + try {
  231 + checkActive(envelope.getMsg());
  232 + if (firstNode != null) {
  233 + pushMsgToNode(firstNode, envelope.getMsg(), envelope.getFromRelationType());
  234 + } else {
  235 + envelope.getMsg().getCallback().onSuccess();
  236 + }
  237 + } catch (RuleNodeException e) {
  238 + log.debug("Rule Chain is not active. Current state [{}] for processor [{}][{}] tenant [{}]", state, entityId.getEntityType(), entityId, tenantId);
233 239 }
234 240 }
235 241
... ... @@ -239,7 +245,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
239 245
240 246 private void onTellNext(TbMsg msg, RuleNodeId originatorNodeId, Set<String> relationTypes, String failureMessage) {
241 247 try {
242   - checkActive();
  248 + checkActive(msg);
243 249 EntityId entityId = msg.getOriginator();
244 250 TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId);
245 251 List<RuleNodeRelation> relations = nodeRoutes.get(originatorNodeId).stream()
... ... @@ -272,6 +278,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
272 278 putToQueue(tpi, msg, callbackWrapper, target);
273 279 }
274 280 }
  281 + } catch (RuleNodeException rne) {
  282 + msg.getCallback().onFailure(rne);
275 283 } catch (Exception e) {
276 284 msg.getCallback().onFailure(new RuleEngineException("onTellNext - " + e.getMessage()));
277 285 }
... ... @@ -333,4 +341,9 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
333 341 }
334 342 }
335 343
  344 + @Override
  345 + protected RuleNodeException getInactiveException() {
  346 + RuleNode firstRuleNode = firstNode != null ? firstNode.getSelf() : null;
  347 + return new RuleNodeException("Rule Chain is not active! Failed to initialize.", ruleChainName, firstRuleNode);
  348 + }
336 349 }
... ...
... ... @@ -27,12 +27,14 @@ import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
27 27
28 28 public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessageProcessor> {
29 29
  30 + private final String ruleChainName;
30 31 private final RuleChainId ruleChainId;
31 32
32   - private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
  33 + private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, String ruleChainName, RuleNodeId ruleNodeId) {
33 34 super(systemContext, tenantId, ruleNodeId);
  35 + this.ruleChainName = ruleChainName;
34 36 this.ruleChainId = ruleChainId;
35   - setProcessor(new RuleNodeActorMessageProcessor(tenantId, ruleChainId, ruleNodeId, systemContext,
  37 + setProcessor(new RuleNodeActorMessageProcessor(tenantId, this.ruleChainName, ruleNodeId, systemContext,
36 38 context().parent(), context().self()));
37 39 }
38 40
... ... @@ -96,19 +98,21 @@ public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessa
96 98
97 99 private final TenantId tenantId;
98 100 private final RuleChainId ruleChainId;
  101 + private final String ruleChainName;
99 102 private final RuleNodeId ruleNodeId;
100 103
101   - public ActorCreator(ActorSystemContext context, TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
  104 + public ActorCreator(ActorSystemContext context, TenantId tenantId, RuleChainId ruleChainId, String ruleChainName, RuleNodeId ruleNodeId) {
102 105 super(context);
103 106 this.tenantId = tenantId;
104 107 this.ruleChainId = ruleChainId;
  108 + this.ruleChainName = ruleChainName;
105 109 this.ruleNodeId = ruleNodeId;
106 110
107 111 }
108 112
109 113 @Override
110 114 public RuleNodeActor create() throws Exception {
111   - return new RuleNodeActor(context, tenantId, ruleChainId, ruleNodeId);
  115 + return new RuleNodeActor(context, tenantId, ruleChainId, ruleChainName, ruleNodeId);
112 116 }
113 117 }
114 118
... ...
... ... @@ -17,37 +17,33 @@ package org.thingsboard.server.actors.ruleChain;
17 17
18 18 import akka.actor.ActorContext;
19 19 import akka.actor.ActorRef;
20   -import org.thingsboard.rule.engine.api.TbContext;
21 20 import org.thingsboard.rule.engine.api.TbNode;
22 21 import org.thingsboard.rule.engine.api.TbNodeConfiguration;
23 22 import org.thingsboard.server.actors.ActorSystemContext;
24 23 import org.thingsboard.server.actors.shared.ComponentMsgProcessor;
25   -import org.thingsboard.server.common.data.id.RuleChainId;
26 24 import org.thingsboard.server.common.data.id.RuleNodeId;
27 25 import org.thingsboard.server.common.data.id.TenantId;
28 26 import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
29 27 import org.thingsboard.server.common.data.rule.RuleNode;
30 28 import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
31   -import org.thingsboard.server.dao.rule.RuleChainService;
  29 +import org.thingsboard.server.common.msg.queue.RuleNodeException;
32 30
33 31 /**
34 32 * @author Andrew Shvayka
35 33 */
36 34 public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNodeId> {
37 35
38   - private final ActorRef parent;
  36 + private final String ruleChainName;
39 37 private final ActorRef self;
40   - private final RuleChainService service;
41 38 private RuleNode ruleNode;
42 39 private TbNode tbNode;
43 40 private DefaultTbContext defaultCtx;
44 41
45   - RuleNodeActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, ActorSystemContext systemContext
  42 + RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext
46 43 , ActorRef parent, ActorRef self) {
47 44 super(systemContext, tenantId, ruleNodeId);
48   - this.parent = parent;
  45 + this.ruleChainName = ruleChainName;
49 46 this.self = self;
50   - this.service = systemContext.getRuleChainService();
51 47 this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId);
52 48 this.defaultCtx = new DefaultTbContext(systemContext, new RuleNodeCtx(tenantId, parent, self, ruleNode));
53 49 }
... ... @@ -63,8 +59,8 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
63 59 @Override
64 60 public void onUpdate(ActorContext context) throws Exception {
65 61 RuleNode newRuleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId);
66   - boolean restartRequired = !(ruleNode.getType().equals(newRuleNode.getType())
67   - && ruleNode.getConfiguration().equals(newRuleNode.getConfiguration()));
  62 + boolean restartRequired = state != ComponentLifecycleState.ACTIVE ||
  63 + !(ruleNode.getType().equals(newRuleNode.getType()) && ruleNode.getConfiguration().equals(newRuleNode.getConfiguration()));
68 64 this.ruleNode = newRuleNode;
69 65 this.defaultCtx.updateSelf(newRuleNode);
70 66 if (restartRequired) {
... ... @@ -91,7 +87,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
91 87 }
92 88
93 89 public void onRuleToSelfMsg(RuleNodeToSelfMsg msg) throws Exception {
94   - checkActive();
  90 + checkActive(msg.getMsg());
95 91 if (ruleNode.isDebugMode()) {
96 92 systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self");
97 93 }
... ... @@ -103,7 +99,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
103 99 }
104 100
105 101 void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception {
106   - checkActive();
  102 + checkActive(msg.getMsg());
107 103 if (ruleNode.isDebugMode()) {
108 104 systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());
109 105 }
... ... @@ -129,4 +125,8 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
129 125 return tbNode;
130 126 }
131 127
  128 + @Override
  129 + protected RuleNodeException getInactiveException() {
  130 + return new RuleNodeException("Rule Node is not active! Failed to initialize.", ruleChainName, ruleNode);
  131 + }
132 132 }
... ...
... ... @@ -22,7 +22,10 @@ import org.thingsboard.server.actors.stats.StatsPersistTick;
22 22 import org.thingsboard.server.common.data.id.EntityId;
23 23 import org.thingsboard.server.common.data.id.TenantId;
24 24 import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
  25 +import org.thingsboard.server.common.msg.TbMsg;
25 26 import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
  27 +import org.thingsboard.server.common.msg.queue.RuleEngineException;
  28 +import org.thingsboard.server.common.msg.queue.RuleNodeException;
26 29
27 30 @Slf4j
28 31 public abstract class ComponentMsgProcessor<T extends EntityId> extends AbstractContextAwareMsgProcessor {
... ... @@ -74,11 +77,17 @@ public abstract class ComponentMsgProcessor<T extends EntityId> extends Abstract
74 77 schedulePeriodicMsgWithDelay(context, new StatsPersistTick(), statsPersistFrequency, statsPersistFrequency);
75 78 }
76 79
77   - protected void checkActive() {
  80 + protected void checkActive(TbMsg tbMsg) throws RuleNodeException {
78 81 if (state != ComponentLifecycleState.ACTIVE) {
79 82 log.debug("Component is not active. Current state [{}] for processor [{}][{}] tenant [{}]", state, entityId.getEntityType(), entityId, tenantId);
80   - throw new IllegalStateException("Rule chain is not active! " + entityId + " - " + tenantId);
  83 + RuleNodeException ruleNodeException = getInactiveException();
  84 + if (tbMsg != null) {
  85 + tbMsg.getCallback().onFailure(ruleNodeException);
  86 + }
  87 + throw ruleNodeException;
81 88 }
82 89 }
83 90
  91 + abstract protected RuleNodeException getInactiveException();
  92 +
84 93 }
... ...
... ... @@ -338,7 +338,7 @@ public class AuthController extends BaseController {
338 338
339 339 @RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST)
340 340 @ResponseBody
341   - public List<OAuth2ClientInfo> getOath2Clients() throws ThingsboardException {
  341 + public List<OAuth2ClientInfo> getOAuth2Clients() throws ThingsboardException {
342 342 try {
343 343 return oauth2Service.getOAuth2Clients();
344 344 } catch (Exception e) {
... ...
... ... @@ -23,6 +23,7 @@ import org.springframework.stereotype.Service;
23 23 import org.thingsboard.rule.engine.api.RpcError;
24 24 import org.thingsboard.server.actors.ActorSystemContext;
25 25 import org.thingsboard.server.common.data.id.TenantId;
  26 +import org.thingsboard.server.common.msg.MsgType;
26 27 import org.thingsboard.server.common.msg.TbActorMsg;
27 28 import org.thingsboard.server.common.msg.queue.ServiceType;
28 29 import org.thingsboard.server.common.msg.queue.TbCallback;
... ... @@ -45,6 +46,7 @@ import org.thingsboard.server.service.encoding.DataDecodingEncodingService;
45 46 import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
46 47 import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
47 48 import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
  49 +import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg;
48 50 import org.thingsboard.server.service.state.DeviceStateService;
49 51 import org.thingsboard.server.service.subscription.SubscriptionManagerService;
50 52 import org.thingsboard.server.service.subscription.TbLocalSubscriptionService;
... ... @@ -100,7 +102,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
100 102 }
101 103
102 104 @PreDestroy
103   - public void destroy(){
  105 + public void destroy() {
104 106 super.destroy();
105 107 }
106 108
... ... @@ -143,8 +145,13 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
143 145 } else if (toCoreMsg.getToDeviceActorNotificationMsg() != null && !toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) {
144 146 Optional<TbActorMsg> actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray());
145 147 if (actorMsg.isPresent()) {
146   - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get());
147   - actorContext.tell(actorMsg.get(), ActorRef.noSender());
  148 + TbActorMsg tbActorMsg = actorMsg.get();
  149 + if (tbActorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) {
  150 + tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) tbActorMsg);
  151 + } else {
  152 + log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get());
  153 + actorContext.tell(actorMsg.get(), ActorRef.noSender());
  154 + }
148 155 }
149 156 callback.onSuccess();
150 157 }
... ...
... ... @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j;
21 21 import org.springframework.beans.factory.annotation.Value;
22 22 import org.springframework.scheduling.annotation.Scheduled;
23 23 import org.springframework.stereotype.Service;
  24 +import org.thingsboard.rule.engine.api.RpcError;
24 25 import org.thingsboard.server.actors.ActorSystemContext;
25 26 import org.thingsboard.server.common.data.id.TenantId;
26 27 import org.thingsboard.server.common.msg.TbActorMsg;
... ... @@ -31,6 +32,7 @@ import org.thingsboard.server.common.msg.queue.ServiceQueue;
31 32 import org.thingsboard.server.common.msg.queue.ServiceType;
32 33 import org.thingsboard.server.common.msg.queue.TbCallback;
33 34 import org.thingsboard.server.common.msg.queue.TbMsgCallback;
  35 +import org.thingsboard.server.gen.transport.TransportProtos;
34 36 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
35 37 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
36 38 import org.thingsboard.server.queue.TbQueueConsumer;
... ... @@ -48,6 +50,9 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStr
48 50 import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
49 51 import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
50 52 import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
  53 +import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
  54 +import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
  55 +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
51 56 import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
52 57
53 58 import javax.annotation.PostConstruct;
... ... @@ -81,6 +86,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
81 86 private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory;
82 87 private final TbQueueRuleEngineSettings ruleEngineSettings;
83 88 private final RuleEngineStatisticsService statisticsService;
  89 + private final TbRuleEngineDeviceRpcService tbDeviceRpcService;
84 90 private final ConcurrentMap<String, TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> consumers = new ConcurrentHashMap<>();
85 91 private final ConcurrentMap<String, TbRuleEngineQueueConfiguration> consumerConfigurations = new ConcurrentHashMap<>();
86 92 private final ConcurrentMap<String, TbRuleEngineConsumerStats> consumerStats = new ConcurrentHashMap<>();
... ... @@ -90,13 +96,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
90 96 TbRuleEngineSubmitStrategyFactory submitStrategyFactory,
91 97 TbQueueRuleEngineSettings ruleEngineSettings,
92 98 TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService,
93   - ActorSystemContext actorContext, DataDecodingEncodingService encodingService) {
  99 + ActorSystemContext actorContext, DataDecodingEncodingService encodingService,
  100 + TbRuleEngineDeviceRpcService tbDeviceRpcService) {
94 101 super(actorContext, encodingService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer());
95 102 this.statisticsService = statisticsService;
96 103 this.ruleEngineSettings = ruleEngineSettings;
97 104 this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory;
98 105 this.submitStrategyFactory = submitStrategyFactory;
99 106 this.processingStrategyFactory = processingStrategyFactory;
  107 + this.tbDeviceRpcService = tbDeviceRpcService;
100 108 }
101 109
102 110 @PostConstruct
... ... @@ -227,7 +235,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
227 235 actorContext.tell(actorMsg.get(), ActorRef.noSender());
228 236 }
229 237 callback.onSuccess();
  238 + } else if (nfMsg.hasFromDeviceRpcResponse()) {
  239 + TransportProtos.FromDeviceRPCResponseProto proto = nfMsg.getFromDeviceRpcResponse();
  240 + RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null;
  241 + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB())
  242 + , proto.getResponse(), error);
  243 + tbDeviceRpcService.processRpcResponseFromDevice(response);
  244 + callback.onSuccess();
230 245 } else {
  246 + log.trace("Received notification with missing handler");
231 247 callback.onSuccess();
232 248 }
233 249 }
... ...
... ... @@ -23,7 +23,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
23 23 import org.thingsboard.server.actors.ActorSystemContext;
24 24 import org.thingsboard.server.common.msg.queue.ServiceType;
25 25 import org.thingsboard.server.common.msg.queue.TbCallback;
26   -import org.thingsboard.server.gen.transport.TransportProtos;
27 26 import org.thingsboard.server.queue.TbQueueConsumer;
28 27 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
29 28 import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
... ...
... ... @@ -23,7 +23,6 @@ import java.util.LinkedHashMap;
23 23 import java.util.Map;
24 24 import java.util.UUID;
25 25 import java.util.concurrent.ConcurrentMap;
26   -import java.util.concurrent.ExecutorService;
27 26 import java.util.concurrent.atomic.AtomicInteger;
28 27 import java.util.function.BiConsumer;
29 28
... ... @@ -77,8 +76,8 @@ public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS
77 76 }
78 77 }
79 78 int submitSize = pendingPack.size();
80   - if (log.isInfoEnabled() && submitSize > 0) {
81   - log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize);
  79 + if (log.isDebugEnabled() && submitSize > 0) {
  80 + log.debug("[{}] submitting [{}] messages to rule engine", queueName, submitSize);
82 81 }
83 82 pendingPack.forEach(msgConsumer);
84 83 }
... ...
... ... @@ -19,14 +19,8 @@ import lombok.extern.slf4j.Slf4j;
19 19 import org.thingsboard.server.gen.transport.TransportProtos;
20 20 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
21 21
22   -import java.util.ArrayList;
23   -import java.util.List;
24 22 import java.util.UUID;
25   -import java.util.concurrent.ConcurrentMap;
26   -import java.util.concurrent.ExecutorService;
27 23 import java.util.function.BiConsumer;
28   -import java.util.function.Function;
29   -import java.util.stream.Collectors;
30 24
31 25 @Slf4j
32 26 public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy {
... ... @@ -37,8 +31,8 @@ public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS
37 31
38 32 @Override
39 33 public void submitAttempt(BiConsumer<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> msgConsumer) {
40   - if (log.isInfoEnabled()) {
41   - log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size());
  34 + if (log.isDebugEnabled()) {
  35 + log.debug("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size());
42 36 }
43 37 orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg));
44 38 }
... ...
... ... @@ -15,26 +15,18 @@
15 15 */
16 16 package org.thingsboard.server.service.queue.processing;
17 17
18   -import com.google.protobuf.InvalidProtocolBufferException;
19 18 import lombok.extern.slf4j.Slf4j;
20 19 import org.thingsboard.server.common.data.id.EntityId;
21   -import org.thingsboard.server.common.data.id.EntityIdFactory;
22   -import org.thingsboard.server.common.msg.TbMsg;
23   -import org.thingsboard.server.common.msg.gen.MsgProtos;
24   -import org.thingsboard.server.common.msg.queue.TbMsgCallback;
25 20 import org.thingsboard.server.gen.transport.TransportProtos;
26 21 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
27 22
28   -import java.util.ArrayList;
29 23 import java.util.LinkedList;
30 24 import java.util.List;
31 25 import java.util.Queue;
32 26 import java.util.UUID;
33 27 import java.util.concurrent.ConcurrentHashMap;
34 28 import java.util.concurrent.ConcurrentMap;
35   -import java.util.concurrent.atomic.AtomicInteger;
36 29 import java.util.function.BiConsumer;
37   -import java.util.stream.Collectors;
38 30
39 31 @Slf4j
40 32 public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy {
... ...
... ... @@ -30,6 +30,5 @@ public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialBy
30 30 @Override
31 31 protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) {
32 32 return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB()));
33   -
34 33 }
35 34 }
... ...
... ... @@ -19,8 +19,6 @@ import lombok.extern.slf4j.Slf4j;
19 19 import org.thingsboard.server.gen.transport.TransportProtos;
20 20 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
21 21
22   -import java.util.LinkedHashMap;
23   -import java.util.Map;
24 22 import java.util.UUID;
25 23 import java.util.concurrent.ConcurrentMap;
26 24 import java.util.concurrent.atomic.AtomicInteger;
... ... @@ -63,8 +61,8 @@ public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSu
63 61 if (idx < listSize) {
64 62 IdMsgPair pair = orderedMsgList.get(idx);
65 63 expectedMsgId = pair.uuid;
66   - if (log.isInfoEnabled()) {
67   - log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg);
  64 + if (log.isDebugEnabled()) {
  65 + log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg);
68 66 }
69 67 msgConsumer.accept(pair.uuid, pair.msg);
70 68 }
... ...
... ... @@ -82,10 +82,10 @@ public class TbRuleEngineProcessingStrategyFactory {
82 82 retryCount++;
83 83 double failedCount = result.getFailedMap().size() + result.getPendingMap().size();
84 84 if (maxRetries > 0 && retryCount > maxRetries) {
85   - log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName);
  85 + log.debug("[{}] Skip reprocess of the rule engine pack due to max retries", queueName);
86 86 return new TbRuleEngineProcessingDecision(true, null);
87 87 } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) {
88   - log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName);
  88 + log.debug("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName);
89 89 return new TbRuleEngineProcessingDecision(true, null);
90 90 } else {
91 91 ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> toReprocess = new ConcurrentHashMap<>(initialTotalCount);
... ... @@ -98,7 +98,7 @@ public class TbRuleEngineProcessingStrategyFactory {
98 98 if (retrySuccessful) {
99 99 result.getSuccessMap().forEach(toReprocess::put);
100 100 }
101   - log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size());
  101 + log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size());
102 102 if (log.isTraceEnabled()) {
103 103 toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
104 104 }
... ... @@ -126,7 +126,7 @@ public class TbRuleEngineProcessingStrategyFactory {
126 126 @Override
127 127 public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) {
128 128 if (!result.isSuccess()) {
129   - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size());
  129 + log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size());
130 130 }
131 131 if (log.isTraceEnabled()) {
132 132 result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
... ...
... ... @@ -51,9 +51,9 @@ public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implemen
51 51 String firstName = getStringAttributeByKey(attributes, config.getBasic().getFirstNameAttributeKey());
52 52 oauth2User.setFirstName(firstName);
53 53 }
54   - if (!StringUtils.isEmpty(config.getBasic().getCustomerNameStrategyPattern())) {
  54 + if (!StringUtils.isEmpty(config.getBasic().getCustomerNamePattern())) {
55 55 StrSubstitutor sub = new StrSubstitutor(attributes, START_PLACEHOLDER_PREFIX, END_PLACEHOLDER_PREFIX);
56   - String customerName = sub.replace(config.getBasic().getCustomerNameStrategyPattern());
  56 + String customerName = sub.replace(config.getBasic().getCustomerNamePattern());
57 57 oauth2User.setCustomerName(customerName);
58 58 }
59 59 return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation());
... ... @@ -68,7 +68,7 @@ public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implemen
68 68 return email.substring(email .indexOf("@") + 1);
69 69 case CUSTOM_TENANT_STRATEGY:
70 70 StrSubstitutor sub = new StrSubstitutor(attributes, START_PLACEHOLDER_PREFIX, END_PLACEHOLDER_PREFIX);
71   - return sub.replace(config.getBasic().getTenantNameStrategyPattern());
  71 + return sub.replace(config.getBasic().getTenantNamePattern());
72 72 default:
73 73 throw new RuntimeException("Tenant Name Strategy with type " + config.getBasic().getTenantNameStrategy() + " is not supported!");
74 74 }
... ... @@ -78,7 +78,6 @@ public class BasicOAuth2ClientMapper extends AbstractOAuth2ClientMapper implemen
78 78 String result = null;
79 79 try {
80 80 result = (String) attributes.get(key);
81   -
82 81 } catch (Exception e) {
83 82 log.warn("Can't convert attribute to String by key " + key);
84 83 }
... ...
... ... @@ -41,7 +41,7 @@ public class CustomOAuth2ClientMapper extends AbstractOAuth2ClientMapper impleme
41 41 return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation());
42 42 }
43 43
44   - public OAuth2User getOAuth2User(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig.CustomOAuth2ClientMapperConfig custom) {
  44 + private synchronized OAuth2User getOAuth2User(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig.CustomOAuth2ClientMapperConfig custom) {
45 45 if (!StringUtils.isEmpty(custom.getUsername()) && !StringUtils.isEmpty(custom.getPassword())) {
46 46 restTemplateBuilder = restTemplateBuilder.basicAuthentication(custom.getUsername(), custom.getPassword());
47 47 }
... ...
... ... @@ -126,8 +126,8 @@ security:
126 126 firstNameAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_FIRST_NAME_ATTRIBUTE_KEY:}"
127 127 lastNameAttributeKey: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_LAST_NAME_ATTRIBUTE_KEY:}"
128 128 tenantNameStrategy: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_TENANT_NAME_STRATEGY:domain}" # domain, email or custom
129   - tenantNameStrategyPattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_TENANT_NAME_STRATEGY_PATTERN:}"
130   - customerNameStrategyPattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_CUSTOMER_NAME_STRATEGY_PATTERN:}"
  129 + tenantNamePattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_TENANT_NAME_PATTERN:}" # %{attribute_key} as placeholder for attributes value by key
  130 + customerNamePattern: "${SECURITY_OAUTH2_DEFAULT_MAPPER_BASIC_CUSTOMER_NAME_PATTERN:}" # %{attribute_key} as placeholder for attributes value by key
131 131 custom:
132 132 url: "${SECURITY_OAUTH2_DEFAULT_MAPPER_CUSTOM_URL:}"
133 133 username: "${SECURITY_OAUTH2_DEFAULT_MAPPER_CUSTOM_USERNAME:}"
... ... @@ -637,7 +637,7 @@ queue:
637 637 pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}"
638 638 stats:
639 639 enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
640   - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}"
  640 + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"
641 641 js:
642 642 # JS Eval request topic
643 643 request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}"
... ... @@ -657,7 +657,7 @@ queue:
657 657 pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}"
658 658 stats:
659 659 enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}"
660   - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:10000}"
  660 + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}"
661 661 queues:
662 662 - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}"
663 663 topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}"
... ...
... ... @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId;
27 27 import org.thingsboard.server.common.msg.gen.MsgProtos;
28 28 import org.thingsboard.server.common.msg.queue.TbMsgCallback;
29 29
  30 +import java.io.IOException;
30 31 import java.io.Serializable;
31 32 import java.util.UUID;
32 33
... ... @@ -47,7 +48,7 @@ public final class TbMsg implements Serializable {
47 48 private final RuleChainId ruleChainId;
48 49 private final RuleNodeId ruleNodeId;
49 50 //This field is not serialized because we use queues and there is no need to do it
50   - private final TbMsgCallback callback;
  51 + transient private final TbMsgCallback callback;
51 52
52 53 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) {
53 54 return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY);
... ... @@ -156,4 +157,13 @@ public final class TbMsg implements Serializable {
156 157 public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
157 158 return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback);
158 159 }
  160 +
  161 + public TbMsgCallback getCallback() {
  162 + //May be null in case of deserialization;
  163 + if (callback != null) {
  164 + return callback;
  165 + } else {
  166 + return TbMsgCallback.EMPTY;
  167 + }
  168 + }
159 169 }
... ...
... ... @@ -36,9 +36,15 @@ public class RuleNodeException extends RuleEngineException {
36 36 public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) {
37 37 super(message);
38 38 this.ruleChainName = ruleChainName;
39   - this.ruleNodeName = ruleNode.getName();
40   - this.ruleChainId = ruleNode.getRuleChainId();
41   - this.ruleNodeId = ruleNode.getId();
  39 + if (ruleNode != null) {
  40 + this.ruleNodeName = ruleNode.getName();
  41 + this.ruleChainId = ruleNode.getRuleChainId();
  42 + this.ruleNodeId = ruleNode.getId();
  43 + } else {
  44 + ruleNodeName = "Unknown";
  45 + ruleChainId = new RuleChainId(RuleChainId.NULL_UUID);
  46 + ruleNodeId = new RuleNodeId(RuleNodeId.NULL_UUID);
  47 + }
42 48 }
43 49
44 50 public String toJsonString() {
... ...
... ... @@ -67,6 +67,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin {
67 67
68 68 try {
69 69 QueueDescription queueDescription = new QueueDescription(topic);
  70 + queueDescription.setRequiresDuplicateDetection(false);
70 71 setQueueConfigs(queueDescription);
71 72
72 73 client.createQueue(queueDescription);
... ...
... ... @@ -31,9 +31,9 @@ import org.apache.qpid.proton.amqp.transport.SenderSettleMode;
31 31 import org.springframework.util.CollectionUtils;
32 32 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
33 33 import org.thingsboard.server.queue.TbQueueAdmin;
34   -import org.thingsboard.server.queue.TbQueueConsumer;
35 34 import org.thingsboard.server.queue.TbQueueMsg;
36 35 import org.thingsboard.server.queue.TbQueueMsgDecoder;
  36 +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate;
37 37 import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
38 38
39 39 import java.time.Duration;
... ... @@ -50,102 +50,72 @@ import java.util.stream.Collectors;
50 50 import java.util.stream.Stream;
51 51
52 52 @Slf4j
53   -public class TbServiceBusConsumerTemplate<T extends TbQueueMsg> implements TbQueueConsumer<T> {
  53 +public class TbServiceBusConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQueueConsumerTemplate<MessageWithDeliveryTag, T> {
54 54 private final TbQueueAdmin admin;
55   - private final String topic;
56 55 private final TbQueueMsgDecoder<T> decoder;
57 56 private final TbServiceBusSettings serviceBusSettings;
58 57
59 58 private final Gson gson = new Gson();
60 59
61 60 private Set<CoreMessageReceiver> receivers;
62   - private volatile Set<TopicPartitionInfo> partitions;
63   - private volatile boolean subscribed;
64   - private volatile boolean stopped = false;
65   - private Map<CoreMessageReceiver, Collection<MessageWithDeliveryTag>> pendingMessages = new ConcurrentHashMap<>();
  61 + private final Map<CoreMessageReceiver, Collection<MessageWithDeliveryTag>> pendingMessages = new ConcurrentHashMap<>();
66 62 private volatile int messagesPerQueue;
67 63
68 64 public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder<T> decoder) {
  65 + super(topic);
69 66 this.admin = admin;
70 67 this.decoder = decoder;
71   - this.topic = topic;
72 68 this.serviceBusSettings = serviceBusSettings;
73 69 }
74 70
75 71 @Override
76   - public String getTopic() {
77   - return topic;
  72 + protected List<MessageWithDeliveryTag> doPoll(long durationInMillis) {
  73 + List<CompletableFuture<Collection<MessageWithDeliveryTag>>> messageFutures =
  74 + receivers.stream()
  75 + .map(receiver -> receiver
  76 + .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis))
  77 + .whenComplete((messages, err) -> {
  78 + if (!CollectionUtils.isEmpty(messages)) {
  79 + pendingMessages.put(receiver, messages);
  80 + } else if (err != null) {
  81 + log.error("Failed to receive messages.", err);
  82 + }
  83 + }))
  84 + .collect(Collectors.toList());
  85 + try {
  86 + return fromList(messageFutures)
  87 + .get()
  88 + .stream()
  89 + .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream())
  90 + .collect(Collectors.toList());
  91 + } catch (InterruptedException | ExecutionException e) {
  92 + if (stopped) {
  93 + log.info("[{}] Service Bus consumer is stopped.", getTopic());
  94 + } else {
  95 + log.error("Failed to receive messages", e);
  96 + }
  97 + return Collections.emptyList();
  98 + }
78 99 }
79 100
80 101 @Override
81   - public void subscribe() {
82   - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true));
83   - subscribed = false;
  102 + protected void doSubscribe(List<String> topicNames) {
  103 + createReceivers();
  104 + messagesPerQueue = receivers.size() / partitions.size();
84 105 }
85 106
86 107 @Override
87   - public void subscribe(Set<TopicPartitionInfo> partitions) {
88   - this.partitions = partitions;
89   - subscribed = false;
  108 + protected void doCommit() {
  109 + pendingMessages.forEach((receiver, msgs) ->
  110 + msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN)));
  111 + pendingMessages.clear();
90 112 }
91 113
92 114 @Override
93   - public void unsubscribe() {
94   - stopped = true;
  115 + protected void doUnsubscribe() {
95 116 receivers.forEach(CoreMessageReceiver::closeAsync);
96 117 }
97 118
98   - @Override
99   - public List<T> poll(long durationInMillis) {
100   - if (!subscribed && partitions == null) {
101   - try {
102   - Thread.sleep(durationInMillis);
103   - } catch (InterruptedException e) {
104   - log.debug("Failed to await subscription", e);
105   - }
106   - } else {
107   - if (!subscribed) {
108   - createReceivers();
109   - messagesPerQueue = receivers.size() / partitions.size();
110   - subscribed = true;
111   - }
112   -
113   - List<CompletableFuture<Collection<MessageWithDeliveryTag>>> messageFutures =
114   - receivers.stream()
115   - .map(receiver -> receiver
116   - .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis))
117   - .whenComplete((messages, err) -> {
118   - if (!CollectionUtils.isEmpty(messages)) {
119   - pendingMessages.put(receiver, messages);
120   - } else if (err != null) {
121   - log.error("Failed to receive messages.", err);
122   - }
123   - }))
124   - .collect(Collectors.toList());
125   - try {
126   - return fromList(messageFutures)
127   - .get()
128   - .stream()
129   - .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream())
130   - .map(message -> {
131   - try {
132   - return decode(message);
133   - } catch (InvalidProtocolBufferException e) {
134   - log.error("Failed to parse message.", e);
135   - throw new RuntimeException("Failed to parse message.", e);
136   - }
137   - }).collect(Collectors.toList());
138   - } catch (InterruptedException | ExecutionException e) {
139   - if (stopped) {
140   - log.info("[{}] Service Bus consumer is stopped.", topic);
141   - } else {
142   - log.error("Failed to receive messages", e);
143   - }
144   - }
145   - }
146   - return Collections.emptyList();
147   - }
148   -
149 119 private void createReceivers() {
150 120 List<CompletableFuture<CoreMessageReceiver>> receiverFutures = partitions.stream()
151 121 .map(TopicPartitionInfo::getFullTopicName)
... ... @@ -167,7 +137,7 @@ public class TbServiceBusConsumerTemplate<T extends TbQueueMsg> implements TbQue
167 137 receivers = new HashSet<>(fromList(receiverFutures).get());
168 138 } catch (InterruptedException | ExecutionException e) {
169 139 if (stopped) {
170   - log.info("[{}] Service Bus consumer is stopped.", topic);
  140 + log.info("[{}] Service Bus consumer is stopped.", getTopic());
171 141 } else {
172 142 log.error("Failed to create receivers", e);
173 143 }
... ... @@ -196,13 +166,7 @@ public class TbServiceBusConsumerTemplate<T extends TbQueueMsg> implements TbQue
196 166 }
197 167
198 168 @Override
199   - public void commit() {
200   - pendingMessages.forEach((receiver, msgs) ->
201   - msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN)));
202   - pendingMessages.clear();
203   - }
204   -
205   - private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException {
  169 + protected T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException {
206 170 DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class);
207 171 return decoder.decode(msg);
208 172 }
... ...
... ... @@ -33,6 +33,7 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
33 33 import java.util.HashMap;
34 34 import java.util.Map;
35 35 import java.util.concurrent.CompletableFuture;
  36 +import java.util.concurrent.ConcurrentHashMap;
36 37 import java.util.concurrent.ExecutorService;
37 38 import java.util.concurrent.Executors;
38 39
... ... @@ -42,14 +43,14 @@ public class TbServiceBusProducerTemplate<T extends TbQueueMsg> implements TbQue
42 43 private final Gson gson = new Gson();
43 44 private final TbQueueAdmin admin;
44 45 private final TbServiceBusSettings serviceBusSettings;
45   - private final Map<String, QueueClient> clients = new HashMap<>();
46   - private ExecutorService executorService;
  46 + private final Map<String, QueueClient> clients = new ConcurrentHashMap<>();
  47 + private final ExecutorService executorService;
47 48
48 49 public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) {
49 50 this.admin = admin;
50 51 this.defaultTopic = defaultTopic;
51 52 this.serviceBusSettings = serviceBusSettings;
52   - executorService = Executors.newSingleThreadExecutor();
  53 + executorService = Executors.newCachedThreadPool();
53 54 }
54 55
55 56 @Override
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.queue.common;
  17 +
  18 +import com.google.common.util.concurrent.ListeningExecutorService;
  19 +import com.google.common.util.concurrent.MoreExecutors;
  20 +import lombok.extern.slf4j.Slf4j;
  21 +import org.thingsboard.server.queue.TbQueueMsg;
  22 +
  23 +import java.util.concurrent.Executors;
  24 +import java.util.concurrent.TimeUnit;
  25 +
  26 +@Slf4j
  27 +public abstract class AbstractParallelTbQueueConsumerTemplate<R, T extends TbQueueMsg> extends AbstractTbQueueConsumerTemplate<R, T> {
  28 +
  29 + protected ListeningExecutorService consumerExecutor;
  30 +
  31 + public AbstractParallelTbQueueConsumerTemplate(String topic) {
  32 + super(topic);
  33 + }
  34 +
  35 + protected void initNewExecutor(int threadPoolSize) {
  36 + if (consumerExecutor != null) {
  37 + consumerExecutor.shutdown();
  38 + try {
  39 + consumerExecutor.awaitTermination(1, TimeUnit.MINUTES);
  40 + } catch (InterruptedException e) {
  41 + log.trace("Interrupted while waiting for consumer executor to stop");
  42 + }
  43 + }
  44 + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threadPoolSize));
  45 + }
  46 +
  47 + protected void shutdownExecutor() {
  48 + if (consumerExecutor != null) {
  49 + consumerExecutor.shutdownNow();
  50 + }
  51 + }
  52 +
  53 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.queue.common;
  17 +
  18 +import lombok.Getter;
  19 +import lombok.extern.slf4j.Slf4j;
  20 +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
  21 +import org.thingsboard.server.queue.TbQueueConsumer;
  22 +import org.thingsboard.server.queue.TbQueueMsg;
  23 +
  24 +import java.io.IOException;
  25 +import java.util.ArrayList;
  26 +import java.util.Collections;
  27 +import java.util.List;
  28 +import java.util.Set;
  29 +import java.util.concurrent.locks.Lock;
  30 +import java.util.concurrent.locks.ReentrantLock;
  31 +import java.util.stream.Collectors;
  32 +
  33 +@Slf4j
  34 +public abstract class AbstractTbQueueConsumerTemplate<R, T extends TbQueueMsg> implements TbQueueConsumer<T> {
  35 +
  36 + private volatile boolean subscribed;
  37 + protected volatile boolean stopped = false;
  38 + protected volatile Set<TopicPartitionInfo> partitions;
  39 + protected final Lock consumerLock = new ReentrantLock();
  40 +
  41 + @Getter
  42 + private final String topic;
  43 +
  44 + public AbstractTbQueueConsumerTemplate(String topic) {
  45 + this.topic = topic;
  46 + }
  47 +
  48 + @Override
  49 + public void subscribe() {
  50 + consumerLock.lock();
  51 + try {
  52 + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true));
  53 + subscribed = false;
  54 + } finally {
  55 + consumerLock.unlock();
  56 + }
  57 + }
  58 +
  59 + @Override
  60 + public void subscribe(Set<TopicPartitionInfo> partitions) {
  61 + consumerLock.lock();
  62 + try {
  63 + this.partitions = partitions;
  64 + subscribed = false;
  65 + } finally {
  66 + consumerLock.unlock();
  67 + }
  68 + }
  69 +
  70 + @Override
  71 + public List<T> poll(long durationInMillis) {
  72 + if (!subscribed && partitions == null) {
  73 + try {
  74 + Thread.sleep(durationInMillis);
  75 + } catch (InterruptedException e) {
  76 + log.debug("Failed to await subscription", e);
  77 + }
  78 + } else {
  79 + long pollStartTs = System.currentTimeMillis();
  80 + consumerLock.lock();
  81 + try {
  82 + if (!subscribed) {
  83 + List<String> topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList());
  84 + doSubscribe(topicNames);
  85 + subscribed = true;
  86 + }
  87 +
  88 + List<R> records = doPoll(durationInMillis);
  89 + if (!records.isEmpty()) {
  90 + List<T> result = new ArrayList<>(records.size());
  91 + records.forEach(record -> {
  92 + try {
  93 + if (record != null) {
  94 + result.add(decode(record));
  95 + }
  96 + } catch (IOException e) {
  97 + log.error("Failed decode record: [{}]", record);
  98 + throw new RuntimeException("Failed to decode record: ", e);
  99 + }
  100 + });
  101 + return result;
  102 + } else {
  103 + long pollDuration = System.currentTimeMillis() - pollStartTs;
  104 + if (pollDuration < durationInMillis) {
  105 + try {
  106 + Thread.sleep(durationInMillis - pollDuration);
  107 + } catch (InterruptedException e) {
  108 + if (!stopped) {
  109 + log.error("Failed to wait.", e);
  110 + }
  111 + }
  112 + }
  113 + }
  114 + } finally {
  115 + consumerLock.unlock();
  116 + }
  117 + }
  118 + return Collections.emptyList();
  119 + }
  120 +
  121 + @Override
  122 + public void commit() {
  123 + consumerLock.lock();
  124 + try {
  125 + doCommit();
  126 + } finally {
  127 + consumerLock.unlock();
  128 + }
  129 + }
  130 +
  131 + @Override
  132 + public void unsubscribe() {
  133 + stopped = true;
  134 + consumerLock.lock();
  135 + try {
  136 + doUnsubscribe();
  137 + } finally {
  138 + consumerLock.unlock();
  139 + }
  140 + }
  141 +
  142 + abstract protected List<R> doPoll(long durationInMillis);
  143 +
  144 + abstract protected T decode(R record) throws IOException;
  145 +
  146 + abstract protected void doSubscribe(List<String> topicNames);
  147 +
  148 + abstract protected void doCommit();
  149 +
  150 + abstract protected void doUnsubscribe();
  151 +
  152 +}
... ...
... ... @@ -16,16 +16,14 @@
16 16 package org.thingsboard.server.queue.kafka;
17 17
18 18 import lombok.Builder;
19   -import lombok.Getter;
20 19 import lombok.extern.slf4j.Slf4j;
21 20 import org.apache.kafka.clients.consumer.ConsumerConfig;
22 21 import org.apache.kafka.clients.consumer.ConsumerRecord;
23 22 import org.apache.kafka.clients.consumer.ConsumerRecords;
24 23 import org.apache.kafka.clients.consumer.KafkaConsumer;
25   -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
26 24 import org.thingsboard.server.queue.TbQueueAdmin;
27   -import org.thingsboard.server.queue.TbQueueConsumer;
28 25 import org.thingsboard.server.queue.TbQueueMsg;
  26 +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate;
29 27
30 28 import java.io.IOException;
31 29 import java.time.Duration;
... ... @@ -33,26 +31,16 @@ import java.util.ArrayList;
33 31 import java.util.Collections;
34 32 import java.util.List;
35 33 import java.util.Properties;
36   -import java.util.Set;
37   -import java.util.concurrent.locks.Lock;
38   -import java.util.concurrent.locks.ReentrantLock;
39   -import java.util.stream.Collectors;
40 34
41 35 /**
42 36 * Created by ashvayka on 24.09.18.
43 37 */
44 38 @Slf4j
45   -public class TbKafkaConsumerTemplate<T extends TbQueueMsg> implements TbQueueConsumer<T> {
  39 +public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQueueConsumerTemplate<ConsumerRecord<String, byte[]>, T> {
46 40
47 41 private final TbQueueAdmin admin;
48 42 private final KafkaConsumer<String, byte[]> consumer;
49 43 private final TbKafkaDecoder<T> decoder;
50   - private volatile boolean subscribed;
51   - private volatile Set<TopicPartitionInfo> partitions;
52   - private final Lock consumerLock;
53   -
54   - @Getter
55   - private final String topic;
56 44
57 45 @Builder
58 46 private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder<T> decoder,
... ... @@ -60,6 +48,7 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> implements TbQueueCon
60 48 boolean autoCommit, int autoCommitIntervalMs,
61 49 int maxPollRecords,
62 50 TbQueueAdmin admin) {
  51 + super(topic);
63 52 Properties props = settings.toProps();
64 53 props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId);
65 54 if (groupId != null) {
... ... @@ -75,94 +64,42 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> implements TbQueueCon
75 64 this.admin = admin;
76 65 this.consumer = new KafkaConsumer<>(props);
77 66 this.decoder = decoder;
78   - this.topic = topic;
79   - this.consumerLock = new ReentrantLock();
80 67 }
81 68
82 69 @Override
83   - public void subscribe() {
84   - consumerLock.lock();
85   - try {
86   - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true));
87   - subscribed = false;
88   - } finally {
89   - consumerLock.unlock();
90   - }
  70 + protected void doSubscribe(List<String> topicNames) {
  71 + topicNames.forEach(admin::createTopicIfNotExists);
  72 + consumer.subscribe(topicNames);
91 73 }
92 74
93 75 @Override
94   - public void subscribe(Set<TopicPartitionInfo> partitions) {
95   - consumerLock.lock();
96   - try {
97   - this.partitions = partitions;
98   - subscribed = false;
99   - } finally {
100   - consumerLock.unlock();
  76 + protected List<ConsumerRecord<String, byte[]>> doPoll(long durationInMillis) {
  77 + ConsumerRecords<String, byte[]> records = consumer.poll(Duration.ofMillis(durationInMillis));
  78 + if (records.isEmpty()) {
  79 + return Collections.emptyList();
  80 + } else {
  81 + List<ConsumerRecord<String, byte[]>> recordList = new ArrayList<>(256);
  82 + records.forEach(recordList::add);
  83 + return recordList;
101 84 }
102 85 }
103 86
104 87 @Override
105   - public List<T> poll(long durationInMillis) {
106   - if (!subscribed && partitions == null) {
107   - try {
108   - Thread.sleep(durationInMillis);
109   - } catch (InterruptedException e) {
110   - log.debug("Failed to await subscription", e);
111   - }
112   - } else {
113   - consumerLock.lock();
114   - try {
115   - if (!subscribed) {
116   - List<String> topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList());
117   - topicNames.forEach(admin::createTopicIfNotExists);
118   - consumer.subscribe(topicNames);
119   - subscribed = true;
120   - }
121   -
122   - ConsumerRecords<String, byte[]> records = consumer.poll(Duration.ofMillis(durationInMillis));
123   - if (records.count() > 0) {
124   - List<T> result = new ArrayList<>();
125   - records.forEach(record -> {
126   - try {
127   - result.add(decode(record));
128   - } catch (IOException e) {
129   - log.error("Failed decode record: [{}]", record);
130   - }
131   - });
132   - return result;
133   - }
134   - } finally {
135   - consumerLock.unlock();
136   - }
137   - }
138   - return Collections.emptyList();
  88 + public T decode(ConsumerRecord<String, byte[]> record) throws IOException {
  89 + return decoder.decode(new KafkaTbQueueMsg(record));
139 90 }
140 91
141 92 @Override
142   - public void commit() {
143   - consumerLock.lock();
144   - try {
145   - consumer.commitAsync();
146   - } finally {
147   - consumerLock.unlock();
148   - }
  93 + protected void doCommit() {
  94 + consumer.commitAsync();
149 95 }
150 96
151 97 @Override
152   - public void unsubscribe() {
153   - consumerLock.lock();
154   - try {
155   - if (consumer != null) {
156   - consumer.unsubscribe();
157   - consumer.close();
158   - }
159   - } finally {
160   - consumerLock.unlock();
  98 + protected void doUnsubscribe() {
  99 + if (consumer != null) {
  100 + consumer.unsubscribe();
  101 + consumer.close();
161 102 }
162 103 }
163 104
164   - public T decode(ConsumerRecord<String, byte[]> record) throws IOException {
165   - return decoder.decode(new KafkaTbQueueMsg(record));
166   - }
167   -
168 105 }
... ...
... ... @@ -15,6 +15,7 @@
15 15 */
16 16 package org.thingsboard.server.queue.pubsub;
17 17
  18 +import com.google.api.gax.rpc.AlreadyExistsException;
18 19 import com.google.cloud.pubsub.v1.SubscriptionAdminClient;
19 20 import com.google.cloud.pubsub.v1.SubscriptionAdminSettings;
20 21 import com.google.cloud.pubsub.v1.TopicAdminClient;
... ... @@ -24,9 +25,9 @@ import com.google.pubsub.v1.ListSubscriptionsRequest;
24 25 import com.google.pubsub.v1.ListTopicsRequest;
25 26 import com.google.pubsub.v1.ProjectName;
26 27 import com.google.pubsub.v1.ProjectSubscriptionName;
27   -import com.google.pubsub.v1.ProjectTopicName;
28 28 import com.google.pubsub.v1.Subscription;
29 29 import com.google.pubsub.v1.Topic;
  30 +import com.google.pubsub.v1.TopicName;
30 31 import lombok.extern.slf4j.Slf4j;
31 32 import org.thingsboard.server.queue.TbQueueAdmin;
32 33
... ... @@ -103,7 +104,10 @@ public class TbPubSubAdmin implements TbQueueAdmin {
103 104
104 105 @Override
105 106 public void createTopicIfNotExists(String partition) {
106   - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition);
  107 + TopicName topicName = TopicName.newBuilder()
  108 + .setTopic(partition)
  109 + .setProject(pubSubSettings.getProjectId())
  110 + .build();
107 111
108 112 if (topicSet.contains(topicName.toString())) {
109 113 createSubscriptionIfNotExists(partition, topicName);
... ... @@ -121,13 +125,18 @@ public class TbPubSubAdmin implements TbQueueAdmin {
121 125 }
122 126 }
123 127
124   - topicAdminClient.createTopic(topicName);
125   - topicSet.add(topicName.toString());
126   - log.info("Created new topic: [{}]", topicName.toString());
  128 + try {
  129 + topicAdminClient.createTopic(topicName);
  130 + log.info("Created new topic: [{}]", topicName.toString());
  131 + } catch (AlreadyExistsException e) {
  132 + log.info("[{}] Topic already exist.", topicName.toString());
  133 + } finally {
  134 + topicSet.add(topicName.toString());
  135 + }
127 136 createSubscriptionIfNotExists(partition, topicName);
128 137 }
129 138
130   - private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) {
  139 + private void createSubscriptionIfNotExists(String partition, TopicName topicName) {
131 140 ProjectSubscriptionName subscriptionName =
132 141 ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition);
133 142
... ... @@ -153,9 +162,14 @@ public class TbPubSubAdmin implements TbQueueAdmin {
153 162 setAckDeadline(subscriptionBuilder);
154 163 setMessageRetention(subscriptionBuilder);
155 164
156   - subscriptionAdminClient.createSubscription(subscriptionBuilder.build());
157   - subscriptionSet.add(subscriptionName.toString());
158   - log.info("Created new subscription: [{}]", subscriptionName.toString());
  165 + try {
  166 + subscriptionAdminClient.createSubscription(subscriptionBuilder.build());
  167 + log.info("Created new subscription: [{}]", subscriptionName.toString());
  168 + } catch (AlreadyExistsException e) {
  169 + log.info("[{}] Subscription already exist.", subscriptionName.toString());
  170 + } finally {
  171 + subscriptionSet.add(subscriptionName.toString());
  172 + }
159 173 }
160 174
161 175 private void setAckDeadline(Subscription.Builder builder) {
... ...
... ... @@ -30,27 +30,25 @@ import com.google.pubsub.v1.PullResponse;
30 30 import com.google.pubsub.v1.ReceivedMessage;
31 31 import lombok.extern.slf4j.Slf4j;
32 32 import org.springframework.util.CollectionUtils;
33   -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
34 33 import org.thingsboard.server.queue.TbQueueAdmin;
35   -import org.thingsboard.server.queue.TbQueueConsumer;
36 34 import org.thingsboard.server.queue.TbQueueMsg;
37 35 import org.thingsboard.server.queue.TbQueueMsgDecoder;
  36 +import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate;
38 37 import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
39 38
40 39 import java.io.IOException;
41 40 import java.util.ArrayList;
42 41 import java.util.Collections;
  42 +import java.util.LinkedHashSet;
43 43 import java.util.List;
44 44 import java.util.Objects;
45 45 import java.util.Set;
46 46 import java.util.concurrent.CopyOnWriteArrayList;
47 47 import java.util.concurrent.ExecutionException;
48   -import java.util.concurrent.ExecutorService;
49   -import java.util.concurrent.Executors;
50 48 import java.util.stream.Collectors;
51 49
52 50 @Slf4j
53   -public class TbPubSubConsumerTemplate<T extends TbQueueMsg> implements TbQueueConsumer<T> {
  51 +public class TbPubSubConsumerTemplate<T extends TbQueueMsg> extends AbstractParallelTbQueueConsumerTemplate<PubsubMessage, T> {
54 52
55 53 private final Gson gson = new Gson();
56 54 private final TbQueueAdmin admin;
... ... @@ -58,23 +56,18 @@ public class TbPubSubConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
58 56 private final TbQueueMsgDecoder<T> decoder;
59 57 private final TbPubSubSettings pubSubSettings;
60 58
61   - private volatile boolean subscribed;
62   - private volatile Set<TopicPartitionInfo> partitions;
63 59 private volatile Set<String> subscriptionNames;
64 60 private final List<AcknowledgeRequest> acknowledgeRequests = new CopyOnWriteArrayList<>();
65 61
66   - private ExecutorService consumerExecutor;
67 62 private final SubscriberStub subscriber;
68   - private volatile boolean stopped;
69   -
70 63 private volatile int messagesPerTopic;
71 64
72 65 public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder<T> decoder) {
  66 + super(topic);
73 67 this.admin = admin;
74 68 this.pubSubSettings = pubSubSettings;
75 69 this.topic = topic;
76 70 this.decoder = decoder;
77   -
78 71 try {
79 72 SubscriberStubSettings subscriberStubSettings =
80 73 SubscriberStubSettings.newBuilder()
... ... @@ -84,89 +77,50 @@ public class TbPubSubConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
84 77 .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize())
85 78 .build())
86 79 .build();
87   -
88 80 this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings);
89 81 } catch (IOException e) {
90 82 log.error("Failed to create subscriber.", e);
91 83 throw new RuntimeException("Failed to create subscriber.", e);
92 84 }
93   - stopped = false;
94 85 }
95 86
96 87 @Override
97   - public String getTopic() {
98   - return topic;
  88 + protected List<PubsubMessage> doPoll(long durationInMillis) {
  89 + try {
  90 + List<ReceivedMessage> messages = receiveMessages();
  91 + if (!messages.isEmpty()) {
  92 + return messages.stream().map(ReceivedMessage::getMessage).collect(Collectors.toList());
  93 + }
  94 + } catch (ExecutionException | InterruptedException e) {
  95 + if (stopped) {
  96 + log.info("[{}] Pub/Sub consumer is stopped.", topic);
  97 + } else {
  98 + log.error("Failed to receive messages", e);
  99 + }
  100 + }
  101 + return Collections.emptyList();
99 102 }
100 103
101 104 @Override
102   - public void subscribe() {
103   - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true));
104   - subscribed = false;
  105 + protected void doSubscribe(List<String> topicNames) {
  106 + subscriptionNames = new LinkedHashSet<>(topicNames);
  107 + subscriptionNames.forEach(admin::createTopicIfNotExists);
  108 + initNewExecutor(subscriptionNames.size() + 1);
  109 + messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size();
105 110 }
106 111
107 112 @Override
108   - public void subscribe(Set<TopicPartitionInfo> partitions) {
109   - this.partitions = partitions;
110   - subscribed = false;
  113 + protected void doCommit() {
  114 + acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall);
  115 + acknowledgeRequests.clear();
111 116 }
112 117
113 118 @Override
114   - public void unsubscribe() {
115   - stopped = true;
116   - if (consumerExecutor != null) {
117   - consumerExecutor.shutdownNow();
118   - }
119   -
  119 + protected void doUnsubscribe() {
120 120 if (subscriber != null) {
121 121 subscriber.close();
122 122 }
123   - }
124   -
125   - @Override
126   - public List<T> poll(long durationInMillis) {
127   - if (!subscribed && partitions == null) {
128   - try {
129   - Thread.sleep(durationInMillis);
130   - } catch (InterruptedException e) {
131   - log.debug("Failed to await subscription", e);
132   - }
133   - } else {
134   - if (!subscribed) {
135   - subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet());
136   - subscriptionNames.forEach(admin::createTopicIfNotExists);
137   - consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size());
138   - messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size();
139   - subscribed = true;
140   - }
141   - List<ReceivedMessage> messages;
142   - try {
143   - messages = receiveMessages();
144   - if (!messages.isEmpty()) {
145   - List<T> result = new ArrayList<>();
146   - messages.forEach(msg -> {
147   - try {
148   - result.add(decode(msg.getMessage()));
149   - } catch (InvalidProtocolBufferException e) {
150   - log.error("Failed decode record: [{}]", msg);
151   - }
152   - });
153   - return result;
154   - }
155   - } catch (ExecutionException | InterruptedException e) {
156   - if (stopped) {
157   - log.info("[{}] Pub/Sub consumer is stopped.", topic);
158   - } else {
159   - log.error("Failed to receive messages", e);
160   - }
161   - }
162   - }
163   - return Collections.emptyList();
164   - }
165   -
166   - @Override
167   - public void commit() {
168   - acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall);
169   - acknowledgeRequests.clear();
  123 + shutdownExecutor();
170 124 }
171 125
172 126 private List<ReceivedMessage> receiveMessages() throws ExecutionException, InterruptedException {
... ... @@ -175,7 +129,7 @@ public class TbPubSubConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
175 129 PullRequest pullRequest =
176 130 PullRequest.newBuilder()
177 131 .setMaxMessages(messagesPerTopic)
178   - .setReturnImmediately(false) // return immediately if messages are not available
  132 +// .setReturnImmediately(false) // return immediately if messages are not available
179 133 .setSubscription(subscriptionName)
180 134 .build();
181 135
... ... @@ -211,6 +165,7 @@ public class TbPubSubConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
211 165 return transform.get();
212 166 }
213 167
  168 + @Override
214 169 public T decode(PubsubMessage message) throws InvalidProtocolBufferException {
215 170 DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class);
216 171 return decoder.decode(msg);
... ...
... ... @@ -49,7 +49,7 @@ public class TbPubSubProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
49 49
50 50 private final Map<String, Publisher> publisherMap = new ConcurrentHashMap<>();
51 51
52   - private ExecutorService pubExecutor = Executors.newCachedThreadPool();
  52 + private final ExecutorService pubExecutor = Executors.newCachedThreadPool();
53 53
54 54 public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) {
55 55 this.defaultTopic = defaultTopic;
... ... @@ -124,8 +124,8 @@ public class TbPubSubProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
124 124 publisherMap.put(topic, publisher);
125 125 return publisher;
126 126 } catch (IOException e) {
127   - log.error("Failed to create topic [{}].", topic, e);
128   - throw new RuntimeException("Failed to create topic.", e);
  127 + log.error("Failed to create Publisher for the topic [{}].", topic, e);
  128 + throw new RuntimeException("Failed to create Publisher for the topic.", e);
129 129 }
130 130 }
131 131
... ...
... ... @@ -27,13 +27,11 @@ import java.util.concurrent.TimeoutException;
27 27 @Slf4j
28 28 public class TbRabbitMqAdmin implements TbQueueAdmin {
29 29
30   - private final TbRabbitMqSettings rabbitMqSettings;
31 30 private final Channel channel;
32 31 private final Connection connection;
33 32 private final Map<String, Object> arguments;
34 33
35 34 public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map<String, Object> arguments) {
36   - this.rabbitMqSettings = rabbitMqSettings;
37 35 this.arguments = arguments;
38 36
39 37 try {
... ...
... ... @@ -23,9 +23,9 @@ import com.rabbitmq.client.GetResponse;
23 23 import lombok.extern.slf4j.Slf4j;
24 24 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
25 25 import org.thingsboard.server.queue.TbQueueAdmin;
26   -import org.thingsboard.server.queue.TbQueueConsumer;
27 26 import org.thingsboard.server.queue.TbQueueMsg;
28 27 import org.thingsboard.server.queue.TbQueueMsgDecoder;
  28 +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate;
29 29 import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
30 30
31 31 import java.io.IOException;
... ... @@ -37,33 +37,26 @@ import java.util.concurrent.TimeoutException;
37 37 import java.util.stream.Collectors;
38 38
39 39 @Slf4j
40   -public class TbRabbitMqConsumerTemplate<T extends TbQueueMsg> implements TbQueueConsumer<T> {
  40 +public class TbRabbitMqConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQueueConsumerTemplate<GetResponse, T> {
41 41
42 42 private final Gson gson = new Gson();
43 43 private final TbQueueAdmin admin;
44   - private final String topic;
45 44 private final TbQueueMsgDecoder<T> decoder;
46   - private final TbRabbitMqSettings rabbitMqSettings;
47 45 private final Channel channel;
48 46 private final Connection connection;
49 47
50   - private volatile Set<TopicPartitionInfo> partitions;
51   - private volatile boolean subscribed;
52 48 private volatile Set<String> queues;
53   - private volatile boolean stopped;
54 49
55 50 public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder<T> decoder) {
  51 + super(topic);
56 52 this.admin = admin;
57 53 this.decoder = decoder;
58   - this.topic = topic;
59   - this.rabbitMqSettings = rabbitMqSettings;
60 54 try {
61 55 connection = rabbitMqSettings.getConnectionFactory().newConnection();
62 56 } catch (IOException | TimeoutException e) {
63 57 log.error("Failed to create connection.", e);
64 58 throw new RuntimeException("Failed to create connection.", e);
65 59 }
66   -
67 60 try {
68 61 channel = connection.createChannel();
69 62 } catch (IOException e) {
... ... @@ -74,25 +67,42 @@ public class TbRabbitMqConsumerTemplate<T extends TbQueueMsg> implements TbQueue
74 67 }
75 68
76 69 @Override
77   - public String getTopic() {
78   - return topic;
  70 + protected List<GetResponse> doPoll(long durationInMillis) {
  71 + List<GetResponse> result = queues.stream()
  72 + .map(queue -> {
  73 + try {
  74 + return channel.basicGet(queue, false);
  75 + } catch (IOException e) {
  76 + log.error("Failed to get messages from queue: [{}]", queue);
  77 + throw new RuntimeException("Failed to get messages from queue.", e);
  78 + }
  79 + }).filter(Objects::nonNull).collect(Collectors.toList());
  80 + if (result.size() > 0) {
  81 + return result;
  82 + } else {
  83 + return Collections.emptyList();
  84 + }
79 85 }
80 86
81 87 @Override
82   - public void subscribe() {
83   - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true));
84   - subscribed = false;
  88 + protected void doSubscribe(List<String> topicNames) {
  89 + queues = partitions.stream()
  90 + .map(TopicPartitionInfo::getFullTopicName)
  91 + .collect(Collectors.toSet());
  92 + queues.forEach(admin::createTopicIfNotExists);
85 93 }
86 94
87 95 @Override
88   - public void subscribe(Set<TopicPartitionInfo> partitions) {
89   - this.partitions = partitions;
90   - subscribed = false;
  96 + protected void doCommit() {
  97 + try {
  98 + channel.basicAck(0, true);
  99 + } catch (IOException e) {
  100 + log.error("Failed to ack messages.", e);
  101 + }
91 102 }
92 103
93 104 @Override
94   - public void unsubscribe() {
95   - stopped = true;
  105 + protected void doUnsubscribe() {
96 106 if (channel != null) {
97 107 try {
98 108 channel.close();
... ... @@ -109,63 +119,6 @@ public class TbRabbitMqConsumerTemplate<T extends TbQueueMsg> implements TbQueue
109 119 }
110 120 }
111 121
112   - @Override
113   - public List<T> poll(long durationInMillis) {
114   - if (!subscribed && partitions == null) {
115   - try {
116   - Thread.sleep(durationInMillis);
117   - } catch (InterruptedException e) {
118   - log.debug("Failed to await subscription", e);
119   - }
120   - } else {
121   - if (!subscribed) {
122   - queues = partitions.stream()
123   - .map(TopicPartitionInfo::getFullTopicName)
124   - .collect(Collectors.toSet());
125   -
126   - queues.forEach(admin::createTopicIfNotExists);
127   - subscribed = true;
128   - }
129   -
130   - List<T> result = queues.stream()
131   - .map(queue -> {
132   - try {
133   - return channel.basicGet(queue, false);
134   - } catch (IOException e) {
135   - log.error("Failed to get messages from queue: [{}]", queue);
136   - throw new RuntimeException("Failed to get messages from queue.", e);
137   - }
138   - }).filter(Objects::nonNull).map(message -> {
139   - try {
140   - return decode(message);
141   - } catch (InvalidProtocolBufferException e) {
142   - log.error("Failed to decode message: [{}].", message);
143   - throw new RuntimeException("Failed to decode message.", e);
144   - }
145   - }).collect(Collectors.toList());
146   - if (result.size() > 0) {
147   - return result;
148   - }
149   - }
150   - try {
151   - Thread.sleep(durationInMillis);
152   - } catch (InterruptedException e) {
153   - if (!stopped) {
154   - log.error("Failed to wait.", e);
155   - }
156   - }
157   - return Collections.emptyList();
158   - }
159   -
160   - @Override
161   - public void commit() {
162   - try {
163   - channel.basicAck(0, true);
164   - } catch (IOException e) {
165   - log.error("Failed to ack messages.", e);
166   - }
167   - }
168   -
169 122 public T decode(GetResponse message) throws InvalidProtocolBufferException {
170 123 DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class);
171 124 return decoder.decode(msg);
... ...
... ... @@ -30,6 +30,8 @@ import org.thingsboard.server.queue.TbQueueProducer;
30 30 import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
31 31
32 32 import java.io.IOException;
  33 +import java.util.Set;
  34 +import java.util.concurrent.ConcurrentHashMap;
33 35 import java.util.concurrent.Executors;
34 36 import java.util.concurrent.TimeoutException;
35 37
... ... @@ -39,10 +41,12 @@ public class TbRabbitMqProducerTemplate<T extends TbQueueMsg> implements TbQueue
39 41 private final Gson gson = new Gson();
40 42 private final TbQueueAdmin admin;
41 43 private final TbRabbitMqSettings rabbitMqSettings;
42   - private ListeningExecutorService producerExecutor;
  44 + private final ListeningExecutorService producerExecutor;
43 45 private final Channel channel;
44 46 private final Connection connection;
45 47
  48 + private final Set<TopicPartitionInfo> topics = ConcurrentHashMap.newKeySet();
  49 +
46 50 public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) {
47 51 this.admin = admin;
48 52 this.defaultTopic = defaultTopic;
... ... @@ -75,6 +79,7 @@ public class TbRabbitMqProducerTemplate<T extends TbQueueMsg> implements TbQueue
75 79
76 80 @Override
77 81 public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) {
  82 + createTopicIfNotExist(tpi);
78 83 AMQP.BasicProperties properties = new AMQP.BasicProperties();
79 84 try {
80 85 channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes());
... ... @@ -110,4 +115,11 @@ public class TbRabbitMqProducerTemplate<T extends TbQueueMsg> implements TbQueue
110 115 }
111 116 }
112 117
  118 + private void createTopicIfNotExist(TopicPartitionInfo tpi) {
  119 + if (topics.contains(tpi)) {
  120 + return;
  121 + }
  122 + admin.createTopicIfNotExists(tpi.getFullTopicName());
  123 + topics.add(tpi);
  124 + }
113 125 }
... ...
... ... @@ -25,21 +25,17 @@ import com.amazonaws.services.sqs.model.Message;
25 25 import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
26 26 import com.google.common.util.concurrent.Futures;
27 27 import com.google.common.util.concurrent.ListenableFuture;
28   -import com.google.common.util.concurrent.ListeningExecutorService;
29   -import com.google.common.util.concurrent.MoreExecutors;
30 28 import com.google.gson.Gson;
31 29 import com.google.protobuf.InvalidProtocolBufferException;
32 30 import lombok.Data;
33 31 import lombok.extern.slf4j.Slf4j;
34 32 import org.springframework.util.CollectionUtils;
35   -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
36 33 import org.thingsboard.server.queue.TbQueueAdmin;
37   -import org.thingsboard.server.queue.TbQueueConsumer;
38 34 import org.thingsboard.server.queue.TbQueueMsg;
39 35 import org.thingsboard.server.queue.TbQueueMsgDecoder;
  36 +import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate;
40 37 import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
41 38
42   -import java.io.IOException;
43 39 import java.util.ArrayList;
44 40 import java.util.Collections;
45 41 import java.util.List;
... ... @@ -47,34 +43,28 @@ import java.util.Objects;
47 43 import java.util.Set;
48 44 import java.util.concurrent.CopyOnWriteArrayList;
49 45 import java.util.concurrent.ExecutionException;
50   -import java.util.concurrent.Executors;
51 46 import java.util.concurrent.TimeUnit;
52 47 import java.util.stream.Collectors;
53 48 import java.util.stream.Stream;
54 49
55 50 @Slf4j
56   -public class TbAwsSqsConsumerTemplate<T extends TbQueueMsg> implements TbQueueConsumer<T> {
  51 +public class TbAwsSqsConsumerTemplate<T extends TbQueueMsg> extends AbstractParallelTbQueueConsumerTemplate<Message, T> {
57 52
58 53 private static final int MAX_NUM_MSGS = 10;
59 54
60 55 private final Gson gson = new Gson();
61 56 private final TbQueueAdmin admin;
62 57 private final AmazonSQS sqsClient;
63   - private final String topic;
64 58 private final TbQueueMsgDecoder<T> decoder;
65 59 private final TbAwsSqsSettings sqsSettings;
66 60
67 61 private final List<AwsSqsMsgWrapper> pendingMessages = new CopyOnWriteArrayList<>();
68 62 private volatile Set<String> queueUrls;
69   - private volatile Set<TopicPartitionInfo> partitions;
70   - private ListeningExecutorService consumerExecutor;
71   - private volatile boolean subscribed;
72   - private volatile boolean stopped = false;
73 63
74 64 public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder<T> decoder) {
  65 + super(topic);
75 66 this.admin = admin;
76 67 this.decoder = decoder;
77   - this.topic = topic;
78 68 this.sqsSettings = sqsSettings;
79 69
80 70 AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey());
... ... @@ -87,81 +77,60 @@ public class TbAwsSqsConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
87 77 }
88 78
89 79 @Override
90   - public String getTopic() {
91   - return topic;
  80 + protected void doSubscribe(List<String> topicNames) {
  81 + queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet());
  82 + initNewExecutor(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1);
92 83 }
93 84
94 85 @Override
95   - public void subscribe() {
96   - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true));
97   - subscribed = false;
  86 + protected List<Message> doPoll(long durationInMillis) {
  87 + int duration = (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis);
  88 + List<ListenableFuture<List<Message>>> futureList = queueUrls
  89 + .stream()
  90 + .map(url -> poll(url, duration))
  91 + .collect(Collectors.toList());
  92 + ListenableFuture<List<List<Message>>> futureResult = Futures.allAsList(futureList);
  93 + try {
  94 + return futureResult.get().stream()
  95 + .flatMap(List::stream)
  96 + .filter(Objects::nonNull)
  97 + .collect(Collectors.toList());
  98 + } catch (InterruptedException | ExecutionException e) {
  99 + if (stopped) {
  100 + log.info("[{}] Aws SQS consumer is stopped.", getTopic());
  101 + } else {
  102 + log.error("Failed to pool messages.", e);
  103 + }
  104 + return Collections.emptyList();
  105 + }
98 106 }
99 107
100 108 @Override
101   - public void subscribe(Set<TopicPartitionInfo> partitions) {
102   - this.partitions = partitions;
103   - subscribed = false;
  109 + public T decode(Message message) throws InvalidProtocolBufferException {
  110 + DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class);
  111 + return decoder.decode(msg);
104 112 }
105 113
106 114 @Override
107   - public void unsubscribe() {
108   - stopped = true;
109   -
110   - if (sqsClient != null) {
111   - sqsClient.shutdown();
112   - }
113   - if (consumerExecutor != null) {
114   - consumerExecutor.shutdownNow();
115   - }
  115 + protected void doCommit() {
  116 + pendingMessages.forEach(msg ->
  117 + consumerExecutor.submit(() -> {
  118 + List<DeleteMessageBatchRequestEntry> entries = msg.getMessages()
  119 + .stream()
  120 + .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle()))
  121 + .collect(Collectors.toList());
  122 + sqsClient.deleteMessageBatch(msg.getUrl(), entries);
  123 + }));
  124 + pendingMessages.clear();
116 125 }
117 126
118 127 @Override
119   - public List<T> poll(long durationInMillis) {
120   - if (!subscribed && partitions == null) {
121   - try {
122   - Thread.sleep(durationInMillis);
123   - } catch (InterruptedException e) {
124   - log.debug("Failed to await subscription", e);
125   - }
126   - } else {
127   - if (!subscribed) {
128   - List<String> topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList());
129   - queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet());
130   - consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1));
131   - subscribed = true;
132   - }
133   -
134   - if (!pendingMessages.isEmpty()) {
135   - log.warn("Present {} non committed messages.", pendingMessages.size());
136   - return Collections.emptyList();
137   - }
138   -
139   - List<ListenableFuture<List<Message>>> futureList = queueUrls
140   - .stream()
141   - .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis)))
142   - .collect(Collectors.toList());
143   - ListenableFuture<List<List<Message>>> futureResult = Futures.allAsList(futureList);
144   - try {
145   - return futureResult.get().stream()
146   - .flatMap(List::stream)
147   - .map(msg -> {
148   - try {
149   - return decode(msg);
150   - } catch (IOException e) {
151   - log.error("Failed to decode message: [{}]", msg);
152   - return null;
153   - }
154   - }).filter(Objects::nonNull)
155   - .collect(Collectors.toList());
156   - } catch (InterruptedException | ExecutionException e) {
157   - if (stopped) {
158   - log.info("[{}] Aws SQS consumer is stopped.", topic);
159   - } else {
160   - log.error("Failed to pool messages.", e);
161   - }
162   - }
  128 + protected void doUnsubscribe() {
  129 + stopped = true;
  130 + if (sqsClient != null) {
  131 + sqsClient.shutdown();
163 132 }
164   - return Collections.emptyList();
  133 + shutdownExecutor();
165 134 }
166 135
167 136 private ListenableFuture<List<Message>> poll(String url, int waitTimeSeconds) {
... ... @@ -172,7 +141,6 @@ public class TbAwsSqsConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
172 141 ReceiveMessageRequest request = new ReceiveMessageRequest();
173 142 request
174 143 .withWaitTimeSeconds(waitTimeSeconds)
175   - .withMessageAttributeNames("headers")
176 144 .withQueueUrl(url)
177 145 .withMaxNumberOfMessages(MAX_NUM_MSGS);
178 146 return sqsClient.receiveMessage(request).getMessages();
... ... @@ -194,25 +162,6 @@ public class TbAwsSqsConsumerTemplate<T extends TbQueueMsg> implements TbQueueCo
194 162 }, consumerExecutor);
195 163 }
196 164
197   - @Override
198   - public void commit() {
199   - pendingMessages.forEach(msg ->
200   - consumerExecutor.submit(() -> {
201   - List<DeleteMessageBatchRequestEntry> entries = msg.getMessages()
202   - .stream()
203   - .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle()))
204   - .collect(Collectors.toList());
205   - sqsClient.deleteMessageBatch(msg.getUrl(), entries);
206   - }));
207   -
208   - pendingMessages.clear();
209   - }
210   -
211   - public T decode(Message message) throws InvalidProtocolBufferException {
212   - DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class);
213   - return decoder.decode(msg);
214   - }
215   -
216 165 @Data
217 166 private static class AwsSqsMsgWrapper {
218 167 private final String url;
... ...
... ... @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.TbQueueProducer;
37 37 import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
38 38
39 39 import java.util.Map;
  40 +import java.util.UUID;
40 41 import java.util.concurrent.ConcurrentHashMap;
41 42 import java.util.concurrent.Executors;
42 43
... ... @@ -80,7 +81,9 @@ public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
80 81 sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName()));
81 82 sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg)));
82 83
83   - sendMsgRequest.withMessageGroupId(msg.getKey().toString());
  84 + sendMsgRequest.withMessageGroupId(tpi.getTopic());
  85 + sendMsgRequest.withMessageDeduplicationId(UUID.randomUUID().toString());
  86 +
84 87 ListenableFuture<SendMessageResult> future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest));
85 88
86 89 Futures.addCallback(future, new FutureCallback<SendMessageResult>() {
... ...
... ... @@ -55,7 +55,6 @@ public class TbAwsSqsQueueAttributes {
55 55 @PostConstruct
56 56 private void init() {
57 57 defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true");
58   - defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true");
59 58
60 59 coreAttributes = getConfigs(coreProperties);
61 60 ruleEngineAttributes = getConfigs(ruleEngineProperties);
... ...
... ... @@ -37,6 +37,7 @@ import org.thingsboard.server.gen.transport.TransportProtos;
37 37 import java.lang.reflect.Field;
38 38 import java.util.List;
39 39 import java.util.Optional;
  40 +import java.util.Set;
40 41 import java.util.UUID;
41 42 import java.util.concurrent.ConcurrentHashMap;
42 43 import java.util.concurrent.ConcurrentMap;
... ... @@ -55,6 +56,8 @@ public class CoapTransportResource extends CoapResource {
55 56 private final Field observerField;
56 57 private final long timeout;
57 58 private final ConcurrentMap<String, TransportProtos.SessionInfoProto> tokenToSessionIdMap = new ConcurrentHashMap<>();
  59 + private final Set<UUID> rpcSubscriptions = ConcurrentHashMap.newKeySet();
  60 + private final Set<UUID> attributeSubscriptions = ConcurrentHashMap.newKeySet();
58 61
59 62 public CoapTransportResource(CoapTransportContext context, String name) {
60 63 super(name);
... ... @@ -149,11 +152,13 @@ public class CoapTransportResource extends CoapResource {
149 152 transportService.process(sessionInfo,
150 153 transportContext.getAdaptor().convertToPostAttributes(sessionId, request),
151 154 new CoapOkCallback(exchange));
  155 + reportActivity(sessionId, sessionInfo);
152 156 break;
153 157 case POST_TELEMETRY_REQUEST:
154 158 transportService.process(sessionInfo,
155 159 transportContext.getAdaptor().convertToPostTelemetry(sessionId, request),
156 160 new CoapOkCallback(exchange));
  161 + reportActivity(sessionId, sessionInfo);
157 162 break;
158 163 case CLAIM_REQUEST:
159 164 transportService.process(sessionInfo,
... ... @@ -161,6 +166,7 @@ public class CoapTransportResource extends CoapResource {
161 166 new CoapOkCallback(exchange));
162 167 break;
163 168 case SUBSCRIBE_ATTRIBUTES_REQUEST:
  169 + attributeSubscriptions.add(sessionId);
164 170 advanced.setObserver(new CoapExchangeObserverProxy((ExchangeObserver) observerField.get(advanced),
165 171 registerAsyncCoapSession(exchange, request, sessionInfo, sessionId)));
166 172 transportService.process(sessionInfo,
... ... @@ -168,6 +174,7 @@ public class CoapTransportResource extends CoapResource {
168 174 new CoapNoOpCallback(exchange));
169 175 break;
170 176 case UNSUBSCRIBE_ATTRIBUTES_REQUEST:
  177 + attributeSubscriptions.remove(sessionId);
171 178 TransportProtos.SessionInfoProto attrSession = lookupAsyncSessionInfo(request);
172 179 if (attrSession != null) {
173 180 transportService.process(attrSession,
... ... @@ -177,6 +184,7 @@ public class CoapTransportResource extends CoapResource {
177 184 }
178 185 break;
179 186 case SUBSCRIBE_RPC_COMMANDS_REQUEST:
  187 + rpcSubscriptions.add(sessionId);
180 188 advanced.setObserver(new CoapExchangeObserverProxy((ExchangeObserver) observerField.get(advanced),
181 189 registerAsyncCoapSession(exchange, request, sessionInfo, sessionId)));
182 190 transportService.process(sessionInfo,
... ... @@ -184,13 +192,13 @@ public class CoapTransportResource extends CoapResource {
184 192 new CoapNoOpCallback(exchange));
185 193 break;
186 194 case UNSUBSCRIBE_RPC_COMMANDS_REQUEST:
  195 + rpcSubscriptions.remove(sessionId);
187 196 TransportProtos.SessionInfoProto rpcSession = lookupAsyncSessionInfo(request);
188 197 if (rpcSession != null) {
189 198 transportService.process(rpcSession,
190 199 TransportProtos.SubscribeToRPCMsg.newBuilder().setUnsubscribe(true).build(),
191 200 new CoapOkCallback(exchange));
192   - transportService.process(sessionInfo, getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null);
193   - transportService.deregisterSession(rpcSession);
  201 + closeAndDeregister(sessionInfo);
194 202 }
195 203 break;
196 204 case TO_DEVICE_RPC_RESPONSE:
... ... @@ -221,6 +229,14 @@ public class CoapTransportResource extends CoapResource {
221 229 }));
222 230 }
223 231
  232 + private void reportActivity(UUID sessionId, TransportProtos.SessionInfoProto sessionInfo) {
  233 + transportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder()
  234 + .setAttributeSubscription(attributeSubscriptions.contains(sessionId))
  235 + .setRpcSubscription(rpcSubscriptions.contains(sessionId))
  236 + .setLastActivityTime(System.currentTimeMillis())
  237 + .build(), TransportServiceCallback.EMPTY);
  238 + }
  239 +
224 240 private TransportProtos.SessionInfoProto lookupAsyncSessionInfo(Request request) {
225 241 String token = request.getSource().getHostAddress() + ":" + request.getSourcePort() + ":" + request.getTokenString();
226 242 return tokenToSessionIdMap.remove(token);
... ... @@ -438,6 +454,9 @@ public class CoapTransportResource extends CoapResource {
438 454 private void closeAndDeregister(TransportProtos.SessionInfoProto session) {
439 455 transportService.process(session, getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null);
440 456 transportService.deregisterSession(session);
  457 + UUID sessionId = new UUID(session.getSessionIdMSB(), session.getSessionIdLSB());
  458 + rpcSubscriptions.remove(sessionId);
  459 + attributeSubscriptions.remove(sessionId);
441 460 }
442 461
443 462 }
... ...
... ... @@ -36,6 +36,7 @@ import org.thingsboard.server.common.transport.TransportContext;
36 36 import org.thingsboard.server.common.transport.TransportService;
37 37 import org.thingsboard.server.common.transport.TransportServiceCallback;
38 38 import org.thingsboard.server.common.transport.adaptor.JsonConverter;
  39 +import org.thingsboard.server.gen.transport.TransportProtos;
39 40 import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
40 41 import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
41 42 import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg;
... ... @@ -102,6 +103,7 @@ public class DeviceApiController {
102 103 TransportService transportService = transportContext.getTransportService();
103 104 transportService.process(sessionInfo, JsonConverter.convertToAttributesProto(new JsonParser().parse(json)),
104 105 new HttpOkCallback(responseWriter));
  106 + reportActivity(sessionInfo);
105 107 }));
106 108 return responseWriter;
107 109 }
... ... @@ -115,6 +117,7 @@ public class DeviceApiController {
115 117 TransportService transportService = transportContext.getTransportService();
116 118 transportService.process(sessionInfo, JsonConverter.convertToTelemetryProto(new JsonParser().parse(json)),
117 119 new HttpOkCallback(responseWriter));
  120 + reportActivity(sessionInfo);
118 121 }));
119 122 return responseWriter;
120 123 }
... ... @@ -274,7 +277,6 @@ public class DeviceApiController {
274 277 }
275 278 }
276 279
277   -
278 280 private static class HttpSessionListener implements SessionMsgListener {
279 281
280 282 private final DeferredResult<ResponseEntity> responseWriter;
... ... @@ -308,4 +310,13 @@ public class DeviceApiController {
308 310 responseWriter.setResult(new ResponseEntity<>(JsonConverter.toJson(msg).toString(), HttpStatus.OK));
309 311 }
310 312 }
  313 +
  314 + private void reportActivity(SessionInfoProto sessionInfo) {
  315 + transportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder()
  316 + .setAttributeSubscription(false)
  317 + .setRpcSubscription(false)
  318 + .setLastActivityTime(System.currentTimeMillis())
  319 + .build(), TransportServiceCallback.EMPTY);
  320 + }
  321 +
311 322 }
... ...
... ... @@ -20,7 +20,20 @@ package org.thingsboard.server.common.transport;
20 20 */
21 21 public interface TransportServiceCallback<T> {
22 22
  23 + TransportServiceCallback<Void> EMPTY = new TransportServiceCallback<Void>() {
  24 + @Override
  25 + public void onSuccess(Void msg) {
  26 +
  27 + }
  28 +
  29 + @Override
  30 + public void onError(Throwable e) {
  31 +
  32 + }
  33 + };
  34 +
23 35 void onSuccess(T msg);
  36 +
24 37 void onError(Throwable e);
25 38
26 39 }
... ...
... ... @@ -31,8 +31,8 @@ public class OAuth2ClientMapperConfig {
31 31 private String firstNameAttributeKey;
32 32 private String lastNameAttributeKey;
33 33 private String tenantNameStrategy;
34   - private String tenantNameStrategyPattern;
35   - private String customerNameStrategyPattern;
  34 + private String tenantNamePattern;
  35 + private String customerNamePattern;
36 36 }
37 37
38 38 @Data
... ...
... ... @@ -44,17 +44,19 @@ services:
44 44 - cassandra
45 45 tb-core2:
46 46 env_file:
47   - - tb-node.cassandra.env
  47 + - tb-node.hybrid.env
48 48 depends_on:
49 49 - kafka
50 50 - redis
  51 + - postgres
51 52 - cassandra
52 53 tb-rule-engine1:
53 54 env_file:
54   - - tb-node.cassandra.env
  55 + - tb-node.hybrid.env
55 56 depends_on:
56 57 - kafka
57 58 - redis
  59 + - postgres
58 60 - cassandra
59 61 tb-rule-engine2:
60 62 env_file:
... ...
1   -
  1 +TB_QUEUE_TYPE=kafka
2 2 REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests
3 3 TB_KAFKA_SERVERS=kafka:9092
4 4 LOGGER_LEVEL=info
... ...
... ... @@ -14,7 +14,7 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -service-type: "TB_SERVICE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ)
  17 +queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ)
18 18 request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC"
19 19
20 20 js:
... ... @@ -25,18 +25,18 @@ kafka:
25 25 # Kafka Bootstrap Servers
26 26 servers: "TB_KAFKA_SERVERS"
27 27 replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR"
28   - topic-properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES"
  28 + topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES"
29 29
30 30 pubsub:
31 31 project_id: "TB_QUEUE_PUBSUB_PROJECT_ID"
32 32 service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT"
33   - queue-properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES"
  33 + queue_properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES"
34 34
35 35 aws_sqs:
36 36 access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID"
37 37 secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY"
38 38 region: "TB_QUEUE_AWS_SQS_REGION"
39   - queue-properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES"
  39 + queue_properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES"
40 40
41 41 rabbitmq:
42 42 host: "TB_QUEUE_RABBIT_MQ_HOST"
... ... @@ -44,14 +44,14 @@ rabbitmq:
44 44 virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST"
45 45 username: "TB_QUEUE_RABBIT_MQ_USERNAME"
46 46 password: "TB_QUEUE_RABBIT_MQ_PASSWORD"
47   - queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES"
  47 + queue_properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES"
48 48
49 49 service_bus:
50 50 namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME"
51 51 sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME"
52 52 sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY"
53 53 max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES"
54   - queue-properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES"
  54 + queue_properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES"
55 55
56 56 logger:
57 57 level: "LOGGER_LEVEL"
... ...
... ... @@ -14,7 +14,7 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -service-type: "kafka"
  17 +queue_type: "kafka"
18 18 request_topic: "js_eval.requests"
19 19
20 20 js:
... ... @@ -25,13 +25,13 @@ kafka:
25 25 # Kafka Bootstrap Servers
26 26 servers: "localhost:9092"
27 27 replication_factor: "1"
28   - topic-properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600"
  28 + topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600"
29 29
30 30 pubsub:
31   - queue-properties: "ackDeadlineInSec:30;messageRetentionInSec:604800"
  31 + queue_properties: "ackDeadlineInSec:30;messageRetentionInSec:604800"
32 32
33 33 aws_sqs:
34   - queue-properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800"
  34 + queue_properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800"
35 35
36 36 rabbitmq:
37 37 host: "localhost"
... ... @@ -39,10 +39,10 @@ rabbitmq:
39 39 virtual_host: "/"
40 40 username: "admin"
41 41 password: "password"
42   - queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000"
  42 + queue_properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000"
43 43
44 44 service_bus:
45   - queue-properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800"
  45 + queue_properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800"
46 46
47 47 logger:
48 48 level: "info"
... ...
... ... @@ -4,6 +4,152 @@
4 4 "lockfileVersion": 1,
5 5 "requires": true,
6 6 "dependencies": {
  7 + "@azure/abort-controller": {
  8 + "version": "1.0.1",
  9 + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.0.1.tgz",
  10 + "integrity": "sha512-wP2Jw6uPp8DEDy0n4KNidvwzDjyVV2xnycEIq7nPzj1rHyb/r+t3OPeNT1INZePP2wy5ZqlwyuyOMTi0ePyY1A==",
  11 + "requires": {
  12 + "tslib": "^1.9.3"
  13 + }
  14 + },
  15 + "@azure/amqp-common": {
  16 + "version": "1.0.0-preview.13",
  17 + "resolved": "https://registry.npmjs.org/@azure/amqp-common/-/amqp-common-1.0.0-preview.13.tgz",
  18 + "integrity": "sha512-v19NGXFm8Hzr2bj/DSWYc2anaDcoAeFQXJGuBT8QO7eS13vaELQNGaynOGipEcI313A1778R/FFCk4o+dylIiw==",
  19 + "requires": {
  20 + "@types/async-lock": "^1.1.0",
  21 + "@types/is-buffer": "^2.0.0",
  22 + "async-lock": "^1.1.3",
  23 + "buffer": "^5.2.1",
  24 + "debug": "^3.1.0",
  25 + "events": "^3.0.0",
  26 + "is-buffer": "^2.0.3",
  27 + "jssha": "^2.3.1",
  28 + "process": "^0.11.10",
  29 + "rhea": "^1.0.18",
  30 + "rhea-promise": "^0.1.15",
  31 + "stream-browserify": "^2.0.2",
  32 + "tslib": "^1.9.3",
  33 + "url": "^0.11.0",
  34 + "util": "^0.11.1"
  35 + },
  36 + "dependencies": {
  37 + "is-buffer": {
  38 + "version": "2.0.4",
  39 + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz",
  40 + "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A=="
  41 + }
  42 + }
  43 + },
  44 + "@azure/core-auth": {
  45 + "version": "1.1.2",
  46 + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.1.2.tgz",
  47 + "integrity": "sha512-IUbP/f3v96dpHgXUwsAjUwDzjlUjawyUhWhGKKB6Qxy+iqppC/pVBPyc6kdpyTe7H30HN+4H3f0lar7Wp9Hx6A==",
  48 + "requires": {
  49 + "@azure/abort-controller": "^1.0.0",
  50 + "@azure/core-tracing": "1.0.0-preview.8",
  51 + "@opentelemetry/api": "^0.6.1",
  52 + "tslib": "^1.10.0"
  53 + }
  54 + },
  55 + "@azure/core-http": {
  56 + "version": "1.1.1",
  57 + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-1.1.1.tgz",
  58 + "integrity": "sha512-yBxH5CtYaCj0f1CKoi3OjQw5C5Go8TbgNA6Q2rX7XsDpN2eeKu0n3kRvzZnKW+brtO1u3YnBBuBLF2KcGoZv6g==",
  59 + "requires": {
  60 + "@azure/abort-controller": "^1.0.0",
  61 + "@azure/core-auth": "^1.1.2",
  62 + "@azure/core-tracing": "1.0.0-preview.8",
  63 + "@azure/logger": "^1.0.0",
  64 + "@opentelemetry/api": "^0.6.1",
  65 + "@types/node-fetch": "^2.5.0",
  66 + "@types/tunnel": "^0.0.1",
  67 + "cross-env": "^6.0.3",
  68 + "form-data": "^3.0.0",
  69 + "node-fetch": "^2.6.0",
  70 + "process": "^0.11.10",
  71 + "tough-cookie": "^3.0.1",
  72 + "tslib": "^1.10.0",
  73 + "tunnel": "^0.0.6",
  74 + "uuid": "^3.3.2",
  75 + "xml2js": "^0.4.19"
  76 + },
  77 + "dependencies": {
  78 + "form-data": {
  79 + "version": "3.0.0",
  80 + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
  81 + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
  82 + "requires": {
  83 + "asynckit": "^0.4.0",
  84 + "combined-stream": "^1.0.8",
  85 + "mime-types": "^2.1.12"
  86 + }
  87 + },
  88 + "tough-cookie": {
  89 + "version": "3.0.1",
  90 + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz",
  91 + "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==",
  92 + "requires": {
  93 + "ip-regex": "^2.1.0",
  94 + "psl": "^1.1.28",
  95 + "punycode": "^2.1.1"
  96 + }
  97 + }
  98 + }
  99 + },
  100 + "@azure/core-tracing": {
  101 + "version": "1.0.0-preview.8",
  102 + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.0-preview.8.tgz",
  103 + "integrity": "sha512-ZKUpCd7Dlyfn7bdc+/zC/sf0aRIaNQMDuSj2RhYRFe3p70hVAnYGp3TX4cnG2yoEALp/LTj/XnZGQ8Xzf6Ja/Q==",
  104 + "requires": {
  105 + "@opencensus/web-types": "0.0.7",
  106 + "@opentelemetry/api": "^0.6.1",
  107 + "tslib": "^1.10.0"
  108 + }
  109 + },
  110 + "@azure/logger": {
  111 + "version": "1.0.0",
  112 + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.0.tgz",
  113 + "integrity": "sha512-g2qLDgvmhyIxR3JVS8N67CyIOeFRKQlX/llxYJQr1OSGQqM3HTpVP8MjmjcEKbL/OIt2N9C9UFaNQuKOw1laOA==",
  114 + "requires": {
  115 + "tslib": "^1.9.3"
  116 + }
  117 + },
  118 + "@azure/service-bus": {
  119 + "version": "1.1.6",
  120 + "resolved": "https://registry.npmjs.org/@azure/service-bus/-/service-bus-1.1.6.tgz",
  121 + "integrity": "sha512-eCJXcJZGWdlVwLEqMcoIqtUrh/NtyFcDDfq/y8gdCOy3Dzuv8JkPTxjdjcxDthwG9mc5Qter3dGOTwh0U8gwiw==",
  122 + "requires": {
  123 + "@azure/amqp-common": "1.0.0-preview.13",
  124 + "@azure/core-http": "^1.0.0",
  125 + "@opentelemetry/types": "^0.2.0",
  126 + "@types/is-buffer": "^2.0.0",
  127 + "@types/long": "^4.0.0",
  128 + "buffer": "^5.2.1",
  129 + "debug": "^4.1.1",
  130 + "is-buffer": "^2.0.3",
  131 + "long": "^4.0.0",
  132 + "process": "^0.11.10",
  133 + "rhea": "^1.0.18",
  134 + "rhea-promise": "^0.1.15",
  135 + "tslib": "^1.10.0"
  136 + },
  137 + "dependencies": {
  138 + "debug": {
  139 + "version": "4.1.1",
  140 + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
  141 + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
  142 + "requires": {
  143 + "ms": "^2.1.1"
  144 + }
  145 + },
  146 + "is-buffer": {
  147 + "version": "2.0.4",
  148 + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz",
  149 + "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A=="
  150 + }
  151 + }
  152 + },
7 153 "@babel/parser": {
8 154 "version": "7.8.4",
9 155 "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.8.4.tgz",
... ... @@ -19,6 +165,69 @@
19 165 "regenerator-runtime": "^0.13.2"
20 166 }
21 167 },
  168 + "@google-cloud/paginator": {
  169 + "version": "2.0.3",
  170 + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-2.0.3.tgz",
  171 + "integrity": "sha512-kp/pkb2p/p0d8/SKUu4mOq8+HGwF8NPzHWkj+VKrIPQPyMRw8deZtrO/OcSiy9C/7bpfU5Txah5ltUNfPkgEXg==",
  172 + "requires": {
  173 + "arrify": "^2.0.0",
  174 + "extend": "^3.0.2"
  175 + }
  176 + },
  177 + "@google-cloud/precise-date": {
  178 + "version": "1.0.3",
  179 + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-1.0.3.tgz",
  180 + "integrity": "sha512-wWnDGh9y3cJHLuVEY8t6un78vizzMWsS7oIWKeFtPj+Ndy+dXvHW0HTx29ZUhen+tswSlQYlwFubvuRP5kKdzQ=="
  181 + },
  182 + "@google-cloud/projectify": {
  183 + "version": "1.0.4",
  184 + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-1.0.4.tgz",
  185 + "integrity": "sha512-ZdzQUN02eRsmTKfBj9FDL0KNDIFNjBn/d6tHQmA/+FImH5DO6ZV8E7FzxMgAUiVAUq41RFAkb25p1oHOZ8psfg=="
  186 + },
  187 + "@google-cloud/promisify": {
  188 + "version": "1.0.4",
  189 + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-1.0.4.tgz",
  190 + "integrity": "sha512-VccZDcOql77obTnFh0TbNED/6ZbbmHDf8UMNnzO1d5g9V0Htfm4k5cllY8P1tJsRKC3zWYGRLaViiupcgVjBoQ=="
  191 + },
  192 + "@google-cloud/pubsub": {
  193 + "version": "1.7.2",
  194 + "resolved": "https://registry.npmjs.org/@google-cloud/pubsub/-/pubsub-1.7.2.tgz",
  195 + "integrity": "sha512-/TziioDSV4FS4wKF1sIaQ+1gvE+um83oHz1nRsZ3L87uWSoOciBjJAcocgPjqrpnW441+Nuw4w0QdSUV1Lka/g==",
  196 + "requires": {
  197 + "@google-cloud/paginator": "^2.0.0",
  198 + "@google-cloud/precise-date": "^1.0.0",
  199 + "@google-cloud/projectify": "^1.0.0",
  200 + "@google-cloud/promisify": "^1.0.0",
  201 + "@types/duplexify": "^3.6.0",
  202 + "@types/long": "^4.0.0",
  203 + "arrify": "^2.0.0",
  204 + "async-each": "^1.0.1",
  205 + "extend": "^3.0.2",
  206 + "google-auth-library": "^5.5.0",
  207 + "google-gax": "^1.14.2",
  208 + "is-stream-ended": "^0.1.4",
  209 + "lodash.snakecase": "^4.1.1",
  210 + "p-defer": "^3.0.0",
  211 + "protobufjs": "^6.8.1"
  212 + }
  213 + },
  214 + "@grpc/grpc-js": {
  215 + "version": "1.0.3",
  216 + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.0.3.tgz",
  217 + "integrity": "sha512-JKV3f5Bv2TZxK6eJSB9EarsZrnLxrvcFNwI9goq0YRXa3S6NNoCSnI3cG3lkXVIJ03Wng1WXe76kc2JQtRe7AQ==",
  218 + "requires": {
  219 + "semver": "^6.2.0"
  220 + }
  221 + },
  222 + "@grpc/proto-loader": {
  223 + "version": "0.5.4",
  224 + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.5.4.tgz",
  225 + "integrity": "sha512-HTM4QpI9B2XFkPz7pjwMyMgZchJ93TVkL3kWPW8GDMDKYxsMnmf4w2TNMJK7+KNiYHS5cJrCEAFlF+AwtXWVPA==",
  226 + "requires": {
  227 + "lodash.camelcase": "^4.3.0",
  228 + "protobufjs": "^6.8.6"
  229 + }
  230 + },
22 231 "@nodelib/fs.scandir": {
23 232 "version": "2.1.3",
24 233 "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz",
... ... @@ -45,23 +254,193 @@
45 254 "fastq": "^1.6.0"
46 255 }
47 256 },
  257 + "@opencensus/web-types": {
  258 + "version": "0.0.7",
  259 + "resolved": "https://registry.npmjs.org/@opencensus/web-types/-/web-types-0.0.7.tgz",
  260 + "integrity": "sha512-xB+w7ZDAu3YBzqH44rCmG9/RlrOmFuDPt/bpf17eJr8eZSrLt7nc7LnWdxM9Mmoj/YKMHpxRg28txu3TcpiL+g=="
  261 + },
  262 + "@opentelemetry/api": {
  263 + "version": "0.6.1",
  264 + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-0.6.1.tgz",
  265 + "integrity": "sha512-wpufGZa7tTxw7eAsjXJtiyIQ42IWQdX9iUQp7ACJcKo1hCtuhLU+K2Nv1U6oRwT1oAlZTE6m4CgWKZBhOiau3Q==",
  266 + "requires": {
  267 + "@opentelemetry/context-base": "^0.6.1"
  268 + }
  269 + },
  270 + "@opentelemetry/context-base": {
  271 + "version": "0.6.1",
  272 + "resolved": "https://registry.npmjs.org/@opentelemetry/context-base/-/context-base-0.6.1.tgz",
  273 + "integrity": "sha512-5bHhlTBBq82ti3qPT15TRxkYTFPPQWbnkkQkmHPtqiS1XcTB69cEKd3Jm7Cfi/vkPoyxapmePE9tyA7EzLt8SQ=="
  274 + },
  275 + "@opentelemetry/types": {
  276 + "version": "0.2.0",
  277 + "resolved": "https://registry.npmjs.org/@opentelemetry/types/-/types-0.2.0.tgz",
  278 + "integrity": "sha512-GtwNB6BNDdsIPAYEdpp3JnOGO/3AJxjPvny53s3HERBdXSJTGQw8IRhiaTEX0b3w9P8+FwFZde4k+qkjn67aVw=="
  279 + },
  280 + "@protobufjs/aspromise": {
  281 + "version": "1.1.2",
  282 + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
  283 + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78="
  284 + },
  285 + "@protobufjs/base64": {
  286 + "version": "1.1.2",
  287 + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
  288 + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
  289 + },
  290 + "@protobufjs/codegen": {
  291 + "version": "2.0.4",
  292 + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
  293 + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
  294 + },
  295 + "@protobufjs/eventemitter": {
  296 + "version": "1.1.0",
  297 + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
  298 + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A="
  299 + },
  300 + "@protobufjs/fetch": {
  301 + "version": "1.1.0",
  302 + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
  303 + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=",
  304 + "requires": {
  305 + "@protobufjs/aspromise": "^1.1.1",
  306 + "@protobufjs/inquire": "^1.1.0"
  307 + }
  308 + },
  309 + "@protobufjs/float": {
  310 + "version": "1.0.2",
  311 + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
  312 + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E="
  313 + },
  314 + "@protobufjs/inquire": {
  315 + "version": "1.1.0",
  316 + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
  317 + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik="
  318 + },
  319 + "@protobufjs/path": {
  320 + "version": "1.1.2",
  321 + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
  322 + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0="
  323 + },
  324 + "@protobufjs/pool": {
  325 + "version": "1.1.0",
  326 + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
  327 + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q="
  328 + },
  329 + "@protobufjs/utf8": {
  330 + "version": "1.1.0",
  331 + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
  332 + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA="
  333 + },
  334 + "@types/async-lock": {
  335 + "version": "1.1.2",
  336 + "resolved": "https://registry.npmjs.org/@types/async-lock/-/async-lock-1.1.2.tgz",
  337 + "integrity": "sha512-j9n4bb6RhgFIydBe0+kpjnBPYumDaDyU8zvbWykyVMkku+c2CSu31MZkLeaBfqIwU+XCxlDpYDfyMQRkM0AkeQ=="
  338 + },
48 339 "@types/color-name": {
49 340 "version": "1.1.1",
50 341 "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
51 342 "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==",
52 343 "dev": true
53 344 },
  345 + "@types/duplexify": {
  346 + "version": "3.6.0",
  347 + "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz",
  348 + "integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==",
  349 + "requires": {
  350 + "@types/node": "*"
  351 + }
  352 + },
  353 + "@types/fs-extra": {
  354 + "version": "8.1.0",
  355 + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.0.tgz",
  356 + "integrity": "sha512-UoOfVEzAUpeSPmjm7h1uk5MH6KZma2z2O7a75onTGjnNvAvMVrPzPL/vBbT65iIGHWj6rokwfmYcmxmlSf2uwg==",
  357 + "requires": {
  358 + "@types/node": "*"
  359 + }
  360 + },
  361 + "@types/is-buffer": {
  362 + "version": "2.0.0",
  363 + "resolved": "https://registry.npmjs.org/@types/is-buffer/-/is-buffer-2.0.0.tgz",
  364 + "integrity": "sha512-0f7N/e3BAz32qDYvgB4d2cqv1DqUwvGxHkXsrucICn8la1Vb6Yl6Eg8mPScGwUiqHJeE7diXlzaK+QMA9m4Gxw==",
  365 + "requires": {
  366 + "@types/node": "*"
  367 + }
  368 + },
  369 + "@types/long": {
  370 + "version": "4.0.1",
  371 + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz",
  372 + "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w=="
  373 + },
  374 + "@types/node": {
  375 + "version": "13.13.4",
  376 + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.4.tgz",
  377 + "integrity": "sha512-x26ur3dSXgv5AwKS0lNfbjpCakGIduWU1DU91Zz58ONRWrIKGunmZBNv4P7N+e27sJkiGDsw/3fT4AtsqQBrBA=="
  378 + },
  379 + "@types/node-fetch": {
  380 + "version": "2.5.7",
  381 + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz",
  382 + "integrity": "sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw==",
  383 + "requires": {
  384 + "@types/node": "*",
  385 + "form-data": "^3.0.0"
  386 + },
  387 + "dependencies": {
  388 + "form-data": {
  389 + "version": "3.0.0",
  390 + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
  391 + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
  392 + "requires": {
  393 + "asynckit": "^0.4.0",
  394 + "combined-stream": "^1.0.8",
  395 + "mime-types": "^2.1.12"
  396 + }
  397 + }
  398 + }
  399 + },
  400 + "@types/tunnel": {
  401 + "version": "0.0.1",
  402 + "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.1.tgz",
  403 + "integrity": "sha512-AOqu6bQu5MSWwYvehMXLukFHnupHrpZ8nvgae5Ggie9UwzDR1CCwoXgSSWNZJuyOlCdfdsWMA5F2LlmvyoTv8A==",
  404 + "requires": {
  405 + "@types/node": "*"
  406 + }
  407 + },
54 408 "abbrev": {
55 409 "version": "1.1.1",
56 410 "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
57 411 "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
58 412 "dev": true
59 413 },
  414 + "abort-controller": {
  415 + "version": "3.0.0",
  416 + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
  417 + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
  418 + "requires": {
  419 + "event-target-shim": "^5.0.0"
  420 + }
  421 + },
  422 + "agent-base": {
  423 + "version": "6.0.0",
  424 + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz",
  425 + "integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==",
  426 + "requires": {
  427 + "debug": "4"
  428 + },
  429 + "dependencies": {
  430 + "debug": {
  431 + "version": "4.1.1",
  432 + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
  433 + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
  434 + "requires": {
  435 + "ms": "^2.1.1"
  436 + }
  437 + }
  438 + }
  439 + },
60 440 "ajv": {
61 441 "version": "6.11.0",
62 442 "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.11.0.tgz",
63 443 "integrity": "sha512-nCprB/0syFYy9fVYU1ox1l2KN8S9I+tziH8D4zdZuLT3N6RMlGSGt5FSTpAiHB/Whv8Qs1cWHma1aMKZyaHRKA==",
64   - "dev": true,
65 444 "requires": {
66 445 "fast-deep-equal": "^3.1.1",
67 446 "fast-json-stable-stringify": "^2.0.0",
... ... @@ -69,6 +448,47 @@
69 448 "uri-js": "^4.2.2"
70 449 }
71 450 },
  451 + "amqplib": {
  452 + "version": "0.5.5",
  453 + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.5.5.tgz",
  454 + "integrity": "sha512-sWx1hbfHbyKMw6bXOK2k6+lHL8TESWxjAx5hG8fBtT7wcxoXNIsFxZMnFyBjxt3yL14vn7WqBDe5U6BGOadtLg==",
  455 + "requires": {
  456 + "bitsyntax": "~0.1.0",
  457 + "bluebird": "^3.5.2",
  458 + "buffer-more-ints": "~1.0.0",
  459 + "readable-stream": "1.x >=1.1.9",
  460 + "safe-buffer": "~5.1.2",
  461 + "url-parse": "~1.4.3"
  462 + },
  463 + "dependencies": {
  464 + "isarray": {
  465 + "version": "0.0.1",
  466 + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
  467 + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
  468 + },
  469 + "readable-stream": {
  470 + "version": "1.1.14",
  471 + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
  472 + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
  473 + "requires": {
  474 + "core-util-is": "~1.0.0",
  475 + "inherits": "~2.0.1",
  476 + "isarray": "0.0.1",
  477 + "string_decoder": "~0.10.x"
  478 + }
  479 + },
  480 + "safe-buffer": {
  481 + "version": "5.1.2",
  482 + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
  483 + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
  484 + },
  485 + "string_decoder": {
  486 + "version": "0.10.31",
  487 + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
  488 + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
  489 + }
  490 + }
  491 + },
72 492 "ansi-align": {
73 493 "version": "2.0.0",
74 494 "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz",
... ... @@ -152,11 +572,15 @@
152 572 "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=",
153 573 "dev": true
154 574 },
  575 + "arrify": {
  576 + "version": "2.0.1",
  577 + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz",
  578 + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug=="
  579 + },
155 580 "asn1": {
156 581 "version": "0.2.4",
157 582 "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
158 583 "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
159   - "dev": true,
160 584 "requires": {
161 585 "safer-buffer": "~2.1.0"
162 586 }
... ... @@ -164,8 +588,7 @@
164 588 "assert-plus": {
165 589 "version": "1.0.0",
166 590 "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
167   - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
168   - "dev": true
  591 + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
169 592 },
170 593 "assign-symbols": {
171 594 "version": "1.0.0",
... ... @@ -184,14 +607,17 @@
184 607 "async-each": {
185 608 "version": "1.0.3",
186 609 "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz",
187   - "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==",
188   - "dev": true
  610 + "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ=="
  611 + },
  612 + "async-lock": {
  613 + "version": "1.2.2",
  614 + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.2.2.tgz",
  615 + "integrity": "sha512-uczz62z2fMWOFbyo6rG4NlV2SdxugJT6sZA2QcfB1XaSjEiOh8CuOb/TttyMnYQCda6nkWecJe465tGQDPJiKw=="
189 616 },
190 617 "asynckit": {
191 618 "version": "0.4.0",
192 619 "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
193   - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
194   - "dev": true
  620 + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
195 621 },
196 622 "atob": {
197 623 "version": "2.1.2",
... ... @@ -199,17 +625,126 @@
199 625 "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==",
200 626 "dev": true
201 627 },
  628 + "aws-sdk": {
  629 + "version": "2.669.0",
  630 + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.669.0.tgz",
  631 + "integrity": "sha512-kuVcSRpDzvkgmeSmMX6Q32eTOb8UeihhUdavMrvUOP6fzSU19cNWS9HAIkYOi/jrEDK85cCZxXjxqE3JGZIGcw==",
  632 + "requires": {
  633 + "buffer": "4.9.1",
  634 + "events": "1.1.1",
  635 + "ieee754": "1.1.13",
  636 + "jmespath": "0.15.0",
  637 + "querystring": "0.2.0",
  638 + "sax": "1.2.1",
  639 + "url": "0.10.3",
  640 + "uuid": "3.3.2",
  641 + "xml2js": "0.4.19"
  642 + },
  643 + "dependencies": {
  644 + "buffer": {
  645 + "version": "4.9.1",
  646 + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
  647 + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
  648 + "requires": {
  649 + "base64-js": "^1.0.2",
  650 + "ieee754": "^1.1.4",
  651 + "isarray": "^1.0.0"
  652 + }
  653 + },
  654 + "events": {
  655 + "version": "1.1.1",
  656 + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
  657 + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ="
  658 + },
  659 + "punycode": {
  660 + "version": "1.3.2",
  661 + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
  662 + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
  663 + },
  664 + "sax": {
  665 + "version": "1.2.1",
  666 + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
  667 + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
  668 + },
  669 + "url": {
  670 + "version": "0.10.3",
  671 + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
  672 + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
  673 + "requires": {
  674 + "punycode": "1.3.2",
  675 + "querystring": "0.2.0"
  676 + }
  677 + },
  678 + "uuid": {
  679 + "version": "3.3.2",
  680 + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
  681 + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
  682 + },
  683 + "xml2js": {
  684 + "version": "0.4.19",
  685 + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
  686 + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
  687 + "requires": {
  688 + "sax": ">=0.6.0",
  689 + "xmlbuilder": "~9.0.1"
  690 + }
  691 + },
  692 + "xmlbuilder": {
  693 + "version": "9.0.7",
  694 + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
  695 + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0="
  696 + }
  697 + }
  698 + },
202 699 "aws-sign2": {
203 700 "version": "0.7.0",
204 701 "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
205   - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
206   - "dev": true
  702 + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
207 703 },
208 704 "aws4": {
209 705 "version": "1.9.1",
210 706 "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz",
211   - "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==",
212   - "dev": true
  707 + "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug=="
  708 + },
  709 + "azure-common": {
  710 + "version": "0.9.22",
  711 + "resolved": "https://registry.npmjs.org/azure-common/-/azure-common-0.9.22.tgz",
  712 + "integrity": "sha512-0r9tK9D+1xl2/VPVtfmGmtkMqfooiBLS87fX+Ab0hOCPVVe/6CgVC4in0wSf2Ta8r65DbvxV5P4/t8fp8Q3EsQ==",
  713 + "requires": {
  714 + "dateformat": "1.0.2-1.2.3",
  715 + "duplexer": "~0.1.1",
  716 + "envconf": "~0.0.4",
  717 + "request": "^2.81.0",
  718 + "through": "~2.3.4",
  719 + "tunnel": "~0.0.2",
  720 + "underscore": "1.4.x",
  721 + "validator": "^9.4.1",
  722 + "xml2js": "^0.4.19",
  723 + "xmlbuilder": "0.4.3"
  724 + },
  725 + "dependencies": {
  726 + "underscore": {
  727 + "version": "1.4.4",
  728 + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.4.4.tgz",
  729 + "integrity": "sha1-YaajIBBiKvoHljvzJSA88SI51gQ="
  730 + },
  731 + "xmlbuilder": {
  732 + "version": "0.4.3",
  733 + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.3.tgz",
  734 + "integrity": "sha1-xGFLp04K0ZbmCcknLNnh3bKKilg="
  735 + }
  736 + }
  737 + },
  738 + "azure-sb": {
  739 + "version": "0.11.1",
  740 + "resolved": "https://registry.npmjs.org/azure-sb/-/azure-sb-0.11.1.tgz",
  741 + "integrity": "sha512-ZYgPeSDMD99i/Em+6wT78zvBkJ/dbh2ypb4DbqQ1Flaif5vWJFzC/iKxxcq/vq+THWoO3+UbqWa0JNXnW3zAvw==",
  742 + "requires": {
  743 + "azure-common": "^0.9.22",
  744 + "mpns": "2.1.3",
  745 + "underscore": "^1.8.3",
  746 + "wns": "~0.5.3"
  747 + }
213 748 },
214 749 "balanced-match": {
215 750 "version": "1.0.0",
... ... @@ -272,15 +807,24 @@
272 807 }
273 808 }
274 809 },
  810 + "base64-js": {
  811 + "version": "1.3.1",
  812 + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
  813 + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
  814 + },
275 815 "bcrypt-pbkdf": {
276 816 "version": "1.0.2",
277 817 "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
278 818 "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
279   - "dev": true,
280 819 "requires": {
281 820 "tweetnacl": "^0.14.3"
282 821 }
283 822 },
  823 + "bignumber.js": {
  824 + "version": "7.2.1",
  825 + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz",
  826 + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ=="
  827 + },
284 828 "binary-extensions": {
285 829 "version": "1.13.1",
286 830 "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",
... ... @@ -297,6 +841,41 @@
297 841 "file-uri-to-path": "1.0.0"
298 842 }
299 843 },
  844 + "bitsyntax": {
  845 + "version": "0.1.0",
  846 + "resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.1.0.tgz",
  847 + "integrity": "sha512-ikAdCnrloKmFOugAfxWws89/fPc+nw0OOG1IzIE72uSOg/A3cYptKCjSUhDTuj7fhsJtzkzlv7l3b8PzRHLN0Q==",
  848 + "requires": {
  849 + "buffer-more-ints": "~1.0.0",
  850 + "debug": "~2.6.9",
  851 + "safe-buffer": "~5.1.2"
  852 + },
  853 + "dependencies": {
  854 + "debug": {
  855 + "version": "2.6.9",
  856 + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
  857 + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
  858 + "requires": {
  859 + "ms": "2.0.0"
  860 + }
  861 + },
  862 + "ms": {
  863 + "version": "2.0.0",
  864 + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
  865 + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
  866 + },
  867 + "safe-buffer": {
  868 + "version": "5.1.2",
  869 + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
  870 + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
  871 + }
  872 + }
  873 + },
  874 + "bluebird": {
  875 + "version": "3.7.2",
  876 + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
  877 + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
  878 + },
300 879 "boxen": {
301 880 "version": "1.3.0",
302 881 "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz",
... ... @@ -351,6 +930,25 @@
351 930 }
352 931 }
353 932 },
  933 + "buffer": {
  934 + "version": "5.6.0",
  935 + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
  936 + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
  937 + "requires": {
  938 + "base64-js": "^1.0.2",
  939 + "ieee754": "^1.1.4"
  940 + }
  941 + },
  942 + "buffer-equal-constant-time": {
  943 + "version": "1.0.1",
  944 + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
  945 + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
  946 + },
  947 + "buffer-more-ints": {
  948 + "version": "1.0.0",
  949 + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz",
  950 + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg=="
  951 + },
354 952 "byline": {
355 953 "version": "5.0.0",
356 954 "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz",
... ... @@ -389,8 +987,7 @@
389 987 "caseless": {
390 988 "version": "0.12.0",
391 989 "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
392   - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
393   - "dev": true
  990 + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
394 991 },
395 992 "chalk": {
396 993 "version": "2.4.2",
... ... @@ -522,7 +1119,6 @@
522 1119 "version": "1.0.8",
523 1120 "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
524 1121 "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
525   - "dev": true,
526 1122 "requires": {
527 1123 "delayed-stream": "~1.0.0"
528 1124 }
... ... @@ -581,6 +1177,52 @@
581 1177 "capture-stack-trace": "^1.0.0"
582 1178 }
583 1179 },
  1180 + "cross-env": {
  1181 + "version": "6.0.3",
  1182 + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-6.0.3.tgz",
  1183 + "integrity": "sha512-+KqxF6LCvfhWvADcDPqo64yVIB31gv/jQulX2NGzKS/g3GEVz6/pt4wjHFtFWsHMddebWD/sDthJemzM4MaAag==",
  1184 + "requires": {
  1185 + "cross-spawn": "^7.0.0"
  1186 + },
  1187 + "dependencies": {
  1188 + "cross-spawn": {
  1189 + "version": "7.0.2",
  1190 + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.2.tgz",
  1191 + "integrity": "sha512-PD6G8QG3S4FK/XCGFbEQrDqO2AnMMsy0meR7lerlIOHAAbkuavGU/pOqprrlvfTNjvowivTeBsjebAL0NSoMxw==",
  1192 + "requires": {
  1193 + "path-key": "^3.1.0",
  1194 + "shebang-command": "^2.0.0",
  1195 + "which": "^2.0.1"
  1196 + }
  1197 + },
  1198 + "path-key": {
  1199 + "version": "3.1.1",
  1200 + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
  1201 + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="
  1202 + },
  1203 + "shebang-command": {
  1204 + "version": "2.0.0",
  1205 + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
  1206 + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
  1207 + "requires": {
  1208 + "shebang-regex": "^3.0.0"
  1209 + }
  1210 + },
  1211 + "shebang-regex": {
  1212 + "version": "3.0.0",
  1213 + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
  1214 + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
  1215 + },
  1216 + "which": {
  1217 + "version": "2.0.2",
  1218 + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
  1219 + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
  1220 + "requires": {
  1221 + "isexe": "^2.0.0"
  1222 + }
  1223 + }
  1224 + }
  1225 + },
584 1226 "cross-spawn": {
585 1227 "version": "5.1.0",
586 1228 "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
... ... @@ -607,16 +1249,19 @@
607 1249 "version": "1.14.1",
608 1250 "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
609 1251 "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
610   - "dev": true,
611 1252 "requires": {
612 1253 "assert-plus": "^1.0.0"
613 1254 }
614 1255 },
  1256 + "dateformat": {
  1257 + "version": "1.0.2-1.2.3",
  1258 + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-1.0.2-1.2.3.tgz",
  1259 + "integrity": "sha1-sCIMAt6YYXQztyhRz0fePfLNvuk="
  1260 + },
615 1261 "debug": {
616 1262 "version": "3.2.6",
617 1263 "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
618 1264 "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
619   - "dev": true,
620 1265 "requires": {
621 1266 "ms": "^2.1.1"
622 1267 }
... ... @@ -683,8 +1328,7 @@
683 1328 "delayed-stream": {
684 1329 "version": "1.0.0",
685 1330 "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
686   - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
687   - "dev": true
  1331 + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
688 1332 },
689 1333 "diagnostics": {
690 1334 "version": "1.1.1",
... ... @@ -714,22 +1358,74 @@
714 1358 "is-obj": "^1.0.0"
715 1359 }
716 1360 },
717   - "duplexer3": {
718   - "version": "0.1.4",
719   - "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",
720   - "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=",
721   - "dev": true
722   - },
  1361 + "duplexer": {
  1362 + "version": "0.1.1",
  1363 + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz",
  1364 + "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E="
  1365 + },
  1366 + "duplexer3": {
  1367 + "version": "0.1.4",
  1368 + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",
  1369 + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=",
  1370 + "dev": true
  1371 + },
  1372 + "duplexify": {
  1373 + "version": "3.7.1",
  1374 + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
  1375 + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
  1376 + "requires": {
  1377 + "end-of-stream": "^1.0.0",
  1378 + "inherits": "^2.0.1",
  1379 + "readable-stream": "^2.0.0",
  1380 + "stream-shift": "^1.0.0"
  1381 + },
  1382 + "dependencies": {
  1383 + "readable-stream": {
  1384 + "version": "2.3.7",
  1385 + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
  1386 + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
  1387 + "requires": {
  1388 + "core-util-is": "~1.0.0",
  1389 + "inherits": "~2.0.3",
  1390 + "isarray": "~1.0.0",
  1391 + "process-nextick-args": "~2.0.0",
  1392 + "safe-buffer": "~5.1.1",
  1393 + "string_decoder": "~1.1.1",
  1394 + "util-deprecate": "~1.0.1"
  1395 + }
  1396 + },
  1397 + "safe-buffer": {
  1398 + "version": "5.1.2",
  1399 + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
  1400 + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
  1401 + },
  1402 + "string_decoder": {
  1403 + "version": "1.1.1",
  1404 + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
  1405 + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
  1406 + "requires": {
  1407 + "safe-buffer": "~5.1.0"
  1408 + }
  1409 + }
  1410 + }
  1411 + },
723 1412 "ecc-jsbn": {
724 1413 "version": "0.1.2",
725 1414 "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
726 1415 "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
727   - "dev": true,
728 1416 "requires": {
729 1417 "jsbn": "~0.1.0",
730 1418 "safer-buffer": "^2.1.0"
731 1419 }
732 1420 },
  1421 + "ecdsa-sig-formatter": {
  1422 + "version": "1.0.11",
  1423 + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
  1424 + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
  1425 + "requires": {
  1426 + "safe-buffer": "^5.0.1"
  1427 + }
  1428 + },
733 1429 "enabled": {
734 1430 "version": "1.0.2",
735 1431 "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz",
... ... @@ -738,11 +1434,24 @@
738 1434 "env-variable": "0.0.x"
739 1435 }
740 1436 },
  1437 + "end-of-stream": {
  1438 + "version": "1.4.4",
  1439 + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
  1440 + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
  1441 + "requires": {
  1442 + "once": "^1.4.0"
  1443 + }
  1444 + },
741 1445 "env-variable": {
742 1446 "version": "0.0.6",
743 1447 "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.6.tgz",
744 1448 "integrity": "sha512-bHz59NlBbtS0NhftmR8+ExBEekE7br0e01jw+kk0NDro7TtZzBYZ5ScGPs3OmwnpyfHTHOtr1Y6uedCdrIldtg=="
745 1449 },
  1450 + "envconf": {
  1451 + "version": "0.0.4",
  1452 + "resolved": "https://registry.npmjs.org/envconf/-/envconf-0.0.4.tgz",
  1453 + "integrity": "sha1-hWda+6I3xD+Y3i1GrcDlMqTc9Is="
  1454 + },
746 1455 "escape-string-regexp": {
747 1456 "version": "1.0.5",
748 1457 "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
... ... @@ -788,6 +1497,16 @@
788 1497 "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
789 1498 "dev": true
790 1499 },
  1500 + "event-target-shim": {
  1501 + "version": "5.0.1",
  1502 + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
  1503 + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="
  1504 + },
  1505 + "events": {
  1506 + "version": "3.1.0",
  1507 + "resolved": "https://registry.npmjs.org/events/-/events-3.1.0.tgz",
  1508 + "integrity": "sha512-Rv+u8MLHNOdMjTAFeT3nCjHn2aGlx435FP/sDHNaRhDEMwyI/aB22Kj2qIN8R0cw3z28psEQLYwxVKLsKrMgWg=="
  1509 + },
791 1510 "execa": {
792 1511 "version": "0.7.0",
793 1512 "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
... ... @@ -862,8 +1581,7 @@
862 1581 "extend": {
863 1582 "version": "3.0.2",
864 1583 "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
865   - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
866   - "dev": true
  1584 + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
867 1585 },
868 1586 "extend-shallow": {
869 1587 "version": "3.0.2",
... ... @@ -954,14 +1672,12 @@
954 1672 "extsprintf": {
955 1673 "version": "1.3.0",
956 1674 "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
957   - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
958   - "dev": true
  1675 + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
959 1676 },
960 1677 "fast-deep-equal": {
961 1678 "version": "3.1.1",
962 1679 "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz",
963   - "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==",
964   - "dev": true
  1680 + "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA=="
965 1681 },
966 1682 "fast-glob": {
967 1683 "version": "3.1.1",
... ... @@ -1033,8 +1749,7 @@
1033 1749 "fast-json-stable-stringify": {
1034 1750 "version": "2.1.0",
1035 1751 "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
1036   - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
1037   - "dev": true
  1752 + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="
1038 1753 },
1039 1754 "fast-levenshtein": {
1040 1755 "version": "2.0.6",
... ... @@ -1047,6 +1762,11 @@
1047 1762 "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz",
1048 1763 "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA=="
1049 1764 },
  1765 + "fast-text-encoding": {
  1766 + "version": "1.0.2",
  1767 + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.2.tgz",
  1768 + "integrity": "sha512-5rQdinSsycpzvAoHga2EDn+LRX1d5xLFsuNG0Kg61JrAT/tASXcLL0nf/33v+sAxlQcfYmWbTURa1mmAf55jGw=="
  1769 + },
1050 1770 "fastq": {
1051 1771 "version": "1.6.0",
1052 1772 "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.6.0.tgz",
... ... @@ -1108,14 +1828,12 @@
1108 1828 "forever-agent": {
1109 1829 "version": "0.6.1",
1110 1830 "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
1111   - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
1112   - "dev": true
  1831 + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
1113 1832 },
1114 1833 "form-data": {
1115 1834 "version": "2.3.3",
1116 1835 "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
1117 1836 "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
1118   - "dev": true,
1119 1837 "requires": {
1120 1838 "asynckit": "^0.4.0",
1121 1839 "combined-stream": "^1.0.6",
... ... @@ -1226,12 +1944,14 @@
1226 1944 "balanced-match": {
1227 1945 "version": "1.0.0",
1228 1946 "bundled": true,
1229   - "dev": true
  1947 + "dev": true,
  1948 + "optional": true
1230 1949 },
1231 1950 "brace-expansion": {
1232 1951 "version": "1.1.11",
1233 1952 "bundled": true,
1234 1953 "dev": true,
  1954 + "optional": true,
1235 1955 "requires": {
1236 1956 "balanced-match": "^1.0.0",
1237 1957 "concat-map": "0.0.1"
... ... @@ -1251,7 +1971,8 @@
1251 1971 "concat-map": {
1252 1972 "version": "0.0.1",
1253 1973 "bundled": true,
1254   - "dev": true
  1974 + "dev": true,
  1975 + "optional": true
1255 1976 },
1256 1977 "console-control-strings": {
1257 1978 "version": "1.1.0",
... ... @@ -1399,6 +2120,7 @@
1399 2120 "version": "3.0.4",
1400 2121 "bundled": true,
1401 2122 "dev": true,
  2123 + "optional": true,
1402 2124 "requires": {
1403 2125 "brace-expansion": "^1.1.7"
1404 2126 }
... ... @@ -1723,6 +2445,34 @@
1723 2445 }
1724 2446 }
1725 2447 },
  2448 + "gaxios": {
  2449 + "version": "2.3.4",
  2450 + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.3.4.tgz",
  2451 + "integrity": "sha512-US8UMj8C5pRnao3Zykc4AAVr+cffoNKRTg9Rsf2GiuZCW69vgJj38VK2PzlPuQU73FZ/nTk9/Av6/JGcE1N9vA==",
  2452 + "requires": {
  2453 + "abort-controller": "^3.0.0",
  2454 + "extend": "^3.0.2",
  2455 + "https-proxy-agent": "^5.0.0",
  2456 + "is-stream": "^2.0.0",
  2457 + "node-fetch": "^2.3.0"
  2458 + },
  2459 + "dependencies": {
  2460 + "is-stream": {
  2461 + "version": "2.0.0",
  2462 + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
  2463 + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw=="
  2464 + }
  2465 + }
  2466 + },
  2467 + "gcp-metadata": {
  2468 + "version": "3.5.0",
  2469 + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.5.0.tgz",
  2470 + "integrity": "sha512-ZQf+DLZ5aKcRpLzYUyBS3yo3N0JSa82lNDO8rj3nMSlovLcz2riKFBsYgDzeXcv75oo5eqB2lx+B14UvPoCRnA==",
  2471 + "requires": {
  2472 + "gaxios": "^2.1.0",
  2473 + "json-bigint": "^0.3.0"
  2474 + }
  2475 + },
1726 2476 "get-stream": {
1727 2477 "version": "3.0.0",
1728 2478 "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
... ... @@ -1739,7 +2489,6 @@
1739 2489 "version": "0.1.7",
1740 2490 "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
1741 2491 "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
1742   - "dev": true,
1743 2492 "requires": {
1744 2493 "assert-plus": "^1.0.0"
1745 2494 }
... ... @@ -1788,6 +2537,67 @@
1788 2537 "slash": "^3.0.0"
1789 2538 }
1790 2539 },
  2540 + "google-auth-library": {
  2541 + "version": "5.10.1",
  2542 + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.10.1.tgz",
  2543 + "integrity": "sha512-rOlaok5vlpV9rSiUu5EpR0vVpc+PhN62oF4RyX/6++DG1VsaulAFEMlDYBLjJDDPI6OcNOCGAKy9UVB/3NIDXg==",
  2544 + "requires": {
  2545 + "arrify": "^2.0.0",
  2546 + "base64-js": "^1.3.0",
  2547 + "ecdsa-sig-formatter": "^1.0.11",
  2548 + "fast-text-encoding": "^1.0.0",
  2549 + "gaxios": "^2.1.0",
  2550 + "gcp-metadata": "^3.4.0",
  2551 + "gtoken": "^4.1.0",
  2552 + "jws": "^4.0.0",
  2553 + "lru-cache": "^5.0.0"
  2554 + },
  2555 + "dependencies": {
  2556 + "lru-cache": {
  2557 + "version": "5.1.1",
  2558 + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
  2559 + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
  2560 + "requires": {
  2561 + "yallist": "^3.0.2"
  2562 + }
  2563 + },
  2564 + "yallist": {
  2565 + "version": "3.1.1",
  2566 + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
  2567 + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
  2568 + }
  2569 + }
  2570 + },
  2571 + "google-gax": {
  2572 + "version": "1.15.3",
  2573 + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.15.3.tgz",
  2574 + "integrity": "sha512-3JKJCRumNm3x2EksUTw4P1Rad43FTpqrtW9jzpf3xSMYXx+ogaqTM1vGo7VixHB4xkAyATXVIa3OcNSh8H9zsQ==",
  2575 + "requires": {
  2576 + "@grpc/grpc-js": "~1.0.3",
  2577 + "@grpc/proto-loader": "^0.5.1",
  2578 + "@types/fs-extra": "^8.0.1",
  2579 + "@types/long": "^4.0.0",
  2580 + "abort-controller": "^3.0.0",
  2581 + "duplexify": "^3.6.0",
  2582 + "google-auth-library": "^5.0.0",
  2583 + "is-stream-ended": "^0.1.4",
  2584 + "lodash.at": "^4.6.0",
  2585 + "lodash.has": "^4.5.2",
  2586 + "node-fetch": "^2.6.0",
  2587 + "protobufjs": "^6.8.9",
  2588 + "retry-request": "^4.0.0",
  2589 + "semver": "^6.0.0",
  2590 + "walkdir": "^0.4.0"
  2591 + }
  2592 + },
  2593 + "google-p12-pem": {
  2594 + "version": "2.0.4",
  2595 + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.4.tgz",
  2596 + "integrity": "sha512-S4blHBQWZRnEW44OcR7TL9WR+QCqByRvhNDZ/uuQfpxywfupikf/miba8js1jZi6ZOGv5slgSuoshCWh6EMDzg==",
  2597 + "requires": {
  2598 + "node-forge": "^0.9.0"
  2599 + }
  2600 + },
1791 2601 "got": {
1792 2602 "version": "6.7.1",
1793 2603 "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
... ... @@ -1813,17 +2623,26 @@
1813 2623 "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==",
1814 2624 "dev": true
1815 2625 },
  2626 + "gtoken": {
  2627 + "version": "4.1.4",
  2628 + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.4.tgz",
  2629 + "integrity": "sha512-VxirzD0SWoFUo5p8RDP8Jt2AGyOmyYcT/pOUgDKJCK+iSw0TMqwrVfY37RXTNmoKwrzmDHSk0GMT9FsgVmnVSA==",
  2630 + "requires": {
  2631 + "gaxios": "^2.1.0",
  2632 + "google-p12-pem": "^2.0.0",
  2633 + "jws": "^4.0.0",
  2634 + "mime": "^2.2.0"
  2635 + }
  2636 + },
1816 2637 "har-schema": {
1817 2638 "version": "2.0.0",
1818 2639 "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
1819   - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=",
1820   - "dev": true
  2640 + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
1821 2641 },
1822 2642 "har-validator": {
1823 2643 "version": "5.1.3",
1824 2644 "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
1825 2645 "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
1826   - "dev": true,
1827 2646 "requires": {
1828 2647 "ajv": "^6.5.5",
1829 2648 "har-schema": "^2.0.0"
... ... @@ -1871,13 +2690,36 @@
1871 2690 "version": "1.2.0",
1872 2691 "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
1873 2692 "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
1874   - "dev": true,
1875 2693 "requires": {
1876 2694 "assert-plus": "^1.0.0",
1877 2695 "jsprim": "^1.2.2",
1878 2696 "sshpk": "^1.7.0"
1879 2697 }
1880 2698 },
  2699 + "https-proxy-agent": {
  2700 + "version": "5.0.0",
  2701 + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
  2702 + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
  2703 + "requires": {
  2704 + "agent-base": "6",
  2705 + "debug": "4"
  2706 + },
  2707 + "dependencies": {
  2708 + "debug": {
  2709 + "version": "4.1.1",
  2710 + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
  2711 + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
  2712 + "requires": {
  2713 + "ms": "^2.1.1"
  2714 + }
  2715 + }
  2716 + }
  2717 + },
  2718 + "ieee754": {
  2719 + "version": "1.1.13",
  2720 + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
  2721 + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
  2722 + },
1881 2723 "ignore": {
1882 2724 "version": "5.1.4",
1883 2725 "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.4.tgz",
... ... @@ -1923,6 +2765,11 @@
1923 2765 "p-is-promise": "^3.0.0"
1924 2766 }
1925 2767 },
  2768 + "ip-regex": {
  2769 + "version": "2.1.0",
  2770 + "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
  2771 + "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk="
  2772 + },
1926 2773 "is-accessor-descriptor": {
1927 2774 "version": "0.1.6",
1928 2775 "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
... ... @@ -2115,11 +2962,15 @@
2115 2962 "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
2116 2963 "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
2117 2964 },
  2965 + "is-stream-ended": {
  2966 + "version": "0.1.4",
  2967 + "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz",
  2968 + "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw=="
  2969 + },
2118 2970 "is-typedarray": {
2119 2971 "version": "1.0.0",
2120 2972 "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
2121   - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
2122   - "dev": true
  2973 + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
2123 2974 },
2124 2975 "is-windows": {
2125 2976 "version": "1.0.2",
... ... @@ -2135,8 +2986,7 @@
2135 2986 "isexe": {
2136 2987 "version": "2.0.0",
2137 2988 "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
2138   - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
2139   - "dev": true
  2989 + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
2140 2990 },
2141 2991 "isobject": {
2142 2992 "version": "3.0.1",
... ... @@ -2147,8 +2997,12 @@
2147 2997 "isstream": {
2148 2998 "version": "0.1.2",
2149 2999 "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
2150   - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
2151   - "dev": true
  3000 + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
  3001 + },
  3002 + "jmespath": {
  3003 + "version": "0.15.0",
  3004 + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
  3005 + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
2152 3006 },
2153 3007 "js-yaml": {
2154 3008 "version": "3.13.1",
... ... @@ -2162,26 +3016,30 @@
2162 3016 "jsbn": {
2163 3017 "version": "0.1.1",
2164 3018 "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
2165   - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
2166   - "dev": true
  3019 + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
  3020 + },
  3021 + "json-bigint": {
  3022 + "version": "0.3.0",
  3023 + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz",
  3024 + "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=",
  3025 + "requires": {
  3026 + "bignumber.js": "^7.0.0"
  3027 + }
2167 3028 },
2168 3029 "json-schema": {
2169 3030 "version": "0.2.3",
2170 3031 "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
2171   - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=",
2172   - "dev": true
  3032 + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
2173 3033 },
2174 3034 "json-schema-traverse": {
2175 3035 "version": "0.4.1",
2176 3036 "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
2177   - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
2178   - "dev": true
  3037 + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
2179 3038 },
2180 3039 "json-stringify-safe": {
2181 3040 "version": "5.0.1",
2182 3041 "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
2183   - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=",
2184   - "dev": true
  3042 + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
2185 3043 },
2186 3044 "json5": {
2187 3045 "version": "1.0.1",
... ... @@ -2204,7 +3062,6 @@
2204 3062 "version": "1.4.1",
2205 3063 "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
2206 3064 "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
2207   - "dev": true,
2208 3065 "requires": {
2209 3066 "assert-plus": "1.0.0",
2210 3067 "extsprintf": "1.3.0",
... ... @@ -2212,6 +3069,30 @@
2212 3069 "verror": "1.10.0"
2213 3070 }
2214 3071 },
  3072 + "jssha": {
  3073 + "version": "2.4.2",
  3074 + "resolved": "https://registry.npmjs.org/jssha/-/jssha-2.4.2.tgz",
  3075 + "integrity": "sha512-/jsi/9C0S70zfkT/4UlKQa5E1xKurDnXcQizcww9JSR/Fv+uIbWM2btG+bFcL3iNoK9jIGS0ls9HWLr1iw0kFg=="
  3076 + },
  3077 + "jwa": {
  3078 + "version": "2.0.0",
  3079 + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
  3080 + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
  3081 + "requires": {
  3082 + "buffer-equal-constant-time": "1.0.1",
  3083 + "ecdsa-sig-formatter": "1.0.11",
  3084 + "safe-buffer": "^5.0.1"
  3085 + }
  3086 + },
  3087 + "jws": {
  3088 + "version": "4.0.0",
  3089 + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
  3090 + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
  3091 + "requires": {
  3092 + "jwa": "^2.0.0",
  3093 + "safe-buffer": "^5.0.1"
  3094 + }
  3095 + },
2215 3096 "kafkajs": {
2216 3097 "version": "1.12.0",
2217 3098 "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-1.12.0.tgz",
... ... @@ -2258,6 +3139,26 @@
2258 3139 "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
2259 3140 "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
2260 3141 },
  3142 + "lodash.at": {
  3143 + "version": "4.6.0",
  3144 + "resolved": "https://registry.npmjs.org/lodash.at/-/lodash.at-4.6.0.tgz",
  3145 + "integrity": "sha1-k83OZk8KGZTqM9181A4jr9EbD/g="
  3146 + },
  3147 + "lodash.camelcase": {
  3148 + "version": "4.3.0",
  3149 + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
  3150 + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY="
  3151 + },
  3152 + "lodash.has": {
  3153 + "version": "4.5.2",
  3154 + "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz",
  3155 + "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI="
  3156 + },
  3157 + "lodash.snakecase": {
  3158 + "version": "4.1.1",
  3159 + "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz",
  3160 + "integrity": "sha1-OdcUo1NXFHg3rv1ktdy7Fr7Nj40="
  3161 + },
2261 3162 "logform": {
2262 3163 "version": "2.1.2",
2263 3164 "resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz",
... ... @@ -2342,17 +3243,20 @@
2342 3243 "to-regex": "^3.0.2"
2343 3244 }
2344 3245 },
  3246 + "mime": {
  3247 + "version": "2.4.5",
  3248 + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.5.tgz",
  3249 + "integrity": "sha512-3hQhEUF027BuxZjQA3s7rIv/7VCQPa27hN9u9g87sEkWaKwQPuXOkVKtOeiyUrnWqTDiOs8Ed2rwg733mB0R5w=="
  3250 + },
2345 3251 "mime-db": {
2346 3252 "version": "1.43.0",
2347 3253 "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz",
2348   - "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==",
2349   - "dev": true
  3254 + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ=="
2350 3255 },
2351 3256 "mime-types": {
2352 3257 "version": "2.1.26",
2353 3258 "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz",
2354 3259 "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==",
2355   - "dev": true,
2356 3260 "requires": {
2357 3261 "mime-db": "1.43.0"
2358 3262 }
... ... @@ -2414,6 +3318,11 @@
2414 3318 "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
2415 3319 "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
2416 3320 },
  3321 + "mpns": {
  3322 + "version": "2.1.3",
  3323 + "resolved": "https://registry.npmjs.org/mpns/-/mpns-2.1.3.tgz",
  3324 + "integrity": "sha512-gPLNoVqwYoKUmNYZ2shMSdaE2XvHSRxWNzyG4DUi6Av7MSujyeOw/nj61nnQeuV/vke5E0Dni468xn0qxTHIZQ=="
  3325 + },
2417 3326 "ms": {
2418 3327 "version": "2.1.2",
2419 3328 "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
... ... @@ -2487,6 +3396,16 @@
2487 3396 "to-regex": "^3.0.1"
2488 3397 }
2489 3398 },
  3399 + "node-fetch": {
  3400 + "version": "2.6.0",
  3401 + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz",
  3402 + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA=="
  3403 + },
  3404 + "node-forge": {
  3405 + "version": "0.9.1",
  3406 + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz",
  3407 + "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ=="
  3408 + },
2490 3409 "nodemon": {
2491 3410 "version": "1.19.4",
2492 3411 "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.4.tgz",
... ... @@ -2540,8 +3459,7 @@
2540 3459 "oauth-sign": {
2541 3460 "version": "0.9.0",
2542 3461 "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
2543   - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
2544   - "dev": true
  3462 + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
2545 3463 },
2546 3464 "object-copy": {
2547 3465 "version": "0.1.0",
... ... @@ -2597,6 +3515,14 @@
2597 3515 "isobject": "^3.0.1"
2598 3516 }
2599 3517 },
  3518 + "once": {
  3519 + "version": "1.4.0",
  3520 + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
  3521 + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
  3522 + "requires": {
  3523 + "wrappy": "1"
  3524 + }
  3525 + },
2600 3526 "one-time": {
2601 3527 "version": "0.0.4",
2602 3528 "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz",
... ... @@ -2622,6 +3548,11 @@
2622 3548 "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
2623 3549 "dev": true
2624 3550 },
  3551 + "p-defer": {
  3552 + "version": "3.0.0",
  3553 + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz",
  3554 + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw=="
  3555 + },
2625 3556 "p-finally": {
2626 3557 "version": "1.0.0",
2627 3558 "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
... ... @@ -2699,8 +3630,7 @@
2699 3630 "performance-now": {
2700 3631 "version": "2.1.0",
2701 3632 "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
2702   - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
2703   - "dev": true
  3633 + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
2704 3634 },
2705 3635 "picomatch": {
2706 3636 "version": "2.2.1",
... ... @@ -2898,6 +3828,11 @@
2898 3828 "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=",
2899 3829 "dev": true
2900 3830 },
  3831 + "process": {
  3832 + "version": "0.11.10",
  3833 + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
  3834 + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI="
  3835 + },
2901 3836 "process-nextick-args": {
2902 3837 "version": "2.0.1",
2903 3838 "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
... ... @@ -2909,6 +3844,26 @@
2909 3844 "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
2910 3845 "dev": true
2911 3846 },
  3847 + "protobufjs": {
  3848 + "version": "6.9.0",
  3849 + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.9.0.tgz",
  3850 + "integrity": "sha512-LlGVfEWDXoI/STstRDdZZKb/qusoAWUnmLg9R8OLSO473mBLWHowx8clbX5/+mKDEI+v7GzjoK9tRPZMMcoTrg==",
  3851 + "requires": {
  3852 + "@protobufjs/aspromise": "^1.1.2",
  3853 + "@protobufjs/base64": "^1.1.2",
  3854 + "@protobufjs/codegen": "^2.0.4",
  3855 + "@protobufjs/eventemitter": "^1.1.0",
  3856 + "@protobufjs/fetch": "^1.1.0",
  3857 + "@protobufjs/float": "^1.0.2",
  3858 + "@protobufjs/inquire": "^1.1.0",
  3859 + "@protobufjs/path": "^1.1.2",
  3860 + "@protobufjs/pool": "^1.1.0",
  3861 + "@protobufjs/utf8": "^1.1.0",
  3862 + "@types/long": "^4.0.1",
  3863 + "@types/node": "^13.7.0",
  3864 + "long": "^4.0.0"
  3865 + }
  3866 + },
2912 3867 "pseudomap": {
2913 3868 "version": "1.0.2",
2914 3869 "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
... ... @@ -2918,8 +3873,7 @@
2918 3873 "psl": {
2919 3874 "version": "1.7.0",
2920 3875 "resolved": "https://registry.npmjs.org/psl/-/psl-1.7.0.tgz",
2921   - "integrity": "sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==",
2922   - "dev": true
  3876 + "integrity": "sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ=="
2923 3877 },
2924 3878 "pstree.remy": {
2925 3879 "version": "1.1.7",
... ... @@ -2930,14 +3884,22 @@
2930 3884 "punycode": {
2931 3885 "version": "2.1.1",
2932 3886 "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
2933   - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
2934   - "dev": true
  3887 + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
2935 3888 },
2936 3889 "qs": {
2937 3890 "version": "6.5.2",
2938 3891 "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
2939   - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
2940   - "dev": true
  3892 + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
  3893 + },
  3894 + "querystring": {
  3895 + "version": "0.2.0",
  3896 + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
  3897 + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
  3898 + },
  3899 + "querystringify": {
  3900 + "version": "2.1.1",
  3901 + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz",
  3902 + "integrity": "sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA=="
2941 3903 },
2942 3904 "rc": {
2943 3905 "version": "1.2.8",
... ... @@ -3061,7 +4023,6 @@
3061 4023 "version": "2.88.2",
3062 4024 "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
3063 4025 "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==",
3064   - "dev": true,
3065 4026 "requires": {
3066 4027 "aws-sign2": "~0.7.0",
3067 4028 "aws4": "^1.8.0",
... ... @@ -3094,6 +4055,11 @@
3094 4055 "throttleit": "^1.0.0"
3095 4056 }
3096 4057 },
  4058 + "requires-port": {
  4059 + "version": "1.0.0",
  4060 + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
  4061 + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
  4062 + },
3097 4063 "resolve": {
3098 4064 "version": "1.15.1",
3099 4065 "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.1.tgz",
... ... @@ -3115,12 +4081,49 @@
3115 4081 "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==",
3116 4082 "dev": true
3117 4083 },
  4084 + "retry-request": {
  4085 + "version": "4.1.1",
  4086 + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz",
  4087 + "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==",
  4088 + "requires": {
  4089 + "debug": "^4.1.1",
  4090 + "through2": "^3.0.1"
  4091 + },
  4092 + "dependencies": {
  4093 + "debug": {
  4094 + "version": "4.1.1",
  4095 + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
  4096 + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
  4097 + "requires": {
  4098 + "ms": "^2.1.1"
  4099 + }
  4100 + }
  4101 + }
  4102 + },
3118 4103 "reusify": {
3119 4104 "version": "1.0.4",
3120 4105 "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
3121 4106 "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
3122 4107 "dev": true
3123 4108 },
  4109 + "rhea": {
  4110 + "version": "1.0.20",
  4111 + "resolved": "https://registry.npmjs.org/rhea/-/rhea-1.0.20.tgz",
  4112 + "integrity": "sha512-qj4LSEykJ0SEYESQLg9Vee6VXH5xHN1pYj7ozPeUk+l+S1OaGKx1FugAu+g+3pPwK46WXV1PJD9XiRx8+tS4cw==",
  4113 + "requires": {
  4114 + "debug": "0.8.0 - 3.5.0"
  4115 + }
  4116 + },
  4117 + "rhea-promise": {
  4118 + "version": "0.1.15",
  4119 + "resolved": "https://registry.npmjs.org/rhea-promise/-/rhea-promise-0.1.15.tgz",
  4120 + "integrity": "sha512-+6uilZXSJGyiqVeHQI3Krv6NTAd8cWRCY2uyCxmzR4/5IFtBqqFem1HV2OiwSj0Gu7OFChIJDfH2JyjN7J0vRA==",
  4121 + "requires": {
  4122 + "debug": "^3.1.0",
  4123 + "rhea": "^1.0.4",
  4124 + "tslib": "^1.9.3"
  4125 + }
  4126 + },
3124 4127 "run-parallel": {
3125 4128 "version": "1.1.9",
3126 4129 "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz",
... ... @@ -3144,8 +4147,12 @@
3144 4147 "safer-buffer": {
3145 4148 "version": "2.1.2",
3146 4149 "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
3147   - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
3148   - "dev": true
  4150 + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
  4151 + },
  4152 + "sax": {
  4153 + "version": "1.2.4",
  4154 + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
  4155 + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
3149 4156 },
3150 4157 "semver": {
3151 4158 "version": "6.3.0",
... ... @@ -3392,7 +4399,6 @@
3392 4399 "version": "1.16.1",
3393 4400 "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
3394 4401 "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==",
3395   - "dev": true,
3396 4402 "requires": {
3397 4403 "asn1": "~0.2.3",
3398 4404 "assert-plus": "^1.0.0",
... ... @@ -3431,6 +4437,44 @@
3431 4437 }
3432 4438 }
3433 4439 },
  4440 + "stream-browserify": {
  4441 + "version": "2.0.2",
  4442 + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",
  4443 + "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==",
  4444 + "requires": {
  4445 + "inherits": "~2.0.1",
  4446 + "readable-stream": "^2.0.2"
  4447 + },
  4448 + "dependencies": {
  4449 + "readable-stream": {
  4450 + "version": "2.3.7",
  4451 + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
  4452 + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
  4453 + "requires": {
  4454 + "core-util-is": "~1.0.0",
  4455 + "inherits": "~2.0.3",
  4456 + "isarray": "~1.0.0",
  4457 + "process-nextick-args": "~2.0.0",
  4458 + "safe-buffer": "~5.1.1",
  4459 + "string_decoder": "~1.1.1",
  4460 + "util-deprecate": "~1.0.1"
  4461 + }
  4462 + },
  4463 + "safe-buffer": {
  4464 + "version": "5.1.2",
  4465 + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
  4466 + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
  4467 + },
  4468 + "string_decoder": {
  4469 + "version": "1.1.1",
  4470 + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
  4471 + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
  4472 + "requires": {
  4473 + "safe-buffer": "~5.1.0"
  4474 + }
  4475 + }
  4476 + }
  4477 + },
3434 4478 "stream-meter": {
3435 4479 "version": "1.0.4",
3436 4480 "resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz",
... ... @@ -3472,6 +4516,11 @@
3472 4516 }
3473 4517 }
3474 4518 },
  4519 + "stream-shift": {
  4520 + "version": "1.0.1",
  4521 + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
  4522 + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ=="
  4523 + },
3475 4524 "string-width": {
3476 4525 "version": "2.1.1",
3477 4526 "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
... ... @@ -3540,6 +4589,19 @@
3540 4589 "integrity": "sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=",
3541 4590 "dev": true
3542 4591 },
  4592 + "through": {
  4593 + "version": "2.3.8",
  4594 + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
  4595 + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
  4596 + },
  4597 + "through2": {
  4598 + "version": "3.0.1",
  4599 + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz",
  4600 + "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==",
  4601 + "requires": {
  4602 + "readable-stream": "2 || 3"
  4603 + }
  4604 + },
3543 4605 "timed-out": {
3544 4606 "version": "4.0.1",
3545 4607 "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz",
... ... @@ -3601,7 +4663,6 @@
3601 4663 "version": "2.5.0",
3602 4664 "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
3603 4665 "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
3604   - "dev": true,
3605 4666 "requires": {
3606 4667 "psl": "^1.1.28",
3607 4668 "punycode": "^2.1.1"
... ... @@ -3612,11 +4673,20 @@
3612 4673 "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
3613 4674 "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw=="
3614 4675 },
  4676 + "tslib": {
  4677 + "version": "1.11.1",
  4678 + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz",
  4679 + "integrity": "sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA=="
  4680 + },
  4681 + "tunnel": {
  4682 + "version": "0.0.6",
  4683 + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
  4684 + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
  4685 + },
3615 4686 "tunnel-agent": {
3616 4687 "version": "0.6.0",
3617 4688 "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
3618 4689 "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
3619   - "dev": true,
3620 4690 "requires": {
3621 4691 "safe-buffer": "^5.0.1"
3622 4692 }
... ... @@ -3624,8 +4694,7 @@
3624 4694 "tweetnacl": {
3625 4695 "version": "0.14.5",
3626 4696 "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
3627   - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
3628   - "dev": true
  4697 + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
3629 4698 },
3630 4699 "type-check": {
3631 4700 "version": "0.3.2",
... ... @@ -3668,6 +4737,11 @@
3668 4737 }
3669 4738 }
3670 4739 },
  4740 + "underscore": {
  4741 + "version": "1.10.2",
  4742 + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.10.2.tgz",
  4743 + "integrity": "sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg=="
  4744 + },
3671 4745 "union-value": {
3672 4746 "version": "1.0.1",
3673 4747 "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
... ... @@ -3780,7 +4854,6 @@
3780 4854 "version": "4.2.2",
3781 4855 "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
3782 4856 "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
3783   - "dev": true,
3784 4857 "requires": {
3785 4858 "punycode": "^2.1.0"
3786 4859 }
... ... @@ -3791,6 +4864,31 @@
3791 4864 "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=",
3792 4865 "dev": true
3793 4866 },
  4867 + "url": {
  4868 + "version": "0.11.0",
  4869 + "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
  4870 + "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
  4871 + "requires": {
  4872 + "punycode": "1.3.2",
  4873 + "querystring": "0.2.0"
  4874 + },
  4875 + "dependencies": {
  4876 + "punycode": {
  4877 + "version": "1.3.2",
  4878 + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
  4879 + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
  4880 + }
  4881 + }
  4882 + },
  4883 + "url-parse": {
  4884 + "version": "1.4.7",
  4885 + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz",
  4886 + "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==",
  4887 + "requires": {
  4888 + "querystringify": "^2.1.1",
  4889 + "requires-port": "^1.0.0"
  4890 + }
  4891 + },
3794 4892 "url-parse-lax": {
3795 4893 "version": "1.0.0",
3796 4894 "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz",
... ... @@ -3806,6 +4904,21 @@
3806 4904 "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==",
3807 4905 "dev": true
3808 4906 },
  4907 + "util": {
  4908 + "version": "0.11.1",
  4909 + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
  4910 + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==",
  4911 + "requires": {
  4912 + "inherits": "2.0.3"
  4913 + },
  4914 + "dependencies": {
  4915 + "inherits": {
  4916 + "version": "2.0.3",
  4917 + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
  4918 + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
  4919 + }
  4920 + }
  4921 + },
3809 4922 "util-deprecate": {
3810 4923 "version": "1.0.2",
3811 4924 "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
... ... @@ -3814,25 +4927,38 @@
3814 4927 "uuid": {
3815 4928 "version": "3.4.0",
3816 4929 "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
3817   - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
3818   - "dev": true
  4930 + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
3819 4931 },
3820 4932 "uuid-parse": {
3821 4933 "version": "1.1.0",
3822 4934 "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz",
3823 4935 "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A=="
3824 4936 },
  4937 + "uuid-random": {
  4938 + "version": "1.3.0",
  4939 + "resolved": "https://registry.npmjs.org/uuid-random/-/uuid-random-1.3.0.tgz",
  4940 + "integrity": "sha512-FSIlv8RFRPOjcHeDYStV7u6aJRfp+THrcWkbAJpw51JCyQLDxsFz+4dHgTYP8hSpZeSMXBpb/1qrK4bodXpSRA=="
  4941 + },
  4942 + "validator": {
  4943 + "version": "9.4.1",
  4944 + "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz",
  4945 + "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA=="
  4946 + },
3825 4947 "verror": {
3826 4948 "version": "1.10.0",
3827 4949 "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
3828 4950 "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
3829   - "dev": true,
3830 4951 "requires": {
3831 4952 "assert-plus": "^1.0.0",
3832 4953 "core-util-is": "1.0.2",
3833 4954 "extsprintf": "^1.2.0"
3834 4955 }
3835 4956 },
  4957 + "walkdir": {
  4958 + "version": "0.4.1",
  4959 + "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.1.tgz",
  4960 + "integrity": "sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ=="
  4961 + },
3836 4962 "which": {
3837 4963 "version": "1.3.1",
3838 4964 "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
... ... @@ -3942,12 +5068,22 @@
3942 5068 }
3943 5069 }
3944 5070 },
  5071 + "wns": {
  5072 + "version": "0.5.4",
  5073 + "resolved": "https://registry.npmjs.org/wns/-/wns-0.5.4.tgz",
  5074 + "integrity": "sha512-WYiJ7khIwUGBD5KAm+YYmwJDDRzFRs4YGAjtbFSoRIdbn9Jcix3p9khJmpvBTXGommaKkvduAn+pc9l4d9yzVQ=="
  5075 + },
3945 5076 "word-wrap": {
3946 5077 "version": "1.2.3",
3947 5078 "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
3948 5079 "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
3949 5080 "dev": true
3950 5081 },
  5082 + "wrappy": {
  5083 + "version": "1.0.2",
  5084 + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
  5085 + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
  5086 + },
3951 5087 "write-file-atomic": {
3952 5088 "version": "2.4.3",
3953 5089 "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz",
... ... @@ -3965,6 +5101,20 @@
3965 5101 "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=",
3966 5102 "dev": true
3967 5103 },
  5104 + "xml2js": {
  5105 + "version": "0.4.23",
  5106 + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
  5107 + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
  5108 + "requires": {
  5109 + "sax": ">=0.6.0",
  5110 + "xmlbuilder": "~11.0.0"
  5111 + }
  5112 + },
  5113 + "xmlbuilder": {
  5114 + "version": "11.0.1",
  5115 + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
  5116 + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
  5117 + },
3968 5118 "yallist": {
3969 5119 "version": "2.1.2",
3970 5120 "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
... ...
... ... @@ -22,6 +22,7 @@
22 22 "azure-sb": "^0.11.1",
23 23 "long": "^4.0.0",
24 24 "uuid-parse": "^1.0.0",
  25 + "uuid-random": "^1.3.0",
25 26 "winston": "^3.0.0",
26 27 "winston-daily-rotate-file": "^3.2.1"
27 28 },
... ...
... ... @@ -19,6 +19,7 @@
19 19 const config = require('config'),
20 20 JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'),
21 21 logger = require('../config/logger')._logger('awsSqsTemplate');
  22 +const uuid = require('uuid-random');
22 23
23 24 const requestTopic = config.get('request_topic');
24 25
... ... @@ -26,10 +27,10 @@ const accessKeyId = config.get('aws_sqs.access_key_id');
26 27 const secretAccessKey = config.get('aws_sqs.secret_access_key');
27 28 const region = config.get('aws_sqs.region');
28 29 const AWS = require('aws-sdk');
29   -const queueProperties = config.get('aws_sqs.queue-properties');
30   -const poolInterval = config.get('js.response_poll_interval');
  30 +const queueProperties = config.get('aws_sqs.queue_properties');
  31 +const pollInterval = config.get('js.response_poll_interval');
31 32
32   -let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'};
  33 +let queueAttributes = {FifoQueue: 'true'};
33 34 let sqsClient;
34 35 let requestQueueURL;
35 36 const queueUrls = new Map();
... ... @@ -51,7 +52,12 @@ function AwsSqsProducer() {
51 52 queueUrls.set(responseTopic, responseQueueUrl);
52 53 }
53 54
54   - let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId};
  55 + let params = {
  56 + MessageBody: msgBody,
  57 + QueueUrl: responseQueueUrl,
  58 + MessageGroupId: 'js_eval',
  59 + MessageDeduplicationId: uuid()
  60 + };
55 61
56 62 return new Promise((resolve, reject) => {
57 63 sqsClient.sendMessage(params, function (err, data) {
... ... @@ -74,11 +80,13 @@ function AwsSqsProducer() {
74 80
75 81 const queues = await getQueues();
76 82
77   - queues.forEach(queueUrl => {
78   - const delimiterPosition = queueUrl.lastIndexOf('/');
79   - const queueName = queueUrl.substring(delimiterPosition + 1);
80   - queueUrls.set(queueName, queueUrl);
81   - })
  83 + if (queues) {
  84 + queues.forEach(queueUrl => {
  85 + const delimiterPosition = queueUrl.lastIndexOf('/');
  86 + const queueName = queueUrl.substring(delimiterPosition + 1);
  87 + queueUrls.set(queueName, queueUrl);
  88 + });
  89 + }
82 90
83 91 parseQueueProperties();
84 92
... ... @@ -95,6 +103,7 @@ function AwsSqsProducer() {
95 103 WaitTimeSeconds: poolInterval / 1000
96 104 };
97 105 while (!stopped) {
  106 + let pollStartTs = new Date().getTime();
98 107 const messages = await new Promise((resolve, reject) => {
99 108 sqsClient.receiveMessage(params, function (err, data) {
100 109 if (err) {
... ... @@ -127,6 +136,11 @@ function AwsSqsProducer() {
127 136 //do nothing
128 137 }
129 138 });
  139 + } else {
  140 + let pollDuration = new Date().getTime() - pollStartTs;
  141 + if (pollDuration < pollInterval) {
  142 + await sleep(pollInterval - pollDuration);
  143 + }
130 144 }
131 145 }
132 146 } catch (e) {
... ... @@ -175,6 +189,12 @@ function parseQueueProperties() {
175 189 });
176 190 }
177 191
  192 +function sleep(ms) {
  193 + return new Promise((resolve) => {
  194 + setTimeout(resolve, ms);
  195 + });
  196 +}
  197 +
178 198 process.on('exit', () => {
179 199 stopped = true;
180 200 logger.info('Aws Sqs client stopped.');
... ...
... ... @@ -20,7 +20,7 @@ const config = require('config'),
20 20 logger = require('../config/logger')._logger('kafkaTemplate'),
21 21 KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator;
22 22 const replicationFactor = config.get('kafka.replication_factor');
23   -const topicProperties = config.get('kafka.topic-properties');
  23 +const topicProperties = config.get('kafka.topic_properties');
24 24
25 25 let kafkaClient;
26 26 let kafkaAdmin;
... ...
... ... @@ -24,7 +24,7 @@ const {PubSub} = require('@google-cloud/pubsub');
24 24 const projectId = config.get('pubsub.project_id');
25 25 const credentials = JSON.parse(config.get('pubsub.service_account'));
26 26 const requestTopic = config.get('request_topic');
27   -const queueProperties = config.get('pubsub.queue-properties');
  27 +const queueProperties = config.get('pubsub.queue_properties');
28 28
29 29 let pubSubClient;
30 30
... ... @@ -98,23 +98,32 @@ function PubSubProducer() {
98 98
99 99 async function createTopic(topic) {
100 100 if (!topics.includes(topic)) {
101   - await pubSubClient.createTopic(topic);
  101 + try {
  102 + await pubSubClient.createTopic(topic);
  103 + logger.info('Created new Pub/Sub topic: %s', topic);
  104 + } catch (e) {
  105 + logger.info('Pub/Sub topic already exists');
  106 + }
102 107 topics.push(topic);
103   - logger.info('Created new Pub/Sub topic: %s', topic);
104 108 }
105 109 await createSubscription(topic)
106 110 }
107 111
108 112 async function createSubscription(topic) {
109 113 if (!subscriptions.includes(topic)) {
110   - await pubSubClient.createSubscription(topic, topic, {
111   - topic: topic,
112   - subscription: topic,
113   - ackDeadlineSeconds: queueProps['ackDeadlineInSec'],
114   - messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']}
115   - });
  114 + try {
  115 + await pubSubClient.createSubscription(topic, topic, {
  116 + topic: topic,
  117 + subscription: topic,
  118 + ackDeadlineSeconds: queueProps['ackDeadlineInSec'],
  119 + messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']}
  120 + });
  121 + logger.info('Created new Pub/Sub subscription: %s', topic);
  122 + } catch (e) {
  123 + logger.info('Pub/Sub subscription already exists.');
  124 + }
  125 +
116 126 subscriptions.push(topic);
117   - logger.info('Created new Pub/Sub subscription: %s', topic);
118 127 }
119 128 }
120 129
... ...
... ... @@ -26,23 +26,23 @@ const port = config.get('rabbitmq.port');
26 26 const vhost = config.get('rabbitmq.virtual_host');
27 27 const username = config.get('rabbitmq.username');
28 28 const password = config.get('rabbitmq.password');
29   -const queueProperties = config.get('rabbitmq.queue-properties');
30   -const poolInterval = config.get('js.response_poll_interval');
  29 +const queueProperties = config.get('rabbitmq.queue_properties');
  30 +const pollInterval = config.get('js.response_poll_interval');
31 31
32 32 const amqp = require('amqplib/callback_api');
33 33
34   -let queueParams = {durable: false, exclusive: false, autoDelete: false};
  34 +let queueOptions = {durable: false, exclusive: false, autoDelete: false};
35 35 let connection;
36 36 let channel;
37 37 let stopped = false;
38   -const responseTopics = [];
  38 +let queues = [];
39 39
40 40 function RabbitMqProducer() {
41 41 this.send = async (responseTopic, scriptId, rawResponse, headers) => {
42 42
43   - if (!responseTopics.includes(responseTopic)) {
  43 + if (!queues.includes(responseTopic)) {
44 44 await createQueue(responseTopic);
45   - responseTopics.push(responseTopic);
  45 + queues.push(responseTopic);
46 46 }
47 47
48 48 let data = JSON.stringify(
... ... @@ -98,6 +98,7 @@ function RabbitMqProducer() {
98 98 const messageProcessor = new JsInvokeMessageProcessor(new RabbitMqProducer());
99 99
100 100 while (!stopped) {
  101 + let pollStartTs = new Date().getTime();
101 102 let message = await new Promise((resolve, reject) => {
102 103 channel.get(requestTopic, {}, function (err, msg) {
103 104 if (err) {
... ... @@ -112,7 +113,10 @@ function RabbitMqProducer() {
112 113 messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8')));
113 114 channel.ack(message);
114 115 } else {
115   - await sleep(poolInterval);
  116 + let pollDuration = new Date().getTime() - pollStartTs;
  117 + if (pollDuration < pollInterval) {
  118 + await sleep(pollInterval - pollDuration);
  119 + }
116 120 }
117 121 }
118 122 } catch (e) {
... ... @@ -123,16 +127,18 @@ function RabbitMqProducer() {
123 127 })();
124 128
125 129 function parseQueueProperties() {
  130 + let args = {};
126 131 const props = queueProperties.split(';');
127 132 props.forEach(p => {
128 133 const delimiterPosition = p.indexOf(':');
129   - queueParams[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
  134 + args[p.substring(0, delimiterPosition)] = +p.substring(delimiterPosition + 1);
130 135 });
  136 + queueOptions['arguments'] = args;
131 137 }
132 138
133   -function createQueue(topic) {
  139 +async function createQueue(topic) {
134 140 return new Promise((resolve, reject) => {
135   - channel.assertQueue(topic, queueParams, function (err) {
  141 + channel.assertQueue(topic, queueOptions, function (err) {
136 142 if (err) {
137 143 reject(err);
138 144 } else {
... ...
... ... @@ -26,7 +26,7 @@ const requestTopic = config.get('request_topic');
26 26 const namespaceName = config.get('service_bus.namespace_name');
27 27 const sasKeyName = config.get('service_bus.sas_key_name');
28 28 const sasKey = config.get('service_bus.sas_key');
29   -const queueProperties = config.get('service_bus.queue-properties');
  29 +const queueProperties = config.get('service_bus.queue_properties');
30 30
31 31 let sbClient;
32 32 let receiverClient;
... ... @@ -140,6 +140,7 @@ function parseQueueProperties() {
140 140 properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1);
141 141 });
142 142 queueOptions = {
  143 + DuplicateDetection: 'false',
143 144 MaxSizeInMegabytes: properties['maxSizeInMb'],
144 145 DefaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`,
145 146 LockDuration: `PT${properties['lockDurationInSec']}S`
... ...
... ... @@ -16,7 +16,7 @@
16 16
17 17 const config = require('config'), logger = require('./config/logger')._logger('main');
18 18
19   -const serviceType = config.get('service-type');
  19 +const serviceType = config.get('queue_type');
20 20 switch (serviceType) {
21 21 case 'kafka':
22 22 logger.info('Starting kafka template.');
... ...
... ... @@ -30,10 +30,10 @@
30 30 <properties>
31 31 <main.dir>${basedir}</main.dir>
32 32 <pkg.user>thingsboard</pkg.user>
33   - <spring-boot.version>2.2.4.RELEASE</spring-boot.version>
  33 + <spring-boot.version>2.2.6.RELEASE</spring-boot.version>
34 34 <spring-oauth2.version>2.1.2.RELEASE</spring-oauth2.version>
35   - <spring.version>5.2.2.RELEASE</spring.version>
36   - <spring-security.version>5.2.2.RELEASE</spring-security.version>
  35 + <spring.version>5.2.6.RELEASE</spring.version>
  36 + <spring-security.version>5.2.3.RELEASE</spring-security.version>
37 37 <spring-data-redis.version>2.2.4.RELEASE</spring-data-redis.version>
38 38 <jedis.version>3.1.0</jedis.version>
39 39 <jjwt.version>0.7.0</jjwt.version>
... ... @@ -62,14 +62,14 @@
62 62 <gson.version>2.6.2</gson.version>
63 63 <velocity.version>1.7</velocity.version>
64 64 <velocity-tools.version>2.0</velocity-tools.version>
65   - <mail.version>1.4.3</mail.version>
  65 + <mail.version>1.6.2</mail.version>
66 66 <curator.version>4.2.0</curator.version>
67 67 <zookeeper.version>3.5.5</zookeeper.version>
68 68 <protobuf.version>3.11.4</protobuf.version>
69 69 <grpc.version>1.22.1</grpc.version>
70 70 <lombok.version>1.16.18</lombok.version>
71 71 <paho.client.version>1.1.0</paho.client.version>
72   - <netty.version>4.1.45.Final</netty.version>
  72 + <netty.version>4.1.49.Final</netty.version>
73 73 <os-maven-plugin.version>1.5.0</os-maven-plugin.version>
74 74 <rabbitmq.version>4.8.0</rabbitmq.version>
75 75 <surfire.version>2.19.1</surfire.version>
... ... @@ -96,7 +96,7 @@
96 96 <snakeyaml.version>1.25</snakeyaml.version>
97 97 <struts.version>1.3.10</struts.version>
98 98 <amazonaws.sqs.version>1.11.747</amazonaws.sqs.version>
99   - <pubsub.client.version>1.84.0</pubsub.client.version>
  99 + <pubsub.client.version>1.105.0</pubsub.client.version>
100 100 <azure-servicebus.version>3.2.0</azure-servicebus.version>
101 101 <passay.version>1.5.0</passay.version>
102 102 <ua-parser.version>1.4.3</ua-parser.version>
... ... @@ -471,12 +471,12 @@
471 471 <dependency>
472 472 <groupId>org.springframework.security</groupId>
473 473 <artifactId>spring-security-oauth2-client</artifactId>
474   - <version>${spring.version}</version>
  474 + <version>${spring-security.version}</version>
475 475 </dependency>
476 476 <dependency>
477 477 <groupId>org.springframework.security</groupId>
478 478 <artifactId>spring-security-oauth2-jose</artifactId>
479   - <version>${spring.version}</version>
  479 + <version>${spring-security.version}</version>
480 480 </dependency>
481 481 <dependency>
482 482 <groupId>org.springframework.boot</groupId>
... ... @@ -591,8 +591,8 @@
591 591 <version>${rabbitmq.version}</version>
592 592 </dependency>
593 593 <dependency>
594   - <groupId>javax.mail</groupId>
595   - <artifactId>mail</artifactId>
  594 + <groupId>com.sun.mail</groupId>
  595 + <artifactId>javax.mail</artifactId>
596 596 <version>${mail.version}</version>
597 597 </dependency>
598 598 <dependency>
... ... @@ -610,6 +610,12 @@
610 610 <groupId>org.apache.zookeeper</groupId>
611 611 <artifactId>zookeeper</artifactId>
612 612 <version>${zookeeper.version}</version>
  613 + <exclusions>
  614 + <exclusion>
  615 + <groupId>log4j</groupId>
  616 + <artifactId>log4j</artifactId>
  617 + </exclusion>
  618 + </exclusions>
613 619 </dependency>
614 620 <dependency>
615 621 <groupId>com.jayway.jsonpath</groupId>
... ... @@ -692,6 +698,12 @@
692 698 <groupId>com.github.fge</groupId>
693 699 <artifactId>json-schema-validator</artifactId>
694 700 <version>${json-schema-validator.version}</version>
  701 + <exclusions>
  702 + <exclusion>
  703 + <groupId>javax.mail</groupId>
  704 + <artifactId>mailapi</artifactId>
  705 + </exclusion>
  706 + </exclusions>
695 707 </dependency>
696 708 <dependency>
697 709 <groupId>com.typesafe.akka</groupId>
... ... @@ -928,12 +940,6 @@
928 940 <groupId>com.microsoft.azure</groupId>
929 941 <artifactId>azure-servicebus</artifactId>
930 942 <version>${azure-servicebus.version}</version>
931   - <exclusions>
932   - <exclusion>
933   - <groupId>com.microsoft.azure</groupId>
934   - <artifactId>adal4j</artifactId>
935   - </exclusion>
936   - </exclusions>
937 943 </dependency>
938 944 <dependency>
939 945 <groupId>org.passay</groupId>
... ...
... ... @@ -93,5 +93,10 @@
93 93 <artifactId>spring-data-redis</artifactId>
94 94 <scope>provided</scope>
95 95 </dependency>
  96 + <dependency>
  97 + <groupId>com.sun.mail</groupId>
  98 + <artifactId>javax.mail</artifactId>
  99 + <scope>provided</scope>
  100 + </dependency>
96 101 </dependencies>
97 102 </project>
... ...
... ... @@ -81,8 +81,8 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode<TbClearAlarmNodeConfig
81 81 }
82 82 alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK);
83 83 return Futures.immediateFuture(new AlarmResult(false, false, true, alarm));
84   - }, MoreExecutors.directExecutor());
85   - }, MoreExecutors.directExecutor());
  84 + }, ctx.getDbCallbackExecutor());
  85 + }, ctx.getDbCallbackExecutor());
86 86 }, ctx.getDbCallbackExecutor());
87 87 }
88 88 }
... ...