简柏林 / thingskit · Commits

Commit 2fb756ba5b897eb78a568b1b684f9945a05b2f38

Authored by Oleg Kolesnik
2 parents 88b93419 69263e3a

Merge branch 'master' of https://github.com/thingsboard/thingsboard

Inline Side-by-side
Showing 93 changed files with 1984 additions and 482 deletions

Too many changes to show.
Reload with full diff Plain diff Email patch

To preserve performance only 93 of 113 files are displayed.

... ... @@ -2,6 +2,7 @@ before_install:
2 2 - sudo rm -f /etc/mavenrc
3 3 - export M2_HOME=/usr/local/maven
4 4 - export MAVEN_OPTS="-Dmaven.repo.local=$HOME/.m2/repository -Xms1024m -Xmx3072m"
  5 + - export HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false
5 6 jdk:
6 7 - oraclejdk8
7 8 language: java
... ...
... ... @@ -61,6 +61,7 @@ import org.thingsboard.server.service.cluster.routing.ClusterRoutingService;
61 61 import org.thingsboard.server.service.cluster.rpc.ClusterRpcService;
62 62 import org.thingsboard.server.service.component.ComponentDiscoveryService;
63 63 import org.thingsboard.server.service.encoding.DataDecodingEncodingService;
  64 +import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService;
64 65 import org.thingsboard.server.service.executors.DbCallbackExecutorService;
65 66 import org.thingsboard.server.service.executors.ExternalCallExecutorService;
66 67 import org.thingsboard.server.service.mail.MailExecutorService;
... ... @@ -188,6 +189,10 @@ public class ActorSystemContext {
188 189
189 190 @Autowired
190 191 @Getter
  192 + private ClusterRpcCallbackExecutorService clusterRpcCallbackExecutor;
  193 +
  194 + @Autowired
  195 + @Getter
191 196 private DbCallbackExecutorService dbCallbackExecutor;
192 197
193 198 @Autowired
... ...
... ... @@ -25,15 +25,21 @@ import akka.actor.Terminated;
25 25 import akka.event.Logging;
26 26 import akka.event.LoggingAdapter;
27 27 import akka.japi.Function;
  28 +import com.google.common.collect.BiMap;
  29 +import com.google.common.collect.HashBiMap;
  30 +import lombok.extern.slf4j.Slf4j;
28 31 import org.thingsboard.server.actors.ActorSystemContext;
29 32 import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor;
30 33 import org.thingsboard.server.actors.service.ContextBasedCreator;
31 34 import org.thingsboard.server.actors.service.DefaultActorService;
32 35 import org.thingsboard.server.actors.shared.rulechain.SystemRuleChainManager;
33 36 import org.thingsboard.server.actors.tenant.TenantActor;
  37 +import org.thingsboard.server.common.data.EntityType;
34 38 import org.thingsboard.server.common.data.Tenant;
35 39 import org.thingsboard.server.common.data.id.TenantId;
36 40 import org.thingsboard.server.common.data.page.PageDataIterable;
  41 +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
  42 +import org.thingsboard.server.common.msg.MsgType;
37 43 import org.thingsboard.server.common.msg.TbActorMsg;
38 44 import org.thingsboard.server.common.msg.aware.TenantAwareMsg;
39 45 import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
... ... @@ -50,16 +56,15 @@ import java.util.Optional;
50 56
51 57 public class AppActor extends RuleChainManagerActor {
52 58
53   - private final LoggingAdapter logger = Logging.getLogger(getContext().system(), this);
54   -
55   - public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID);
  59 + private static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID);
56 60 private final TenantService tenantService;
57   - private final Map<TenantId, ActorRef> tenantActors;
  61 + private final BiMap<TenantId, ActorRef> tenantActors;
  62 + private boolean ruleChainsInitialized;
58 63
59 64 private AppActor(ActorSystemContext systemContext) {
60 65 super(systemContext, new SystemRuleChainManager(systemContext));
61 66 this.tenantService = systemContext.getTenantService();
62   - this.tenantActors = new HashMap<>();
  67 + this.tenantActors = HashBiMap.create();
63 68 }
64 69
65 70 @Override
... ... @@ -69,28 +74,20 @@ public class AppActor extends RuleChainManagerActor {
69 74
70 75 @Override
71 76 public void preStart() {
72   - logger.info("Starting main system actor.");
73   - try {
74   - initRuleChains();
75   -
76   - if (systemContext.isTenantComponentsInitEnabled()) {
77   - PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT);
78   - for (Tenant tenant : tenantIterator) {
79   - logger.debug("[{}] Creating tenant actor", tenant.getId());
80   - getOrCreateTenantActor(tenant.getId());
81   - logger.debug("Tenant actor created.");
82   - }
83   - }
84   -
85   - logger.info("Main system actor started.");
86   - } catch (Exception e) {
87   - logger.error(e, "Unknown failure");
88   - }
89 77 }
90 78
91 79 @Override
92 80 protected boolean process(TbActorMsg msg) {
  81 + if (!ruleChainsInitialized) {
  82 + initRuleChainsAndTenantActors();
  83 + ruleChainsInitialized = true;
  84 + if (msg.getMsgType() != MsgType.APP_INIT_MSG) {
  85 + log.warn("Rule Chains initialized by unexpected message: {}", msg);
  86 + }
  87 + }
93 88 switch (msg.getMsgType()) {
  89 + case APP_INIT_MSG:
  90 + break;
94 91 case SEND_TO_CLUSTER_MSG:
95 92 onPossibleClusterMsg((SendToClusterMsg) msg);
96 93 break;
... ... @@ -118,6 +115,24 @@ public class AppActor extends RuleChainManagerActor {
118 115 return true;
119 116 }
120 117
  118 + private void initRuleChainsAndTenantActors() {
  119 + log.info("Starting main system actor.");
  120 + try {
  121 + initRuleChains();
  122 + if (systemContext.isTenantComponentsInitEnabled()) {
  123 + PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT);
  124 + for (Tenant tenant : tenantIterator) {
  125 + log.debug("[{}] Creating tenant actor", tenant.getId());
  126 + getOrCreateTenantActor(tenant.getId());
  127 + log.debug("Tenant actor created.");
  128 + }
  129 + }
  130 + log.info("Main system actor started.");
  131 + } catch (Exception e) {
  132 + log.warn("Unknown failure", e);
  133 + }
  134 + }
  135 +
121 136 private void onPossibleClusterMsg(SendToClusterMsg msg) {
122 137 Optional<ServerAddress> address = systemContext.getRoutingService().resolveById(msg.getEntityId());
123 138 if (address.isPresent()) {
... ... @@ -130,7 +145,8 @@ public class AppActor extends RuleChainManagerActor {
130 145
131 146 private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) {
132 147 if (SYSTEM_TENANT.equals(msg.getTenantId())) {
133   - //TODO: ashvayka handle this.
  148 +// this may be a notification about system entities created.
  149 +// log.warn("[{}] Invalid service to rule engine msg called. System messages are not supported yet: {}", SYSTEM_TENANT, msg);
134 150 } else {
135 151 getOrCreateTenantActor(msg.getTenantId()).tell(msg, self());
136 152 }
... ... @@ -143,16 +159,26 @@ public class AppActor extends RuleChainManagerActor {
143 159 }
144 160
145 161 private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) {
146   - ActorRef target;
  162 + ActorRef target = null;
147 163 if (SYSTEM_TENANT.equals(msg.getTenantId())) {
148 164 target = getEntityActorRef(msg.getEntityId());
149 165 } else {
150   - target = getOrCreateTenantActor(msg.getTenantId());
  166 + if (msg.getEntityId().getEntityType() == EntityType.TENANT
  167 + && msg.getEvent() == ComponentLifecycleEvent.DELETED) {
  168 + log.debug("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg);
  169 + ActorRef tenantActor = tenantActors.remove(new TenantId(msg.getEntityId().getId()));
  170 + if (tenantActor != null) {
  171 + log.debug("[{}] Deleting tenant actor: {}", msg.getTenantId(), tenantActor);
  172 + context().stop(tenantActor);
  173 + }
  174 + } else {
  175 + target = getOrCreateTenantActor(msg.getTenantId());
  176 + }
151 177 }
152 178 if (target != null) {
153 179 target.tell(msg, ActorRef.noSender());
154 180 } else {
155   - logger.debug("Invalid component lifecycle msg: {}", msg);
  181 + log.debug("[{}] Invalid component lifecycle msg: {}", msg.getTenantId(), msg);
156 182 }
157 183 }
158 184
... ... @@ -161,14 +187,24 @@ public class AppActor extends RuleChainManagerActor {
161 187 }
162 188
163 189 private ActorRef getOrCreateTenantActor(TenantId tenantId) {
164   - return tenantActors.computeIfAbsent(tenantId, k -> context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId))
165   - .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString()));
  190 + return tenantActors.computeIfAbsent(tenantId, k -> {
  191 + log.debug("[{}] Creating tenant actor.", tenantId);
  192 + ActorRef tenantActor = context().actorOf(Props.create(new TenantActor.ActorCreator(systemContext, tenantId))
  193 + .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), tenantId.toString());
  194 + context().watch(tenantActor);
  195 + log.debug("[{}] Created tenant actor: {}.", tenantId, tenantActor);
  196 + return tenantActor;
  197 + });
166 198 }
167 199
168   - private void processTermination(Terminated message) {
  200 + @Override
  201 + protected void processTermination(Terminated message) {
169 202 ActorRef terminated = message.actor();
170 203 if (terminated instanceof LocalActorRef) {
171   - logger.debug("Removed actor: {}", terminated);
  204 + boolean removed = tenantActors.inverse().remove(terminated) != null;
  205 + if (removed) {
  206 + log.debug("[{}] Removed actor:", terminated);
  207 + }
172 208 } else {
173 209 throw new IllegalStateException("Remote actors are not supported!");
174 210 }
... ... @@ -182,20 +218,17 @@ public class AppActor extends RuleChainManagerActor {
182 218 }
183 219
184 220 @Override
185   - public AppActor create() throws Exception {
  221 + public AppActor create() {
186 222 return new AppActor(context);
187 223 }
188 224 }
189 225
190   - private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function<Throwable, Directive>() {
191   - @Override
192   - public Directive apply(Throwable t) {
193   - logger.error(t, "Unknown failure");
194   - if (t instanceof RuntimeException) {
195   - return SupervisorStrategy.restart();
196   - } else {
197   - return SupervisorStrategy.stop();
198   - }
  226 + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> {
  227 + log.warn("Unknown failure", t);
  228 + if (t instanceof RuntimeException) {
  229 + return SupervisorStrategy.restart();
  230 + } else {
  231 + return SupervisorStrategy.stop();
199 232 }
200 233 });
201 234 }
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.actors.app;
  17 +
  18 +import org.thingsboard.server.common.msg.MsgType;
  19 +import org.thingsboard.server.common.msg.TbActorMsg;
  20 +
  21 +public class AppInitMsg implements TbActorMsg {
  22 +
  23 + @Override
  24 + public MsgType getMsgType() {
  25 + return MsgType.APP_INIT_MSG;
  26 + }
  27 +}
... ...
... ... @@ -15,17 +15,13 @@
15 15 */
16 16 package org.thingsboard.server.actors.device;
17 17
18   -import akka.event.Logging;
19   -import akka.event.LoggingAdapter;
20 18 import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg;
21 19 import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg;
22 20 import org.thingsboard.server.actors.ActorSystemContext;
23 21 import org.thingsboard.server.actors.service.ContextAwareActor;
24   -import org.thingsboard.server.actors.service.ContextBasedCreator;
25 22 import org.thingsboard.server.common.data.id.DeviceId;
26 23 import org.thingsboard.server.common.data.id.TenantId;
27 24 import org.thingsboard.server.common.msg.TbActorMsg;
28   -import org.thingsboard.server.common.msg.cluster.ClusterEventMsg;
29 25 import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg;
30 26 import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg;
31 27 import org.thingsboard.server.service.rpc.ToDeviceRpcRequestActorMsg;
... ... @@ -34,23 +30,21 @@ import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWra
34 30
35 31 public class DeviceActor extends ContextAwareActor {
36 32
37   - private final LoggingAdapter logger = Logging.getLogger(getContext().system(), this);
38   -
39 33 private final DeviceActorMessageProcessor processor;
40 34
41   - private DeviceActor(ActorSystemContext systemContext, TenantId tenantId, DeviceId deviceId) {
  35 + DeviceActor(ActorSystemContext systemContext, TenantId tenantId, DeviceId deviceId) {
42 36 super(systemContext);
43   - this.processor = new DeviceActorMessageProcessor(systemContext, logger, tenantId, deviceId);
  37 + this.processor = new DeviceActorMessageProcessor(systemContext, tenantId, deviceId);
44 38 }
45 39
46 40 @Override
47 41 public void preStart() {
48   - logger.debug("[{}][{}] Starting device actor.", processor.tenantId, processor.deviceId);
  42 + log.debug("[{}][{}] Starting device actor.", processor.tenantId, processor.deviceId);
49 43 try {
50 44 processor.initSessionTimeout(context());
51   - logger.debug("[{}][{}] Device actor started.", processor.tenantId, processor.deviceId);
  45 + log.debug("[{}][{}] Device actor started.", processor.tenantId, processor.deviceId);
52 46 } catch (Exception e) {
53   - logger.error(e, "[{}][{}] Unknown failure", processor.tenantId, processor.deviceId);
  47 + log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.deviceId, e);
54 48 }
55 49 }
56 50
... ... @@ -90,22 +84,4 @@ public class DeviceActor extends ContextAwareActor {
90 84 return true;
91 85 }
92 86
93   - public static class ActorCreator extends ContextBasedCreator<DeviceActor> {
94   - private static final long serialVersionUID = 1L;
95   -
96   - private final TenantId tenantId;
97   - private final DeviceId deviceId;
98   -
99   - public ActorCreator(ActorSystemContext context, TenantId tenantId, DeviceId deviceId) {
100   - super(context);
101   - this.tenantId = tenantId;
102   - this.deviceId = deviceId;
103   - }
104   -
105   - @Override
106   - public DeviceActor create() throws Exception {
107   - return new DeviceActor(context, tenantId, deviceId);
108   - }
109   - }
110   -
111 87 }
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.actors.device;
  17 +
  18 +import org.thingsboard.server.actors.ActorSystemContext;
  19 +import org.thingsboard.server.actors.service.ContextBasedCreator;
  20 +import org.thingsboard.server.common.data.id.DeviceId;
  21 +import org.thingsboard.server.common.data.id.TenantId;
  22 +
  23 +public class DeviceActorCreator extends ContextBasedCreator<DeviceActor> {
  24 + private static final long serialVersionUID = 1L;
  25 +
  26 + private final TenantId tenantId;
  27 + private final DeviceId deviceId;
  28 +
  29 + public DeviceActorCreator(ActorSystemContext context, TenantId tenantId, DeviceId deviceId) {
  30 + super(context);
  31 + this.tenantId = tenantId;
  32 + this.deviceId = deviceId;
  33 + }
  34 +
  35 + @Override
  36 + public DeviceActor create() {
  37 + return new DeviceActor(context, tenantId, deviceId);
  38 + }
  39 +}
... ...
... ... @@ -24,6 +24,8 @@ import com.google.common.util.concurrent.ListenableFuture;
24 24 import com.google.gson.Gson;
25 25 import com.google.gson.JsonObject;
26 26 import com.google.gson.JsonParser;
  27 +import com.google.protobuf.InvalidProtocolBufferException;
  28 +import lombok.extern.slf4j.Slf4j;
27 29 import org.thingsboard.rule.engine.api.RpcError;
28 30 import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg;
29 31 import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg;
... ... @@ -88,6 +90,7 @@ import java.util.stream.Collectors;
88 90 /**
89 91 * @author Andrew Shvayka
90 92 */
  93 +@Slf4j
91 94 class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
92 95
93 96 final TenantId tenantId;
... ... @@ -106,8 +109,8 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
106 109 private String deviceType;
107 110 private TbMsgMetaData defaultMetaData;
108 111
109   - DeviceActorMessageProcessor(ActorSystemContext systemContext, LoggingAdapter logger, TenantId tenantId, DeviceId deviceId) {
110   - super(systemContext, logger);
  112 + DeviceActorMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, DeviceId deviceId) {
  113 + super(systemContext);
111 114 this.tenantId = tenantId;
112 115 this.deviceId = deviceId;
113 116 this.sessions = new LinkedHashMap<>();
... ... @@ -136,30 +139,30 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
136 139
137 140 long timeout = request.getExpirationTime() - System.currentTimeMillis();
138 141 if (timeout <= 0) {
139   - logger.debug("[{}][{}] Ignoring message due to exp time reached", deviceId, request.getId(), request.getExpirationTime());
  142 + log.debug("[{}][{}] Ignoring message due to exp time reached, {}", deviceId, request.getId(), request.getExpirationTime());
140 143 return;
141 144 }
142 145
143 146 boolean sent = rpcSubscriptions.size() > 0;
144 147 Set<UUID> syncSessionSet = new HashSet<>();
145   - rpcSubscriptions.entrySet().forEach(sub -> {
146   - sendToTransport(rpcRequest, sub.getKey(), sub.getValue().getNodeId());
147   - if (TransportProtos.SessionType.SYNC == sub.getValue().getType()) {
148   - syncSessionSet.add(sub.getKey());
  148 + rpcSubscriptions.forEach((key, value) -> {
  149 + sendToTransport(rpcRequest, key, value.getNodeId());
  150 + if (TransportProtos.SessionType.SYNC == value.getType()) {
  151 + syncSessionSet.add(key);
149 152 }
150 153 });
151 154 syncSessionSet.forEach(rpcSubscriptions::remove);
152 155
153 156 if (request.isOneway() && sent) {
154   - logger.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId());
  157 + log.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId());
155 158 systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null));
156 159 } else {
157 160 registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout);
158 161 }
159 162 if (sent) {
160   - logger.debug("[{}] RPC request {} is sent!", deviceId, request.getId());
  163 + log.debug("[{}] RPC request {} is sent!", deviceId, request.getId());
161 164 } else {
162   - logger.debug("[{}] RPC request {} is NOT sent!", deviceId, request.getId());
  165 + log.debug("[{}] RPC request {} is NOT sent!", deviceId, request.getId());
163 166 }
164 167 }
165 168
... ... @@ -172,7 +175,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
172 175 void processServerSideRpcTimeout(ActorContext context, DeviceActorServerSideRpcTimeoutMsg msg) {
173 176 ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId());
174 177 if (requestMd != null) {
175   - logger.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId());
  178 + log.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId());
176 179 systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
177 180 null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION));
178 181 }
... ... @@ -181,13 +184,13 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
181 184 private void sendPendingRequests(ActorContext context, UUID sessionId, SessionInfoProto sessionInfo) {
182 185 TransportProtos.SessionType sessionType = getSessionType(sessionId);
183 186 if (!toDeviceRpcPendingMap.isEmpty()) {
184   - logger.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId);
  187 + log.debug("[{}] Pushing {} pending RPC messages to new async session [{}]", deviceId, toDeviceRpcPendingMap.size(), sessionId);
185 188 if (sessionType == TransportProtos.SessionType.SYNC) {
186   - logger.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId);
  189 + log.debug("[{}] Cleanup sync rpc session [{}]", deviceId, sessionId);
187 190 rpcSubscriptions.remove(sessionId);
188 191 }
189 192 } else {
190   - logger.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId);
  193 + log.debug("[{}] No pending RPC messages for new async session [{}]", deviceId, sessionId);
191 194 }
192 195 Set<Integer> sentOneWayIds = new HashSet<>();
193 196 if (sessionType == TransportProtos.SessionType.ASYNC) {
... ... @@ -335,7 +338,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
335 338 void processClientSideRpcTimeout(ActorContext context, DeviceActorClientSideRpcTimeoutMsg msg) {
336 339 ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(msg.getId());
337 340 if (data != null) {
338   - logger.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId());
  341 + log.debug("[{}] Client side RPC request [{}] timeout detected!", deviceId, msg.getId());
339 342 sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder()
340 343 .setRequestId(msg.getId()).setError("timeout").build()
341 344 , data.getSessionId(), data.getNodeId());
... ... @@ -346,9 +349,12 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
346 349 int requestId = msg.getMsg().getRequestId();
347 350 ToServerRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId);
348 351 if (data != null) {
  352 + log.debug("[{}] Pushing reply to [{}][{}]!", deviceId, data.getNodeId(), data.getSessionId());
349 353 sendToTransport(TransportProtos.ToServerRpcResponseMsg.newBuilder()
350 354 .setRequestId(requestId).setPayload(msg.getMsg().getData()).build()
351 355 , data.getSessionId(), data.getNodeId());
  356 + } else {
  357 + log.debug("[{}][{}] Pending RPC request to server not found!", deviceId, requestId);
352 358 }
353 359 }
354 360
... ... @@ -380,7 +386,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
380 386 hasNotificationData = true;
381 387 }
382 388 } else {
383   - logger.debug("[{}] No public server side attributes changed!", deviceId);
  389 + log.debug("[{}] No public server side attributes changed!", deviceId);
384 390 }
385 391 }
386 392 }
... ... @@ -391,27 +397,27 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
391 397 });
392 398 }
393 399 } else {
394   - logger.debug("[{}] No registered attributes subscriptions to process!", deviceId);
  400 + log.debug("[{}] No registered attributes subscriptions to process!", deviceId);
395 401 }
396 402 }
397 403
398 404 private void processRpcResponses(ActorContext context, SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg responseMsg) {
399 405 UUID sessionId = getSessionId(sessionInfo);
400   - logger.debug("[{}] Processing rpc command response [{}]", deviceId, sessionId);
  406 + log.debug("[{}] Processing rpc command response [{}]", deviceId, sessionId);
401 407 ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId());
402 408 boolean success = requestMd != null;
403 409 if (success) {
404 410 systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
405 411 responseMsg.getPayload(), null));
406 412 } else {
407   - logger.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId());
  413 + log.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId());
408 414 }
409 415 }
410 416
411 417 private void processSubscriptionCommands(ActorContext context, SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg subscribeCmd) {
412 418 UUID sessionId = getSessionId(sessionInfo);
413 419 if (subscribeCmd.getUnsubscribe()) {
414   - logger.debug("[{}] Canceling attributes subscription for session [{}]", deviceId, sessionId);
  420 + log.debug("[{}] Canceling attributes subscription for session [{}]", deviceId, sessionId);
415 421 attributeSubscriptions.remove(sessionId);
416 422 } else {
417 423 SessionInfoMetaData sessionMD = sessions.get(sessionId);
... ... @@ -419,7 +425,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
419 425 sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId()));
420 426 }
421 427 sessionMD.setSubscribedToAttributes(true);
422   - logger.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId);
  428 + log.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId);
423 429 attributeSubscriptions.put(sessionId, sessionMD.getSessionInfo());
424 430 dumpSessions();
425 431 }
... ... @@ -432,7 +438,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
432 438 private void processSubscriptionCommands(ActorContext context, SessionInfoProto sessionInfo, SubscribeToRPCMsg subscribeCmd) {
433 439 UUID sessionId = getSessionId(sessionInfo);
434 440 if (subscribeCmd.getUnsubscribe()) {
435   - logger.debug("[{}] Canceling rpc subscription for session [{}]", deviceId, sessionId);
  441 + log.debug("[{}] Canceling rpc subscription for session [{}]", deviceId, sessionId);
436 442 rpcSubscriptions.remove(sessionId);
437 443 } else {
438 444 SessionInfoMetaData sessionMD = sessions.get(sessionId);
... ... @@ -440,7 +446,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
440 446 sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId()));
441 447 }
442 448 sessionMD.setSubscribedToRPC(true);
443   - logger.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId);
  449 + log.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId);
444 450 rpcSubscriptions.put(sessionId, sessionMD.getSessionInfo());
445 451 sendPendingRequests(context, sessionId, sessionInfo);
446 452 dumpSessions();
... ... @@ -451,10 +457,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
451 457 UUID sessionId = getSessionId(sessionInfo);
452 458 if (msg.getEvent() == SessionEvent.OPEN) {
453 459 if (sessions.containsKey(sessionId)) {
454   - logger.debug("[{}] Received duplicate session open event [{}]", deviceId, sessionId);
  460 + log.debug("[{}] Received duplicate session open event [{}]", deviceId, sessionId);
455 461 return;
456 462 }
457   - logger.debug("[{}] Processing new session [{}]", deviceId, sessionId);
  463 + log.debug("[{}] Processing new session [{}]", deviceId, sessionId);
458 464 if (sessions.size() >= systemContext.getMaxConcurrentSessionsPerDevice()) {
459 465 UUID sessionIdToRemove = sessions.keySet().stream().findFirst().orElse(null);
460 466 if (sessionIdToRemove != null) {
... ... @@ -467,7 +473,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
467 473 }
468 474 dumpSessions();
469 475 } else if (msg.getEvent() == SessionEvent.CLOSED) {
470   - logger.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId);
  476 + log.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId);
471 477 sessions.remove(sessionId);
472 478 attributeSubscriptions.remove(sessionId);
473 479 rpcSubscriptions.remove(sessionId);
... ... @@ -478,19 +484,19 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
478 484 }
479 485 }
480 486
481   - private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto subscriptionInfo) {
482   - UUID sessionId = getSessionId(sessionInfo);
483   - SessionInfoMetaData sessionMD = sessions.get(sessionId);
484   - if (sessionMD != null) {
485   - sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime());
486   - sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription());
487   - sessionMD.setSubscribedToRPC(subscriptionInfo.getRpcSubscription());
488   - if (subscriptionInfo.getAttributeSubscription()) {
489   - attributeSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo());
490   - }
491   - if (subscriptionInfo.getRpcSubscription()) {
492   - rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo());
493   - }
  487 + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfoProto, TransportProtos.SubscriptionInfoProto subscriptionInfo) {
  488 + UUID sessionId = getSessionId(sessionInfoProto);
  489 + SessionInfoMetaData sessionMD = sessions.computeIfAbsent(sessionId,
  490 + id -> new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId()), 0L));
  491 +
  492 + sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime());
  493 + sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription());
  494 + sessionMD.setSubscribedToRPC(subscriptionInfo.getRpcSubscription());
  495 + if (subscriptionInfo.getAttributeSubscription()) {
  496 + attributeSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo());
  497 + }
  498 + if (subscriptionInfo.getRpcSubscription()) {
  499 + rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo());
494 500 }
495 501 dumpSessions();
496 502 }
... ... @@ -623,10 +629,16 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
623 629 }
624 630
625 631 private void restoreSessions() {
626   - logger.debug("[{}] Restoring sessions from cache", deviceId);
627   - TransportProtos.DeviceSessionsCacheEntry sessionsDump = systemContext.getDeviceSessionCacheService().get(deviceId);
628   - if (sessionsDump.getSerializedSize() == 0) {
629   - logger.debug("[{}] No session information found", deviceId);
  632 + log.debug("[{}] Restoring sessions from cache", deviceId);
  633 + TransportProtos.DeviceSessionsCacheEntry sessionsDump = null;
  634 + try {
  635 + sessionsDump = TransportProtos.DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId));
  636 + } catch (InvalidProtocolBufferException e) {
  637 + log.warn("[{}] Failed to decode device sessions from cache", deviceId);
  638 + return;
  639 + }
  640 + if (sessionsDump.getSessionsCount() == 0) {
  641 + log.debug("[{}] No session information found", deviceId);
630 642 return;
631 643 }
632 644 for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) {
... ... @@ -644,13 +656,13 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
644 656 rpcSubscriptions.put(sessionId, sessionInfo);
645 657 sessionMD.setSubscribedToRPC(true);
646 658 }
647   - logger.debug("[{}] Restored session: {}", deviceId, sessionMD);
  659 + log.debug("[{}] Restored session: {}", deviceId, sessionMD);
648 660 }
649   - logger.debug("[{}] Restored sessions: {}, rpc subscriptions: {}, attribute subscriptions: {}", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size());
  661 + log.debug("[{}] Restored sessions: {}, rpc subscriptions: {}, attribute subscriptions: {}", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size());
650 662 }
651 663
652 664 private void dumpSessions() {
653   - logger.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size());
  665 + log.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size());
654 666 List<TransportProtos.SessionSubscriptionInfoProto> sessionsList = new ArrayList<>(sessions.size());
655 667 sessions.forEach((uuid, sessionMD) -> {
656 668 if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) {
... ... @@ -668,11 +680,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
668 680 sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder()
669 681 .setSessionInfo(sessionInfoProto)
670 682 .setSubscriptionInfo(subscriptionInfoProto).build());
671   - logger.debug("[{}] Dumping session: {}", deviceId, sessionMD);
  683 + log.debug("[{}] Dumping session: {}", deviceId, sessionMD);
672 684 });
673 685 systemContext.getDeviceSessionCacheService()
674 686 .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder()
675   - .addAllSessions(sessionsList).build());
  687 + .addAllSessions(sessionsList).build().toByteArray());
676 688 }
677 689
678 690 void initSessionTimeout(ActorContext context) {
... ...
... ... @@ -17,10 +17,12 @@ package org.thingsboard.server.actors.rpc;
17 17
18 18 import akka.actor.ActorRef;
19 19 import lombok.extern.slf4j.Slf4j;
  20 +import org.thingsboard.server.actors.ActorSystemContext;
20 21 import org.thingsboard.server.actors.service.ActorService;
21 22 import org.thingsboard.server.gen.cluster.ClusterAPIProtos;
22 23 import org.thingsboard.server.service.cluster.rpc.GrpcSession;
23 24 import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener;
  25 +import org.thingsboard.server.service.executors.ClusterRpcCallbackExecutorService;
24 26
25 27 /**
26 28 * @author Andrew Shvayka
... ... @@ -28,19 +30,21 @@ import org.thingsboard.server.service.cluster.rpc.GrpcSessionListener;
28 30 @Slf4j
29 31 public class BasicRpcSessionListener implements GrpcSessionListener {
30 32
  33 + private final ClusterRpcCallbackExecutorService callbackExecutorService;
31 34 private final ActorService service;
32 35 private final ActorRef manager;
33 36 private final ActorRef self;
34 37
35   - BasicRpcSessionListener(ActorService service, ActorRef manager, ActorRef self) {
36   - this.service = service;
  38 + BasicRpcSessionListener(ActorSystemContext context, ActorRef manager, ActorRef self) {
  39 + this.service = context.getActorService();
  40 + this.callbackExecutorService = context.getClusterRpcCallbackExecutor();
37 41 this.manager = manager;
38 42 this.self = self;
39 43 }
40 44
41 45 @Override
42 46 public void onConnected(GrpcSession session) {
43   - log.info("{} session started -> {}", getType(session), session.getRemoteServer());
  47 + log.info("[{}][{}] session started", session.getRemoteServer(), getType(session));
44 48 if (!session.isClient()) {
45 49 manager.tell(new RpcSessionConnectedMsg(session.getRemoteServer(), session.getSessionId()), self);
46 50 }
... ... @@ -48,21 +52,25 @@ public class BasicRpcSessionListener implements GrpcSessionListener {
48 52
49 53 @Override
50 54 public void onDisconnected(GrpcSession session) {
51   - log.info("{} session closed -> {}", getType(session), session.getRemoteServer());
  55 + log.info("[{}][{}] session closed", session.getRemoteServer(), getType(session));
52 56 manager.tell(new RpcSessionDisconnectedMsg(session.isClient(), session.getRemoteServer()), self);
53 57 }
54 58
55 59 @Override
56 60 public void onReceiveClusterGrpcMsg(GrpcSession session, ClusterAPIProtos.ClusterMessage clusterMessage) {
57   - log.trace("{} Service [{}] received session actor msg {}", getType(session),
58   - session.getRemoteServer(),
59   - clusterMessage);
60   - service.onReceivedMsg(session.getRemoteServer(), clusterMessage);
  61 + log.trace("Received session actor msg from [{}][{}]: {}", session.getRemoteServer(), getType(session), clusterMessage);
  62 + callbackExecutorService.execute(() -> {
  63 + try {
  64 + service.onReceivedMsg(session.getRemoteServer(), clusterMessage);
  65 + } catch (Exception e) {
  66 + log.debug("[{}][{}] Failed to process cluster message: {}", session.getRemoteServer(), getType(session), clusterMessage, e);
  67 + }
  68 + });
61 69 }
62 70
63 71 @Override
64 72 public void onError(GrpcSession session, Throwable t) {
65   - log.warn("{} session got error -> {}", getType(session), session.getRemoteServer(), t);
  73 + log.warn("[{}][{}] session got error -> {}", session.getRemoteServer(), getType(session), t);
66 74 manager.tell(new RpcSessionClosedMsg(session.isClient(), session.getRemoteServer()), self);
67 75 session.close();
68 76 }
... ...
... ... @@ -16,9 +16,12 @@
16 16 package org.thingsboard.server.actors.rpc;
17 17
18 18 import akka.actor.ActorRef;
  19 +import akka.actor.OneForOneStrategy;
19 20 import akka.actor.Props;
  21 +import akka.actor.SupervisorStrategy;
20 22 import akka.event.Logging;
21 23 import akka.event.LoggingAdapter;
  24 +import lombok.extern.slf4j.Slf4j;
22 25 import org.thingsboard.server.actors.ActorSystemContext;
23 26 import org.thingsboard.server.actors.service.ContextAwareActor;
24 27 import org.thingsboard.server.actors.service.ContextBasedCreator;
... ... @@ -29,6 +32,7 @@ import org.thingsboard.server.common.msg.cluster.ServerAddress;
29 32 import org.thingsboard.server.common.msg.cluster.ServerType;
30 33 import org.thingsboard.server.gen.cluster.ClusterAPIProtos;
31 34 import org.thingsboard.server.service.cluster.discovery.ServerInstance;
  35 +import scala.concurrent.duration.Duration;
32 36
33 37 import java.util.*;
34 38
... ... @@ -37,15 +41,11 @@ import java.util.*;
37 41 */
38 42 public class RpcManagerActor extends ContextAwareActor {
39 43
40   - private final LoggingAdapter log = Logging.getLogger(getContext().system(), this);
41   -
42 44 private final Map<ServerAddress, SessionActorInfo> sessionActors;
43   -
44 45 private final Map<ServerAddress, Queue<ClusterAPIProtos.ClusterMessage>> pendingMsgs;
45   -
46 46 private final ServerAddress instance;
47 47
48   - RpcManagerActor(ActorSystemContext systemContext) {
  48 + private RpcManagerActor(ActorSystemContext systemContext) {
49 49 super(systemContext);
50 50 this.sessionActors = new HashMap<>();
51 51 this.pendingMsgs = new HashMap<>();
... ... @@ -64,7 +64,7 @@ public class RpcManagerActor extends ContextAwareActor {
64 64 }
65 65
66 66 @Override
67   - public void onReceive(Object msg) throws Exception {
  67 + public void onReceive(Object msg) {
68 68 if (msg instanceof ClusterAPIProtos.ClusterMessage) {
69 69 onMsg((ClusterAPIProtos.ClusterMessage) msg);
70 70 } else if (msg instanceof RpcBroadcastMsg) {
... ... @@ -116,7 +116,7 @@ public class RpcManagerActor extends ContextAwareActor {
116 116 queue.add(msg);
117 117 }
118 118 } else {
119   - logger.warning("Cluster msg doesn't have server address [{}]", msg);
  119 + log.warn("Cluster msg doesn't have server address [{}]", msg);
120 120 }
121 121 }
122 122
... ... @@ -164,6 +164,7 @@ public class RpcManagerActor extends ContextAwareActor {
164 164 log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect);
165 165 SessionActorInfo sessionRef = sessionActors.get(remoteAddress);
166 166 if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) {
  167 + context().stop(sessionRef.actor);
167 168 sessionActors.remove(remoteAddress);
168 169 pendingMsgs.remove(remoteAddress);
169 170 if (reconnect) {
... ... @@ -173,9 +174,13 @@ public class RpcManagerActor extends ContextAwareActor {
173 174 }
174 175
175 176 private void onCreateSessionRequest(RpcSessionCreateRequestMsg msg) {
176   - ActorRef actorRef = createSessionActor(msg);
177 177 if (msg.getRemoteAddress() != null) {
178   - register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef);
  178 + if (!sessionActors.containsKey(msg.getRemoteAddress())) {
  179 + ActorRef actorRef = createSessionActor(msg);
  180 + register(msg.getRemoteAddress(), msg.getMsgUid(), actorRef);
  181 + }
  182 + } else {
  183 + createSessionActor(msg);
179 184 }
180 185 }
181 186
... ... @@ -194,7 +199,8 @@ public class RpcManagerActor extends ContextAwareActor {
194 199 private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) {
195 200 log.info("[{}] Creating session actor.", msg.getMsgUid());
196 201 ActorRef actor = context().actorOf(
197   - Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())).withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME));
  202 + Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid()))
  203 + .withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME));
198 204 actor.tell(msg, context().self());
199 205 return actor;
200 206 }
... ... @@ -207,8 +213,18 @@ public class RpcManagerActor extends ContextAwareActor {
207 213 }
208 214
209 215 @Override
210   - public RpcManagerActor create() throws Exception {
  216 + public RpcManagerActor create() {
211 217 return new RpcManagerActor(context);
212 218 }
213 219 }
  220 +
  221 + @Override
  222 + public SupervisorStrategy supervisorStrategy() {
  223 + return strategy;
  224 + }
  225 +
  226 + private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), t -> {
  227 + log.warn("Unknown failure", t);
  228 + return SupervisorStrategy.resume();
  229 + });
214 230 }
... ...
... ... @@ -15,12 +15,10 @@
15 15 */
16 16 package org.thingsboard.server.actors.rpc;
17 17
18   -import akka.event.Logging;
19   -import akka.event.LoggingAdapter;
20   -import io.grpc.Channel;
21 18 import io.grpc.ManagedChannel;
22 19 import io.grpc.ManagedChannelBuilder;
23 20 import io.grpc.stub.StreamObserver;
  21 +import lombok.extern.slf4j.Slf4j;
24 22 import org.thingsboard.server.actors.ActorSystemContext;
25 23 import org.thingsboard.server.actors.service.ContextAwareActor;
26 24 import org.thingsboard.server.actors.service.ContextBasedCreator;
... ... @@ -38,15 +36,15 @@ import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CO
38 36 /**
39 37 * @author Andrew Shvayka
40 38 */
  39 +@Slf4j
41 40 public class RpcSessionActor extends ContextAwareActor {
42 41
43   - private final LoggingAdapter log = Logging.getLogger(getContext().system(), this);
44 42
45 43 private final UUID sessionId;
46 44 private GrpcSession session;
47 45 private GrpcSessionListener listener;
48 46
49   - public RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) {
  47 + private RpcSessionActor(ActorSystemContext systemContext, UUID sessionId) {
50 48 super(systemContext);
51 49 this.sessionId = sessionId;
52 50 }
... ... @@ -58,7 +56,7 @@ public class RpcSessionActor extends ContextAwareActor {
58 56 }
59 57
60 58 @Override
61   - public void onReceive(Object msg) throws Exception {
  59 + public void onReceive(Object msg) {
62 60 if (msg instanceof ClusterAPIProtos.ClusterMessage) {
63 61 tell((ClusterAPIProtos.ClusterMessage) msg);
64 62 } else if (msg instanceof RpcSessionCreateRequestMsg) {
... ... @@ -67,19 +65,29 @@ public class RpcSessionActor extends ContextAwareActor {
67 65 }
68 66
69 67 private void tell(ClusterAPIProtos.ClusterMessage msg) {
70   - session.sendMsg(msg);
  68 + if (session != null) {
  69 + session.sendMsg(msg);
  70 + } else {
  71 + log.trace("Failed to send message due to missing session!");
  72 + }
71 73 }
72 74
73 75 @Override
74 76 public void postStop() {
75   - log.info("Closing session -> {}", session.getRemoteServer());
76   - session.close();
  77 + if (session != null) {
  78 + log.info("Closing session -> {}", session.getRemoteServer());
  79 + try {
  80 + session.close();
  81 + } catch (RuntimeException e) {
  82 + log.trace("Failed to close session!", e);
  83 + }
  84 + }
77 85 }
78 86
79 87 private void initSession(RpcSessionCreateRequestMsg msg) {
80 88 log.info("[{}] Initializing session", context().self());
81 89 ServerAddress remoteServer = msg.getRemoteAddress();
82   - listener = new BasicRpcSessionListener(systemContext.getActorService(), context().parent(), context().self());
  90 + listener = new BasicRpcSessionListener(systemContext, context().parent(), context().self());
83 91 if (msg.getRemoteAddress() == null) {
84 92 // Server session
85 93 session = new GrpcSession(listener);
... ... @@ -113,7 +121,7 @@ public class RpcSessionActor extends ContextAwareActor {
113 121 }
114 122
115 123 @Override
116   - public RpcSessionActor create() throws Exception {
  124 + public RpcSessionActor create() {
117 125 return new RpcSessionActor(context, sessionId);
118 126 }
119 127 }
... ...
... ... @@ -15,6 +15,7 @@
15 15 */
16 16 package org.thingsboard.server.actors.ruleChain;
17 17
  18 +import akka.actor.ActorInitializationException;
18 19 import akka.actor.OneForOneStrategy;
19 20 import akka.actor.SupervisorStrategy;
20 21 import org.thingsboard.server.actors.ActorSystemContext;
... ... @@ -33,7 +34,7 @@ public class RuleChainActor extends ComponentActor<RuleChainId, RuleChainActorMe
33 34 private RuleChainActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId) {
34 35 super(systemContext, tenantId, ruleChainId);
35 36 setProcessor(new RuleChainActorMessageProcessor(tenantId, ruleChainId, systemContext,
36   - logger, context().parent(), context().self()));
  37 + context().parent(), context().self()));
37 38 }
38 39
39 40 @Override
... ... @@ -79,7 +80,7 @@ public class RuleChainActor extends ComponentActor<RuleChainId, RuleChainActorMe
79 80 }
80 81
81 82 @Override
82   - public RuleChainActor create() throws Exception {
  83 + public RuleChainActor create() {
83 84 return new RuleChainActor(context, tenantId, ruleChainId);
84 85 }
85 86 }
... ...
... ... @@ -23,6 +23,7 @@ import com.datastax.driver.core.utils.UUIDs;
23 23
24 24 import java.util.Optional;
25 25
  26 +import lombok.extern.slf4j.Slf4j;
26 27 import org.thingsboard.server.actors.ActorSystemContext;
27 28 import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg;
28 29 import org.thingsboard.server.actors.service.DefaultActorService;
... ... @@ -55,6 +56,7 @@ import java.util.stream.Collectors;
55 56 /**
56 57 * @author Andrew Shvayka
57 58 */
  59 +@Slf4j
58 60 public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleChainId> {
59 61
60 62 private static final long DEFAULT_CLUSTER_PARTITION = 0L;
... ... @@ -67,24 +69,34 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
67 69 private RuleNodeId firstId;
68 70 private RuleNodeCtx firstNode;
69 71 private boolean started;
  72 + private String ruleChainName;
70 73
71 74 RuleChainActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, ActorSystemContext systemContext
72   - , LoggingAdapter logger, ActorRef parent, ActorRef self) {
73   - super(systemContext, logger, tenantId, ruleChainId);
  75 + , ActorRef parent, ActorRef self) {
  76 + super(systemContext, tenantId, ruleChainId);
74 77 this.parent = parent;
75 78 this.self = self;
76 79 this.nodeActors = new HashMap<>();
77 80 this.nodeRoutes = new HashMap<>();
78 81 this.service = systemContext.getRuleChainService();
  82 + this.ruleChainName = ruleChainId.toString();
79 83 }
80 84
81 85 @Override
82   - public void start(ActorContext context) throws Exception {
  86 + public String getComponentName() {
  87 + return null;
  88 + }
  89 +
  90 + @Override
  91 + public void start(ActorContext context) {
83 92 if (!started) {
84 93 RuleChain ruleChain = service.findRuleChainById(entityId);
  94 + ruleChainName = ruleChain.getName();
85 95 List<RuleNode> ruleNodeList = service.getRuleChainNodes(entityId);
  96 + log.trace("[{}][{}] Starting rule chain with {} nodes", tenantId, entityId, ruleNodeList.size());
86 97 // Creating and starting the actors;
87 98 for (RuleNode ruleNode : ruleNodeList) {
  99 + log.trace("[{}][{}] Creating rule node [{}]: {}", entityId, ruleNode.getId(), ruleNode.getName(), ruleNode);
88 100 ActorRef ruleNodeActor = createRuleNodeActor(context, ruleNode);
89 101 nodeActors.put(ruleNode.getId(), new RuleNodeCtx(tenantId, self, ruleNodeActor, ruleNode));
90 102 }
... ... @@ -96,16 +108,19 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
96 108 }
97 109
98 110 @Override
99   - public void onUpdate(ActorContext context) throws Exception {
  111 + public void onUpdate(ActorContext context) {
100 112 RuleChain ruleChain = service.findRuleChainById(entityId);
  113 + ruleChainName = ruleChain.getName();
101 114 List<RuleNode> ruleNodeList = service.getRuleChainNodes(entityId);
102   -
  115 + log.trace("[{}][{}] Updating rule chain with {} nodes", tenantId, entityId, ruleNodeList.size());
103 116 for (RuleNode ruleNode : ruleNodeList) {
104 117 RuleNodeCtx existing = nodeActors.get(ruleNode.getId());
105 118 if (existing == null) {
  119 + log.trace("[{}][{}] Creating rule node [{}]: {}", entityId, ruleNode.getId(), ruleNode.getName(), ruleNode);
106 120 ActorRef ruleNodeActor = createRuleNodeActor(context, ruleNode);
107 121 nodeActors.put(ruleNode.getId(), new RuleNodeCtx(tenantId, self, ruleNodeActor, ruleNode));
108 122 } else {
  123 + log.trace("[{}][{}] Updating rule node [{}]: {}", entityId, ruleNode.getId(), ruleNode.getName(), ruleNode);
109 124 existing.setSelf(ruleNode);
110 125 existing.getSelfActor().tell(new ComponentLifecycleMsg(tenantId, existing.getSelf().getId(), ComponentLifecycleEvent.UPDATED), self);
111 126 }
... ... @@ -114,6 +129,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
114 129 Set<RuleNodeId> existingNodes = ruleNodeList.stream().map(RuleNode::getId).collect(Collectors.toSet());
115 130 List<RuleNodeId> removedRules = nodeActors.keySet().stream().filter(node -> !existingNodes.contains(node)).collect(Collectors.toList());
116 131 removedRules.forEach(ruleNodeId -> {
  132 + log.trace("[{}][{}] Removing rule node [{}]", tenantId, entityId, ruleNodeId);
117 133 RuleNodeCtx removed = nodeActors.remove(ruleNodeId);
118 134 removed.getSelfActor().tell(new ComponentLifecycleMsg(tenantId, removed.getSelf().getId(), ComponentLifecycleEvent.DELETED), self);
119 135 });
... ... @@ -122,7 +138,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
122 138 }
123 139
124 140 @Override
125   - public void stop(ActorContext context) throws Exception {
  141 + public void stop(ActorContext context) {
  142 + log.trace("[{}][{}] Stopping rule chain with {} nodes", tenantId, entityId, nodeActors.size());
126 143 nodeActors.values().stream().map(RuleNodeCtx::getSelfActor).forEach(context::stop);
127 144 nodeActors.clear();
128 145 nodeRoutes.clear();
... ... @@ -131,7 +148,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
131 148 }
132 149
133 150 @Override
134   - public void onClusterEventMsg(ClusterEventMsg msg) throws Exception {
  151 + public void onClusterEventMsg(ClusterEventMsg msg) {
135 152
136 153 }
137 154
... ... @@ -148,10 +165,12 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
148 165 // Populating the routes map;
149 166 for (RuleNode ruleNode : ruleNodeList) {
150 167 List<EntityRelation> relations = service.getRuleNodeRelations(ruleNode.getId());
  168 + log.trace("[{}][{}][{}] Processing rule node relations [{}]", tenantId, entityId, ruleNode.getId(), relations.size());
151 169 if (relations.size() == 0) {
152 170 nodeRoutes.put(ruleNode.getId(), Collections.emptyList());
153 171 } else {
154 172 for (EntityRelation relation : relations) {
  173 + log.trace("[{}][{}][{}] Processing rule node relation [{}]", tenantId, entityId, ruleNode.getId(), relation.getTo());
155 174 if (relation.getTo().getEntityType() == EntityType.RULE_NODE) {
156 175 RuleNodeCtx ruleNodeCtx = nodeActors.get(new RuleNodeId(relation.getTo().getId()));
157 176 if (ruleNodeCtx == null) {
... ... @@ -165,13 +184,15 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
165 184 }
166 185
167 186 firstId = ruleChain.getFirstRuleNodeId();
168   - firstNode = nodeActors.get(ruleChain.getFirstRuleNodeId());
  187 + firstNode = nodeActors.get(firstId);
169 188 state = ComponentLifecycleState.ACTIVE;
170 189 }
171 190
172 191 void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg envelope) {
  192 + log.trace("[{}][{}] Processing message [{}]: {}", entityId, firstId, envelope.getTbMsg().getId(), envelope.getTbMsg());
173 193 checkActive();
174 194 if (firstNode != null) {
  195 + log.trace("[{}][{}] Pushing message to first rule node", entityId, firstId);
175 196 pushMsgToNode(firstNode, enrichWithRuleChainId(envelope.getTbMsg()), "");
176 197 }
177 198 }
... ... @@ -216,7 +237,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
216 237
217 238 private void onRemoteTellNext(ServerAddress serverAddress, RuleNodeToRuleChainTellNextMsg envelope) {
218 239 TbMsg msg = envelope.getMsg();
219   - logger.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator());
  240 + log.debug("Forwarding [{}] msg to remote server [{}] due to changed originator id: [{}]", msg.getId(), serverAddress, msg.getOriginator());
220 241 envelope = new RemoteToRuleChainTellNextMsg(envelope, tenantId, entityId);
221 242 systemContext.getRpcService().tell(systemContext.getEncodingService().convertToProtoDataMessage(serverAddress, envelope));
222 243 }
... ... @@ -230,17 +251,20 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
230 251 int relationsCount = relations.size();
231 252 EntityId ackId = msg.getRuleNodeId() != null ? msg.getRuleNodeId() : msg.getRuleChainId();
232 253 if (relationsCount == 0) {
  254 + log.trace("[{}][{}][{}] No outbound relations to process", tenantId, entityId, msg.getId());
233 255 if (ackId != null) {
234 256 // TODO: Ack this message in Kafka
235 257 // queue.ack(tenantId, msg, ackId.getId(), msg.getClusterPartition());
236 258 }
237 259 } else if (relationsCount == 1) {
238 260 for (RuleNodeRelation relation : relations) {
  261 + log.trace("[{}][{}][{}] Pushing message to single target: [{}]", tenantId, entityId, msg.getId(), relation.getOut());
239 262 pushToTarget(msg, relation.getOut(), relation.getType());
240 263 }
241 264 } else {
242 265 for (RuleNodeRelation relation : relations) {
243 266 EntityId target = relation.getOut();
  267 + log.trace("[{}][{}][{}] Pushing message to multiple targets: [{}]", tenantId, entityId, msg.getId(), relation.getOut());
244 268 switch (target.getEntityType()) {
245 269 case RULE_NODE:
246 270 enqueueAndForwardMsgCopyToNode(msg, target, relation.getType());
... ...
... ... @@ -32,7 +32,7 @@ public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessa
32 32 super(systemContext, tenantId, ruleNodeId);
33 33 this.ruleChainId = ruleChainId;
34 34 setProcessor(new RuleNodeActorMessageProcessor(tenantId, ruleChainId, ruleNodeId, systemContext,
35   - logger, context().parent(), context().self()));
  35 + context().parent(), context().self()));
36 36 }
37 37
38 38 @Override
... ... @@ -60,7 +60,9 @@ public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessa
60 60 }
61 61
62 62 private void onRuleNodeToSelfMsg(RuleNodeToSelfMsg msg) {
63   - logger.debug("[{}] Going to process rule msg: {}", id, msg.getMsg());
  63 + if (log.isDebugEnabled()) {
  64 + log.debug("[{}][{}][{}] Going to process rule msg: {}", ruleChainId, id, processor.getComponentName(), msg.getMsg());
  65 + }
64 66 try {
65 67 processor.onRuleToSelfMsg(msg);
66 68 increaseMessagesProcessedCount();
... ... @@ -70,7 +72,9 @@ public class RuleNodeActor extends ComponentActor<RuleNodeId, RuleNodeActorMessa
70 72 }
71 73
72 74 private void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) {
73   - logger.debug("[{}] Going to process rule msg: {}", id, msg.getMsg());
  75 + if (log.isDebugEnabled()) {
  76 + log.debug("[{}][{}][{}] Going to process rule msg: {}", ruleChainId, id, processor.getComponentName(), msg.getMsg());
  77 + }
74 78 try {
75 79 processor.onRuleChainToRuleNodeMsg(msg);
76 80 increaseMessagesProcessedCount();
... ...
... ... @@ -44,8 +44,8 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
44 44 private TbContext defaultCtx;
45 45
46 46 RuleNodeActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, ActorSystemContext systemContext
47   - , LoggingAdapter logger, ActorRef parent, ActorRef self) {
48   - super(systemContext, logger, tenantId, ruleNodeId);
  47 + , ActorRef parent, ActorRef self) {
  48 + super(systemContext, tenantId, ruleNodeId);
49 49 this.parent = parent;
50 50 this.self = self;
51 51 this.service = systemContext.getRuleChainService();
... ... @@ -75,7 +75,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
75 75 }
76 76
77 77 @Override
78   - public void stop(ActorContext context) throws Exception {
  78 + public void stop(ActorContext context) {
79 79 if (tbNode != null) {
80 80 tbNode.destroy();
81 81 }
... ... @@ -83,7 +83,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
83 83 }
84 84
85 85 @Override
86   - public void onClusterEventMsg(ClusterEventMsg msg) throws Exception {
  86 + public void onClusterEventMsg(ClusterEventMsg msg) {
87 87
88 88 }
89 89
... ... @@ -111,6 +111,11 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
111 111 }
112 112 }
113 113
  114 + @Override
  115 + public String getComponentName() {
  116 + return ruleNode.getName();
  117 + }
  118 +
114 119 private TbNode initComponent(RuleNode ruleNode) throws Exception {
115 120 Class<?> componentClazz = Class.forName(ruleNode.getType());
116 121 TbNode tbNode = (TbNode) (componentClazz.newInstance());
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.actors.service;
18 18 import akka.actor.ActorRef;
19 19 import akka.event.Logging;
20 20 import akka.event.LoggingAdapter;
  21 +import lombok.extern.slf4j.Slf4j;
21 22 import org.thingsboard.server.actors.ActorSystemContext;
22 23 import org.thingsboard.server.actors.shared.ComponentMsgProcessor;
23 24 import org.thingsboard.server.actors.stats.StatsPersistMsg;
... ... @@ -32,8 +33,6 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
32 33 */
33 34 public abstract class ComponentActor<T extends EntityId, P extends ComponentMsgProcessor<T>> extends ContextAwareActor {
34 35
35   - protected final LoggingAdapter logger = Logging.getLogger(getContext().system(), this);
36   -
37 36 private long lastPersistedErrorTs = 0L;
38 37 protected final TenantId tenantId;
39 38 protected final T id;
... ... @@ -54,13 +53,14 @@ public abstract class ComponentActor<T extends EntityId, P extends ComponentMsgP
54 53 @Override
55 54 public void preStart() {
56 55 try {
  56 + log.debug("[{}][{}][{}] Starting processor.", tenantId, id, id.getEntityType());
57 57 processor.start(context());
58 58 logLifecycleEvent(ComponentLifecycleEvent.STARTED);
59 59 if (systemContext.isStatisticsEnabled()) {
60 60 scheduleStatsPersistTick();
61 61 }
62 62 } catch (Exception e) {
63   - logger.warning("[{}][{}] Failed to start {} processor: {}", tenantId, id, id.getEntityType(), e);
  63 + log.warn("[{}][{}] Failed to start {} processor: {}", tenantId, id, id.getEntityType(), e);
64 64 logAndPersist("OnStart", e, true);
65 65 logLifecycleEvent(ComponentLifecycleEvent.STARTED, e);
66 66 }
... ... @@ -70,7 +70,7 @@ public abstract class ComponentActor<T extends EntityId, P extends ComponentMsgP
70 70 try {
71 71 processor.scheduleStatsPersistTick(context(), systemContext.getStatisticsPersistFrequency());
72 72 } catch (Exception e) {
73   - logger.error("[{}][{}] Failed to schedule statistics store message. No statistics is going to be stored: {}", tenantId, id, e.getMessage());
  73 + log.error("[{}][{}] Failed to schedule statistics store message. No statistics is going to be stored: {}", tenantId, id, e.getMessage());
74 74 logAndPersist("onScheduleStatsPersistMsg", e);
75 75 }
76 76 }
... ... @@ -78,16 +78,18 @@ public abstract class ComponentActor<T extends EntityId, P extends ComponentMsgP
78 78 @Override
79 79 public void postStop() {
80 80 try {
  81 + log.debug("[{}][{}] Stopping processor.", tenantId, id, id.getEntityType());
81 82 processor.stop(context());
82 83 logLifecycleEvent(ComponentLifecycleEvent.STOPPED);
83 84 } catch (Exception e) {
84   - logger.warning("[{}][{}] Failed to stop {} processor: {}", tenantId, id, id.getEntityType(), e.getMessage());
  85 + log.warn("[{}][{}] Failed to stop {} processor: {}", tenantId, id, id.getEntityType(), e.getMessage());
85 86 logAndPersist("OnStop", e, true);
86 87 logLifecycleEvent(ComponentLifecycleEvent.STOPPED, e);
87 88 }
88 89 }
89 90
90 91 protected void onComponentLifecycleMsg(ComponentLifecycleMsg msg) {
  92 + log.debug("[{}][{}][{}] onComponentLifecycleMsg: [{}]", tenantId, id, id.getEntityType(), msg.getEvent());
91 93 try {
92 94 switch (msg.getEvent()) {
93 95 case CREATED:
... ... @@ -148,9 +150,9 @@ public abstract class ComponentActor<T extends EntityId, P extends ComponentMsgP
148 150 private void logAndPersist(String method, Exception e, boolean critical) {
149 151 errorsOccurred++;
150 152 if (critical) {
151   - logger.warning("[{}][{}] Failed to process {} msg: {}", id, tenantId, method, e);
  153 + log.warn("[{}][{}][{}] Failed to process {} msg: {}", id, tenantId, processor.getComponentName(), method, e);
152 154 } else {
153   - logger.debug("[{}][{}] Failed to process {} msg: {}", id, tenantId, method, e);
  155 + log.debug("[{}][{}][{}] Failed to process {} msg: {}", id, tenantId, processor.getComponentName(), method, e);
154 156 }
155 157 long ts = System.currentTimeMillis();
156 158 if (ts - lastPersistedErrorTs > getErrorPersistFrequency()) {
... ...
... ... @@ -15,14 +15,17 @@
15 15 */
16 16 package org.thingsboard.server.actors.service;
17 17
  18 +import akka.actor.Terminated;
18 19 import akka.actor.UntypedActor;
19   -import akka.event.Logging;
20   -import akka.event.LoggingAdapter;
  20 +import org.slf4j.Logger;
  21 +import org.slf4j.LoggerFactory;
21 22 import org.thingsboard.server.actors.ActorSystemContext;
22 23 import org.thingsboard.server.common.msg.TbActorMsg;
23 24
  25 +
24 26 public abstract class ContextAwareActor extends UntypedActor {
25   - protected final LoggingAdapter logger = Logging.getLogger(getContext().system(), this);
  27 +
  28 + protected final Logger log = LoggerFactory.getLogger(getClass());
26 29
27 30 public static final int ENTITY_PACK_LIMIT = 1024;
28 31
... ... @@ -34,22 +37,27 @@ public abstract class ContextAwareActor extends UntypedActor {
34 37 }
35 38
36 39 @Override
37   - public void onReceive(Object msg) throws Exception {
38   - if (logger.isDebugEnabled()) {
39   - logger.debug("Processing msg: {}", msg);
  40 + public void onReceive(Object msg) {
  41 + if (log.isDebugEnabled()) {
  42 + log.debug("Processing msg: {}", msg);
40 43 }
41 44 if (msg instanceof TbActorMsg) {
42 45 try {
43 46 if (!process((TbActorMsg) msg)) {
44   - logger.warning("Unknown message: {}!", msg);
  47 + log.warn("Unknown message: {}!", msg);
45 48 }
46 49 } catch (Exception e) {
47 50 throw e;
48 51 }
  52 + } else if (msg instanceof Terminated) {
  53 + processTermination((Terminated) msg);
49 54 } else {
50   - logger.warning("Unknown message: {}!", msg);
  55 + log.warn("Unknown message: {}!", msg);
51 56 }
52 57 }
53 58
  59 + protected void processTermination(Terminated msg) {
  60 + }
  61 +
54 62 protected abstract boolean process(TbActorMsg msg);
55 63 }
... ...
... ... @@ -22,11 +22,14 @@ import akka.actor.Terminated;
22 22 import com.google.protobuf.ByteString;
23 23 import lombok.extern.slf4j.Slf4j;
24 24 import org.springframework.beans.factory.annotation.Autowired;
  25 +import org.springframework.boot.context.event.ApplicationReadyEvent;
  26 +import org.springframework.context.event.EventListener;
25 27 import org.springframework.stereotype.Service;
26 28 import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg;
27 29 import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg;
28 30 import org.thingsboard.server.actors.ActorSystemContext;
29 31 import org.thingsboard.server.actors.app.AppActor;
  32 +import org.thingsboard.server.actors.app.AppInitMsg;
30 33 import org.thingsboard.server.actors.rpc.RpcBroadcastMsg;
31 34 import org.thingsboard.server.actors.rpc.RpcManagerActor;
32 35 import org.thingsboard.server.actors.rpc.RpcSessionCreateRequestMsg;
... ... @@ -54,6 +57,12 @@ import scala.concurrent.duration.Duration;
54 57 import javax.annotation.PostConstruct;
55 58 import javax.annotation.PreDestroy;
56 59
  60 +import java.util.Arrays;
  61 +import java.util.UUID;
  62 +import java.util.concurrent.Executors;
  63 +import java.util.concurrent.ScheduledExecutorService;
  64 +import java.util.concurrent.TimeUnit;
  65 +
57 66 import static org.thingsboard.server.gen.cluster.ClusterAPIProtos.MessageType.CLUSTER_ACTOR_MESSAGE;
58 67
59 68 @Service
... ... @@ -86,6 +95,8 @@ public class DefaultActorService implements ActorService {
86 95
87 96 private ActorRef rpcManagerActor;
88 97
  98 + private ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
  99 +
89 100 @PostConstruct
90 101 public void initActorSystem() {
91 102 log.info("Initializing Actor system. {}", actorContext.getRuleChainService());
... ... @@ -106,6 +117,12 @@ public class DefaultActorService implements ActorService {
106 117 log.info("Actor system initialized.");
107 118 }
108 119
  120 + @EventListener(ApplicationReadyEvent.class)
  121 + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
  122 + log.info("Received application ready event. Sending application init message to actor system");
  123 + appActor.tell(new AppInitMsg(), ActorRef.noSender());
  124 + }
  125 +
109 126 @PreDestroy
110 127 public void stopActorSystem() {
111 128 Future<Terminated> status = system.terminate();
... ...
... ... @@ -22,22 +22,22 @@ import akka.event.LoggingAdapter;
22 22 import com.fasterxml.jackson.databind.ObjectMapper;
23 23 import lombok.AllArgsConstructor;
24 24 import lombok.Data;
  25 +import lombok.extern.slf4j.Slf4j;
25 26 import org.thingsboard.server.actors.ActorSystemContext;
26 27 import scala.concurrent.ExecutionContextExecutor;
27 28 import scala.concurrent.duration.Duration;
28 29
29 30 import java.util.concurrent.TimeUnit;
30 31
  32 +@Slf4j
31 33 public abstract class AbstractContextAwareMsgProcessor {
32 34
33 35 protected final ActorSystemContext systemContext;
34   - protected final LoggingAdapter logger;
35 36 protected final ObjectMapper mapper = new ObjectMapper();
36 37
37   - protected AbstractContextAwareMsgProcessor(ActorSystemContext systemContext, LoggingAdapter logger) {
  38 + protected AbstractContextAwareMsgProcessor(ActorSystemContext systemContext) {
38 39 super();
39 40 this.systemContext = systemContext;
40   - this.logger = logger;
41 41 }
42 42
43 43 private Scheduler getScheduler() {
... ... @@ -53,7 +53,7 @@ public abstract class AbstractContextAwareMsgProcessor {
53 53 }
54 54
55 55 private void schedulePeriodicMsgWithDelay(Object msg, long delayInMs, long periodInMs, ActorRef target) {
56   - logger.debug("Scheduling periodic msg {} every {} ms with delay {} ms", msg, periodInMs, delayInMs);
  56 + log.debug("Scheduling periodic msg {} every {} ms with delay {} ms", msg, periodInMs, delayInMs);
57 57 getScheduler().schedule(Duration.create(delayInMs, TimeUnit.MILLISECONDS), Duration.create(periodInMs, TimeUnit.MILLISECONDS), target, msg, getSystemDispatcher(), null);
58 58 }
59 59
... ... @@ -62,7 +62,7 @@ public abstract class AbstractContextAwareMsgProcessor {
62 62 }
63 63
64 64 private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) {
65   - logger.debug("Scheduling msg {} with delay {} ms", msg, delayInMs);
  65 + log.debug("Scheduling msg {} with delay {} ms", msg, delayInMs);
66 66 getScheduler().scheduleOnce(Duration.create(delayInMs, TimeUnit.MILLISECONDS), target, msg, getSystemDispatcher(), null);
67 67 }
68 68
... ...
... ... @@ -19,6 +19,7 @@ import akka.actor.ActorContext;
19 19 import akka.event.LoggingAdapter;
20 20 import com.google.common.util.concurrent.FutureCallback;
21 21 import com.google.common.util.concurrent.Futures;
  22 +import lombok.extern.slf4j.Slf4j;
22 23 import org.thingsboard.server.actors.ActorSystemContext;
23 24 import org.thingsboard.server.actors.stats.StatsPersistTick;
24 25 import org.thingsboard.server.common.data.id.EntityId;
... ... @@ -30,18 +31,21 @@ import org.thingsboard.server.common.msg.cluster.ClusterEventMsg;
30 31 import javax.annotation.Nullable;
31 32 import java.util.function.Consumer;
32 33
  34 +@Slf4j
33 35 public abstract class ComponentMsgProcessor<T extends EntityId> extends AbstractContextAwareMsgProcessor {
34 36
35 37 protected final TenantId tenantId;
36 38 protected final T entityId;
37 39 protected ComponentLifecycleState state;
38 40
39   - protected ComponentMsgProcessor(ActorSystemContext systemContext, LoggingAdapter logger, TenantId tenantId, T id) {
40   - super(systemContext, logger);
  41 + protected ComponentMsgProcessor(ActorSystemContext systemContext, TenantId tenantId, T id) {
  42 + super(systemContext);
41 43 this.tenantId = tenantId;
42 44 this.entityId = id;
43 45 }
44 46
  47 + public abstract String getComponentName();
  48 +
45 49 public abstract void start(ActorContext context) throws Exception;
46 50
47 51 public abstract void stop(ActorContext context) throws Exception;
... ... @@ -79,7 +83,7 @@ public abstract class ComponentMsgProcessor<T extends EntityId> extends Abstract
79 83
80 84 protected void checkActive() {
81 85 if (state != ComponentLifecycleState.ACTIVE) {
82   - logger.warning("Rule chain is not active. Current state [{}] for processor [{}] tenant [{}]", state, tenantId, entityId);
  86 + log.warn("Rule chain is not active. Current state [{}] for processor [{}] tenant [{}]", state, tenantId, entityId);
83 87 throw new IllegalStateException("Rule chain is not active! " + entityId + " - " + tenantId);
84 88 }
85 89 }
... ...
... ... @@ -20,6 +20,8 @@ import akka.actor.ActorRef;
20 20 import akka.actor.Props;
21 21 import akka.actor.UntypedActor;
22 22 import akka.japi.Creator;
  23 +import com.google.common.collect.BiMap;
  24 +import com.google.common.collect.HashBiMap;
23 25 import lombok.extern.slf4j.Slf4j;
24 26 import org.thingsboard.server.actors.ActorSystemContext;
25 27 import org.thingsboard.server.actors.service.ContextAwareActor;
... ... @@ -39,11 +41,11 @@ import java.util.Map;
39 41 public abstract class EntityActorsManager<T extends EntityId, A extends UntypedActor, M extends SearchTextBased<? extends UUIDBased>> {
40 42
41 43 protected final ActorSystemContext systemContext;
42   - protected final Map<T, ActorRef> actors;
  44 + protected final BiMap<T, ActorRef> actors;
43 45
44 46 public EntityActorsManager(ActorSystemContext systemContext) {
45 47 this.systemContext = systemContext;
46   - this.actors = new HashMap<>();
  48 + this.actors = HashBiMap.create();
47 49 }
48 50
49 51 protected abstract TenantId getTenantId();
... ... @@ -65,7 +67,8 @@ public abstract class EntityActorsManager<T extends EntityId, A extends UntypedA
65 67 }
66 68 }
67 69
68   - public void visit(M entity, ActorRef actorRef) {}
  70 + public void visit(M entity, ActorRef actorRef) {
  71 + }
69 72
70 73 public ActorRef getOrCreateActor(ActorContext context, T entityId) {
71 74 return actors.computeIfAbsent(entityId, eId ->
... ...
... ... @@ -15,10 +15,9 @@
15 15 */
16 16 package org.thingsboard.server.actors.stats;
17 17
18   -import akka.event.Logging;
19   -import akka.event.LoggingAdapter;
20 18 import com.fasterxml.jackson.databind.JsonNode;
21 19 import com.fasterxml.jackson.databind.ObjectMapper;
  20 +import lombok.extern.slf4j.Slf4j;
22 21 import org.thingsboard.server.actors.ActorSystemContext;
23 22 import org.thingsboard.server.actors.service.ContextAwareActor;
24 23 import org.thingsboard.server.actors.service.ContextBasedCreator;
... ... @@ -27,9 +26,9 @@ import org.thingsboard.server.common.data.Event;
27 26 import org.thingsboard.server.common.msg.TbActorMsg;
28 27 import org.thingsboard.server.common.msg.cluster.ServerAddress;
29 28
  29 +@Slf4j
30 30 public class StatsActor extends ContextAwareActor {
31 31
32   - private final LoggingAdapter logger = Logging.getLogger(getContext().system(), this);
33 32 private final ObjectMapper mapper = new ObjectMapper();
34 33
35 34 public StatsActor(ActorSystemContext context) {
... ... @@ -43,13 +42,13 @@ public class StatsActor extends ContextAwareActor {
43 42 }
44 43
45 44 @Override
46   - public void onReceive(Object msg) throws Exception {
47   - logger.debug("Received message: {}", msg);
  45 + public void onReceive(Object msg) {
  46 + log.debug("Received message: {}", msg);
48 47 if (msg instanceof StatsPersistMsg) {
49 48 try {
50 49 onStatsPersistMsg((StatsPersistMsg) msg);
51 50 } catch (Exception e) {
52   - logger.warning("Failed to persist statistics: {}", msg, e);
  51 + log.warn("Failed to persist statistics: {}", msg, e);
53 52 }
54 53 }
55 54 }
... ... @@ -75,7 +74,7 @@ public class StatsActor extends ContextAwareActor {
75 74 }
76 75
77 76 @Override
78   - public StatsActor create() throws Exception {
  77 + public StatsActor create() {
79 78 return new StatsActor(context);
80 79 }
81 80 }
... ...
... ... @@ -17,15 +17,19 @@ package org.thingsboard.server.actors.tenant;
17 17
18 18 import akka.actor.ActorInitializationException;
19 19 import akka.actor.ActorRef;
  20 +import akka.actor.LocalActorRef;
20 21 import akka.actor.OneForOneStrategy;
21 22 import akka.actor.Props;
22 23 import akka.actor.SupervisorStrategy;
  24 +import akka.actor.Terminated;
23 25 import akka.japi.Function;
  26 +import com.google.common.collect.BiMap;
  27 +import com.google.common.collect.HashBiMap;
  28 +import lombok.extern.slf4j.Slf4j;
24 29 import org.thingsboard.server.actors.ActorSystemContext;
25   -import org.thingsboard.server.actors.device.DeviceActor;
  30 +import org.thingsboard.server.actors.device.DeviceActorCreator;
26 31 import org.thingsboard.server.actors.device.DeviceActorToRuleEngineMsg;
27 32 import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor;
28   -import org.thingsboard.server.actors.ruleChain.RuleChainToRuleChainMsg;
29 33 import org.thingsboard.server.actors.service.ContextBasedCreator;
30 34 import org.thingsboard.server.actors.service.DefaultActorService;
31 35 import org.thingsboard.server.actors.shared.rulechain.TenantRuleChainManager;
... ... @@ -33,6 +37,7 @@ import org.thingsboard.server.common.data.EntityType;
33 37 import org.thingsboard.server.common.data.id.DeviceId;
34 38 import org.thingsboard.server.common.data.id.RuleChainId;
35 39 import org.thingsboard.server.common.data.id.TenantId;
  40 +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
36 41 import org.thingsboard.server.common.data.rule.RuleChain;
37 42 import org.thingsboard.server.common.msg.TbActorMsg;
38 43 import org.thingsboard.server.common.msg.aware.DeviceAwareMsg;
... ... @@ -47,15 +52,14 @@ import java.util.Map;
47 52 public class TenantActor extends RuleChainManagerActor {
48 53
49 54 private final TenantId tenantId;
50   - private final Map<DeviceId, ActorRef> deviceActors;
  55 + private final BiMap<DeviceId, ActorRef> deviceActors;
51 56
52 57 private TenantActor(ActorSystemContext systemContext, TenantId tenantId) {
53 58 super(systemContext, new TenantRuleChainManager(systemContext, tenantId));
54 59 this.tenantId = tenantId;
55   - this.deviceActors = new HashMap<>();
  60 + this.deviceActors = HashBiMap.create();
56 61 }
57 62
58   -
59 63 @Override
60 64 public SupervisorStrategy supervisorStrategy() {
61 65 return strategy;
... ... @@ -63,16 +67,21 @@ public class TenantActor extends RuleChainManagerActor {
63 67
64 68 @Override
65 69 public void preStart() {
66   - logger.info("[{}] Starting tenant actor.", tenantId);
  70 + log.info("[{}] Starting tenant actor.", tenantId);
67 71 try {
68 72 initRuleChains();
69   - logger.info("[{}] Tenant actor started.", tenantId);
  73 + log.info("[{}] Tenant actor started.", tenantId);
70 74 } catch (Exception e) {
71   - logger.error(e, "[{}] Unknown failure", tenantId);
  75 + log.warn("[{}] Unknown failure", tenantId, e);
72 76 }
73 77 }
74 78
75 79 @Override
  80 + public void postStop() {
  81 + log.info("[{}] Stopping tenant actor.", tenantId);
  82 + }
  83 +
  84 + @Override
76 85 protected boolean process(TbActorMsg msg) {
77 86 switch (msg.getMsgType()) {
78 87 case CLUSTER_EVENT_MSG:
... ... @@ -105,22 +114,20 @@ public class TenantActor extends RuleChainManagerActor {
105 114 return true;
106 115 }
107 116
108   - @Override
109   - protected void broadcast(Object msg) {
110   - super.broadcast(msg);
111   -// deviceActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender()));
112   - }
113   -
114 117 private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) {
115   - if (ruleChainManager.getRootChainActor()!=null)
116   - ruleChainManager.getRootChainActor().tell(msg, self());
117   - else logger.info("[{}] No Root Chain", msg);
  118 + if (ruleChainManager.getRootChainActor() != null) {
  119 + ruleChainManager.getRootChainActor().tell(msg, self());
  120 + } else {
  121 + log.info("[{}] No Root Chain: {}", tenantId, msg);
  122 + }
118 123 }
119 124
120 125 private void onDeviceActorToRuleEngineMsg(DeviceActorToRuleEngineMsg msg) {
121   - if (ruleChainManager.getRootChainActor()!=null)
122   - ruleChainManager.getRootChainActor().tell(msg, self());
123   - else logger.info("[{}] No Root Chain", msg);
  126 + if (ruleChainManager.getRootChainActor() != null) {
  127 + ruleChainManager.getRootChainActor().tell(msg, self());
  128 + } else {
  129 + log.info("[{}] No Root Chain: {}", tenantId, msg);
  130 + }
124 131 }
125 132
126 133 private void onRuleChainMsg(RuleChainAwareMsg msg) {
... ... @@ -141,13 +148,35 @@ public class TenantActor extends RuleChainManagerActor {
141 148 }
142 149 target.tell(msg, ActorRef.noSender());
143 150 } else {
144   - logger.debug("Invalid component lifecycle msg: {}", msg);
  151 + log.debug("[{}] Invalid component lifecycle msg: {}", tenantId, msg);
145 152 }
146 153 }
147 154
148 155 private ActorRef getOrCreateDeviceActor(DeviceId deviceId) {
149   - return deviceActors.computeIfAbsent(deviceId, k -> context().actorOf(Props.create(new DeviceActor.ActorCreator(systemContext, tenantId, deviceId))
150   - .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME), deviceId.toString()));
  156 + return deviceActors.computeIfAbsent(deviceId, k -> {
  157 + log.debug("[{}][{}] Creating device actor.", tenantId, deviceId);
  158 + ActorRef deviceActor = context().actorOf(Props.create(new DeviceActorCreator(systemContext, tenantId, deviceId))
  159 + .withDispatcher(DefaultActorService.CORE_DISPATCHER_NAME)
  160 + , deviceId.toString());
  161 + context().watch(deviceActor);
  162 + log.debug("[{}][{}] Created device actor: {}.", tenantId, deviceId, deviceActor);
  163 + return deviceActor;
  164 + });
  165 + }
  166 +
  167 + @Override
  168 + protected void processTermination(Terminated message) {
  169 + ActorRef terminated = message.actor();
  170 + if (terminated instanceof LocalActorRef) {
  171 + boolean removed = deviceActors.inverse().remove(terminated) != null;
  172 + if (removed) {
  173 + log.debug("[{}] Removed actor:", terminated);
  174 + } else {
  175 + log.warn("[{}] Removed actor was not found in the device map!");
  176 + }
  177 + } else {
  178 + throw new IllegalStateException("Remote actors are not supported!");
  179 + }
151 180 }
152 181
153 182 public static class ActorCreator extends ContextBasedCreator<TenantActor> {
... ... @@ -161,7 +190,7 @@ public class TenantActor extends RuleChainManagerActor {
161 190 }
162 191
163 192 @Override
164   - public TenantActor create() throws Exception {
  193 + public TenantActor create() {
165 194 return new TenantActor(context, tenantId);
166 195 }
167 196 }
... ... @@ -169,8 +198,8 @@ public class TenantActor extends RuleChainManagerActor {
169 198 private final SupervisorStrategy strategy = new OneForOneStrategy(3, Duration.create("1 minute"), new Function<Throwable, SupervisorStrategy.Directive>() {
170 199 @Override
171 200 public SupervisorStrategy.Directive apply(Throwable t) {
172   - logger.error(t, "Unknown failure");
173   - if(t instanceof ActorInitializationException){
  201 + log.warn("[{}] Unknown failure", tenantId, t);
  202 + if (t instanceof ActorInitializationException) {
174 203 return SupervisorStrategy.stop();
175 204 } else {
176 205 return SupervisorStrategy.resume();
... ...
... ... @@ -19,9 +19,11 @@ import com.datastax.driver.core.utils.UUIDs;
19 19 import com.fasterxml.jackson.databind.ObjectMapper;
20 20 import com.fasterxml.jackson.databind.node.ArrayNode;
21 21 import com.fasterxml.jackson.databind.node.ObjectNode;
  22 +import lombok.Getter;
22 23 import lombok.extern.slf4j.Slf4j;
23 24 import org.apache.commons.lang3.StringUtils;
24 25 import org.springframework.beans.factory.annotation.Autowired;
  26 +import org.springframework.beans.factory.annotation.Value;
25 27 import org.springframework.security.core.Authentication;
26 28 import org.springframework.security.core.context.SecurityContextHolder;
27 29 import org.springframework.web.bind.annotation.ExceptionHandler;
... ... @@ -152,6 +154,11 @@ public abstract class BaseController {
152 154 @Autowired
153 155 protected AttributesService attributesService;
154 156
  157 + @Value("${server.log_controller_error_stack_trace}")
  158 + @Getter
  159 + private boolean logControllerErrorStackTrace;
  160 +
  161 +
155 162 @ExceptionHandler(ThingsboardException.class)
156 163 public void handleThingsboardException(ThingsboardException ex, HttpServletResponse response) {
157 164 errorResponseHandler.handle(ex, response);
... ... @@ -162,7 +169,7 @@ public abstract class BaseController {
162 169 }
163 170
164 171 private ThingsboardException handleException(Exception exception, boolean logException) {
165   - if (logException) {
  172 + if (logException && logControllerErrorStackTrace) {
166 173 log.error("Error [{}]", exception.getMessage(), exception);
167 174 }
168 175
... ...
... ... @@ -52,7 +52,6 @@ public class DashboardController extends BaseController {
52 52 public static final String DASHBOARD_ID = "dashboardId";
53 53
54 54 @Value("${dashboard.max_datapoints_limit}")
55   - @Getter
56 55 private long maxDatapointsLimit;
57 56
58 57
... ...
... ... @@ -29,7 +29,6 @@ import org.springframework.web.bind.annotation.RequestParam;
29 29 import org.springframework.web.bind.annotation.ResponseBody;
30 30 import org.springframework.web.bind.annotation.ResponseStatus;
31 31 import org.springframework.web.bind.annotation.RestController;
32   -import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg;
33 32 import org.thingsboard.server.common.data.Customer;
34 33 import org.thingsboard.server.common.data.DataConstants;
35 34 import org.thingsboard.server.common.data.EntitySubtype;
... ... @@ -39,7 +38,6 @@ import org.thingsboard.server.common.data.audit.ActionType;
39 38 import org.thingsboard.server.common.data.entityview.EntityViewSearchQuery;
40 39 import org.thingsboard.server.common.data.exception.ThingsboardException;
41 40 import org.thingsboard.server.common.data.id.CustomerId;
42   -import org.thingsboard.server.common.data.id.DeviceId;
43 41 import org.thingsboard.server.common.data.id.EntityId;
44 42 import org.thingsboard.server.common.data.id.EntityViewId;
45 43 import org.thingsboard.server.common.data.id.TenantId;
... ... @@ -47,7 +45,6 @@ import org.thingsboard.server.common.data.id.UUIDBased;
47 45 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
48 46 import org.thingsboard.server.common.data.page.TextPageData;
49 47 import org.thingsboard.server.common.data.page.TextPageLink;
50   -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
51 48 import org.thingsboard.server.dao.exception.IncorrectParameterException;
52 49 import org.thingsboard.server.dao.model.ModelConstants;
53 50 import org.thingsboard.server.service.security.model.SecurityUser;
... ... @@ -174,7 +171,7 @@ public class EntityViewController extends BaseController {
174 171 EntityView entityView = checkEntityViewId(entityViewId);
175 172 entityViewService.deleteEntityView(entityViewId);
176 173 logEntityAction(entityViewId, entityView, entityView.getCustomerId(),
177   - ActionType.DELETED,null, strEntityViewId);
  174 + ActionType.DELETED, null, strEntityViewId);
178 175 } catch (Exception e) {
179 176 logEntityAction(emptyId(EntityType.ENTITY_VIEW),
180 177 null,
... ... @@ -185,10 +182,23 @@ public class EntityViewController extends BaseController {
185 182 }
186 183
187 184 @PreAuthorize("hasAuthority('TENANT_ADMIN')")
  185 + @RequestMapping(value = "/tenant/entityViews", params = {"entityViewName"}, method = RequestMethod.GET)
  186 + @ResponseBody
  187 + public EntityView getTenantEntityView(
  188 + @RequestParam String entityViewName) throws ThingsboardException {
  189 + try {
  190 + TenantId tenantId = getCurrentUser().getTenantId();
  191 + return checkNotNull(entityViewService.findEntityViewByTenantIdAndName(tenantId, entityViewName));
  192 + } catch (Exception e) {
  193 + throw handleException(e);
  194 + }
  195 + }
  196 +
  197 + @PreAuthorize("hasAuthority('TENANT_ADMIN')")
188 198 @RequestMapping(value = "/customer/{customerId}/entityView/{entityViewId}", method = RequestMethod.POST)
189 199 @ResponseBody
190 200 public EntityView assignEntityViewToCustomer(@PathVariable(CUSTOMER_ID) String strCustomerId,
191   - @PathVariable(ENTITY_VIEW_ID) String strEntityViewId) throws ThingsboardException {
  201 + @PathVariable(ENTITY_VIEW_ID) String strEntityViewId) throws ThingsboardException {
192 202 checkParameter(CUSTOMER_ID, strCustomerId);
193 203 checkParameter(ENTITY_VIEW_ID, strEntityViewId);
194 204 try {
... ...
... ... @@ -49,9 +49,11 @@ import org.thingsboard.server.common.data.kv.Aggregation;
49 49 import org.thingsboard.server.common.data.kv.AttributeKey;
50 50 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
51 51 import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
  52 +import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery;
52 53 import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
53 54 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
54 55 import org.thingsboard.server.common.data.kv.BooleanDataEntry;
  56 +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
55 57 import org.thingsboard.server.common.data.kv.DoubleDataEntry;
56 58 import org.thingsboard.server.common.data.kv.KvEntry;
57 59 import org.thingsboard.server.common.data.kv.LongDataEntry;
... ... @@ -60,12 +62,10 @@ import org.thingsboard.server.common.data.kv.StringDataEntry;
60 62 import org.thingsboard.server.common.data.kv.TsKvEntry;
61 63 import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
62 64 import org.thingsboard.server.common.transport.adaptor.JsonConverter;
63   -import org.thingsboard.server.dao.attributes.AttributesService;
64 65 import org.thingsboard.server.dao.timeseries.TimeseriesService;
65 66 import org.thingsboard.server.service.security.AccessValidator;
66 67 import org.thingsboard.server.service.security.model.SecurityUser;
67 68 import org.thingsboard.server.service.telemetry.AttributeData;
68   -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
69 69 import org.thingsboard.server.service.telemetry.TsData;
70 70 import org.thingsboard.server.service.telemetry.exception.InvalidParametersException;
71 71 import org.thingsboard.server.service.telemetry.exception.UncheckedApiException;
... ... @@ -250,6 +250,60 @@ public class TelemetryController extends BaseController {
250 250 }
251 251
252 252 @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
  253 + @RequestMapping(value = "/{entityType}/{entityId}/timeseries/delete", method = RequestMethod.DELETE)
  254 + @ResponseBody
  255 + public DeferredResult<ResponseEntity> deleteEntityTimeseries(@PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr,
  256 + @RequestParam(name = "keys") String keysStr,
  257 + @RequestParam(name = "deleteAllDataForKeys", defaultValue = "false") boolean deleteAllDataForKeys,
  258 + @RequestParam(name = "startTs", required = false) Long startTs,
  259 + @RequestParam(name = "endTs", required = false) Long endTs,
  260 + @RequestParam(name = "rewriteLatestIfDeleted", defaultValue = "false") boolean rewriteLatestIfDeleted) throws ThingsboardException {
  261 + EntityId entityId = EntityIdFactory.getByTypeAndId(entityType, entityIdStr);
  262 + return deleteTimeseries(entityId, keysStr, deleteAllDataForKeys, startTs, endTs, rewriteLatestIfDeleted);
  263 + }
  264 +
  265 + private DeferredResult<ResponseEntity> deleteTimeseries(EntityId entityIdStr, String keysStr, boolean deleteAllDataForKeys,
  266 + Long startTs, Long endTs, boolean rewriteLatestIfDeleted) throws ThingsboardException {
  267 + List<String> keys = toKeysList(keysStr);
  268 + if (keys.isEmpty()) {
  269 + return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST);
  270 + }
  271 + SecurityUser user = getCurrentUser();
  272 +
  273 + long deleteFromTs;
  274 + long deleteToTs;
  275 + if (deleteAllDataForKeys) {
  276 + deleteFromTs = 0L;
  277 + deleteToTs = System.currentTimeMillis();
  278 + } else {
  279 + deleteFromTs = startTs;
  280 + deleteToTs = endTs;
  281 + }
  282 +
  283 + return accessValidator.validateEntityAndCallback(user, entityIdStr, (result, entityId) -> {
  284 + List<DeleteTsKvQuery> deleteTsKvQueries = new ArrayList<>();
  285 + for (String key : keys) {
  286 + deleteTsKvQueries.add(new BaseDeleteTsKvQuery(key, deleteFromTs, deleteToTs, rewriteLatestIfDeleted));
  287 + }
  288 +
  289 + ListenableFuture<List<Void>> future = tsService.remove(entityId, deleteTsKvQueries);
  290 + Futures.addCallback(future, new FutureCallback<List<Void>>() {
  291 + @Override
  292 + public void onSuccess(@Nullable List<Void> tmp) {
  293 + logTimeseriesDeleted(user, entityId, keys, null);
  294 + result.setResult(new ResponseEntity<>(HttpStatus.OK));
  295 + }
  296 +
  297 + @Override
  298 + public void onFailure(Throwable t) {
  299 + logTimeseriesDeleted(user, entityId, keys, t);
  300 + result.setResult(new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR));
  301 + }
  302 + }, executor);
  303 + });
  304 + }
  305 +
  306 + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
253 307 @RequestMapping(value = "/{deviceId}/{scope}", method = RequestMethod.DELETE)
254 308 @ResponseBody
255 309 public DeferredResult<ResponseEntity> deleteEntityAttributes(@PathVariable("deviceId") String deviceIdStr,
... ... @@ -506,6 +560,15 @@ public class TelemetryController extends BaseController {
506 560 };
507 561 }
508 562
  563 + private void logTimeseriesDeleted(SecurityUser user, EntityId entityId, List<String> keys, Throwable e) {
  564 + try {
  565 + logEntityAction(user, (UUIDBased & EntityId) entityId, null, null, ActionType.TIMESERIES_DELETED, toException(e),
  566 + keys);
  567 + } catch (ThingsboardException te) {
  568 + log.warn("Failed to log timeseries delete", te);
  569 + }
  570 + }
  571 +
509 572 private void logAttributesDeleted(SecurityUser user, EntityId entityId, String scope, List<String> keys, Throwable e) {
510 573 try {
511 574 logEntityAction(user, (UUIDBased & EntityId) entityId, null, null, ActionType.ATTRIBUTES_DELETED, toException(e),
... ...
... ... @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
32 32 import org.thingsboard.server.common.data.id.TenantId;
33 33 import org.thingsboard.server.common.data.page.TextPageData;
34 34 import org.thingsboard.server.common.data.page.TextPageLink;
  35 +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
35 36 import org.thingsboard.server.dao.tenant.TenantService;
36 37 import org.thingsboard.server.service.install.InstallScripts;
37 38
... ... @@ -84,6 +85,8 @@ public class TenantController extends BaseController {
84 85 try {
85 86 TenantId tenantId = new TenantId(toUUID(strTenantId));
86 87 tenantService.deleteTenant(tenantId);
  88 +
  89 + actorService.onEntityStateChange(tenantId, tenantId, ComponentLifecycleEvent.DELETED);
87 90 } catch (Exception e) {
88 91 throw handleException(e);
89 92 }
... ...
... ... @@ -21,6 +21,7 @@ import org.apache.commons.lang3.SerializationException;
21 21 import org.apache.commons.lang3.SerializationUtils;
22 22 import org.apache.curator.framework.CuratorFramework;
23 23 import org.apache.curator.framework.CuratorFrameworkFactory;
  24 +import org.apache.curator.framework.imps.CuratorFrameworkState;
24 25 import org.apache.curator.framework.recipes.cache.ChildData;
25 26 import org.apache.curator.framework.recipes.cache.PathChildrenCache;
26 27 import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
... ... @@ -30,12 +31,14 @@ import org.apache.curator.framework.state.ConnectionStateListener;
30 31 import org.apache.curator.retry.RetryForever;
31 32 import org.apache.curator.utils.CloseableUtils;
32 33 import org.apache.zookeeper.CreateMode;
  34 +import org.apache.zookeeper.KeeperException;
33 35 import org.springframework.beans.factory.annotation.Autowired;
34 36 import org.springframework.beans.factory.annotation.Value;
35 37 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
36 38 import org.springframework.boot.context.event.ApplicationReadyEvent;
37 39 import org.springframework.context.ApplicationListener;
38 40 import org.springframework.context.annotation.Lazy;
  41 +import org.springframework.context.event.EventListener;
39 42 import org.springframework.stereotype.Service;
40 43 import org.springframework.util.Assert;
41 44 import org.thingsboard.server.actors.service.ActorService;
... ... @@ -51,13 +54,15 @@ import java.util.List;
51 54 import java.util.NoSuchElementException;
52 55 import java.util.stream.Collectors;
53 56
  57 +import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED;
  58 +
54 59 /**
55 60 * @author Andrew Shvayka
56 61 */
57 62 @Service
58 63 @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true", matchIfMissing = false)
59 64 @Slf4j
60   -public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheListener, ApplicationListener<ApplicationReadyEvent> {
  65 +public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheListener {
61 66
62 67 @Value("${zk.url}")
63 68 private String zkUrl;
... ... @@ -95,6 +100,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
95 100 private PathChildrenCache cache;
96 101 private String nodePath;
97 102
  103 + private volatile boolean stopped = false;
  104 +
98 105 @PostConstruct
99 106 public void init() {
100 107 log.info("Initializing...");
... ... @@ -115,6 +122,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
115 122 cache.start();
116 123 } catch (Exception e) {
117 124 log.error("Failed to connect to ZK: {}", e.getMessage(), e);
  125 + CloseableUtils.closeQuietly(cache);
118 126 CloseableUtils.closeQuietly(client);
119 127 throw new RuntimeException(e);
120 128 }
... ... @@ -122,25 +130,50 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
122 130
123 131 @PreDestroy
124 132 public void destroy() {
  133 + stopped = true;
125 134 unpublishCurrentServer();
  135 + CloseableUtils.closeQuietly(cache);
126 136 CloseableUtils.closeQuietly(client);
127 137 log.info("Stopped discovery service");
128 138 }
129 139
130 140 @Override
131   - public void publishCurrentServer() {
  141 + public synchronized void publishCurrentServer() {
  142 + ServerInstance self = this.serverInstance.getSelf();
  143 + if (currentServerExists()) {
  144 + log.info("[{}:{}] ZK node for current instance already exists, NOT created new one: {}", self.getHost(), self.getPort(), nodePath);
  145 + } else {
  146 + try {
  147 + log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort());
  148 + nodePath = client.create()
  149 + .creatingParentsIfNeeded()
  150 + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress()));
  151 + log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath);
  152 + client.getConnectionStateListenable().addListener(checkReconnect(self));
  153 + } catch (Exception e) {
  154 + log.error("Failed to create ZK node", e);
  155 + throw new RuntimeException(e);
  156 + }
  157 + }
  158 + }
  159 +
  160 + private boolean currentServerExists() {
  161 + if (nodePath == null) {
  162 + return false;
  163 + }
132 164 try {
133 165 ServerInstance self = this.serverInstance.getSelf();
134   - log.info("[{}:{}] Creating ZK node for current instance", self.getHost(), self.getPort());
135   - nodePath = client.create()
136   - .creatingParentsIfNeeded()
137   - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(zkNodesDir + "/", SerializationUtils.serialize(self.getServerAddress()));
138   - log.info("[{}:{}] Created ZK node for current instance: {}", self.getHost(), self.getPort(), nodePath);
139   - client.getConnectionStateListenable().addListener(checkReconnect(self));
  166 + ServerAddress registeredServerAdress = null;
  167 + registeredServerAdress = SerializationUtils.deserialize(client.getData().forPath(nodePath));
  168 + if (self.getServerAddress() != null && self.getServerAddress().equals(registeredServerAdress)) {
  169 + return true;
  170 + }
  171 + } catch (KeeperException.NoNodeException e) {
  172 + log.info("ZK node does not exist: {}", nodePath);
140 173 } catch (Exception e) {
141   - log.error("Failed to create ZK node", e);
142   - throw new RuntimeException(e);
  174 + log.error("Couldn't check if ZK node exists", e);
143 175 }
  176 + return false;
144 177 }
145 178
146 179 private ConnectionStateListener checkReconnect(ServerInstance self) {
... ... @@ -200,8 +233,17 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
200 233 .collect(Collectors.toList());
201 234 }
202 235
203   - @Override
  236 + @EventListener(ApplicationReadyEvent.class)
204 237 public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
  238 + log.info("Received application ready event. Starting current ZK node.");
  239 + if (stopped) {
  240 + log.debug("Ignoring application ready event. Service is stopped.");
  241 + return;
  242 + }
  243 + if (client.getState() != CuratorFrameworkState.STARTED) {
  244 + log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState());
  245 + return;
  246 + }
205 247 publishCurrentServer();
206 248 getOtherServers().forEach(
207 249 server -> log.info("Found active server: [{}:{}]", server.getHost(), server.getPort())
... ... @@ -210,6 +252,14 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
210 252
211 253 @Override
212 254 public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception {
  255 + if (stopped) {
  256 + log.debug("Ignoring {}. Service is stopped.", pathChildrenCacheEvent);
  257 + return;
  258 + }
  259 + if (client.getState() != CuratorFrameworkState.STARTED) {
  260 + log.debug("Ignoring {}, ZK client is not started, ZK client state [{}]", pathChildrenCacheEvent, client.getState());
  261 + return;
  262 + }
213 263 ChildData data = pathChildrenCacheEvent.getData();
214 264 if (data == null) {
215 265 log.debug("Ignoring {} due to empty child data", pathChildrenCacheEvent);
... ... @@ -218,6 +268,10 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
218 268 log.debug("Ignoring {} due to empty child's data", pathChildrenCacheEvent);
219 269 return;
220 270 } else if (nodePath != null && nodePath.equals(data.getPath())) {
  271 + if (pathChildrenCacheEvent.getType() == CHILD_REMOVED) {
  272 + log.info("ZK node for current instance is somehow deleted.");
  273 + publishCurrentServer();
  274 + }
221 275 log.debug("Ignoring event about current server {}", pathChildrenCacheEvent);
222 276 return;
223 277 }
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.executors;
  17 +
  18 +import org.springframework.beans.factory.annotation.Value;
  19 +import org.springframework.stereotype.Component;
  20 +
  21 +@Component
  22 +public class ClusterRpcCallbackExecutorService extends AbstractListeningExecutor {
  23 +
  24 + @Value("${actors.cluster.grpc_callback_thread_pool_size}")
  25 + private int grpcCallbackExecutorThreadPoolSize;
  26 +
  27 + @Override
  28 + protected int getThreadPollSize() {
  29 + return grpcCallbackExecutorThreadPoolSize;
  30 + }
  31 +
  32 +}
... ...
... ... @@ -71,7 +71,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
71 71 private int maxErrors;
72 72
73 73 private TbKafkaRequestTemplate<JsInvokeProtos.RemoteJsRequest, JsInvokeProtos.RemoteJsResponse> kafkaTemplate;
74   - protected Map<UUID, String> scriptIdToBodysMap = new ConcurrentHashMap<>();
  74 + private Map<UUID, String> scriptIdToBodysMap = new ConcurrentHashMap<>();
75 75
76 76 @PostConstruct
77 77 public void init() {
... ... @@ -100,7 +100,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
100 100 responseBuilder.settings(kafkaSettings);
101 101 responseBuilder.topic(responseTopicPrefix + "." + nodeIdProvider.getNodeId());
102 102 responseBuilder.clientId("js-" + nodeIdProvider.getNodeId());
103   - responseBuilder.groupId("rule-engine-node");
  103 + responseBuilder.groupId("rule-engine-node-" + nodeIdProvider.getNodeId());
104 104 responseBuilder.autoCommit(true);
105 105 responseBuilder.autoCommitIntervalMs(autoCommitInterval);
106 106 responseBuilder.decoder(new RemoteJsResponseDecoder());
... ... @@ -136,6 +136,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
136 136 .setCompileRequest(jsRequest)
137 137 .build();
138 138
  139 + log.trace("Post compile request for scriptId [{}]", scriptId);
139 140 ListenableFuture<JsInvokeProtos.RemoteJsResponse> future = kafkaTemplate.post(scriptId.toString(), jsRequestWrapper);
140 141 return Futures.transform(future, response -> {
141 142 JsInvokeProtos.JsCompileResponse compilationResult = response.getCompileResponse();
... ...
... ... @@ -22,6 +22,7 @@ import io.jsonwebtoken.Jwts;
22 22 import io.jsonwebtoken.MalformedJwtException;
23 23 import io.jsonwebtoken.SignatureException;
24 24 import io.jsonwebtoken.UnsupportedJwtException;
  25 +import lombok.extern.slf4j.Slf4j;
25 26 import org.slf4j.Logger;
26 27 import org.slf4j.LoggerFactory;
27 28 import org.springframework.security.authentication.BadCredentialsException;
... ... @@ -29,12 +30,11 @@ import org.thingsboard.server.service.security.exception.JwtExpiredTokenExceptio
29 30
30 31 import java.io.Serializable;
31 32
  33 +@Slf4j
32 34 public class RawAccessJwtToken implements JwtToken, Serializable {
33 35
34 36 private static final long serialVersionUID = -797397445703066079L;
35 37
36   - private static Logger logger = LoggerFactory.getLogger(RawAccessJwtToken.class);
37   -
38 38 private String token;
39 39
40 40 public RawAccessJwtToken(String token) {
... ... @@ -52,10 +52,10 @@ public class RawAccessJwtToken implements JwtToken, Serializable {
52 52 try {
53 53 return Jwts.parser().setSigningKey(signingKey).parseClaimsJws(this.token);
54 54 } catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException | SignatureException ex) {
55   - logger.error("Invalid JWT Token", ex);
  55 + log.error("Invalid JWT Token", ex);
56 56 throw new BadCredentialsException("Invalid JWT token: ", ex);
57 57 } catch (ExpiredJwtException expiredEx) {
58   - logger.info("JWT Token is expired", expiredEx);
  58 + log.info("JWT Token is expired", expiredEx);
59 59 throw new JwtExpiredTokenException(this, "JWT Token expired", expiredEx);
60 60 }
61 61 }
... ...
... ... @@ -22,8 +22,8 @@ import org.springframework.stereotype.Service;
22 22 import org.thingsboard.server.common.data.id.DeviceId;
23 23 import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry;
24 24
25   -import java.util.ArrayList;
26 25 import java.util.Collections;
  26 +import java.util.UUID;
27 27
28 28 import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE;
29 29
... ... @@ -35,16 +35,23 @@ import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE;
35 35 public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService {
36 36
37 37 @Override
38   - @Cacheable(cacheNames = SESSIONS_CACHE, key = "#deviceId")
39   - public DeviceSessionsCacheEntry get(DeviceId deviceId) {
  38 + @Cacheable(cacheNames = SESSIONS_CACHE, key = "#deviceId.toString()")
  39 + public byte[] get(DeviceId deviceId) {
40 40 log.debug("[{}] Fetching session data from cache", deviceId);
41   - return DeviceSessionsCacheEntry.newBuilder().addAllSessions(Collections.emptyList()).build();
  41 + return DeviceSessionsCacheEntry.newBuilder().addAllSessions(Collections.emptyList()).build().toByteArray();
42 42 }
43 43
44 44 @Override
45   - @CachePut(cacheNames = SESSIONS_CACHE, key = "#deviceId")
46   - public DeviceSessionsCacheEntry put(DeviceId deviceId, DeviceSessionsCacheEntry sessions) {
47   - log.debug("[{}] Pushing session data from cache: {}", deviceId, sessions);
  45 + @CachePut(cacheNames = SESSIONS_CACHE, key = "#deviceId.toString()")
  46 + public byte[] put(DeviceId deviceId, byte[] sessions) {
  47 + log.debug("[{}] Pushing session data to cache: {}", deviceId, sessions);
48 48 return sessions;
49 49 }
  50 +
  51 + public static void main (String[] args){
  52 + UUID uuid = UUID.fromString("d5db434e-9cd2-4903-8b3b-421b2d93664d");
  53 + System.out.println(uuid.getMostSignificantBits());
  54 + System.out.println(uuid.getLeastSignificantBits());
  55 + }
  56 +
50 57 }
... ...
... ... @@ -23,8 +23,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheE
23 23 */
24 24 public interface DeviceSessionCacheService {
25 25
26   - DeviceSessionsCacheEntry get(DeviceId deviceId);
  26 + byte[] get(DeviceId deviceId);
27 27
28   - DeviceSessionsCacheEntry put(DeviceId deviceId, DeviceSessionsCacheEntry sessions);
  28 + byte[] put(DeviceId deviceId, byte[] sessions);
29 29
30 30 }
... ...
... ... @@ -35,23 +35,11 @@ import org.thingsboard.server.common.data.id.EntityId;
35 35 import org.thingsboard.server.common.data.id.EntityIdFactory;
36 36 import org.thingsboard.server.common.data.id.EntityViewId;
37 37 import org.thingsboard.server.common.data.id.TenantId;
38   -import org.thingsboard.server.common.data.kv.AttributeKvEntry;
39   -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
40   -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
41   -import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
42   -import org.thingsboard.server.common.data.kv.BooleanDataEntry;
43   -import org.thingsboard.server.common.data.kv.DataType;
44   -import org.thingsboard.server.common.data.kv.DoubleDataEntry;
45   -import org.thingsboard.server.common.data.kv.KvEntry;
46   -import org.thingsboard.server.common.data.kv.LongDataEntry;
47   -import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
48   -import org.thingsboard.server.common.data.kv.StringDataEntry;
49   -import org.thingsboard.server.common.data.kv.TsKvEntry;
  38 +import org.thingsboard.server.common.data.kv.*;
50 39 import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
51 40 import org.thingsboard.server.common.msg.cluster.ServerAddress;
52 41 import org.thingsboard.server.dao.attributes.AttributesService;
53 42 import org.thingsboard.server.dao.entityview.EntityViewService;
54   -import org.thingsboard.server.dao.model.ModelConstants;
55 43 import org.thingsboard.server.dao.timeseries.TimeseriesService;
56 44 import org.thingsboard.server.gen.cluster.ClusterAPIProtos;
57 45 import org.thingsboard.server.service.cluster.routing.ClusterRoutingService;
... ... @@ -68,7 +56,6 @@ import javax.annotation.PostConstruct;
68 56 import javax.annotation.PreDestroy;
69 57 import java.util.ArrayList;
70 58 import java.util.Collections;
71   -import java.util.HashMap;
72 59 import java.util.HashSet;
73 60 import java.util.Iterator;
74 61 import java.util.List;
... ... @@ -339,9 +326,9 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
339 326 Set<Subscription> subscriptions = e.getValue();
340 327 Optional<ServerAddress> newAddressOptional = routingService.resolveById(e.getKey());
341 328 if (newAddressOptional.isPresent()) {
342   - newAddressOptional.ifPresent(serverAddress -> checkSubsciptionsNewAddress(serverAddress, subscriptions));
  329 + newAddressOptional.ifPresent(serverAddress -> checkSubscriptionsNewAddress(serverAddress, subscriptions));
343 330 } else {
344   - checkSubsciptionsPrevAddress(subscriptions);
  331 + checkSubscriptionsPrevAddress(subscriptions);
345 332 }
346 333 if (subscriptions.size() == 0) {
347 334 log.trace("[{}] No more subscriptions for this device on current server.", e.getKey());
... ... @@ -350,7 +337,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
350 337 }
351 338 }
352 339
353   - private void checkSubsciptionsNewAddress(ServerAddress newAddress, Set<Subscription> subscriptions) {
  340 + private void checkSubscriptionsNewAddress(ServerAddress newAddress, Set<Subscription> subscriptions) {
354 341 Iterator<Subscription> subscriptionIterator = subscriptions.iterator();
355 342 while (subscriptionIterator.hasNext()) {
356 343 Subscription s = subscriptionIterator.next();
... ... @@ -368,7 +355,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
368 355 }
369 356 }
370 357
371   - private void checkSubsciptionsPrevAddress(Set<Subscription> subscriptions) {
  358 + private void checkSubscriptionsPrevAddress(Set<Subscription> subscriptions) {
372 359 for (Subscription s : subscriptions) {
373 360 if (s.isLocal() && s.getServer() != null) {
374 361 log.trace("[{}] Local subscription is no longer handled on remote server address [{}]", s.getWsSessionId(), s.getServer());
... ... @@ -381,7 +368,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
381 368
382 369 private void addRemoteWsSubscription(ServerAddress address, String sessionId, Subscription subscription) {
383 370 EntityId entityId = subscription.getEntityId();
384   - log.trace("[{}] Registering remote subscription [{}] for device [{}] to [{}]", sessionId, subscription.getSubscriptionId(), entityId, address);
  371 + log.trace("[{}] Registering remote subscription [{}] for entity [{}] to [{}]", sessionId, subscription.getSubscriptionId(), entityId, address);
385 372 registerSubscription(sessionId, entityId, subscription);
386 373 if (subscription.getType() == TelemetryFeature.ATTRIBUTES) {
387 374 final Map<String, Long> keyStates = subscription.getKeyStates();
... ... @@ -401,17 +388,22 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
401 388 long curTs = System.currentTimeMillis();
402 389 List<ReadTsKvQuery> queries = new ArrayList<>();
403 390 subscription.getKeyStates().entrySet().forEach(e -> {
404   - queries.add(new BaseReadTsKvQuery(e.getKey(), e.getValue() + 1L, curTs));
  391 + if (curTs > e.getValue()) {
  392 + queries.add(new BaseReadTsKvQuery(e.getKey(), e.getValue() + 1L, curTs, 0, 1000, Aggregation.NONE));
  393 + } else {
  394 + log.debug("[{}] Invalid subscription [{}], entityId [{}] curTs [{}]", sessionId, subscription, entityId, curTs);
  395 + }
405 396 });
406   -
407   - DonAsynchron.withCallback(tsService.findAll(entityId, queries),
408   - missedUpdates -> {
409   - if (missedUpdates != null && !missedUpdates.isEmpty()) {
410   - tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates));
411   - }
412   - },
413   - e -> log.error("Failed to fetch missed updates.", e),
414   - tsCallBackExecutor);
  397 + if (!queries.isEmpty()) {
  398 + DonAsynchron.withCallback(tsService.findAll(entityId, queries),
  399 + missedUpdates -> {
  400 + if (missedUpdates != null && !missedUpdates.isEmpty()) {
  401 + tellRemoteSubUpdate(address, sessionId, new SubscriptionUpdate(subscription.getSubscriptionId(), missedUpdates));
  402 + }
  403 + },
  404 + e -> log.error("Failed to fetch missed updates.", e),
  405 + tsCallBackExecutor);
  406 + }
415 407 }
416 408 }
417 409
... ...
... ... @@ -29,6 +29,9 @@ import org.apache.kafka.clients.producer.RecordMetadata;
29 29 import org.springframework.beans.factory.annotation.Autowired;
30 30 import org.springframework.beans.factory.annotation.Value;
31 31 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
  32 +import org.springframework.boot.context.event.ApplicationReadyEvent;
  33 +import org.springframework.context.event.ContextRefreshedEvent;
  34 +import org.springframework.context.event.EventListener;
32 35 import org.springframework.stereotype.Service;
33 36 import org.thingsboard.server.actors.ActorSystemContext;
34 37 import org.thingsboard.server.actors.service.ActorService;
... ... @@ -127,7 +130,11 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ
127 130
128 131 ruleEngineConsumer = ruleEngineConsumerBuilder.build();
129 132 ruleEngineConsumer.subscribe();
  133 + }
130 134
  135 + @EventListener(ApplicationReadyEvent.class)
  136 + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
  137 + log.info("Received application ready event. Starting polling for events.");
131 138 LocalBucketBuilder builder = Bucket4j.builder();
132 139 builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1)));
133 140 builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1)));
... ... @@ -149,6 +156,7 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ
149 156 records.forEach(record -> {
150 157 try {
151 158 ToRuleEngineMsg toRuleEngineMsg = ruleEngineConsumer.decode(record);
  159 + log.trace("Forwarding message to rule engine {}", toRuleEngineMsg);
152 160 if (toRuleEngineMsg.hasToDeviceActorMsg()) {
153 161 forwardToDeviceActor(toRuleEngineMsg.getToDeviceActorMsg());
154 162 }
... ... @@ -175,18 +183,21 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ
175 183
176 184 @Override
177 185 public void process(String nodeId, DeviceActorToTransportMsg msg, Runnable onSuccess, Consumer<Throwable> onFailure) {
178   - notificationsProducer.send(notificationsTopic + "." + nodeId,
179   - new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB()).toString(),
180   - ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build()
181   - , new QueueCallbackAdaptor(onSuccess, onFailure));
  186 + String topic = notificationsTopic + "." + nodeId;
  187 + UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB());
  188 + ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setToDeviceSessionMsg(msg).build();
  189 + log.trace("[{}][{}] Pushing session data to topic: {}", topic, sessionId, transportMsg);
  190 + notificationsProducer.send(topic, sessionId.toString(), transportMsg, new QueueCallbackAdaptor(onSuccess, onFailure));
182 191 }
183 192
184 193 private void forwardToDeviceActor(TransportToDeviceActorMsg toDeviceActorMsg) {
185 194 TransportToDeviceActorMsgWrapper wrapper = new TransportToDeviceActorMsgWrapper(toDeviceActorMsg);
186 195 Optional<ServerAddress> address = routingService.resolveById(wrapper.getDeviceId());
187 196 if (address.isPresent()) {
  197 + log.trace("[{}] Pushing message to remote server: {}", address.get(), toDeviceActorMsg);
188 198 rpcService.tell(encodingService.convertToProtoDataMessage(address.get(), wrapper));
189 199 } else {
  200 + log.trace("Pushing message to local server: {}", toDeviceActorMsg);
190 201 actorContext.getAppActor().tell(wrapper, ActorRef.noSender());
191 202 }
192 203 }
... ...
... ... @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j;
19 19 import org.springframework.beans.factory.annotation.Autowired;
20 20 import org.springframework.beans.factory.annotation.Value;
21 21 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
  22 +import org.springframework.boot.context.event.ApplicationReadyEvent;
  23 +import org.springframework.context.event.EventListener;
22 24 import org.springframework.stereotype.Component;
23 25 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
24 26 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
... ... @@ -93,6 +95,11 @@ public class RemoteTransportApiService {
93 95 builder.executor(transportCallbackExecutor);
94 96 builder.handler(transportApiService);
95 97 transportApiTemplate = builder.build();
  98 + }
  99 +
  100 + @EventListener(ApplicationReadyEvent.class)
  101 + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
  102 + log.info("Received application ready event. Starting polling for events.");
96 103 transportApiTemplate.init();
97 104 }
98 105
... ...
... ... @@ -19,7 +19,7 @@ akka {
19 19 # JVM shutdown, System.exit(-1), in case of a fatal error,
20 20 # such as OutOfMemoryError
21 21 jvm-exit-on-fatal-error = off
22   - loglevel = "DEBUG"
  22 + loglevel = "INFO"
23 23 loggers = ["akka.event.slf4j.Slf4jLogger"]
24 24 }
25 25
... ...
... ... @@ -31,6 +31,7 @@ server:
31 31 key-store-type: "${SSL_KEY_STORE_TYPE:PKCS12}"
32 32 # Alias that identifies the key in the key store
33 33 key-alias: "${SSL_KEY_ALIAS:tomcat}"
  34 + log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:true}"
34 35
35 36 # Zookeeper connection parameters. Used for service discovery.
36 37 zk:
... ... @@ -63,7 +64,7 @@ cluster:
63 64
64 65 # Plugins configuration parameters
65 66 plugins:
66   - # Comma seperated package list used during classpath scanning for plugins
  67 + # Comma separated package list used during classpath scanning for plugins
67 68 scan_packages: "${PLUGINS_SCAN_PACKAGES:org.thingsboard.server.extensions,org.thingsboard.rule.engine}"
68 69
69 70 # Security parameters
... ... @@ -83,6 +84,7 @@ dashboard:
83 84 max_datapoints_limit: "${DASHBOARD_MAX_DATAPOINTS_LIMIT:50000}"
84 85
85 86 database:
  87 + ts_max_intervals: "${DATABASE_TS_MAX_INTERVALS:700}" # Max number of DB queries generated by single API call to fetch telemetry records
86 88 entities:
87 89 type: "${DATABASE_ENTITIES_TYPE:sql}" # cassandra OR sql
88 90 ts:
... ... @@ -105,7 +107,7 @@ cassandra:
105 107 metrics: "${CASSANDRA_DISABLE_METRICS:true}"
106 108 # NONE SNAPPY LZ4
107 109 compression: "${CASSANDRA_COMPRESSION:none}"
108   - # Specify cassandra claster initialization timeout (if no hosts available during startup)
  110 + # Specify cassandra cluster initialization timeout in milliseconds (if no hosts available during startup)
109 111 init_timeout_ms: "${CASSANDRA_CLUSTER_INIT_TIMEOUT_MS:300000}"
110 112 # Specify cassandra claster initialization retry interval (if no hosts available during startup)
111 113 init_retry_interval_ms: "${CASSANDRA_CLUSTER_INIT_RETRY_INTERVAL_MS:3000}"
... ... @@ -151,6 +153,8 @@ sql:
151 153
152 154 # Actor system parameters
153 155 actors:
  156 + cluster:
  157 + grpc_callback_thread_pool_size: "${ACTORS_CLUSTER_GRPC_CALLBACK_THREAD_POOL_SIZE:10}"
154 158 tenant:
155 159 create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}"
156 160 session:
... ... @@ -306,7 +310,7 @@ audit_log:
306 310 "user": "${AUDIT_LOG_MASK_USER:W}"
307 311 "rule_chain": "${AUDIT_LOG_MASK_RULE_CHAIN:W}"
308 312 "alarm": "${AUDIT_LOG_MASK_ALARM:W}"
309   - "entity_view": "${AUDIT_LOG_MASK_RULE_CHAIN:W}"
  313 + "entity_view": "${AUDIT_LOG_MASK_ENTITY_VIEW:W}"
310 314 sink:
311 315 # Type of external sink. possible options: none, elasticsearch
312 316 type: "${AUDIT_LOG_SINK_TYPE:none}"
... ... @@ -320,7 +324,7 @@ audit_log:
320 324 date_format: "${AUDIT_LOG_SINK_DATE_FORMAT:YYYY.MM.DD}"
321 325 scheme_name: "${AUDIT_LOG_SINK_SCHEME_NAME:http}" # http or https
322 326 host: "${AUDIT_LOG_SINK_HOST:localhost}"
323   - port: "${AUDIT_LOG_SINK_POST:9200}"
  327 + port: "${AUDIT_LOG_SINK_PORT:9200}"
324 328 user_name: "${AUDIT_LOG_SINK_USER_NAME:}"
325 329 password: "${AUDIT_LOG_SINK_PASSWORD:}"
326 330
... ... @@ -340,7 +344,7 @@ kafka:
340 344 requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}"
341 345 responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}"
342 346 max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}"
343   - request_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}"
  347 + max_requests_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}"
344 348 request_poll_interval: "${TB_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}"
345 349 request_auto_commit_interval: "${TB_TRANSPORT_REQUEST_AUTO_COMMIT_INTERVAL_MS:100}"
346 350 rule_engine:
... ... @@ -367,7 +371,7 @@ js:
367 371 # JS Eval request topic
368 372 request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}"
369 373 # JS Eval responses topic prefix that is combined with node id
370   - response_topic_prefix: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.responses}"
  374 + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}"
371 375 # JS Eval max pending requests
372 376 max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}"
373 377 # JS Eval max request timeout
... ... @@ -405,6 +409,9 @@ transport:
405 409 enabled: "${TB_TRANSPORT_RATE_LIMITS_ENABLED:false}"
406 410 tenant: "${TB_TRANSPORT_RATE_LIMITS_TENANT:1000:1,20000:60}"
407 411 device: "${TB_TRANSPORT_RATE_LIMITS_DEVICE:10:1,300:60}"
  412 + json:
  413 + # Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON
  414 + type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}"
408 415 # Local HTTP transport parameters
409 416 http:
410 417 enabled: "${HTTP_ENABLED:true}"
... ...
... ... @@ -9,7 +9,7 @@
9 9
10 10 <logger name="org.thingsboard.server" level="WARN"/>
11 11 <logger name="org.springframework" level="WARN"/>
12   - <logger name="org.springframework.boot.test" level="DEBUG"/>
  12 + <logger name="org.springframework.boot.test" level="WARN"/>
13 13 <logger name="org.apache.cassandra" level="WARN"/>
14 14 <logger name="org.cassandraunit" level="INFO"/>
15 15
... ...
... ... @@ -24,6 +24,7 @@ public enum ActionType {
24 24 UPDATED(false), // log entity
25 25 ATTRIBUTES_UPDATED(false), // log attributes/values
26 26 ATTRIBUTES_DELETED(false), // log attributes
  27 + TIMESERIES_DELETED(false), // log timeseries
27 28 RPC_CALL(false), // log method and params
28 29 CREDENTIALS_UPDATED(false), // log new credentials
29 30 ASSIGNED_TO_CUSTOMER(false), // log customer name
... ... @@ -32,11 +33,11 @@ public enum ActionType {
32 33 SUSPENDED(false), // log string id
33 34 CREDENTIALS_READ(true), // log device id
34 35 ATTRIBUTES_READ(true), // log attributes
35   - RELATION_ADD_OR_UPDATE (false),
36   - RELATION_DELETED (false),
37   - RELATIONS_DELETED (false),
38   - ALARM_ACK (false),
39   - ALARM_CLEAR (false);
  36 + RELATION_ADD_OR_UPDATE(false),
  37 + RELATION_DELETED(false),
  38 + RELATIONS_DELETED(false),
  39 + ALARM_ACK(false),
  40 + ALARM_CLEAR(false);
40 41
41 42 private final boolean isRead;
42 43
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.objects;
18 18 import lombok.Data;
19 19 import lombok.NoArgsConstructor;
20 20
  21 +import java.io.Serializable;
21 22 import java.util.ArrayList;
22 23 import java.util.List;
23 24
... ... @@ -26,7 +27,7 @@ import java.util.List;
26 27 */
27 28 @Data
28 29 @NoArgsConstructor
29   -public class AttributesEntityView {
  30 +public class AttributesEntityView implements Serializable {
30 31
31 32 private List<String> cs = new ArrayList<>();
32 33 private List<String> ss = new ArrayList<>();
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.objects;
18 18 import lombok.Data;
19 19 import lombok.NoArgsConstructor;
20 20
  21 +import java.io.Serializable;
21 22 import java.util.ArrayList;
22 23 import java.util.List;
23 24
... ... @@ -26,7 +27,7 @@ import java.util.List;
26 27 */
27 28 @Data
28 29 @NoArgsConstructor
29   -public class TelemetryEntityView {
  30 +public class TelemetryEntityView implements Serializable {
30 31
31 32 private List<String> timeseries;
32 33 private AttributesEntityView attributes;
... ...
... ... @@ -28,6 +28,8 @@ public enum MsgType {
28 28 */
29 29 CLUSTER_EVENT_MSG,
30 30
  31 + APP_INIT_MSG,
  32 +
31 33 /**
32 34 * All messages, could be send to cluster
33 35 */
... ...
... ... @@ -77,9 +77,10 @@ public class TBKafkaProducerTemplate<T> {
77 77 result.all().get();
78 78 } catch (Exception e) {
79 79 if ((e instanceof TopicExistsException) || (e.getCause() != null && e.getCause() instanceof TopicExistsException)) {
80   - log.trace("[{}] Topic already exists: ", defaultTopic);
  80 + log.trace("[{}] Topic already exists.", defaultTopic);
81 81 } else {
82   - log.trace("[{}] Failed to create topic: {}", defaultTopic, e.getMessage(), e);
  82 + log.info("[{}] Failed to create topic: {}", defaultTopic, e.getMessage(), e);
  83 + throw new RuntimeException(e);
83 84 }
84 85 }
85 86 //Maybe this should not be cached, but we don't plan to change size of partitions
... ...
... ... @@ -23,6 +23,9 @@ import lombok.extern.slf4j.Slf4j;
23 23 import org.apache.kafka.clients.admin.CreateTopicsResult;
24 24 import org.apache.kafka.clients.admin.NewTopic;
25 25 import org.apache.kafka.clients.consumer.ConsumerRecords;
  26 +import org.apache.kafka.clients.producer.Callback;
  27 +import org.apache.kafka.clients.producer.RecordMetadata;
  28 +import org.apache.kafka.common.errors.TopicExistsException;
26 29 import org.apache.kafka.common.header.Header;
27 30 import org.apache.kafka.common.header.internals.RecordHeader;
28 31
... ... @@ -83,7 +86,13 @@ public class TbKafkaRequestTemplate<Request, Response> extends AbstractTbKafkaTe
83 86 CreateTopicsResult result = admin.createTopic(new NewTopic(responseTemplate.getTopic(), 1, (short) 1));
84 87 result.all().get();
85 88 } catch (Exception e) {
86   - log.trace("Failed to create topic: {}", e.getMessage(), e);
  89 + if ((e instanceof TopicExistsException) || (e.getCause() != null && e.getCause() instanceof TopicExistsException)) {
  90 + log.trace("[{}] Topic already exists. ", responseTemplate.getTopic());
  91 + } else {
  92 + log.info("[{}] Failed to create topic: {}", responseTemplate.getTopic(), e.getMessage(), e);
  93 + throw new RuntimeException(e);
  94 + }
  95 +
87 96 }
88 97 this.requestTemplate.init();
89 98 tickTs = System.currentTimeMillis();
... ... @@ -92,7 +101,11 @@ public class TbKafkaRequestTemplate<Request, Response> extends AbstractTbKafkaTe
92 101 long nextCleanupMs = 0L;
93 102 while (!stopped) {
94 103 ConsumerRecords<String, byte[]> responses = responseTemplate.poll(Duration.ofMillis(pollInterval));
  104 + if (responses.count() > 0) {
  105 + log.trace("Polling responses completed, consumer records count [{}]", responses.count());
  106 + }
95 107 responses.forEach(response -> {
  108 + log.trace("Received response to Kafka Template request: {}", response);
96 109 Header requestIdHeader = response.headers().lastHeader(TbKafkaSettings.REQUEST_ID_HEADER);
97 110 Response decodedResponse = null;
98 111 UUID requestId = null;
... ... @@ -109,6 +122,7 @@ public class TbKafkaRequestTemplate<Request, Response> extends AbstractTbKafkaTe
109 122 if (requestId == null) {
110 123 log.error("[{}] Missing requestId in header and body", response);
111 124 } else {
  125 + log.trace("[{}] Response received", requestId);
112 126 ResponseMetaData<Response> expectedResponse = pendingRequests.remove(requestId);
113 127 if (expectedResponse == null) {
114 128 log.trace("[{}] Invalid or stale request", requestId);
... ... @@ -132,6 +146,7 @@ public class TbKafkaRequestTemplate<Request, Response> extends AbstractTbKafkaTe
132 146 if (kv.getValue().expTime < tickTs) {
133 147 ResponseMetaData<Response> staleRequest = pendingRequests.remove(kv.getKey());
134 148 if (staleRequest != null) {
  149 + log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", kv.getKey(), staleRequest.expTime, tickTs);
135 150 staleRequest.future.setException(new TimeoutException());
136 151 }
137 152 }
... ... @@ -158,9 +173,17 @@ public class TbKafkaRequestTemplate<Request, Response> extends AbstractTbKafkaTe
158 173 headers.add(new RecordHeader(TbKafkaSettings.REQUEST_ID_HEADER, uuidToBytes(requestId)));
159 174 headers.add(new RecordHeader(TbKafkaSettings.RESPONSE_TOPIC_HEADER, stringToBytes(responseTemplate.getTopic())));
160 175 SettableFuture<Response> future = SettableFuture.create();
161   - pendingRequests.putIfAbsent(requestId, new ResponseMetaData<>(tickTs + maxRequestTimeout, future));
  176 + ResponseMetaData<Response> responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future);
  177 + pendingRequests.putIfAbsent(requestId, responseMetaData);
162 178 request = requestTemplate.enrich(request, responseTemplate.getTopic(), requestId);
163   - requestTemplate.send(key, request, headers, null);
  179 + log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime);
  180 + requestTemplate.send(key, request, headers, (metadata, exception) -> {
  181 + if (exception != null) {
  182 + log.trace("[{}] Failed to post the request", requestId, exception);
  183 + } else {
  184 + log.trace("[{}] Posted the request", requestId, metadata);
  185 + }
  186 + });
164 187 return future;
165 188 }
166 189
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.kafka;
18 18 import lombok.Builder;
19 19 import lombok.extern.slf4j.Slf4j;
20 20 import org.apache.kafka.clients.consumer.ConsumerRecords;
  21 +import org.apache.kafka.common.errors.InterruptException;
21 22 import org.apache.kafka.common.header.Header;
22 23 import org.apache.kafka.common.header.internals.RecordHeader;
23 24
... ... @@ -127,6 +128,10 @@ public class TbKafkaResponseTemplate<Request, Response> extends AbstractTbKafkaT
127 128 log.warn("[{}] Failed to process the request: {}", requestId, request, e);
128 129 }
129 130 });
  131 + } catch (InterruptException ie) {
  132 + if (!stopped) {
  133 + log.warn("Fetching data from kafka was interrupted.", ie);
  134 + }
130 135 } catch (Throwable e) {
131 136 log.warn("Failed to obtain messages from queue.", e);
132 137 try {
... ...
... ... @@ -141,7 +141,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
141 141 processUnsubscribe(ctx, (MqttUnsubscribeMessage) msg);
142 142 break;
143 143 case PINGREQ:
144   - if (checkConnected(ctx)) {
  144 + if (checkConnected(ctx, msg)) {
145 145 ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0)));
146 146 transportService.reportActivity(sessionInfo);
147 147 if (gatewaySessionHandler != null) {
... ... @@ -150,7 +150,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
150 150 }
151 151 break;
152 152 case DISCONNECT:
153   - if (checkConnected(ctx)) {
  153 + if (checkConnected(ctx, msg)) {
154 154 processDisconnect(ctx);
155 155 }
156 156 break;
... ... @@ -161,12 +161,12 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
161 161 }
162 162
163 163 private void processPublish(ChannelHandlerContext ctx, MqttPublishMessage mqttMsg) {
164   - if (!checkConnected(ctx)) {
  164 + if (!checkConnected(ctx, mqttMsg)) {
165 165 return;
166 166 }
167 167 String topicName = mqttMsg.variableHeader().topicName();
168 168 int msgId = mqttMsg.variableHeader().packetId();
169   - log.trace("[{}] Processing publish msg [{}][{}]!", sessionId, topicName, msgId);
  169 + log.trace("[{}][{}] Processing publish msg [{}][{}]!", sessionId, deviceSessionCtx.getDeviceId(), topicName, msgId);
170 170
171 171 if (topicName.startsWith(MqttTopics.BASE_GATEWAY_API_TOPIC)) {
172 172 if (gatewaySessionHandler != null) {
... ... @@ -248,7 +248,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
248 248 }
249 249
250 250 private void processSubscribe(ChannelHandlerContext ctx, MqttSubscribeMessage mqttMsg) {
251   - if (!checkConnected(ctx)) {
  251 + if (!checkConnected(ctx, mqttMsg)) {
252 252 return;
253 253 }
254 254 log.trace("[{}] Processing subscription [{}]!", sessionId, mqttMsg.variableHeader().messageId());
... ... @@ -293,7 +293,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
293 293 }
294 294
295 295 private void processUnsubscribe(ChannelHandlerContext ctx, MqttUnsubscribeMessage mqttMsg) {
296   - if (!checkConnected(ctx)) {
  296 + if (!checkConnected(ctx, mqttMsg)) {
297 297 return;
298 298 }
299 299 log.trace("[{}] Processing subscription [{}]!", sessionId, mqttMsg.variableHeader().messageId());
... ... @@ -336,6 +336,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
336 336
337 337 private void processAuthTokenConnect(ChannelHandlerContext ctx, MqttConnectMessage msg) {
338 338 String userName = msg.payload().userName();
  339 + log.info("[{}] Processing connect msg for client with user name: {}!", sessionId, userName);
339 340 if (StringUtils.isEmpty(userName)) {
340 341 ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_BAD_USER_NAME_OR_PASSWORD));
341 342 ctx.close();
... ... @@ -444,11 +445,11 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
444 445 return new MqttPubAckMessage(mqttFixedHeader, mqttMsgIdVariableHeader);
445 446 }
446 447
447   - private boolean checkConnected(ChannelHandlerContext ctx) {
  448 + private boolean checkConnected(ChannelHandlerContext ctx, MqttMessage msg) {
448 449 if (deviceSessionCtx.isConnected()) {
449 450 return true;
450 451 } else {
451   - log.info("[{}] Closing current session due to invalid msg order [{}][{}]", sessionId);
  452 + log.info("[{}] Closing current session due to invalid msg order: {}", sessionId, msg);
452 453 ctx.close();
453 454 return false;
454 455 }
... ... @@ -496,6 +497,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
496 497 transportService.registerAsyncSession(sessionInfo, this);
497 498 checkGatewaySession();
498 499 ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED));
  500 + log.info("[{}] Client connected!", sessionId);
499 501 }
500 502 }
501 503
... ...
... ... @@ -119,10 +119,10 @@ public class GatewaySessionHandler {
119 119 GatewayDeviceSessionCtx deviceSessionCtx = new GatewayDeviceSessionCtx(GatewaySessionHandler.this, msg.getDeviceInfo(), mqttQoSMap);
120 120 if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) {
121 121 SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo();
  122 + transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx);
122 123 transportService.process(deviceSessionInfo, AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null);
123 124 transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null);
124 125 transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null);
125   - transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx);
126 126 }
127 127 future.set(devices.get(deviceName));
128 128 }
... ...
... ... @@ -43,7 +43,7 @@ public interface TransportService {
43 43 void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg msg,
44 44 TransportServiceCallback<TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg> callback);
45 45
46   - boolean checkLimits(SessionInfoProto sessionInfo, TransportServiceCallback<Void> callback);
  46 + boolean checkLimits(SessionInfoProto sessionInfo, Object msg, TransportServiceCallback<Void> callback);
47 47
48 48 void process(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback<Void> callback);
49 49
... ...
... ... @@ -22,6 +22,7 @@ import com.google.gson.JsonObject;
22 22 import com.google.gson.JsonParser;
23 23 import com.google.gson.JsonPrimitive;
24 24 import com.google.gson.JsonSyntaxException;
  25 +import org.apache.commons.lang3.math.NumberUtils;
25 26 import org.springframework.util.StringUtils;
26 27 import org.thingsboard.server.common.data.kv.AttributeKey;
27 28 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
... ... @@ -58,6 +59,8 @@ public class JsonConverter {
58 59 private static final String CAN_T_PARSE_VALUE = "Can't parse value: ";
59 60 private static final String DEVICE_PROPERTY = "device";
60 61
  62 + private static boolean isTypeCastEnabled = true;
  63 +
61 64 public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonObject) throws JsonSyntaxException {
62 65 long systemTs = System.currentTimeMillis();
63 66 PostTelemetryMsg.Builder builder = PostTelemetryMsg.newBuilder();
... ... @@ -128,24 +131,22 @@ public class JsonConverter {
128 131 if (element.isJsonPrimitive()) {
129 132 JsonPrimitive value = element.getAsJsonPrimitive();
130 133 if (value.isString()) {
131   - result.add(KeyValueProto.newBuilder().setKey(valueEntry.getKey()).setType(KeyValueType.STRING_V)
132   - .setStringV(value.getAsString()).build());
  134 + if(isTypeCastEnabled && NumberUtils.isParsable(value.getAsString())) {
  135 + try {
  136 + result.add(buildNumericKeyValueProto(value, valueEntry.getKey()));
  137 + } catch (RuntimeException th) {
  138 + result.add(KeyValueProto.newBuilder().setKey(valueEntry.getKey()).setType(KeyValueType.STRING_V)
  139 + .setStringV(value.getAsString()).build());
  140 + }
  141 + } else {
  142 + result.add(KeyValueProto.newBuilder().setKey(valueEntry.getKey()).setType(KeyValueType.STRING_V)
  143 + .setStringV(value.getAsString()).build());
  144 + }
133 145 } else if (value.isBoolean()) {
134 146 result.add(KeyValueProto.newBuilder().setKey(valueEntry.getKey()).setType(KeyValueType.BOOLEAN_V)
135 147 .setBoolV(value.getAsBoolean()).build());
136 148 } else if (value.isNumber()) {
137   - if (value.getAsString().contains(".")) {
138   - result.add(KeyValueProto.newBuilder().setKey(valueEntry.getKey()).setType(KeyValueType.DOUBLE_V)
139   - .setDoubleV(value.getAsDouble()).build());
140   - } else {
141   - try {
142   - long longValue = Long.parseLong(value.getAsString());
143   - result.add(KeyValueProto.newBuilder().setKey(valueEntry.getKey()).setType(KeyValueType.LONG_V)
144   - .setLongV(longValue).build());
145   - } catch (NumberFormatException e) {
146   - throw new JsonSyntaxException("Big integer values are not supported!");
147   - }
148   - }
  149 + result.add(buildNumericKeyValueProto(value, valueEntry.getKey()));
149 150 } else {
150 151 throw new JsonSyntaxException(CAN_T_PARSE_VALUE + value);
151 152 }
... ... @@ -156,6 +157,24 @@ public class JsonConverter {
156 157 return result;
157 158 }
158 159
  160 + private static KeyValueProto buildNumericKeyValueProto(JsonPrimitive value, String key) {
  161 + if (value.getAsString().contains(".")) {
  162 + return KeyValueProto.newBuilder()
  163 + .setKey(key)
  164 + .setType(KeyValueType.DOUBLE_V)
  165 + .setDoubleV(value.getAsDouble())
  166 + .build();
  167 + } else {
  168 + try {
  169 + long longValue = Long.parseLong(value.getAsString());
  170 + return KeyValueProto.newBuilder().setKey(key).setType(KeyValueType.LONG_V)
  171 + .setLongV(longValue).build();
  172 + } catch (NumberFormatException e) {
  173 + throw new JsonSyntaxException("Big integer values are not supported!");
  174 + }
  175 + }
  176 + }
  177 +
159 178 public static TransportProtos.ToServerRpcRequestMsg convertToServerRpcRequest(JsonElement json, int requestId) throws JsonSyntaxException {
160 179 JsonObject object = json.getAsJsonObject();
161 180 return TransportProtos.ToServerRpcRequestMsg.newBuilder().setRequestId(requestId).setMethodName(object.get("method").getAsString()).setParams(GSON.toJson(object.get("params"))).build();
... ... @@ -370,7 +389,15 @@ public class JsonConverter {
370 389 if (element.isJsonPrimitive()) {
371 390 JsonPrimitive value = element.getAsJsonPrimitive();
372 391 if (value.isString()) {
373   - result.add(new StringDataEntry(valueEntry.getKey(), value.getAsString()));
  392 + if(isTypeCastEnabled && NumberUtils.isParsable(value.getAsString())) {
  393 + try {
  394 + parseNumericValue(result, valueEntry, value);
  395 + } catch (RuntimeException th) {
  396 + result.add(new StringDataEntry(valueEntry.getKey(), value.getAsString()));
  397 + }
  398 + } else {
  399 + result.add(new StringDataEntry(valueEntry.getKey(), value.getAsString()));
  400 + }
374 401 } else if (value.isBoolean()) {
375 402 result.add(new BooleanDataEntry(valueEntry.getKey(), value.getAsBoolean()));
376 403 } else if (value.isNumber()) {
... ... @@ -426,5 +453,7 @@ public class JsonConverter {
426 453 }
427 454 }
428 455
429   -
  456 + public static void setTypeCastEnabled(boolean enabled) {
  457 + isTypeCastEnabled = enabled;
  458 + }
430 459 }
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.transport.adaptor;
  17 +
  18 +import lombok.extern.slf4j.Slf4j;
  19 +import org.springframework.beans.factory.annotation.Value;
  20 +import org.springframework.context.annotation.Configuration;
  21 +
  22 +@Configuration
  23 +@Slf4j
  24 +public class JsonConverterConfig {
  25 +
  26 + @Value("${transport.json.type_cast_enabled:true}")
  27 + public void setIsJsonTypeCastEnabled(boolean jsonTypeCastEnabled) {
  28 + JsonConverter.setTypeCastEnabled(jsonTypeCastEnabled);
  29 + log.info("JSON type cast enabled = {}", jsonTypeCastEnabled);
  30 + }
  31 +}
... ...
... ... @@ -68,7 +68,7 @@ public abstract class AbstractTransportService implements TransportService {
68 68
69 69 @Override
70 70 public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback<Void> callback) {
71   - if (checkLimits(sessionInfo, callback)) {
  71 + if (checkLimits(sessionInfo, msg, callback)) {
72 72 reportActivityInternal(sessionInfo);
73 73 doProcess(sessionInfo, msg, callback);
74 74 }
... ... @@ -76,7 +76,7 @@ public abstract class AbstractTransportService implements TransportService {
76 76
77 77 @Override
78 78 public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback<Void> callback) {
79   - if (checkLimits(sessionInfo, callback)) {
  79 + if (checkLimits(sessionInfo, msg, callback)) {
80 80 reportActivityInternal(sessionInfo);
81 81 doProcess(sessionInfo, msg, callback);
82 82 }
... ... @@ -84,7 +84,7 @@ public abstract class AbstractTransportService implements TransportService {
84 84
85 85 @Override
86 86 public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback<Void> callback) {
87   - if (checkLimits(sessionInfo, callback)) {
  87 + if (checkLimits(sessionInfo, msg, callback)) {
88 88 reportActivityInternal(sessionInfo);
89 89 doProcess(sessionInfo, msg, callback);
90 90 }
... ... @@ -92,7 +92,7 @@ public abstract class AbstractTransportService implements TransportService {
92 92
93 93 @Override
94 94 public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback) {
95   - if (checkLimits(sessionInfo, callback)) {
  95 + if (checkLimits(sessionInfo, msg, callback)) {
96 96 reportActivityInternal(sessionInfo);
97 97 doProcess(sessionInfo, msg, callback);
98 98 }
... ... @@ -100,7 +100,7 @@ public abstract class AbstractTransportService implements TransportService {
100 100
101 101 @Override
102 102 public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback<Void> callback) {
103   - if (checkLimits(sessionInfo, callback)) {
  103 + if (checkLimits(sessionInfo, msg, callback)) {
104 104 SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo);
105 105 sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe());
106 106 doProcess(sessionInfo, msg, callback);
... ... @@ -109,7 +109,7 @@ public abstract class AbstractTransportService implements TransportService {
109 109
110 110 @Override
111 111 public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback<Void> callback) {
112   - if (checkLimits(sessionInfo, callback)) {
  112 + if (checkLimits(sessionInfo, msg, callback)) {
113 113 SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo);
114 114 sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe());
115 115 doProcess(sessionInfo, msg, callback);
... ... @@ -118,7 +118,7 @@ public abstract class AbstractTransportService implements TransportService {
118 118
119 119 @Override
120 120 public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback) {
121   - if (checkLimits(sessionInfo, callback)) {
  121 + if (checkLimits(sessionInfo, msg, callback)) {
122 122 reportActivityInternal(sessionInfo);
123 123 doProcess(sessionInfo, msg, callback);
124 124 }
... ... @@ -126,7 +126,7 @@ public abstract class AbstractTransportService implements TransportService {
126 126
127 127 @Override
128 128 public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback) {
129   - if (checkLimits(sessionInfo, callback)) {
  129 + if (checkLimits(sessionInfo, msg, callback)) {
130 130 reportActivityInternal(sessionInfo);
131 131 doProcess(sessionInfo, msg, callback);
132 132 }
... ... @@ -196,7 +196,10 @@ public abstract class AbstractTransportService implements TransportService {
196 196 }
197 197
198 198 @Override
199   - public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, TransportServiceCallback<Void> callback) {
  199 + public boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback<Void> callback) {
  200 + if (log.isTraceEnabled()) {
  201 + log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg);
  202 + }
200 203 if (!rateLimitEnabled) {
201 204 return true;
202 205 }
... ... @@ -206,6 +209,9 @@ public abstract class AbstractTransportService implements TransportService {
206 209 if (callback != null) {
207 210 callback.onError(new TbRateLimitsException(EntityType.TENANT));
208 211 }
  212 + if (log.isTraceEnabled()) {
  213 + log.trace("[{}][{}] Tenant level rate limit detected: {}", toId(sessionInfo), tenantId, msg);
  214 + }
209 215 return false;
210 216 }
211 217 DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()));
... ... @@ -214,8 +220,12 @@ public abstract class AbstractTransportService implements TransportService {
214 220 if (callback != null) {
215 221 callback.onError(new TbRateLimitsException(EntityType.DEVICE));
216 222 }
  223 + if (log.isTraceEnabled()) {
  224 + log.trace("[{}][{}] Device level rate limit detected: {}", toId(sessionInfo), deviceId, msg);
  225 + }
217 226 return false;
218 227 }
  228 +
219 229 return true;
220 230 }
221 231
... ... @@ -250,11 +260,11 @@ public abstract class AbstractTransportService implements TransportService {
250 260 }
251 261 }
252 262
253   - private UUID toId(TransportProtos.SessionInfoProto sessionInfo) {
  263 + protected UUID toId(TransportProtos.SessionInfoProto sessionInfo) {
254 264 return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB());
255 265 }
256 266
257   - String getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) {
  267 + protected String getRoutingKey(TransportProtos.SessionInfoProto sessionInfo) {
258 268 return new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()).toString();
259 269 }
260 270
... ...
... ... @@ -197,6 +197,7 @@ public class RemoteTransportService extends AbstractTransportService {
197 197
198 198 @Override
199 199 public void process(ValidateDeviceTokenRequestMsg msg, TransportServiceCallback<ValidateDeviceCredentialsResponseMsg> callback) {
  200 + log.trace("Processing msg: {}", msg);
200 201 AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getToken(),
201 202 TransportApiRequestMsg.newBuilder().setValidateTokenRequestMsg(msg).build()),
202 203 response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor);
... ... @@ -204,6 +205,7 @@ public class RemoteTransportService extends AbstractTransportService {
204 205
205 206 @Override
206 207 public void process(ValidateDeviceX509CertRequestMsg msg, TransportServiceCallback<ValidateDeviceCredentialsResponseMsg> callback) {
  208 + log.trace("Processing msg: {}", msg);
207 209 AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getHash(),
208 210 TransportApiRequestMsg.newBuilder().setValidateX509CertRequestMsg(msg).build()),
209 211 response -> callback.onSuccess(response.getValidateTokenResponseMsg()), callback::onError, transportCallbackExecutor);
... ... @@ -211,6 +213,7 @@ public class RemoteTransportService extends AbstractTransportService {
211 213
212 214 @Override
213 215 public void process(GetOrCreateDeviceFromGatewayRequestMsg msg, TransportServiceCallback<GetOrCreateDeviceFromGatewayResponseMsg> callback) {
  216 + log.trace("Processing msg: {}", msg);
214 217 AsyncCallbackTemplate.withCallback(transportApiTemplate.post(msg.getDeviceName(),
215 218 TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(msg).build()),
216 219 response -> callback.onSuccess(response.getGetOrCreateDeviceResponseMsg()), callback::onError, transportCallbackExecutor);
... ... @@ -218,6 +221,9 @@ public class RemoteTransportService extends AbstractTransportService {
218 221
219 222 @Override
220 223 public void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback<Void> callback) {
  224 + if (log.isTraceEnabled()) {
  225 + log.trace("[{}] Processing msg: {}", toId(sessionInfo), msg);
  226 + }
221 227 ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
222 228 TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
223 229 .setSubscriptionInfo(msg).build()
... ...
... ... @@ -45,7 +45,6 @@ public abstract class DeviceAwareSessionContext implements SessionContext {
45 45 this.deviceId = new DeviceId(new UUID(deviceInfo.getDeviceIdMSB(), deviceInfo.getDeviceIdLSB()));
46 46 }
47 47
48   -
49 48 public boolean isConnected() {
50 49 return deviceInfo != null;
51 50 }
... ...
... ... @@ -43,6 +43,8 @@ public interface EntityViewService {
43 43
44 44 EntityView findEntityViewById(EntityViewId entityViewId);
45 45
  46 + EntityView findEntityViewByTenantIdAndName(TenantId tenantId, String name);
  47 +
46 48 TextPageData<EntityView> findEntityViewByTenantId(TenantId tenantId, TextPageLink pageLink);
47 49
48 50 TextPageData<EntityView> findEntityViewByTenantIdAndType(TenantId tenantId, TextPageLink pageLink, String type);
... ...
... ... @@ -29,8 +29,6 @@ import org.springframework.cache.annotation.Cacheable;
29 29 import org.springframework.cache.annotation.Caching;
30 30 import org.springframework.stereotype.Service;
31 31 import org.thingsboard.server.common.data.Customer;
32   -import org.thingsboard.server.common.data.DataConstants;
33   -import org.thingsboard.server.common.data.Device;
34 32 import org.thingsboard.server.common.data.EntitySubtype;
35 33 import org.thingsboard.server.common.data.EntityType;
36 34 import org.thingsboard.server.common.data.EntityView;
... ... @@ -40,12 +38,10 @@ import org.thingsboard.server.common.data.id.CustomerId;
40 38 import org.thingsboard.server.common.data.id.EntityId;
41 39 import org.thingsboard.server.common.data.id.EntityViewId;
42 40 import org.thingsboard.server.common.data.id.TenantId;
43   -import org.thingsboard.server.common.data.kv.AttributeKvEntry;
44 41 import org.thingsboard.server.common.data.page.TextPageData;
45 42 import org.thingsboard.server.common.data.page.TextPageLink;
46 43 import org.thingsboard.server.common.data.relation.EntityRelation;
47 44 import org.thingsboard.server.common.data.relation.EntitySearchDirection;
48   -import org.thingsboard.server.dao.attributes.AttributesService;
49 45 import org.thingsboard.server.dao.customer.CustomerDao;
50 46 import org.thingsboard.server.dao.entity.AbstractEntityService;
51 47 import org.thingsboard.server.dao.exception.DataValidationException;
... ... @@ -56,15 +52,13 @@ import org.thingsboard.server.dao.tenant.TenantDao;
56 52 import javax.annotation.Nullable;
57 53 import java.util.ArrayList;
58 54 import java.util.Arrays;
59   -import java.util.Collection;
60 55 import java.util.Collections;
61 56 import java.util.Comparator;
62 57 import java.util.List;
63   -import java.util.concurrent.ExecutionException;
  58 +import java.util.Optional;
64 59 import java.util.stream.Collectors;
65 60
66 61 import static org.thingsboard.server.common.data.CacheConstants.ENTITY_VIEW_CACHE;
67   -import static org.thingsboard.server.common.data.CacheConstants.RELATIONS_CACHE;
68 62 import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
69 63 import static org.thingsboard.server.dao.service.Validator.validateId;
70 64 import static org.thingsboard.server.dao.service.Validator.validatePageLink;
... ... @@ -96,6 +90,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
96 90
97 91 @Caching(evict = {
98 92 @CacheEvict(cacheNames = ENTITY_VIEW_CACHE, key = "{#entityView.tenantId, #entityView.entityId}"),
  93 + @CacheEvict(cacheNames = ENTITY_VIEW_CACHE, key = "{#entityView.tenantId, #entityView.name}"),
99 94 @CacheEvict(cacheNames = ENTITY_VIEW_CACHE, key = "{#entityView.id}")})
100 95 @Override
101 96 public EntityView saveEntityView(EntityView entityView) {
... ... @@ -137,6 +132,15 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
137 132 return entityViewDao.findById(entityViewId.getId());
138 133 }
139 134
  135 + @Cacheable(cacheNames = ENTITY_VIEW_CACHE, key = "{#tenantId, #name}")
  136 + @Override
  137 + public EntityView findEntityViewByTenantIdAndName(TenantId tenantId, String name) {
  138 + log.trace("Executing findEntityViewByTenantIdAndName [{}][{}]", tenantId, name);
  139 + validateId(tenantId, INCORRECT_TENANT_ID + tenantId);
  140 + Optional<EntityView> entityViewOpt = entityViewDao.findEntityViewByTenantIdAndName(tenantId.getId(), name);
  141 + return entityViewOpt.orElse(null);
  142 + }
  143 +
140 144 @Override
141 145 public TextPageData<EntityView> findEntityViewByTenantId(TenantId tenantId, TextPageLink pageLink) {
142 146 log.trace("Executing findEntityViewsByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink);
... ... @@ -255,6 +259,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
255 259 deleteEntityRelations(entityViewId);
256 260 EntityView entityView = entityViewDao.findById(entityViewId.getId());
257 261 cacheManager.getCache(ENTITY_VIEW_CACHE).evict(Arrays.asList(entityView.getTenantId(), entityView.getEntityId()));
  262 + cacheManager.getCache(ENTITY_VIEW_CACHE).evict(Arrays.asList(entityView.getTenantId(), entityView.getName()));
258 263 entityViewDao.removeById(entityViewId.getId());
259 264 }
260 265
... ...
... ... @@ -20,14 +20,29 @@ import lombok.Data;
20 20 import lombok.NoArgsConstructor;
21 21 import org.thingsboard.server.common.data.EntityType;
22 22
  23 +import javax.persistence.Column;
  24 +import javax.persistence.Embeddable;
  25 +import javax.persistence.EnumType;
  26 +import javax.persistence.Enumerated;
23 27 import java.io.Serializable;
24 28
  29 +import static org.thingsboard.server.dao.model.ModelConstants.ATTRIBUTE_KEY_COLUMN;
  30 +import static org.thingsboard.server.dao.model.ModelConstants.ATTRIBUTE_TYPE_COLUMN;
  31 +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN;
  32 +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_TYPE_COLUMN;
  33 +
25 34 @Data
26 35 @AllArgsConstructor
27 36 @NoArgsConstructor
  37 +@Embeddable
28 38 public class AttributeKvCompositeKey implements Serializable {
  39 + @Enumerated(EnumType.STRING)
  40 + @Column(name = ENTITY_TYPE_COLUMN)
29 41 private EntityType entityType;
  42 + @Column(name = ENTITY_ID_COLUMN)
30 43 private String entityId;
  44 + @Column(name = ATTRIBUTE_TYPE_COLUMN)
31 45 private String attributeType;
  46 + @Column(name = ATTRIBUTE_KEY_COLUMN)
32 47 private String attributeKey;
33 48 }
... ...
... ... @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry;
27 27 import org.thingsboard.server.dao.model.ToData;
28 28
29 29 import javax.persistence.Column;
  30 +import javax.persistence.EmbeddedId;
30 31 import javax.persistence.Entity;
31 32 import javax.persistence.EnumType;
32 33 import javax.persistence.Enumerated;
... ... @@ -48,25 +49,10 @@ import static org.thingsboard.server.dao.model.ModelConstants.STRING_VALUE_COLUM
48 49 @Data
49 50 @Entity
50 51 @Table(name = "attribute_kv")
51   -@IdClass(AttributeKvCompositeKey.class)
52 52 public class AttributeKvEntity implements ToData<AttributeKvEntry>, Serializable {
53 53
54   - @Id
55   - @Enumerated(EnumType.STRING)
56   - @Column(name = ENTITY_TYPE_COLUMN)
57   - private EntityType entityType;
58   -
59   - @Id
60   - @Column(name = ENTITY_ID_COLUMN)
61   - private String entityId;
62   -
63   - @Id
64   - @Column(name = ATTRIBUTE_TYPE_COLUMN)
65   - private String attributeType;
66   -
67   - @Id
68   - @Column(name = ATTRIBUTE_KEY_COLUMN)
69   - private String attributeKey;
  54 + @EmbeddedId
  55 + private AttributeKvCompositeKey id;
70 56
71 57 @Column(name = BOOLEAN_VALUE_COLUMN)
72 58 private Boolean booleanValue;
... ... @@ -87,13 +73,13 @@ public class AttributeKvEntity implements ToData<AttributeKvEntry>, Serializable
87 73 public AttributeKvEntry toData() {
88 74 KvEntry kvEntry = null;
89 75 if (strValue != null) {
90   - kvEntry = new StringDataEntry(attributeKey, strValue);
  76 + kvEntry = new StringDataEntry(id.getAttributeKey(), strValue);
91 77 } else if (booleanValue != null) {
92   - kvEntry = new BooleanDataEntry(attributeKey, booleanValue);
  78 + kvEntry = new BooleanDataEntry(id.getAttributeKey(), booleanValue);
93 79 } else if (doubleValue != null) {
94   - kvEntry = new DoubleDataEntry(attributeKey, doubleValue);
  80 + kvEntry = new DoubleDataEntry(id.getAttributeKey(), doubleValue);
95 81 } else if (longValue != null) {
96   - kvEntry = new LongDataEntry(attributeKey, longValue);
  82 + kvEntry = new LongDataEntry(id.getAttributeKey(), longValue);
97 83 }
98 84 return new BaseAttributeKvEntry(kvEntry, lastUpdateTs);
99 85 }
... ...
... ... @@ -15,7 +15,9 @@
15 15 */
16 16 package org.thingsboard.server.dao.sql.attributes;
17 17
  18 +import org.springframework.data.jpa.repository.Query;
18 19 import org.springframework.data.repository.CrudRepository;
  20 +import org.springframework.data.repository.query.Param;
19 21 import org.thingsboard.server.common.data.EntityType;
20 22 import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey;
21 23 import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
... ... @@ -26,8 +28,11 @@ import java.util.List;
26 28 @SqlDao
27 29 public interface AttributeKvRepository extends CrudRepository<AttributeKvEntity, AttributeKvCompositeKey> {
28 30
29   - List<AttributeKvEntity> findAllByEntityTypeAndEntityIdAndAttributeType(EntityType entityType,
30   - String entityId,
31   - String attributeType);
  31 + @Query("SELECT a FROM AttributeKvEntity a WHERE a.id.entityType = :entityType " +
  32 + "AND a.id.entityId = :entityId " +
  33 + "AND a.id.attributeType = :attributeType")
  34 + List<AttributeKvEntity> findAllByEntityTypeAndEntityIdAndAttributeType(@Param("entityType") EntityType entityType,
  35 + @Param("entityId") String entityId,
  36 + @Param("attributeType") String attributeType);
32 37 }
33 38
... ...
... ... @@ -79,10 +79,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
79 79 @Override
80 80 public ListenableFuture<Void> save(EntityId entityId, String attributeType, AttributeKvEntry attribute) {
81 81 AttributeKvEntity entity = new AttributeKvEntity();
82   - entity.setEntityType(entityId.getEntityType());
83   - entity.setEntityId(fromTimeUUID(entityId.getId()));
84   - entity.setAttributeType(attributeType);
85   - entity.setAttributeKey(attribute.getKey());
  82 + entity.setId(new AttributeKvCompositeKey(entityId.getEntityType(), fromTimeUUID(entityId.getId()), attributeType, attribute.getKey()));
86 83 entity.setLastUpdateTs(attribute.getLastUpdateTs());
87 84 entity.setStrValue(attribute.getStrValue().orElse(null));
88 85 entity.setDoubleValue(attribute.getDoubleValue().orElse(null));
... ... @@ -100,10 +97,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
100 97 .stream()
101 98 .map(key -> {
102 99 AttributeKvEntity entityToDelete = new AttributeKvEntity();
103   - entityToDelete.setEntityType(entityId.getEntityType());
104   - entityToDelete.setEntityId(fromTimeUUID(entityId.getId()));
105   - entityToDelete.setAttributeType(attributeType);
106   - entityToDelete.setAttributeKey(key);
  100 + entityToDelete.setId(new AttributeKvCompositeKey(entityId.getEntityType(), fromTimeUUID(entityId.getId()), attributeType, key));
107 101 return entityToDelete;
108 102 }).collect(Collectors.toList());
109 103
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.timeseries;
17 17
18 18 import com.google.common.base.Function;
19 19 import com.google.common.collect.Lists;
  20 +import com.google.common.util.concurrent.FutureCallback;
20 21 import com.google.common.util.concurrent.Futures;
21 22 import com.google.common.util.concurrent.ListenableFuture;
22 23 import com.google.common.util.concurrent.ListeningExecutorService;
... ... @@ -31,6 +32,7 @@ import org.springframework.stereotype.Component;
31 32 import org.thingsboard.server.common.data.UUIDConverter;
32 33 import org.thingsboard.server.common.data.id.EntityId;
33 34 import org.thingsboard.server.common.data.kv.Aggregation;
  35 +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
34 36 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
35 37 import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
36 38 import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
... ... @@ -41,9 +43,9 @@ import org.thingsboard.server.dao.model.sql.TsKvEntity;
41 43 import org.thingsboard.server.dao.model.sql.TsKvLatestCompositeKey;
42 44 import org.thingsboard.server.dao.model.sql.TsKvLatestEntity;
43 45 import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
  46 +import org.thingsboard.server.dao.timeseries.SimpleListenableFuture;
44 47 import org.thingsboard.server.dao.timeseries.TimeseriesDao;
45 48 import org.thingsboard.server.dao.timeseries.TsInsertExecutorType;
46   -import org.thingsboard.server.dao.util.SqlDao;
47 49 import org.thingsboard.server.dao.util.SqlTsDao;
48 50
49 51 import javax.annotation.Nullable;
... ... @@ -53,6 +55,7 @@ import java.util.ArrayList;
53 55 import java.util.List;
54 56 import java.util.Optional;
55 57 import java.util.concurrent.CompletableFuture;
  58 +import java.util.concurrent.ExecutionException;
56 59 import java.util.concurrent.Executors;
57 60 import java.util.stream.Collectors;
58 61
... ... @@ -64,6 +67,8 @@ import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
64 67 @SqlTsDao
65 68 public class JpaTimeseriesDao extends JpaAbstractDaoListeningExecutorService implements TimeseriesDao {
66 69
  70 + private static final String DESC_ORDER = "DESC";
  71 +
67 72 @Value("${sql.ts_inserts_executor_type}")
68 73 private String insertExecutorType;
69 74
... ... @@ -326,14 +331,72 @@ public class JpaTimeseriesDao extends JpaAbstractDaoListeningExecutorService imp
326 331
327 332 @Override
328 333 public ListenableFuture<Void> removeLatest(EntityId entityId, DeleteTsKvQuery query) {
329   - TsKvLatestEntity latestEntity = new TsKvLatestEntity();
330   - latestEntity.setEntityType(entityId.getEntityType());
331   - latestEntity.setEntityId(fromTimeUUID(entityId.getId()));
332   - latestEntity.setKey(query.getKey());
333   - return service.submit(() -> {
334   - tsKvLatestRepository.delete(latestEntity);
335   - return null;
  334 + ListenableFuture<TsKvEntry> latestFuture = findLatest(entityId, query.getKey());
  335 +
  336 + ListenableFuture<Boolean> booleanFuture = Futures.transform(latestFuture, tsKvEntry -> {
  337 + long ts = tsKvEntry.getTs();
  338 + return ts > query.getStartTs() && ts <= query.getEndTs();
  339 + }, service);
  340 +
  341 + ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
  342 + if (isRemove) {
  343 + TsKvLatestEntity latestEntity = new TsKvLatestEntity();
  344 + latestEntity.setEntityType(entityId.getEntityType());
  345 + latestEntity.setEntityId(fromTimeUUID(entityId.getId()));
  346 + latestEntity.setKey(query.getKey());
  347 + return service.submit(() -> {
  348 + tsKvLatestRepository.delete(latestEntity);
  349 + return null;
  350 + });
  351 + }
  352 + return Futures.immediateFuture(null);
  353 + }, service);
  354 +
  355 + final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>();
  356 + Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() {
  357 + @Override
  358 + public void onSuccess(@Nullable Void result) {
  359 + if (query.getRewriteLatestIfDeleted()) {
  360 + ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
  361 + if (isRemove) {
  362 + return getNewLatestEntryFuture(entityId, query);
  363 + }
  364 + return Futures.immediateFuture(null);
  365 + }, service);
  366 +
  367 + try {
  368 + resultFuture.set(savedLatestFuture.get());
  369 + } catch (InterruptedException | ExecutionException e) {
  370 + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e);
  371 + }
  372 + } else {
  373 + resultFuture.set(null);
  374 + }
  375 + }
  376 +
  377 + @Override
  378 + public void onFailure(Throwable t) {
  379 + log.warn("[{}] Failed to process remove of the latest value", entityId, t);
  380 + }
336 381 });
  382 + return resultFuture;
  383 + }
  384 +
  385 + private ListenableFuture<Void> getNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) {
  386 + long startTs = 0;
  387 + long endTs = query.getStartTs() - 1;
  388 + ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1,
  389 + Aggregation.NONE, DESC_ORDER);
  390 + ListenableFuture<List<TsKvEntry>> future = findAllAsync(entityId, findNewLatestQuery);
  391 +
  392 + return Futures.transformAsync(future, entryList -> {
  393 + if (entryList.size() == 1) {
  394 + return saveLatest(entityId, entryList.get(0));
  395 + } else {
  396 + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey());
  397 + }
  398 + return Futures.immediateFuture(null);
  399 + }, service);
337 400 }
338 401
339 402 @Override
... ...
... ... @@ -47,7 +47,7 @@ public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvComposite
47 47 @Modifying
48 48 @Query("DELETE FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " +
49 49 "AND tskv.entityType = :entityType AND tskv.key = :entityKey " +
50   - "AND tskv.ts > :startTs AND tskv.ts < :endTs")
  50 + "AND tskv.ts > :startTs AND tskv.ts <= :endTs")
51 51 void delete(@Param("entityId") String entityId,
52 52 @Param("entityType") EntityType entityType,
53 53 @Param("entityKey") String key,
... ...
... ... @@ -20,11 +20,13 @@ import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
21 21 import lombok.extern.slf4j.Slf4j;
22 22 import org.springframework.beans.factory.annotation.Autowired;
  23 +import org.springframework.beans.factory.annotation.Value;
23 24 import org.springframework.stereotype.Service;
24 25 import org.thingsboard.server.common.data.EntityType;
25 26 import org.thingsboard.server.common.data.EntityView;
26 27 import org.thingsboard.server.common.data.id.EntityId;
27 28 import org.thingsboard.server.common.data.id.EntityViewId;
  29 +import org.thingsboard.server.common.data.kv.Aggregation;
28 30 import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
29 31 import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
30 32 import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
... ... @@ -47,8 +49,11 @@ import static org.apache.commons.lang3.StringUtils.isBlank;
47 49 @Slf4j
48 50 public class BaseTimeseriesService implements TimeseriesService {
49 51
50   - public static final int INSERTS_PER_ENTRY = 3;
51   - public static final int DELETES_PER_ENTRY = INSERTS_PER_ENTRY;
  52 + private static final int INSERTS_PER_ENTRY = 3;
  53 + private static final int DELETES_PER_ENTRY = INSERTS_PER_ENTRY;
  54 +
  55 + @Value("${database.ts_max_intervals}")
  56 + private long maxTsIntervals;
52 57
53 58 @Autowired
54 59 private TimeseriesDao timeseriesDao;
... ... @@ -59,7 +64,7 @@ public class BaseTimeseriesService implements TimeseriesService {
59 64 @Override
60 65 public ListenableFuture<List<TsKvEntry>> findAll(EntityId entityId, List<ReadTsKvQuery> queries) {
61 66 validate(entityId);
62   - queries.forEach(BaseTimeseriesService::validate);
  67 + queries.forEach(this::validate);
63 68 if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) {
64 69 EntityView entityView = entityViewService.findEntityViewById((EntityViewId) entityId);
65 70 List<ReadTsKvQuery> filteredQueries =
... ... @@ -189,7 +194,7 @@ public class BaseTimeseriesService implements TimeseriesService {
189 194 Validator.validateEntityId(entityId, "Incorrect entityId " + entityId);
190 195 }
191 196
192   - private static void validate(ReadTsKvQuery query) {
  197 + private void validate(ReadTsKvQuery query) {
193 198 if (query == null) {
194 199 throw new IncorrectParameterException("ReadTsKvQuery can't be null");
195 200 } else if (isBlank(query.getKey())) {
... ... @@ -197,6 +202,14 @@ public class BaseTimeseriesService implements TimeseriesService {
197 202 } else if (query.getAggregation() == null) {
198 203 throw new IncorrectParameterException("Incorrect ReadTsKvQuery. Aggregation can't be empty");
199 204 }
  205 + if(!Aggregation.NONE.equals(query.getAggregation())) {
  206 + long step = Math.max(query.getInterval(), 1000);
  207 + long intervalCounts = (query.getEndTs() - query.getStartTs()) / step;
  208 + if (intervalCounts > maxTsIntervals || intervalCounts < 0) {
  209 + throw new IncorrectParameterException("Incorrect TsKvQuery. Number of intervals is to high - " + intervalCounts + ". " +
  210 + "Please increase 'interval' parameter for your query or reduce the time range of the query.");
  211 + }
  212 + }
200 213 }
201 214
202 215 private static void validate(DeleteTsKvQuery query) {
... ...
... ... @@ -48,7 +48,6 @@ import org.thingsboard.server.common.data.kv.StringDataEntry;
48 48 import org.thingsboard.server.common.data.kv.TsKvEntry;
49 49 import org.thingsboard.server.dao.model.ModelConstants;
50 50 import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao;
51   -import org.thingsboard.server.dao.util.NoSqlDao;
52 51 import org.thingsboard.server.dao.util.NoSqlTsDao;
53 52
54 53 import javax.annotation.Nullable;
... ... @@ -62,6 +61,7 @@ import java.util.Arrays;
62 61 import java.util.Collections;
63 62 import java.util.List;
64 63 import java.util.Optional;
  64 +import java.util.concurrent.ExecutionException;
65 65 import java.util.stream.Collectors;
66 66
67 67 import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
... ... @@ -434,14 +434,14 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
434 434 public ListenableFuture<Void> removeLatest(EntityId entityId, DeleteTsKvQuery query) {
435 435 ListenableFuture<TsKvEntry> latestEntryFuture = findLatest(entityId, query.getKey());
436 436
437   - ListenableFuture<Boolean> booleanFuture = Futures.transformAsync(latestEntryFuture, latestEntry -> {
  437 + ListenableFuture<Boolean> booleanFuture = Futures.transform(latestEntryFuture, latestEntry -> {
438 438 long ts = latestEntry.getTs();
439   - if (ts >= query.getStartTs() && ts <= query.getEndTs()) {
440   - return Futures.immediateFuture(true);
  439 + if (ts > query.getStartTs() && ts <= query.getEndTs()) {
  440 + return true;
441 441 } else {
442 442 log.trace("Won't be deleted latest value for [{}], key - {}", entityId, query.getKey());
443 443 }
444   - return Futures.immediateFuture(false);
  444 + return false;
445 445 }, readResultsProcessingExecutor);
446 446
447 447 ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
... ... @@ -451,18 +451,34 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
451 451 return Futures.immediateFuture(null);
452 452 }, readResultsProcessingExecutor);
453 453
454   - if (query.getRewriteLatestIfDeleted()) {
455   - ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
456   - if (isRemove) {
457   - return getNewLatestEntryFuture(entityId, query);
  454 + final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>();
  455 + Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() {
  456 + @Override
  457 + public void onSuccess(@Nullable Void result) {
  458 + if (query.getRewriteLatestIfDeleted()) {
  459 + ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
  460 + if (isRemove) {
  461 + return getNewLatestEntryFuture(entityId, query);
  462 + }
  463 + return Futures.immediateFuture(null);
  464 + }, readResultsProcessingExecutor);
  465 +
  466 + try {
  467 + resultFuture.set(savedLatestFuture.get());
  468 + } catch (InterruptedException | ExecutionException e) {
  469 + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e);
  470 + }
  471 + } else {
  472 + resultFuture.set(null);
458 473 }
459   - return Futures.immediateFuture(null);
460   - }, readResultsProcessingExecutor);
  474 + }
461 475
462   - return Futures.transformAsync(Futures.allAsList(Arrays.asList(savedLatestFuture, removedLatestFuture)),
463   - list -> Futures.immediateFuture(null), readResultsProcessingExecutor);
464   - }
465   - return removedLatestFuture;
  476 + @Override
  477 + public void onFailure(Throwable t) {
  478 + log.warn("[{}] Failed to process remove of the latest value", entityId, t);
  479 + }
  480 + });
  481 + return resultFuture;
466 482 }
467 483
468 484 private ListenableFuture<Void> getNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) {
... ...
... ... @@ -152,7 +152,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
152 152 }
153 153
154 154 @Test
155   - public void testDeleteDeviceTsData() throws Exception {
  155 + public void testDeleteDeviceTsDataWithoutOverwritingLatest() throws Exception {
156 156 DeviceId deviceId = new DeviceId(UUIDs.timeBased());
157 157
158 158 saveEntries(deviceId, 10000);
... ... @@ -172,6 +172,26 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
172 172 }
173 173
174 174 @Test
  175 + public void testDeleteDeviceTsDataWithOverwritingLatest() throws Exception {
  176 + DeviceId deviceId = new DeviceId(UUIDs.timeBased());
  177 +
  178 + saveEntries(deviceId, 10000);
  179 + saveEntries(deviceId, 20000);
  180 + saveEntries(deviceId, 30000);
  181 + saveEntries(deviceId, 40000);
  182 +
  183 + tsService.remove(deviceId, Collections.singletonList(
  184 + new BaseDeleteTsKvQuery(STRING_KEY, 25000, 45000, true))).get();
  185 +
  186 + List<TsKvEntry> list = tsService.findAll(deviceId, Collections.singletonList(
  187 + new BaseReadTsKvQuery(STRING_KEY, 5000, 45000, 10000, 10, Aggregation.NONE))).get();
  188 + Assert.assertEquals(2, list.size());
  189 +
  190 + List<TsKvEntry> latest = tsService.findLatest(deviceId, Collections.singletonList(STRING_KEY)).get();
  191 + Assert.assertEquals(20000, latest.get(0).getTs());
  192 + }
  193 +
  194 + @Test
175 195 public void testFindDeviceTsData() throws Exception {
176 196 DeviceId deviceId = new DeviceId(UUIDs.timeBased());
177 197 List<TsKvEntry> entries = new ArrayList<>();
... ...
... ... @@ -35,4 +35,4 @@ redis.connection.port=6379
35 35 redis.connection.db=0
36 36 redis.connection.password=
37 37
38   -rule.queue.type=memory
  38 +database.ts_max_intervals=700
\ No newline at end of file
... ...
... ... @@ -15,4 +15,4 @@ TB_VERSION=latest
15 15
16 16 DATABASE=postgres
17 17
18   -KAFKA_TOPICS="js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1"
  18 +LOAD_BALANCER_NAME=haproxy-certbot
... ...
... ... @@ -4,4 +4,5 @@ tb-node/log/**
4 4 tb-node/db/**
5 5 tb-node/postgres/**
6 6 tb-node/cassandra/**
  7 +tb-transports/*/log
7 8 !.env
... ...
1   -#!/bin/bash
2   -#
3   -# Copyright © 2016-2018 The Thingsboard Authors
4   -#
5   -# Licensed under the Apache License, Version 2.0 (the "License");
6   -# you may not use this file except in compliance with the License.
7   -# You may obtain a copy of the License at
8   -#
9   -# http://www.apache.org/licenses/LICENSE-2.0
10   -#
11   -# Unless required by applicable law or agreed to in writing, software
12   -# distributed under the License is distributed on an "AS IS" BASIS,
13   -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   -# See the License for the specific language governing permissions and
15   -# limitations under the License.
16   -#
17   -
18   -dirsArray=("./haproxy/certs.d" "./haproxy/letsencrypt" "./tb-node/postgres" "./tb-node/cassandra" "./tb-node/log/tb1" "./tb-node/log/tb2")
19   -
20   -for dir in ${dirsArray[@]}
21   -do
22   - if [ ! -d "$dir" ]; then
23   - echo creating dir $dir
24   - mkdir -p $dir
25   - fi
26   -done
  1 +#
  2 +# Copyright © 2016-2018 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +version: '2.2'
  18 +
  19 +services:
  20 + postgres:
  21 + volumes:
  22 + - postgres-db-volume:/var/lib/postgresql/data
  23 + tb1:
  24 + volumes:
  25 + - tb-log-volume:/var/log/thingsboard
  26 + tb2:
  27 + volumes:
  28 + - tb-log-volume:/var/log/thingsboard
  29 + tb-coap-transport:
  30 + volumes:
  31 + - tb-coap-transport-log-volume:/var/log/tb-coap-transport
  32 + tb-http-transport1:
  33 + volumes:
  34 + - tb-http-transport-log-volume:/var/log/tb-http-transport
  35 + tb-http-transport2:
  36 + volumes:
  37 + - tb-http-transport-log-volume:/var/log/tb-http-transport
  38 + tb-mqtt-transport1:
  39 + volumes:
  40 + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
  41 + tb-mqtt-transport2:
  42 + volumes:
  43 + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
  44 +
  45 +volumes:
  46 + postgres-db-volume:
  47 + external: true
  48 + name: ${POSTGRES_DATA_VOLUME}
  49 + tb-log-volume:
  50 + external: true
  51 + name: ${TB_LOG_VOLUME}
  52 + tb-coap-transport-log-volume:
  53 + external: true
  54 + name: ${TB_COAP_TRANSPORT_LOG_VOLUME}
  55 + tb-http-transport-log-volume:
  56 + external: true
  57 + name: ${TB_HTTP_TRANSPORT_LOG_VOLUME}
  58 + tb-mqtt-transport-log-volume:
  59 + external: true
  60 + name: ${TB_MQTT_TRANSPORT_LOG_VOLUME}
... ...
... ... @@ -64,6 +64,7 @@ services:
64 64 depends_on:
65 65 - kafka
66 66 - redis
  67 + - tb-js-executor
67 68 tb2:
68 69 restart: always
69 70 image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}"
... ... @@ -84,13 +85,19 @@ services:
84 85 depends_on:
85 86 - kafka
86 87 - redis
  88 + - tb-js-executor
87 89 tb-mqtt-transport1:
88 90 restart: always
89 91 image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
90 92 ports:
91 93 - "1883"
  94 + environment:
  95 + TB_HOST: tb-mqtt-transport1
92 96 env_file:
93 97 - tb-mqtt-transport.env
  98 + volumes:
  99 + - ./tb-transports/mqtt/conf:/config
  100 + - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport
94 101 depends_on:
95 102 - kafka
96 103 tb-mqtt-transport2:
... ... @@ -98,8 +105,13 @@ services:
98 105 image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
99 106 ports:
100 107 - "1883"
  108 + environment:
  109 + TB_HOST: tb-mqtt-transport2
101 110 env_file:
102 111 - tb-mqtt-transport.env
  112 + volumes:
  113 + - ./tb-transports/mqtt/conf:/config
  114 + - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport
103 115 depends_on:
104 116 - kafka
105 117 tb-http-transport1:
... ... @@ -107,8 +119,13 @@ services:
107 119 image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
108 120 ports:
109 121 - "8081"
  122 + environment:
  123 + TB_HOST: tb-http-transport1
110 124 env_file:
111 125 - tb-http-transport.env
  126 + volumes:
  127 + - ./tb-transports/http/conf:/config
  128 + - ./tb-transports/http/log:/var/log/tb-http-transport
112 129 depends_on:
113 130 - kafka
114 131 tb-http-transport2:
... ... @@ -116,8 +133,13 @@ services:
116 133 image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
117 134 ports:
118 135 - "8081"
  136 + environment:
  137 + TB_HOST: tb-http-transport2
119 138 env_file:
120 139 - tb-http-transport.env
  140 + volumes:
  141 + - ./tb-transports/http/conf:/config
  142 + - ./tb-transports/http/log:/var/log/tb-http-transport
121 143 depends_on:
122 144 - kafka
123 145 tb-coap-transport:
... ... @@ -125,8 +147,13 @@ services:
125 147 image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
126 148 ports:
127 149 - "5683:5683/udp"
  150 + environment:
  151 + TB_HOST: tb-coap-transport
128 152 env_file:
129 153 - tb-coap-transport.env
  154 + volumes:
  155 + - ./tb-transports/coap/conf:/config
  156 + - ./tb-transports/coap/log:/var/log/tb-coap-transport
130 157 depends_on:
131 158 - kafka
132 159 tb-web-ui1:
... ... @@ -145,7 +172,7 @@ services:
145 172 - tb-web-ui.env
146 173 haproxy:
147 174 restart: always
148   - container_name: haproxy-certbot
  175 + container_name: "${LOAD_BALANCER_NAME}"
149 176 image: xalauc/haproxy-certbot:1.7.9
150 177 volumes:
151 178 - ./haproxy/config:/config
... ... @@ -153,7 +180,6 @@ services:
153 180 - ./haproxy/certs.d:/usr/local/etc/haproxy/certs.d
154 181 ports:
155 182 - "80:80"
156   - - "8080"
157 183 - "443:443"
158 184 - "1883:1883"
159 185 - "9999:9999"
... ... @@ -163,7 +189,6 @@ services:
163 189 HTTP_PORT: 80
164 190 HTTPS_PORT: 443
165 191 MQTT_PORT: 1883
166   - TB_API_PORT: 8080
167 192 FORCE_HTTPS_REDIRECT: "false"
168 193 links:
169 194 - tb1
... ...
... ... @@ -15,8 +15,6 @@
15 15 # limitations under the License.
16 16 #
17 17
18   -./check-dirs.sh
19   -
20 18 while [[ $# -gt 0 ]]
21 19 do
22 20 key="$1"
... ...
... ... @@ -15,8 +15,6 @@
15 15 # limitations under the License.
16 16 #
17 17
18   -./check-dirs.sh
19   -
20 18 set -e
21 19
22 20 source compose-utils.sh
... ...
... ... @@ -15,8 +15,6 @@
15 15 # limitations under the License.
16 16 #
17 17
18   -./check-dirs.sh
19   -
20 18 for i in "$@"
21 19 do
22 20 case $i in
... ...
... ... @@ -56,13 +56,20 @@ frontend http-in
56 56
57 57 reqadd X-Forwarded-Proto:\ http
58 58
  59 + acl acl_static path_beg /static/ /index.html
  60 + acl acl_static path /
  61 + acl acl_static_rulenode path_beg /static/rulenode/
  62 +
59 63 acl transport_http_acl path_beg /api/v1/
60 64 acl letsencrypt_http_acl path_beg /.well-known/acme-challenge/
  65 +
61 66 redirect scheme https if !letsencrypt_http_acl !transport_http_acl { env(FORCE_HTTPS_REDIRECT) -m str true }
  67 +
62 68 use_backend letsencrypt_http if letsencrypt_http_acl
63 69 use_backend tb-http-backend if transport_http_acl
  70 + use_backend tb-web-backend if acl_static !acl_static_rulenode
64 71
65   - default_backend tb-web-backend
  72 + default_backend tb-api-backend
66 73
67 74 frontend https_in
68 75 bind *:${HTTPS_PORT} ssl crt /usr/local/etc/haproxy/default.pem crt /usr/local/etc/haproxy/certs.d ciphers ECDHE-RSA-AES256-SHA:RC4-SHA:RC4:HIGH:!MD5:!aNULL:!EDH:!AESGCM
... ... @@ -72,14 +79,15 @@ frontend https_in
72 79 reqadd X-Forwarded-Proto:\ https
73 80
74 81 acl transport_http_acl path_beg /api/v1/
75   - use_backend tb-http-backend if transport_http_acl
76 82
77   - default_backend tb-web-backend
  83 + acl acl_static path_beg /static/ /index.html
  84 + acl acl_static path /
  85 + acl acl_static_rulenode path_beg /static/rulenode/
78 86
79   -frontend http-api-in
80   - bind *:${TB_API_PORT}
  87 + use_backend tb-http-backend if transport_http_acl
  88 + use_backend tb-web-backend if acl_static !acl_static_rulenode
81 89
82   - default_backend tb-api-backend
  90 + default_backend tb-api-backend
83 91
84 92 backend letsencrypt_http
85 93 server letsencrypt_http_srv 127.0.0.1:8080
... ...
... ... @@ -4,7 +4,7 @@ KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092
4 4 KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092
5 5 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
6 6 KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE
7   -KAFKA_CREATE_TOPICS=${KAFKA_TOPICS}
  7 +KAFKA_CREATE_TOPICS=js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1
8 8 KAFKA_AUTO_CREATE_TOPICS_ENABLE=false
9 9 KAFKA_LOG_RETENTION_BYTES=1073741824
10 10 KAFKA_LOG_SEGMENT_BYTES=268435456
... ...
... ... @@ -24,7 +24,7 @@
24 24 <file>/var/log/thingsboard/${TB_HOST}/thingsboard.log</file>
25 25 <rollingPolicy
26 26 class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
27   - <fileNamePattern>/var/log/thingsboard/thingsboard.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
  27 + <fileNamePattern>/var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
28 28 <maxFileSize>100MB</maxFileSize>
29 29 <maxHistory>30</maxHistory>
30 30 <totalSizeCap>3GB</totalSizeCap>
... ...
... ... @@ -15,7 +15,7 @@
15 15 #
16 16
17 17 export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data"
18   -export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
  18 +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
19 19 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
20 20 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
21 21 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"
... ...
docker/tb-transports/coap/conf/logback.xml renamed from msa/transport/coap/docker/logback.xml
... ... @@ -21,10 +21,10 @@
21 21
22 22 <appender name="fileLogAppender"
23 23 class="ch.qos.logback.core.rolling.RollingFileAppender">
24   - <file>/var/log/${pkg.name}/${pkg.name}.log</file>
  24 + <file>/var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.log</file>
25 25 <rollingPolicy
26 26 class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
27   - <fileNamePattern>/var/log/${pkg.name}/${pkg.name}.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
  27 + <fileNamePattern>/var/log/tb-coap-transport/${TB_HOST}/tb-coap-transport.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
28 28 <maxFileSize>100MB</maxFileSize>
29 29 <maxHistory>30</maxHistory>
30 30 <totalSizeCap>3GB</totalSizeCap>
... ...
docker/tb-transports/coap/conf/tb-coap-transport.conf renamed from msa/transport/coap/docker/tb-coap-transport.conf
... ... @@ -14,10 +14,10 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -export JAVA_OPTS="$JAVA_OPTS -Xloggc:@pkg.logFolder@/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
  17 +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
18 18 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
19 19 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
20 20 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"
21 21 export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError"
22   -export LOG_FILENAME=${pkg.name}.out
23   -export LOADER_PATH=${pkg.installFolder}/conf
  22 +export LOG_FILENAME=tb-coap-transport.out
  23 +export LOADER_PATH=/usr/share/tb-coap-transport/conf
... ...
docker/tb-transports/http/conf/logback.xml renamed from msa/transport/http/docker/logback.xml
... ... @@ -21,10 +21,10 @@
21 21
22 22 <appender name="fileLogAppender"
23 23 class="ch.qos.logback.core.rolling.RollingFileAppender">
24   - <file>/var/log/${pkg.name}/${pkg.name}.log</file>
  24 + <file>/var/log/tb-http-transport/${TB_HOST}/tb-http-transport.log</file>
25 25 <rollingPolicy
26 26 class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
27   - <fileNamePattern>/var/log/${pkg.name}/${pkg.name}.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
  27 + <fileNamePattern>/var/log/tb-http-transport/${TB_HOST}/tb-http-transport.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
28 28 <maxFileSize>100MB</maxFileSize>
29 29 <maxHistory>30</maxHistory>
30 30 <totalSizeCap>3GB</totalSizeCap>
... ...
docker/tb-transports/http/conf/tb-http-transport.conf renamed from msa/transport/http/docker/tb-http-transport.conf
... ... @@ -14,10 +14,10 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -export JAVA_OPTS="$JAVA_OPTS -Xloggc:@pkg.logFolder@/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
  17 +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
18 18 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
19 19 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
20 20 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"
21 21 export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError"
22   -export LOG_FILENAME=${pkg.name}.out
23   -export LOADER_PATH=${pkg.installFolder}/conf
  22 +export LOG_FILENAME=tb-http-transport.out
  23 +export LOADER_PATH=/usr/share/tb-http-transport/conf
... ...
docker/tb-transports/mqtt/conf/logback.xml renamed from msa/transport/mqtt/docker/logback.xml
... ... @@ -21,10 +21,10 @@
21 21
22 22 <appender name="fileLogAppender"
23 23 class="ch.qos.logback.core.rolling.RollingFileAppender">
24   - <file>/var/log/${pkg.name}/${pkg.name}.log</file>
  24 + <file>/var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.log</file>
25 25 <rollingPolicy
26 26 class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
27   - <fileNamePattern>/var/log/${pkg.name}/${pkg.name}.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
  27 + <fileNamePattern>/var/log/tb-mqtt-transport/${TB_HOST}/tb-mqtt-transport.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
28 28 <maxFileSize>100MB</maxFileSize>
29 29 <maxHistory>30</maxHistory>
30 30 <totalSizeCap>3GB</totalSizeCap>
... ...
docker/tb-transports/mqtt/conf/tb-mqtt-transport.conf renamed from msa/transport/mqtt/docker/tb-mqtt-transport.conf
... ... @@ -14,10 +14,10 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -export JAVA_OPTS="$JAVA_OPTS -Xloggc:@pkg.logFolder@/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
  17 +export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_HOST}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_HOST}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
18 18 export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10"
19 19 export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark"
20 20 export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled"
21 21 export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError"
22   -export LOG_FILENAME=${pkg.name}.out
23   -export LOADER_PATH=${pkg.installFolder}/conf
  22 +export LOG_FILENAME=tb-mqtt-transport.out
  23 +export LOADER_PATH=/usr/share/tb-mqtt-transport/conf
... ...
1 1
2 2 HTTP_BIND_ADDRESS=0.0.0.0
3 3 HTTP_BIND_PORT=8080
4   -TB_HOST=haproxy
5   -TB_PORT=8080
  4 +TB_ENABLE_PROXY=false
6 5 LOGGER_LEVEL=info
7 6 LOG_FOLDER=logs
8 7 LOGGER_FILENAME=tb-web-ui-%DATE%.log
... ...
  1 +
  2 +## Black box tests execution
  3 +To run the black box tests with using Docker, the local Docker images of Thingsboard's microservices should be built. <br />
  4 +- Build the local Docker images in the directory with the Thingsboard's main [pom.xml](./../../pom.xml):
  5 +
  6 + mvn clean install -Ddockerfile.skip=false
  7 +- Verify that the new local images were built:
  8 +
  9 + docker image ls
  10 +As result, in REPOSITORY column, next images should be present:
  11 +
  12 + thingsboard/tb-coap-transport
  13 + thingsboard/tb-http-transport
  14 + thingsboard/tb-mqtt-transport
  15 + thingsboard/tb-node
  16 + thingsboard/tb-web-ui
  17 + thingsboard/tb-js-executor
  18 +
  19 +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory:
  20 +
  21 + mvn clean install -DblackBoxTests.skip=false
  22 +
  23 +
... ...
  1 +<!--
  2 +
  3 + Copyright © 2016-2018 The Thingsboard Authors
  4 +
  5 + Licensed under the Apache License, Version 2.0 (the "License");
  6 + you may not use this file except in compliance with the License.
  7 + You may obtain a copy of the License at
  8 +
  9 + http://www.apache.org/licenses/LICENSE-2.0
  10 +
  11 + Unless required by applicable law or agreed to in writing, software
  12 + distributed under the License is distributed on an "AS IS" BASIS,
  13 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 + See the License for the specific language governing permissions and
  15 + limitations under the License.
  16 +
  17 +-->
  18 +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  19 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  20 + <modelVersion>4.0.0</modelVersion>
  21 +
  22 + <parent>
  23 + <groupId>org.thingsboard</groupId>
  24 + <version>2.2.0-SNAPSHOT</version>
  25 + <artifactId>msa</artifactId>
  26 + </parent>
  27 + <groupId>org.thingsboard.msa</groupId>
  28 + <artifactId>black-box-tests</artifactId>
  29 +
  30 + <name>ThingsBoard Black Box Tests</name>
  31 + <url>https://thingsboard.io</url>
  32 + <description>Project for ThingsBoard black box testing with using Docker</description>
  33 +
  34 + <properties>
  35 + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  36 + <main.dir>${basedir}/../..</main.dir>
  37 + <blackBoxTests.skip>true</blackBoxTests.skip>
  38 + <testcontainers.version>1.9.1</testcontainers.version>
  39 + <zeroturnaround.version>1.10</zeroturnaround.version>
  40 + <java-websocket.version>1.3.9</java-websocket.version>
  41 + <httpclient.version>4.5.6</httpclient.version>
  42 + </properties>
  43 +
  44 + <dependencies>
  45 + <dependency>
  46 + <groupId>org.testcontainers</groupId>
  47 + <artifactId>testcontainers</artifactId>
  48 + <version>${testcontainers.version}</version>
  49 + </dependency>
  50 + <dependency>
  51 + <groupId>org.zeroturnaround</groupId>
  52 + <artifactId>zt-exec</artifactId>
  53 + <version>${zeroturnaround.version}</version>
  54 + </dependency>
  55 + <dependency>
  56 + <groupId>org.java-websocket</groupId>
  57 + <artifactId>Java-WebSocket</artifactId>
  58 + <version>${java-websocket.version}</version>
  59 + </dependency>
  60 + <dependency>
  61 + <groupId>org.apache.httpcomponents</groupId>
  62 + <artifactId>httpclient</artifactId>
  63 + <version>${httpclient.version}</version>
  64 + </dependency>
  65 + <dependency>
  66 + <groupId>io.takari.junit</groupId>
  67 + <artifactId>takari-cpsuite</artifactId>
  68 + </dependency>
  69 + <dependency>
  70 + <groupId>ch.qos.logback</groupId>
  71 + <artifactId>logback-classic</artifactId>
  72 + </dependency>
  73 + <dependency>
  74 + <groupId>com.google.code.gson</groupId>
  75 + <artifactId>gson</artifactId>
  76 + </dependency>
  77 + <dependency>
  78 + <groupId>org.apache.commons</groupId>
  79 + <artifactId>commons-lang3</artifactId>
  80 + </dependency>
  81 + <dependency>
  82 + <groupId>com.google.guava</groupId>
  83 + <artifactId>guava</artifactId>
  84 + </dependency>
  85 + <dependency>
  86 + <groupId>org.thingsboard</groupId>
  87 + <artifactId>netty-mqtt</artifactId>
  88 + </dependency>
  89 + <dependency>
  90 + <groupId>org.thingsboard</groupId>
  91 + <artifactId>tools</artifactId>
  92 + </dependency>
  93 + </dependencies>
  94 +
  95 + <build>
  96 + <plugins>
  97 + <plugin>
  98 + <groupId>org.apache.maven.plugins</groupId>
  99 + <artifactId>maven-surefire-plugin</artifactId>
  100 + <configuration>
  101 + <includes>
  102 + <include>**/*TestSuite.java</include>
  103 + </includes>
  104 + <skipTests>${blackBoxTests.skip}</skipTests>
  105 + </configuration>
  106 + </plugin>
  107 + </plugins>
  108 + </build>
  109 +
  110 +</project>
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.msa;
  17 +
  18 +import com.fasterxml.jackson.databind.ObjectMapper;
  19 +import com.google.common.collect.ImmutableMap;
  20 +import com.google.gson.JsonArray;
  21 +import com.google.gson.JsonObject;
  22 +import lombok.extern.slf4j.Slf4j;
  23 +import org.apache.commons.lang3.RandomStringUtils;
  24 +import org.apache.http.config.Registry;
  25 +import org.apache.http.config.RegistryBuilder;
  26 +import org.apache.http.conn.socket.ConnectionSocketFactory;
  27 +import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
  28 +import org.apache.http.conn.ssl.TrustStrategy;
  29 +import org.apache.http.conn.ssl.X509HostnameVerifier;
  30 +import org.apache.http.impl.client.CloseableHttpClient;
  31 +import org.apache.http.impl.client.HttpClients;
  32 +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
  33 +import org.apache.http.ssl.SSLContextBuilder;
  34 +import org.apache.http.ssl.SSLContexts;
  35 +import org.junit.*;
  36 +import org.junit.rules.TestRule;
  37 +import org.junit.rules.TestWatcher;
  38 +import org.junit.runner.Description;
  39 +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
  40 +import org.thingsboard.client.tools.RestClient;
  41 +import org.thingsboard.server.common.data.Device;
  42 +import org.thingsboard.server.common.data.EntityType;
  43 +import org.thingsboard.server.common.data.id.DeviceId;
  44 +import org.thingsboard.server.msa.mapper.WsTelemetryResponse;
  45 +
  46 +import javax.net.ssl.*;
  47 +import java.net.URI;
  48 +import java.security.cert.X509Certificate;
  49 +import java.util.List;
  50 +import java.util.Map;
  51 +import java.util.Random;
  52 +
  53 +@Slf4j
  54 +public abstract class AbstractContainerTest {
  55 + protected static final String HTTPS_URL = "https://localhost";
  56 + protected static final String WSS_URL = "wss://localhost";
  57 + protected static RestClient restClient;
  58 + protected ObjectMapper mapper = new ObjectMapper();
  59 +
  60 + @BeforeClass
  61 + public static void before() throws Exception {
  62 + restClient = new RestClient(HTTPS_URL);
  63 + restClient.getRestTemplate().setRequestFactory(getRequestFactoryForSelfSignedCert());
  64 + }
  65 +
  66 + @Rule
  67 + public TestRule watcher = new TestWatcher() {
  68 + protected void starting(Description description) {
  69 + log.info("=================================================");
  70 + log.info("STARTING TEST: {}" , description.getMethodName());
  71 + log.info("=================================================");
  72 + }
  73 +
  74 + /**
  75 + * Invoked when a test succeeds
  76 + */
  77 + protected void succeeded(Description description) {
  78 + log.info("=================================================");
  79 + log.info("SUCCEEDED TEST: {}" , description.getMethodName());
  80 + log.info("=================================================");
  81 + }
  82 +
  83 + /**
  84 + * Invoked when a test fails
  85 + */
  86 + protected void failed(Throwable e, Description description) {
  87 + log.info("=================================================");
  88 + log.info("FAILED TEST: {}" , description.getMethodName(), e);
  89 + log.info("=================================================");
  90 + }
  91 + };
  92 +
  93 + protected Device createDevice(String name) {
  94 + return restClient.createDevice(name + RandomStringUtils.randomAlphanumeric(7), "DEFAULT");
  95 + }
  96 +
  97 + protected WsClient subscribeToWebSocket(DeviceId deviceId, String scope, CmdsType property) throws Exception {
  98 + WsClient wsClient = new WsClient(new URI(WSS_URL + "/api/ws/plugins/telemetry?token=" + restClient.getToken()));
  99 + SSLContextBuilder builder = SSLContexts.custom();
  100 + builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true);
  101 + wsClient.setSocket(builder.build().getSocketFactory().createSocket());
  102 + wsClient.connectBlocking();
  103 +
  104 + JsonObject cmdsObject = new JsonObject();
  105 + cmdsObject.addProperty("entityType", EntityType.DEVICE.name());
  106 + cmdsObject.addProperty("entityId", deviceId.toString());
  107 + cmdsObject.addProperty("scope", scope);
  108 + cmdsObject.addProperty("cmdId", new Random().nextInt(100));
  109 +
  110 + JsonArray cmd = new JsonArray();
  111 + cmd.add(cmdsObject);
  112 + JsonObject wsRequest = new JsonObject();
  113 + wsRequest.add(property.toString(), cmd);
  114 + wsClient.send(wsRequest.toString());
  115 + wsClient.waitForFirstReply();
  116 + return wsClient;
  117 + }
  118 +
  119 + protected Map<String, Long> getExpectedLatestValues(long ts) {
  120 + return ImmutableMap.<String, Long>builder()
  121 + .put("booleanKey", ts)
  122 + .put("stringKey", ts)
  123 + .put("doubleKey", ts)
  124 + .put("longKey", ts)
  125 + .build();
  126 + }
  127 +
  128 + protected boolean verify(WsTelemetryResponse wsTelemetryResponse, String key, Long expectedTs, String expectedValue) {
  129 + List<Object> list = wsTelemetryResponse.getDataValuesByKey(key);
  130 + return expectedTs.equals(list.get(0)) && expectedValue.equals(list.get(1));
  131 + }
  132 +
  133 + protected boolean verify(WsTelemetryResponse wsTelemetryResponse, String key, String expectedValue) {
  134 + List<Object> list = wsTelemetryResponse.getDataValuesByKey(key);
  135 + return expectedValue.equals(list.get(1));
  136 + }
  137 +
  138 + protected JsonObject createPayload(long ts) {
  139 + JsonObject values = createPayload();
  140 + JsonObject payload = new JsonObject();
  141 + payload.addProperty("ts", ts);
  142 + payload.add("values", values);
  143 + return payload;
  144 + }
  145 +
  146 + protected JsonObject createPayload() {
  147 + JsonObject values = new JsonObject();
  148 + values.addProperty("stringKey", "value1");
  149 + values.addProperty("booleanKey", true);
  150 + values.addProperty("doubleKey", 42.0);
  151 + values.addProperty("longKey", 73L);
  152 +
  153 + return values;
  154 + }
  155 +
  156 + protected enum CmdsType {
  157 + TS_SUB_CMDS("tsSubCmds"),
  158 + HISTORY_CMDS("historyCmds"),
  159 + ATTR_SUB_CMDS("attrSubCmds");
  160 +
  161 + private final String text;
  162 +
  163 + CmdsType(final String text) {
  164 + this.text = text;
  165 + }
  166 +
  167 + @Override
  168 + public String toString() {
  169 + return text;
  170 + }
  171 + }
  172 +
  173 + private static HttpComponentsClientHttpRequestFactory getRequestFactoryForSelfSignedCert() throws Exception {
  174 + SSLContextBuilder builder = SSLContexts.custom();
  175 + builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true);
  176 + SSLContext sslContext = builder.build();
  177 + SSLConnectionSocketFactory sslSelfSigned = new SSLConnectionSocketFactory(sslContext, new X509HostnameVerifier() {
  178 + @Override
  179 + public void verify(String host, SSLSocket ssl) {
  180 + }
  181 +
  182 + @Override
  183 + public void verify(String host, X509Certificate cert) {
  184 + }
  185 +
  186 + @Override
  187 + public void verify(String host, String[] cns, String[] subjectAlts) {
  188 + }
  189 +
  190 + @Override
  191 + public boolean verify(String s, SSLSession sslSession) {
  192 + return true;
  193 + }
  194 + });
  195 +
  196 + Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder
  197 + .<ConnectionSocketFactory>create()
  198 + .register("https", sslSelfSigned)
  199 + .build();
  200 +
  201 + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
  202 + CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
  203 + return new HttpComponentsClientHttpRequestFactory(httpClient);
  204 + }
  205 +
  206 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.msa;
  17 +
  18 +import org.junit.ClassRule;
  19 +import org.junit.extensions.cpsuite.ClasspathSuite;
  20 +import org.junit.rules.ExternalResource;
  21 +import org.junit.runner.RunWith;
  22 +import org.testcontainers.containers.DockerComposeContainer;
  23 +import org.testcontainers.containers.wait.strategy.Wait;
  24 +import org.testcontainers.utility.Base58;
  25 +
  26 +import java.io.File;
  27 +import java.time.Duration;
  28 +import java.util.Arrays;
  29 +import java.util.HashMap;
  30 +import java.util.List;
  31 +import java.util.Map;
  32 +
  33 +@RunWith(ClasspathSuite.class)
  34 +@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"})
  35 +public class ContainerTestSuite {
  36 +
  37 + private static DockerComposeContainer testContainer;
  38 +
  39 + @ClassRule
  40 + public static ThingsBoardDbInstaller installTb = new ThingsBoardDbInstaller();
  41 +
  42 + @ClassRule
  43 + public static DockerComposeContainer getTestContainer() {
  44 + if (testContainer == null) {
  45 + testContainer = new DockerComposeContainer(
  46 + new File("./../../docker/docker-compose.yml"),
  47 + new File("./../../docker/docker-compose.postgres.yml"),
  48 + new File("./../../docker/docker-compose.postgres.volumes.yml"))
  49 + .withPull(false)
  50 + .withLocalCompose(true)
  51 + .withTailChildContainers(true)
  52 + .withEnv(installTb.getEnv())
  53 + .withEnv("LOAD_BALANCER_NAME", "")
  54 + .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(Duration.ofSeconds(120)));
  55 + }
  56 + return testContainer;
  57 + }
  58 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +
  17 +package org.thingsboard.server.msa;
  18 +
  19 +
  20 +import com.google.common.base.Splitter;
  21 +import com.google.common.collect.Maps;
  22 +import lombok.extern.slf4j.Slf4j;
  23 +import org.apache.commons.lang3.SystemUtils;
  24 +import org.testcontainers.containers.ContainerLaunchException;
  25 +import org.testcontainers.utility.CommandLine;
  26 +import org.zeroturnaround.exec.InvalidExitValueException;
  27 +import org.zeroturnaround.exec.ProcessExecutor;
  28 +import org.zeroturnaround.exec.stream.slf4j.Slf4jStream;
  29 +
  30 +import java.io.File;
  31 +import java.util.HashMap;
  32 +import java.util.List;
  33 +import java.util.Map;
  34 +import java.util.Objects;
  35 +import java.util.stream.Stream;
  36 +
  37 +import static com.google.common.base.Preconditions.checkArgument;
  38 +import static com.google.common.base.Preconditions.checkNotNull;
  39 +import static java.util.stream.Collectors.joining;
  40 +
  41 +@Slf4j
  42 +public class DockerComposeExecutor {
  43 +
  44 + String ENV_PROJECT_NAME = "COMPOSE_PROJECT_NAME";
  45 + String ENV_COMPOSE_FILE = "COMPOSE_FILE";
  46 +
  47 + private static final String COMPOSE_EXECUTABLE = SystemUtils.IS_OS_WINDOWS ? "docker-compose.exe" : "docker-compose";
  48 + private static final String DOCKER_EXECUTABLE = SystemUtils.IS_OS_WINDOWS ? "docker.exe" : "docker";
  49 +
  50 + private final List<File> composeFiles;
  51 + private final String identifier;
  52 + private String cmd = "";
  53 + private Map<String, String> env = new HashMap<>();
  54 +
  55 + public DockerComposeExecutor(List<File> composeFiles, String identifier) {
  56 + validateFileList(composeFiles);
  57 + this.composeFiles = composeFiles;
  58 + this.identifier = identifier;
  59 + }
  60 +
  61 + public DockerComposeExecutor withCommand(String cmd) {
  62 + this.cmd = cmd;
  63 + return this;
  64 + }
  65 +
  66 + public DockerComposeExecutor withEnv(Map<String, String> env) {
  67 + this.env = env;
  68 + return this;
  69 + }
  70 +
  71 + public void invokeCompose() {
  72 + // bail out early
  73 + if (!CommandLine.executableExists(COMPOSE_EXECUTABLE)) {
  74 + throw new ContainerLaunchException("Local Docker Compose not found. Is " + COMPOSE_EXECUTABLE + " on the PATH?");
  75 + }
  76 + final Map<String, String> environment = Maps.newHashMap(env);
  77 + environment.put(ENV_PROJECT_NAME, identifier);
  78 + final Stream<String> absoluteDockerComposeFilePaths = composeFiles.stream().map(File::getAbsolutePath).map(Objects::toString);
  79 + final String composeFileEnvVariableValue = absoluteDockerComposeFilePaths.collect(joining(File.pathSeparator + ""));
  80 + log.debug("Set env COMPOSE_FILE={}", composeFileEnvVariableValue);
  81 + final File pwd = composeFiles.get(0).getAbsoluteFile().getParentFile().getAbsoluteFile();
  82 + environment.put(ENV_COMPOSE_FILE, composeFileEnvVariableValue);
  83 + log.info("Local Docker Compose is running command: {}", cmd);
  84 + final List<String> command = Splitter.onPattern(" ").omitEmptyStrings().splitToList(COMPOSE_EXECUTABLE + " " + cmd);
  85 + try {
  86 + new ProcessExecutor().command(command).redirectOutput(Slf4jStream.of(log).asInfo()).redirectError(Slf4jStream.of(log).asError()).environment(environment).directory(pwd).exitValueNormal().executeNoTimeout();
  87 + log.info("Docker Compose has finished running");
  88 + } catch (InvalidExitValueException e) {
  89 + throw new ContainerLaunchException("Local Docker Compose exited abnormally with code " + e.getExitValue() + " whilst running command: " + cmd);
  90 + } catch (Exception e) {
  91 + throw new ContainerLaunchException("Error running local Docker Compose command: " + cmd, e);
  92 + }
  93 + }
  94 +
  95 + public void invokeDocker() {
  96 + // bail out early
  97 + if (!CommandLine.executableExists(DOCKER_EXECUTABLE)) {
  98 + throw new ContainerLaunchException("Local Docker not found. Is " + DOCKER_EXECUTABLE + " on the PATH?");
  99 + }
  100 + final File pwd = composeFiles.get(0).getAbsoluteFile().getParentFile().getAbsoluteFile();
  101 + log.info("Local Docker is running command: {}", cmd);
  102 + final List<String> command = Splitter.onPattern(" ").omitEmptyStrings().splitToList(DOCKER_EXECUTABLE + " " + cmd);
  103 + try {
  104 + new ProcessExecutor().command(command).redirectOutput(Slf4jStream.of(log).asInfo()).redirectError(Slf4jStream.of(log).asError()).directory(pwd).exitValueNormal().executeNoTimeout();
  105 + log.info("Docker has finished running");
  106 + } catch (InvalidExitValueException e) {
  107 + throw new ContainerLaunchException("Local Docker exited abnormally with code " + e.getExitValue() + " whilst running command: " + cmd);
  108 + } catch (Exception e) {
  109 + throw new ContainerLaunchException("Error running local Docker command: " + cmd, e);
  110 + }
  111 + }
  112 +
  113 + void validateFileList(List<File> composeFiles) {
  114 + checkNotNull(composeFiles);
  115 + checkArgument(!composeFiles.isEmpty(), "No docker compose file have been provided");
  116 + }
  117 +
  118 +
  119 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.msa;
  17 +
  18 +import org.junit.rules.ExternalResource;
  19 +import org.testcontainers.utility.Base58;
  20 +
  21 +import java.io.File;
  22 +import java.util.Arrays;
  23 +import java.util.HashMap;
  24 +import java.util.List;
  25 +import java.util.Map;
  26 +
  27 +public class ThingsBoardDbInstaller extends ExternalResource {
  28 +
  29 + private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume";
  30 + private final static String TB_LOG_VOLUME = "tb-log-test-volume";
  31 + private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume";
  32 + private final static String TB_HTTP_TRANSPORT_LOG_VOLUME = "tb-http-transport-log-test-volume";
  33 + private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume";
  34 +
  35 + private final DockerComposeExecutor dockerCompose;
  36 +
  37 + private final String postgresDataVolume;
  38 + private final String tbLogVolume;
  39 + private final String tbCoapTransportLogVolume;
  40 + private final String tbHttpTransportLogVolume;
  41 + private final String tbMqttTransportLogVolume;
  42 + private final Map<String, String> env;
  43 +
  44 + public ThingsBoardDbInstaller() {
  45 + List<File> composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"),
  46 + new File("./../../docker/docker-compose.postgres.yml"),
  47 + new File("./../../docker/docker-compose.postgres.volumes.yml"));
  48 +
  49 + String identifier = Base58.randomString(6).toLowerCase();
  50 + String project = identifier + Base58.randomString(6).toLowerCase();
  51 +
  52 + postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME;
  53 + tbLogVolume = project + "_" + TB_LOG_VOLUME;
  54 + tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME;
  55 + tbHttpTransportLogVolume = project + "_" + TB_HTTP_TRANSPORT_LOG_VOLUME;
  56 + tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME;
  57 +
  58 + dockerCompose = new DockerComposeExecutor(composeFiles, project);
  59 +
  60 + env = new HashMap<>();
  61 + env.put("POSTGRES_DATA_VOLUME", postgresDataVolume);
  62 + env.put("TB_LOG_VOLUME", tbLogVolume);
  63 + env.put("TB_COAP_TRANSPORT_LOG_VOLUME", tbCoapTransportLogVolume);
  64 + env.put("TB_HTTP_TRANSPORT_LOG_VOLUME", tbHttpTransportLogVolume);
  65 + env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume);
  66 + dockerCompose.withEnv(env);
  67 + }
  68 +
  69 + public Map<String, String> getEnv() {
  70 + return env;
  71 + }
  72 +
  73 + @Override
  74 + protected void before() throws Throwable {
  75 + try {
  76 +
  77 + dockerCompose.withCommand("volume create " + postgresDataVolume);
  78 + dockerCompose.invokeDocker();
  79 +
  80 + dockerCompose.withCommand("volume create " + tbLogVolume);
  81 + dockerCompose.invokeDocker();
  82 +
  83 + dockerCompose.withCommand("volume create " + tbCoapTransportLogVolume);
  84 + dockerCompose.invokeDocker();
  85 +
  86 + dockerCompose.withCommand("volume create " + tbHttpTransportLogVolume);
  87 + dockerCompose.invokeDocker();
  88 +
  89 + dockerCompose.withCommand("volume create " + tbMqttTransportLogVolume);
  90 + dockerCompose.invokeDocker();
  91 +
  92 + dockerCompose.withCommand("up -d redis postgres");
  93 + dockerCompose.invokeCompose();
  94 +
  95 + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb1");
  96 + dockerCompose.invokeCompose();
  97 +
  98 + } finally {
  99 + try {
  100 + dockerCompose.withCommand("down -v");
  101 + dockerCompose.invokeCompose();
  102 + } catch (Exception e) {}
  103 + }
  104 + }
  105 +
  106 + @Override
  107 + protected void after() {
  108 + copyLogs(tbLogVolume, "./target/tb-logs/");
  109 + copyLogs(tbCoapTransportLogVolume, "./target/tb-coap-transport-logs/");
  110 + copyLogs(tbHttpTransportLogVolume, "./target/tb-http-transport-logs/");
  111 + copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/");
  112 +
  113 + dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume +
  114 + " " + tbCoapTransportLogVolume + " " + tbHttpTransportLogVolume + " " + tbMqttTransportLogVolume);
  115 + dockerCompose.invokeDocker();
  116 + }
  117 +
  118 + private void copyLogs(String volumeName, String targetDir) {
  119 + File tbLogsDir = new File(targetDir);
  120 + tbLogsDir.mkdirs();
  121 +
  122 + dockerCompose.withCommand("run -d --rm --name tb-logs-container -v " + volumeName + ":/root alpine tail -f /dev/null");
  123 + dockerCompose.invokeDocker();
  124 +
  125 + dockerCompose.withCommand("cp tb-logs-container:/root/. "+tbLogsDir.getAbsolutePath());
  126 + dockerCompose.invokeDocker();
  127 +
  128 + dockerCompose.withCommand("rm -f tb-logs-container");
  129 + dockerCompose.invokeDocker();
  130 + }
  131 +
  132 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.msa;
  17 +
  18 +import com.fasterxml.jackson.databind.ObjectMapper;
  19 +import lombok.extern.slf4j.Slf4j;
  20 +import org.java_websocket.client.WebSocketClient;
  21 +import org.java_websocket.handshake.ServerHandshake;
  22 +import org.thingsboard.server.msa.mapper.WsTelemetryResponse;
  23 +
  24 +import java.io.IOException;
  25 +import java.net.URI;
  26 +import java.util.concurrent.CountDownLatch;
  27 +import java.util.concurrent.TimeUnit;
  28 +
  29 +@Slf4j
  30 +public class WsClient extends WebSocketClient {
  31 + private static final ObjectMapper mapper = new ObjectMapper();
  32 + private WsTelemetryResponse message;
  33 +
  34 + private volatile boolean firstReplyReceived;
  35 + private CountDownLatch firstReply = new CountDownLatch(1);
  36 + private CountDownLatch latch = new CountDownLatch(1);
  37 +
  38 + WsClient(URI serverUri) {
  39 + super(serverUri);
  40 + }
  41 +
  42 + @Override
  43 + public void onOpen(ServerHandshake serverHandshake) {
  44 + }
  45 +
  46 + @Override
  47 + public void onMessage(String message) {
  48 + if (!firstReplyReceived) {
  49 + firstReplyReceived = true;
  50 + firstReply.countDown();
  51 + } else {
  52 + try {
  53 + WsTelemetryResponse response = mapper.readValue(message, WsTelemetryResponse.class);
  54 + if (!response.getData().isEmpty()) {
  55 + this.message = response;
  56 + latch.countDown();
  57 + }
  58 + } catch (IOException e) {
  59 + log.error("ws message can't be read");
  60 + }
  61 + }
  62 + }
  63 +
  64 + @Override
  65 + public void onClose(int code, String reason, boolean remote) {
  66 + log.info("ws is closed, due to [{}]", reason);
  67 + }
  68 +
  69 + @Override
  70 + public void onError(Exception ex) {
  71 + ex.printStackTrace();
  72 + }
  73 +
  74 + public WsTelemetryResponse getLastMessage() {
  75 + try {
  76 + latch.await(10, TimeUnit.SECONDS);
  77 + return this.message;
  78 + } catch (InterruptedException e) {
  79 + log.error("Timeout, ws message wasn't received");
  80 + }
  81 + return null;
  82 + }
  83 +
  84 + void waitForFirstReply() {
  85 + try {
  86 + firstReply.await(10, TimeUnit.SECONDS);
  87 + } catch (InterruptedException e) {
  88 + log.error("Timeout, ws message wasn't received");
  89 + throw new RuntimeException(e);
  90 + }
  91 + }
  92 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.msa.connectivity;
  17 +
  18 +import com.google.common.collect.Sets;
  19 +import org.junit.Assert;
  20 +import org.junit.Test;
  21 +import org.springframework.http.ResponseEntity;
  22 +import org.thingsboard.server.common.data.Device;
  23 +import org.thingsboard.server.common.data.security.DeviceCredentials;
  24 +import org.thingsboard.server.msa.AbstractContainerTest;
  25 +import org.thingsboard.server.msa.WsClient;
  26 +import org.thingsboard.server.msa.mapper.WsTelemetryResponse;
  27 +
  28 +public class HttpClientTest extends AbstractContainerTest {
  29 +
  30 + @Test
  31 + public void telemetryUpload() throws Exception {
  32 + restClient.login("tenant@thingsboard.org", "tenant");
  33 +
  34 + Device device = createDevice("http_");
  35 + DeviceCredentials deviceCredentials = restClient.getCredentials(device.getId());
  36 +
  37 + WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS);
  38 + ResponseEntity deviceTelemetryResponse = restClient.getRestTemplate()
  39 + .postForEntity(HTTPS_URL + "/api/v1/{credentialsId}/telemetry",
  40 + mapper.readTree(createPayload().toString()),
  41 + ResponseEntity.class,
  42 + deviceCredentials.getCredentialsId());
  43 + Assert.assertTrue(deviceTelemetryResponse.getStatusCode().is2xxSuccessful());
  44 + WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
  45 + wsClient.closeBlocking();
  46 +
  47 + Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"),
  48 + actualLatestTelemetry.getLatestValues().keySet());
  49 +
  50 + Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString()));
  51 + Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1"));
  52 + Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0)));
  53 + Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73)));
  54 +
  55 + restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId());
  56 + }
  57 +}
... ...