简柏林 / thingskit · Commits

Commit cdc72653dedc669e59c9773cef9f9489324fadc1

Authored by Viacheslav Kukhtyn
2 parents e15fb63b b42b99da

Merge remote-tracking branch 'upstream/master'

Inline Side-by-side
Showing 96 changed files with 1922 additions and 1408 deletions

Too many changes to show.
Reload with full diff Plain diff Email patch

To preserve performance only 96 of 191 files are displayed.

... ... @@ -32,3 +32,4 @@ pom.xml.versionsBackup
32 32 **/Californium.properties
33 33 **/.env
34 34 .instance_id
  35 +rebuild-docker.sh
... ...
... ... @@ -15,7 +15,7 @@
15 15 "resources": [],
16 16 "templateHtml": "",
17 17 "templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}",
18   - "controllerScript": "self.onInit = function() {\n \n self.ctx.datasourceTitleCells = [];\n self.ctx.valueCells = [];\n self.ctx.labelCells = [];\n \n for (var i=0; i < self.ctx.datasources.length; i++) {\n var tbDatasource = self.ctx.datasources[i];\n\n var datasourceId = 'tbDatasource' + i;\n self.ctx.$container.append(\n \"<div id='\" + datasourceId +\n \"' class='tbDatasource-container'></div>\"\n );\n\n var datasourceContainer = $('#' + datasourceId,\n self.ctx.$container);\n\n datasourceContainer.append(\n \"<div class='tbDatasource-title'>\" +\n tbDatasource.name + \"</div>\"\n );\n \n var datasourceTitleCell = $('.tbDatasource-title', datasourceContainer);\n self.ctx.datasourceTitleCells.push(datasourceTitleCell);\n \n var tableId = 'table' + i;\n datasourceContainer.append(\n \"<table id='\" + tableId +\n \"' class='tbDatasource-table'><col width='30%'><col width='70%'></table>\"\n );\n var table = $('#' + tableId, self.ctx.$container);\n\n for (var a = 0; a < tbDatasource.dataKeys.length; a++) {\n var dataKey = tbDatasource.dataKeys[a];\n var labelCellId = 'labelCell' + a;\n var cellId = 'cell' + a;\n table.append(\"<tr><td id='\" + labelCellId + \"'>\" + dataKey.label +\n \"</td><td id='\" + cellId +\n \"'></td></tr>\");\n var labelCell = $('#' + labelCellId, table);\n self.ctx.labelCells.push(labelCell);\n var valueCell = $('#' + cellId, table);\n self.ctx.valueCells.push(valueCell);\n }\n } \n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.valueCells.length; i++) {\n var cellData = self.ctx.data[i];\n console.log(self.ctx); //del\n if (cellData && cellData.data && cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length -\n 1];\n var value = tvPair[1];\n var textValue;\n //toDo -> + IsNumber\n \n if (isNumber(value)) {\n var decimals = self.ctx.decimals;\n var units = self.ctx.units;\n if (cellData.dataKey.decimals || cellData.dataKey.decimals === 0) {\n decimals = cellData.dataKey.decimals;\n }\n if (cellData.dataKey.units) {\n units = cellData.dataKey.units;\n }\n txtValue = self.ctx.utils.formatValue(value, decimals, units, true);\n } else {\n txtValue = value;\n }\n self.ctx.valueCells[i].html(txtValue);\n }\n }\n \n function isNumber(n) {\n return !isNaN(parseFloat(n)) && isFinite(n);\n }\n}\n\nself.onResize = function() {\n var datasoirceTitleFontSize = self.ctx.height/8;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n datasoirceTitleFontSize = self.ctx.width/12;\n }\n datasoirceTitleFontSize = Math.min(datasoirceTitleFontSize, 20);\n for (var i = 0; i < self.ctx.datasourceTitleCells.length; i++) {\n self.ctx.datasourceTitleCells[i].css('font-size', datasoirceTitleFontSize+'px');\n }\n var valueFontSize = self.ctx.height/9;\n var labelFontSize = self.ctx.height/9;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n valueFontSize = self.ctx.width/15;\n labelFontSize = self.ctx.width/15;\n }\n valueFontSize = Math.min(valueFontSize, 18);\n labelFontSize = Math.min(labelFontSize, 18);\n\n for (i = 0; i < self.ctx.valueCells; i++) {\n self.ctx.valueCells[i].css('font-size', valueFontSize+'px');\n self.ctx.valueCells[i].css('height', valueFontSize*2.5+'px');\n self.ctx.valueCells[i].css('padding', '0px ' + valueFontSize + 'px');\n self.ctx.labelCells[i].css('font-size', labelFontSize+'px');\n self.ctx.labelCells[i].css('height', labelFontSize*2.5+'px');\n self.ctx.labelCells[i].css('padding', '0px ' + labelFontSize + 'px');\n } \n}\n\nself.onDestroy = function() {\n}\n",
  18 + "controllerScript": "self.onInit = function() {\n \n self.ctx.datasourceTitleCells = [];\n self.ctx.valueCells = [];\n self.ctx.labelCells = [];\n \n for (var i=0; i < self.ctx.datasources.length; i++) {\n var tbDatasource = self.ctx.datasources[i];\n\n var datasourceId = 'tbDatasource' + i;\n self.ctx.$container.append(\n \"<div id='\" + datasourceId +\n \"' class='tbDatasource-container'></div>\"\n );\n\n var datasourceContainer = $('#' + datasourceId,\n self.ctx.$container);\n\n datasourceContainer.append(\n \"<div class='tbDatasource-title'>\" +\n tbDatasource.name + \"</div>\"\n );\n \n var datasourceTitleCell = $('.tbDatasource-title', datasourceContainer);\n self.ctx.datasourceTitleCells.push(datasourceTitleCell);\n \n var tableId = 'table' + i;\n datasourceContainer.append(\n \"<table id='\" + tableId +\n \"' class='tbDatasource-table'><col width='30%'><col width='70%'></table>\"\n );\n var table = $('#' + tableId, self.ctx.$container);\n\n for (var a = 0; a < tbDatasource.dataKeys.length; a++) {\n var dataKey = tbDatasource.dataKeys[a];\n var labelCellId = 'labelCell' + a;\n var cellId = 'cell' + a;\n table.append(\"<tr><td id='\" + labelCellId + \"'>\" + dataKey.label +\n \"</td><td id='\" + cellId +\n \"'></td></tr>\");\n var labelCell = $('#' + labelCellId, table);\n self.ctx.labelCells.push(labelCell);\n var valueCell = $('#' + cellId, table);\n self.ctx.valueCells.push(valueCell);\n }\n } \n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.valueCells.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData && cellData.data && cellData.data.length > 0) {\n var tvPair = cellData.data[cellData.data.length -\n 1];\n var value = tvPair[1];\n var textValue;\n //toDo -> + IsNumber\n \n if (isNumber(value)) {\n var decimals = self.ctx.decimals;\n var units = self.ctx.units;\n if (cellData.dataKey.decimals || cellData.dataKey.decimals === 0) {\n decimals = cellData.dataKey.decimals;\n }\n if (cellData.dataKey.units) {\n units = cellData.dataKey.units;\n }\n txtValue = self.ctx.utils.formatValue(value, decimals, units, true);\n } else {\n txtValue = value;\n }\n self.ctx.valueCells[i].html(txtValue);\n }\n }\n \n function isNumber(n) {\n return !isNaN(parseFloat(n)) && isFinite(n);\n }\n}\n\nself.onResize = function() {\n var datasoirceTitleFontSize = self.ctx.height/8;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n datasoirceTitleFontSize = self.ctx.width/12;\n }\n datasoirceTitleFontSize = Math.min(datasoirceTitleFontSize, 20);\n for (var i = 0; i < self.ctx.datasourceTitleCells.length; i++) {\n self.ctx.datasourceTitleCells[i].css('font-size', datasoirceTitleFontSize+'px');\n }\n var valueFontSize = self.ctx.height/9;\n var labelFontSize = self.ctx.height/9;\n if (self.ctx.width/self.ctx.height <= 1.5) {\n valueFontSize = self.ctx.width/15;\n labelFontSize = self.ctx.width/15;\n }\n valueFontSize = Math.min(valueFontSize, 18);\n labelFontSize = Math.min(labelFontSize, 18);\n\n for (i = 0; i < self.ctx.valueCells; i++) {\n self.ctx.valueCells[i].css('font-size', valueFontSize+'px');\n self.ctx.valueCells[i].css('height', valueFontSize*2.5+'px');\n self.ctx.valueCells[i].css('padding', '0px ' + valueFontSize + 'px');\n self.ctx.labelCells[i].css('font-size', labelFontSize+'px');\n self.ctx.labelCells[i].css('height', labelFontSize*2.5+'px');\n self.ctx.labelCells[i].css('padding', '0px ' + labelFontSize + 'px');\n } \n}\n\nself.onDestroy = function() {\n}\n",
19 19 "settingsSchema": "{}",
20 20 "dataKeySettingsSchema": "{}\n",
21 21 "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Attributes card\"}"
... ... @@ -27,7 +27,7 @@
27 27 "descriptor": {
28 28 "type": "latest",
29 29 "sizeX": 7.5,
30   - "sizeY": 4.5,
  30 + "sizeY": 6.5,
31 31 "resources": [],
32 32 "templateHtml": "<tb-entities-table-widget \n table-id=\"tableId\"\n ctx=\"ctx\">\n</tb-entities-table-widget>",
33 33 "templateCss": "",
... ...
  1 +--
  2 +-- Copyright © 2016-2018 The Thingsboard Authors
  3 +--
  4 +-- Licensed under the Apache License, Version 2.0 (the "License");
  5 +-- you may not use this file except in compliance with the License.
  6 +-- You may obtain a copy of the License at
  7 +--
  8 +-- http://www.apache.org/licenses/LICENSE-2.0
  9 +--
  10 +-- Unless required by applicable law or agreed to in writing, software
  11 +-- distributed under the License is distributed on an "AS IS" BASIS,
  12 +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +-- See the License for the specific language governing permissions and
  14 +-- limitations under the License.
  15 +--
  16 +
  17 +ALTER TABLE component_descriptor ADD UNIQUE (clazz);
... ...
... ... @@ -67,6 +67,7 @@ import org.thingsboard.server.service.mail.MailExecutorService;
67 67 import org.thingsboard.server.service.rpc.DeviceRpcService;
68 68 import org.thingsboard.server.service.script.JsExecutorService;
69 69 import org.thingsboard.server.service.script.JsInvokeService;
  70 +import org.thingsboard.server.service.session.DeviceSessionCacheService;
70 71 import org.thingsboard.server.service.state.DeviceStateService;
71 72 import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
72 73 import org.thingsboard.server.service.transport.RuleEngineTransportService;
... ... @@ -201,6 +202,10 @@ public class ActorSystemContext {
201 202 @Getter
202 203 private DeviceStateService deviceStateService;
203 204
  205 + @Autowired
  206 + @Getter
  207 + private DeviceSessionCacheService deviceSessionCacheService;
  208 +
204 209 @Lazy
205 210 @Autowired
206 211 @Getter
... ... @@ -254,6 +259,14 @@ public class ActorSystemContext {
254 259 @Getter
255 260 private boolean allowSystemMailService;
256 261
  262 + @Value("${transport.sessions.inactivity_timeout}")
  263 + @Getter
  264 + private long sessionInactivityTimeout;
  265 +
  266 + @Value("${transport.sessions.report_timeout}")
  267 + @Getter
  268 + private long sessionReportTimeout;
  269 +
257 270 @Getter
258 271 @Setter
259 272 private ActorSystem actorSystem;
... ...
... ... @@ -44,11 +44,19 @@ public class DeviceActor extends ContextAwareActor {
44 44 }
45 45
46 46 @Override
  47 + public void preStart() {
  48 + logger.debug("[{}][{}] Starting device actor.", processor.tenantId, processor.deviceId);
  49 + try {
  50 + processor.initSessionTimeout(context());
  51 + logger.debug("[{}][{}] Device actor started.", processor.tenantId, processor.deviceId);
  52 + } catch (Exception e) {
  53 + logger.error(e, "[{}][{}] Unknown failure", processor.tenantId, processor.deviceId);
  54 + }
  55 + }
  56 +
  57 + @Override
47 58 protected boolean process(TbActorMsg msg) {
48 59 switch (msg.getMsgType()) {
49   - case CLUSTER_EVENT_MSG:
50   - processor.processClusterEventMsg((ClusterEventMsg) msg);
51   - break;
52 60 case TRANSPORT_TO_DEVICE_ACTOR_MSG:
53 61 processor.process(context(), (TransportToDeviceActorMsgWrapper) msg);
54 62 break;
... ... @@ -73,6 +81,9 @@ public class DeviceActor extends ContextAwareActor {
73 81 case DEVICE_ACTOR_CLIENT_SIDE_RPC_TIMEOUT_MSG:
74 82 processor.processClientSideRpcTimeout(context(), (DeviceActorClientSideRpcTimeoutMsg) msg);
75 83 break;
  84 + case SESSION_TIMEOUT_MSG:
  85 + processor.checkSessionsTimeout();
  86 + break;
76 87 default:
77 88 return false;
78 89 }
... ...
... ... @@ -41,7 +41,6 @@ import org.thingsboard.server.common.msg.TbMsg;
41 41 import org.thingsboard.server.common.msg.TbMsgDataType;
42 42 import org.thingsboard.server.common.msg.TbMsgMetaData;
43 43 import org.thingsboard.server.common.msg.cluster.ClusterEventMsg;
44   -import org.thingsboard.server.common.msg.cluster.ServerAddress;
45 44 import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
46 45 import org.thingsboard.server.common.msg.session.SessionMsgType;
47 46 import org.thingsboard.server.common.msg.timeout.DeviceActorClientSideRpcTimeoutMsg;
... ... @@ -89,11 +88,11 @@ import java.util.stream.Collectors;
89 88 /**
90 89 * @author Andrew Shvayka
91 90 */
92   -public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
  91 +class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
93 92
94   - private final TenantId tenantId;
95   - private final DeviceId deviceId;
96   - private final Map<UUID, SessionInfo> sessions;
  93 + final TenantId tenantId;
  94 + final DeviceId deviceId;
  95 + private final Map<UUID, SessionInfoMetaData> sessions;
97 96 private final Map<UUID, SessionInfo> attributeSubscriptions;
98 97 private final Map<UUID, SessionInfo> rpcSubscriptions;
99 98 private final Map<Integer, ToDeviceRpcRequestMetadata> toDeviceRpcPendingMap;
... ... @@ -117,6 +116,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
117 116 this.toDeviceRpcPendingMap = new HashMap<>();
118 117 this.toServerRpcPendingMap = new HashMap<>();
119 118 initAttributes();
  119 + restoreSessions();
120 120 }
121 121
122 122 private void initAttributes() {
... ... @@ -152,7 +152,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
152 152
153 153 if (request.isOneway() && sent) {
154 154 logger.debug("[{}] Rpc command response sent [{}]!", deviceId, request.getId());
155   - systemContext.getDeviceRpcService().processRpcResponseFromDevice(new FromDeviceRpcResponse(msg.getMsg().getId(), msg.getServerAddress(), null, null));
  155 + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(msg.getMsg().getId(), null, null));
156 156 } else {
157 157 registerPendingRpcRequest(context, msg, sent, rpcRequest, timeout);
158 158 }
... ... @@ -161,7 +161,6 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
161 161 } else {
162 162 logger.debug("[{}] RPC request {} is NOT sent!", deviceId, request.getId());
163 163 }
164   -
165 164 }
166 165
167 166 private void registerPendingRpcRequest(ActorContext context, ToDeviceRpcRequestActorMsg msg, boolean sent, ToDeviceRpcRequestMsg rpcRequest, long timeout) {
... ... @@ -174,8 +173,8 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
174 173 ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(msg.getId());
175 174 if (requestMd != null) {
176 175 logger.debug("[{}] RPC request [{}] timeout detected!", deviceId, msg.getId());
177   - systemContext.getDeviceRpcService().processRpcResponseFromDevice(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
178   - requestMd.getMsg().getServerAddress(), null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION));
  176 + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
  177 + null, requestMd.isSent() ? RpcError.TIMEOUT : RpcError.NO_ACTIVE_CONNECTION));
179 178 }
180 179 }
181 180
... ... @@ -202,12 +201,11 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
202 201
203 202 private Consumer<Map.Entry<Integer, ToDeviceRpcRequestMetadata>> processPendingRpc(ActorContext context, UUID sessionId, String nodeId, Set<Integer> sentOneWayIds) {
204 203 return entry -> {
205   - ToDeviceRpcRequestActorMsg requestActorMsg = entry.getValue().getMsg();
206 204 ToDeviceRpcRequest request = entry.getValue().getMsg().getMsg();
207 205 ToDeviceRpcRequestBody body = request.getBody();
208 206 if (request.isOneway()) {
209 207 sentOneWayIds.add(entry.getKey());
210   - systemContext.getDeviceRpcService().processRpcResponseFromDevice(new FromDeviceRpcResponse(request.getId(), requestActorMsg.getServerAddress(), null, null));
  208 + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(request.getId(), null, null));
211 209 }
212 210 ToDeviceRpcRequestMsg rpcRequest = ToDeviceRpcRequestMsg.newBuilder().setRequestId(
213 211 entry.getKey()).setMethodName(body.getMethod()).setParams(body.getParams()).build();
... ... @@ -228,11 +226,11 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
228 226 }
229 227 if (msg.hasPostAttributes()) {
230 228 handlePostAttributesRequest(context, msg.getSessionInfo(), msg.getPostAttributes());
231   - reportActivity();
  229 + reportLogicalDeviceActivity();
232 230 }
233 231 if (msg.hasPostTelemetry()) {
234 232 handlePostTelemetryRequest(context, msg.getSessionInfo(), msg.getPostTelemetry());
235   - reportActivity();
  233 + reportLogicalDeviceActivity();
236 234 }
237 235 if (msg.hasGetAttributes()) {
238 236 handleGetAttributesRequest(context, msg.getSessionInfo(), msg.getGetAttributes());
... ... @@ -242,11 +240,14 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
242 240 }
243 241 if (msg.hasToServerRPCCallRequest()) {
244 242 handleClientSideRPCRequest(context, msg.getSessionInfo(), msg.getToServerRPCCallRequest());
245   - reportActivity();
  243 + reportLogicalDeviceActivity();
  244 + }
  245 + if (msg.hasSubscriptionInfo()) {
  246 + handleSessionActivity(context, msg.getSessionInfo(), msg.getSubscriptionInfo());
246 247 }
247 248 }
248 249
249   - private void reportActivity() {
  250 + private void reportLogicalDeviceActivity() {
250 251 systemContext.getDeviceStateService().onDeviceActivity(deviceId);
251 252 }
252 253
... ... @@ -400,35 +401,27 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
400 401 ToDeviceRpcRequestMetadata requestMd = toDeviceRpcPendingMap.remove(responseMsg.getRequestId());
401 402 boolean success = requestMd != null;
402 403 if (success) {
403   - systemContext.getDeviceRpcService().processRpcResponseFromDevice(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
404   - requestMd.getMsg().getServerAddress(), responseMsg.getPayload(), null));
  404 + systemContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromDeviceActor(new FromDeviceRpcResponse(requestMd.getMsg().getMsg().getId(),
  405 + responseMsg.getPayload(), null));
405 406 } else {
406 407 logger.debug("[{}] Rpc command response [{}] is stale!", deviceId, responseMsg.getRequestId());
407 408 }
408 409 }
409 410
410   - void processClusterEventMsg(ClusterEventMsg msg) {
411   -// if (!msg.isAdded()) {
412   -// logger.debug("[{}] Clearing attributes/rpc subscription for server [{}]", deviceId, msg.getServerAddress());
413   -// Predicate<Map.Entry<SessionId, SessionInfo>> filter = e -> e.getValue().getServer()
414   -// .map(serverAddress -> serverAddress.equals(msg.getServerAddress())).orElse(false);
415   -// attributeSubscriptions.entrySet().removeIf(filter);
416   -// rpcSubscriptions.entrySet().removeIf(filter);
417   -// }
418   - }
419   -
420 411 private void processSubscriptionCommands(ActorContext context, SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg subscribeCmd) {
421 412 UUID sessionId = getSessionId(sessionInfo);
422 413 if (subscribeCmd.getUnsubscribe()) {
423 414 logger.debug("[{}] Canceling attributes subscription for session [{}]", deviceId, sessionId);
424 415 attributeSubscriptions.remove(sessionId);
425 416 } else {
426   - SessionInfo session = sessions.get(sessionId);
427   - if (session == null) {
428   - session = new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId());
  417 + SessionInfoMetaData sessionMD = sessions.get(sessionId);
  418 + if (sessionMD == null) {
  419 + sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId()));
429 420 }
  421 + sessionMD.setSubscribedToAttributes(true);
430 422 logger.debug("[{}] Registering attributes subscription for session [{}]", deviceId, sessionId);
431   - attributeSubscriptions.put(sessionId, session);
  423 + attributeSubscriptions.put(sessionId, sessionMD.getSessionInfo());
  424 + dumpSessions();
432 425 }
433 426 }
434 427
... ... @@ -442,20 +435,22 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
442 435 logger.debug("[{}] Canceling rpc subscription for session [{}]", deviceId, sessionId);
443 436 rpcSubscriptions.remove(sessionId);
444 437 } else {
445   - SessionInfo session = sessions.get(sessionId);
446   - if (session == null) {
447   - session = new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId());
  438 + SessionInfoMetaData sessionMD = sessions.get(sessionId);
  439 + if (sessionMD == null) {
  440 + sessionMD = new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.SYNC, sessionInfo.getNodeId()));
448 441 }
  442 + sessionMD.setSubscribedToRPC(true);
449 443 logger.debug("[{}] Registering rpc subscription for session [{}]", deviceId, sessionId);
450   - rpcSubscriptions.put(sessionId, session);
  444 + rpcSubscriptions.put(sessionId, sessionMD.getSessionInfo());
451 445 sendPendingRequests(context, sessionId, sessionInfo);
  446 + dumpSessions();
452 447 }
453 448 }
454 449
455 450 private void processSessionStateMsgs(SessionInfoProto sessionInfo, SessionEventMsg msg) {
456 451 UUID sessionId = getSessionId(sessionInfo);
457 452 if (msg.getEvent() == SessionEvent.OPEN) {
458   - if(sessions.containsKey(sessionId)){
  453 + if (sessions.containsKey(sessionId)) {
459 454 logger.debug("[{}] Received duplicate session open event [{}]", deviceId, sessionId);
460 455 return;
461 456 }
... ... @@ -463,13 +458,14 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
463 458 if (sessions.size() >= systemContext.getMaxConcurrentSessionsPerDevice()) {
464 459 UUID sessionIdToRemove = sessions.keySet().stream().findFirst().orElse(null);
465 460 if (sessionIdToRemove != null) {
466   - closeSession(sessionIdToRemove, sessions.remove(sessionIdToRemove));
  461 + notifyTransportAboutClosedSession(sessionIdToRemove, sessions.remove(sessionIdToRemove));
467 462 }
468 463 }
469   - sessions.put(sessionId, new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId()));
  464 + sessions.put(sessionId, new SessionInfoMetaData(new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfo.getNodeId())));
470 465 if (sessions.size() == 1) {
471 466 reportSessionOpen();
472 467 }
  468 + dumpSessions();
473 469 } else if (msg.getEvent() == SessionEvent.CLOSED) {
474 470 logger.debug("[{}] Canceling subscriptions for closed session [{}]", deviceId, sessionId);
475 471 sessions.remove(sessionId);
... ... @@ -478,21 +474,40 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
478 474 if (sessions.isEmpty()) {
479 475 reportSessionClose();
480 476 }
  477 + dumpSessions();
  478 + }
  479 + }
  480 +
  481 + private void handleSessionActivity(ActorContext context, SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto subscriptionInfo) {
  482 + UUID sessionId = getSessionId(sessionInfo);
  483 + SessionInfoMetaData sessionMD = sessions.get(sessionId);
  484 + if (sessionMD != null) {
  485 + sessionMD.setLastActivityTime(subscriptionInfo.getLastActivityTime());
  486 + sessionMD.setSubscribedToAttributes(subscriptionInfo.getAttributeSubscription());
  487 + sessionMD.setSubscribedToRPC(subscriptionInfo.getRpcSubscription());
  488 + if (subscriptionInfo.getAttributeSubscription()) {
  489 + attributeSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo());
  490 + }
  491 + if (subscriptionInfo.getRpcSubscription()) {
  492 + rpcSubscriptions.putIfAbsent(sessionId, sessionMD.getSessionInfo());
  493 + }
481 494 }
  495 + dumpSessions();
482 496 }
483 497
484 498 void processCredentialsUpdate() {
485   - sessions.forEach(this::closeSession);
  499 + sessions.forEach(this::notifyTransportAboutClosedSession);
486 500 attributeSubscriptions.clear();
487 501 rpcSubscriptions.clear();
  502 + dumpSessions();
488 503 }
489 504
490   - private void closeSession(UUID sessionId, SessionInfo sessionInfo) {
  505 + private void notifyTransportAboutClosedSession(UUID sessionId, SessionInfoMetaData sessionMd) {
491 506 DeviceActorToTransportMsg msg = DeviceActorToTransportMsg.newBuilder()
492 507 .setSessionIdMSB(sessionId.getMostSignificantBits())
493 508 .setSessionIdLSB(sessionId.getLeastSignificantBits())
494 509 .setSessionCloseNotification(SessionCloseNotificationProto.getDefaultInstance()).build();
495   - systemContext.getRuleEngineTransportService().process(sessionInfo.getNodeId(), msg);
  510 + systemContext.getRuleEngineTransportService().process(sessionMd.getSessionInfo().getNodeId(), msg);
496 511 }
497 512
498 513 void processNameOrTypeUpdate(DeviceNameOrTypeUpdateMsg msg) {
... ... @@ -606,4 +621,75 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
606 621 }
607 622 return builder.build();
608 623 }
  624 +
  625 + private void restoreSessions() {
  626 + logger.debug("[{}] Restoring sessions from cache", deviceId);
  627 + TransportProtos.DeviceSessionsCacheEntry sessionsDump = systemContext.getDeviceSessionCacheService().get(deviceId);
  628 + if (sessionsDump.getSerializedSize() == 0) {
  629 + logger.debug("[{}] No session information found", deviceId);
  630 + return;
  631 + }
  632 + for (TransportProtos.SessionSubscriptionInfoProto sessionSubscriptionInfoProto : sessionsDump.getSessionsList()) {
  633 + SessionInfoProto sessionInfoProto = sessionSubscriptionInfoProto.getSessionInfo();
  634 + UUID sessionId = getSessionId(sessionInfoProto);
  635 + SessionInfo sessionInfo = new SessionInfo(TransportProtos.SessionType.ASYNC, sessionInfoProto.getNodeId());
  636 + TransportProtos.SubscriptionInfoProto subInfo = sessionSubscriptionInfoProto.getSubscriptionInfo();
  637 + SessionInfoMetaData sessionMD = new SessionInfoMetaData(sessionInfo, subInfo.getLastActivityTime());
  638 + sessions.put(sessionId, sessionMD);
  639 + if (subInfo.getAttributeSubscription()) {
  640 + attributeSubscriptions.put(sessionId, sessionInfo);
  641 + sessionMD.setSubscribedToAttributes(true);
  642 + }
  643 + if (subInfo.getRpcSubscription()) {
  644 + rpcSubscriptions.put(sessionId, sessionInfo);
  645 + sessionMD.setSubscribedToRPC(true);
  646 + }
  647 + logger.debug("[{}] Restored session: {}", deviceId, sessionMD);
  648 + }
  649 + logger.debug("[{}] Restored sessions: {}, rpc subscriptions: {}, attribute subscriptions: {}", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size());
  650 + }
  651 +
  652 + private void dumpSessions() {
  653 + logger.debug("[{}] Dumping sessions: {}, rpc subscriptions: {}, attribute subscriptions: {} to cache", deviceId, sessions.size(), rpcSubscriptions.size(), attributeSubscriptions.size());
  654 + List<TransportProtos.SessionSubscriptionInfoProto> sessionsList = new ArrayList<>(sessions.size());
  655 + sessions.forEach((uuid, sessionMD) -> {
  656 + if (sessionMD.getSessionInfo().getType() == TransportProtos.SessionType.SYNC) {
  657 + return;
  658 + }
  659 + SessionInfo sessionInfo = sessionMD.getSessionInfo();
  660 + TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder()
  661 + .setLastActivityTime(sessionMD.getLastActivityTime())
  662 + .setAttributeSubscription(sessionMD.isSubscribedToAttributes())
  663 + .setRpcSubscription(sessionMD.isSubscribedToRPC()).build();
  664 + TransportProtos.SessionInfoProto sessionInfoProto = TransportProtos.SessionInfoProto.newBuilder()
  665 + .setSessionIdMSB(uuid.getMostSignificantBits())
  666 + .setSessionIdLSB(uuid.getLeastSignificantBits())
  667 + .setNodeId(sessionInfo.getNodeId()).build();
  668 + sessionsList.add(TransportProtos.SessionSubscriptionInfoProto.newBuilder()
  669 + .setSessionInfo(sessionInfoProto)
  670 + .setSubscriptionInfo(subscriptionInfoProto).build());
  671 + logger.debug("[{}] Dumping session: {}", deviceId, sessionMD);
  672 + });
  673 + systemContext.getDeviceSessionCacheService()
  674 + .put(deviceId, TransportProtos.DeviceSessionsCacheEntry.newBuilder()
  675 + .addAllSessions(sessionsList).build());
  676 + }
  677 +
  678 + void initSessionTimeout(ActorContext context) {
  679 + schedulePeriodicMsgWithDelay(context, SessionTimeoutCheckMsg.instance(), systemContext.getSessionInactivityTimeout(), systemContext.getSessionInactivityTimeout());
  680 + }
  681 +
  682 + void checkSessionsTimeout() {
  683 + long expTime = System.currentTimeMillis() - systemContext.getSessionInactivityTimeout();
  684 + Map<UUID, SessionInfoMetaData> sessionsToRemove = sessions.entrySet().stream().filter(kv -> kv.getValue().getLastActivityTime() < expTime).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
  685 + sessionsToRemove.forEach((sessionId, sessionMD) -> {
  686 + sessions.remove(sessionId);
  687 + rpcSubscriptions.remove(sessionId);
  688 + attributeSubscriptions.remove(sessionId);
  689 + notifyTransportAboutClosedSession(sessionId, sessionMD);
  690 + });
  691 + if (!sessionsToRemove.isEmpty()) {
  692 + dumpSessions();
  693 + }
  694 + }
609 695 }
... ...
... ... @@ -25,4 +25,5 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionType;
25 25 public class SessionInfo {
26 26 private final SessionType type;
27 27 private final String nodeId;
  28 + private long lastActivityTime;
28 29 }
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.actors.device;
  17 +
  18 +import lombok.Data;
  19 +import org.thingsboard.server.gen.transport.TransportProtos.SessionType;
  20 +
  21 +/**
  22 + * @author Andrew Shvayka
  23 + */
  24 +@Data
  25 +class SessionInfoMetaData {
  26 + private final SessionInfo sessionInfo;
  27 + private long lastActivityTime;
  28 + private boolean subscribedToAttributes;
  29 + private boolean subscribedToRPC;
  30 +
  31 + SessionInfoMetaData(SessionInfo sessionInfo) {
  32 + this(sessionInfo, System.currentTimeMillis());
  33 + }
  34 +
  35 + SessionInfoMetaData(SessionInfo sessionInfo, long lastActivityTime) {
  36 + this.sessionInfo = sessionInfo;
  37 + this.lastActivityTime = lastActivityTime;
  38 + }
  39 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.actors.device;
  17 +
  18 +import org.thingsboard.server.common.msg.MsgType;
  19 +import org.thingsboard.server.common.msg.TbActorMsg;
  20 +
  21 +/**
  22 + * Created by ashvayka on 29.10.18.
  23 + */
  24 +public class SessionTimeoutCheckMsg implements TbActorMsg {
  25 +
  26 + private static final SessionTimeoutCheckMsg INSTANCE = new SessionTimeoutCheckMsg();
  27 +
  28 + private SessionTimeoutCheckMsg() {
  29 + }
  30 +
  31 + public static SessionTimeoutCheckMsg instance() {
  32 + return INSTANCE;
  33 + }
  34 +
  35 + @Override
  36 + public MsgType getMsgType() {
  37 + return MsgType.SESSION_TIMEOUT_MSG;
  38 + }
  39 +}
... ...
... ... @@ -32,7 +32,7 @@ public class BasicRpcSessionListener implements GrpcSessionListener {
32 32 private final ActorRef manager;
33 33 private final ActorRef self;
34 34
35   - public BasicRpcSessionListener(ActorService service, ActorRef manager, ActorRef self) {
  35 + BasicRpcSessionListener(ActorService service, ActorRef manager, ActorRef self) {
36 36 this.service = service;
37 37 this.manager = manager;
38 38 this.self = self;
... ...
... ... @@ -45,7 +45,7 @@ public class RpcManagerActor extends ContextAwareActor {
45 45
46 46 private final ServerAddress instance;
47 47
48   - public RpcManagerActor(ActorSystemContext systemContext) {
  48 + RpcManagerActor(ActorSystemContext systemContext) {
49 49 super(systemContext);
50 50 this.sessionActors = new HashMap<>();
51 51 this.pendingMsgs = new HashMap<>();
... ... @@ -55,7 +55,6 @@ public class RpcManagerActor extends ContextAwareActor {
55 55 .filter(otherServer -> otherServer.getServerAddress().compareTo(instance) > 0)
56 56 .forEach(otherServer -> onCreateSessionRequest(
57 57 new RpcSessionCreateRequestMsg(UUID.randomUUID(), otherServer.getServerAddress(), null)));
58   -
59 58 }
60 59
61 60 @Override
... ... @@ -104,10 +103,10 @@ public class RpcManagerActor extends ContextAwareActor {
104 103 ServerAddress address = new ServerAddress(msg.getServerAddress().getHost(), msg.getServerAddress().getPort(), ServerType.CORE);
105 104 SessionActorInfo session = sessionActors.get(address);
106 105 if (session != null) {
107   - log.debug("{} Forwarding msg to session actor", address);
  106 + log.debug("{} Forwarding msg to session actor: {}", address, msg);
108 107 session.getActor().tell(msg, ActorRef.noSender());
109 108 } else {
110   - log.debug("{} Storing msg to pending queue", address);
  109 + log.debug("{} Storing msg to pending queue: {}", address, msg);
111 110 Queue<ClusterAPIProtos.ClusterMessage> queue = pendingMsgs.get(address);
112 111 if (queue == null) {
113 112 queue = new LinkedList<>();
... ... @@ -117,7 +116,7 @@ public class RpcManagerActor extends ContextAwareActor {
117 116 queue.add(msg);
118 117 }
119 118 } else {
120   - logger.warning("Cluster msg doesn't have set Server Address [{}]", msg);
  119 + logger.warning("Cluster msg doesn't have server address [{}]", msg);
121 120 }
122 121 }
123 122
... ... @@ -162,9 +161,9 @@ public class RpcManagerActor extends ContextAwareActor {
162 161 }
163 162
164 163 private void onSessionClose(boolean reconnect, ServerAddress remoteAddress) {
165   - log.debug("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect);
  164 + log.info("[{}] session closed. Should reconnect: {}", remoteAddress, reconnect);
166 165 SessionActorInfo sessionRef = sessionActors.get(remoteAddress);
167   - if (context().sender() != null && context().sender().equals(sessionRef.actor)) {
  166 + if (sessionRef != null && context().sender() != null && context().sender().equals(sessionRef.actor)) {
168 167 sessionActors.remove(remoteAddress);
169 168 pendingMsgs.remove(remoteAddress);
170 169 if (reconnect) {
... ... @@ -182,18 +181,18 @@ public class RpcManagerActor extends ContextAwareActor {
182 181
183 182 private void register(ServerAddress remoteAddress, UUID uuid, ActorRef sender) {
184 183 sessionActors.put(remoteAddress, new SessionActorInfo(uuid, sender));
185   - log.debug("[{}][{}] Registering session actor.", remoteAddress, uuid);
  184 + log.info("[{}][{}] Registering session actor.", remoteAddress, uuid);
186 185 Queue<ClusterAPIProtos.ClusterMessage> data = pendingMsgs.remove(remoteAddress);
187 186 if (data != null) {
188   - log.debug("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size());
  187 + log.info("[{}][{}] Forwarding {} pending messages.", remoteAddress, uuid, data.size());
189 188 data.forEach(msg -> sender.tell(new RpcSessionTellMsg(msg), ActorRef.noSender()));
190 189 } else {
191   - log.debug("[{}][{}] No pending messages to forward.", remoteAddress, uuid);
  190 + log.info("[{}][{}] No pending messages to forward.", remoteAddress, uuid);
192 191 }
193 192 }
194 193
195 194 private ActorRef createSessionActor(RpcSessionCreateRequestMsg msg) {
196   - log.debug("[{}] Creating session actor.", msg.getMsgUid());
  195 + log.info("[{}] Creating session actor.", msg.getMsgUid());
197 196 ActorRef actor = context().actorOf(
198 197 Props.create(new RpcSessionActor.ActorCreator(systemContext, msg.getMsgUid())).withDispatcher(DefaultActorService.RPC_DISPATCHER_NAME));
199 198 actor.tell(msg, context().self());
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.actors.rpc;
18 18 import akka.event.Logging;
19 19 import akka.event.LoggingAdapter;
20 20 import io.grpc.Channel;
  21 +import io.grpc.ManagedChannel;
21 22 import io.grpc.ManagedChannelBuilder;
22 23 import io.grpc.stub.StreamObserver;
23 24 import org.thingsboard.server.actors.ActorSystemContext;
... ... @@ -88,8 +89,8 @@ public class RpcSessionActor extends ContextAwareActor {
88 89 systemContext.getRpcService().onSessionCreated(msg.getMsgUid(), session.getInputStream());
89 90 } else {
90 91 // Client session
91   - Channel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext(true).build();
92   - session = new GrpcSession(remoteServer, listener);
  92 + ManagedChannel channel = ManagedChannelBuilder.forAddress(remoteServer.getHost(), remoteServer.getPort()).usePlaintext().build();
  93 + session = new GrpcSession(remoteServer, listener, channel);
93 94 session.initInputStream();
94 95
95 96 ClusterRpcServiceGrpc.ClusterRpcServiceStub stub = ClusterRpcServiceGrpc.newStub(channel);
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.actors.ruleChain;
17 17
18 18 import akka.actor.ActorRef;
19 19 import com.datastax.driver.core.utils.UUIDs;
  20 +import org.springframework.util.StringUtils;
20 21 import org.thingsboard.rule.engine.api.ListeningExecutor;
21 22 import org.thingsboard.rule.engine.api.MailService;
22 23 import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest;
... ... @@ -35,6 +36,8 @@ import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody;
35 36 import org.thingsboard.server.common.data.rule.RuleNode;
36 37 import org.thingsboard.server.common.msg.TbMsg;
37 38 import org.thingsboard.server.common.msg.TbMsgMetaData;
  39 +import org.thingsboard.server.common.msg.cluster.ServerAddress;
  40 +import org.thingsboard.server.common.msg.cluster.ServerType;
38 41 import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
39 42 import org.thingsboard.server.dao.alarm.AlarmService;
40 43 import org.thingsboard.server.dao.asset.AssetService;
... ... @@ -232,16 +235,22 @@ class DefaultTbContext implements TbContext {
232 235 return new RuleEngineRpcService() {
233 236 @Override
234 237 public void sendRpcReply(DeviceId deviceId, int requestId, String body) {
235   - mainCtx.getDeviceRpcService().sendRpcReplyToDevice(nodeCtx.getTenantId(), deviceId, requestId, body);
  238 + mainCtx.getDeviceRpcService().sendReplyToRpcCallFromDevice(nodeCtx.getTenantId(), deviceId, requestId, body);
236 239 }
237 240
238 241 @Override
239 242 public void sendRpcRequest(RuleEngineDeviceRpcRequest src, Consumer<RuleEngineDeviceRpcResponse> consumer) {
240 243 ToDeviceRpcRequest request = new ToDeviceRpcRequest(src.getRequestUUID(), nodeCtx.getTenantId(), src.getDeviceId(),
241 244 src.isOneway(), src.getExpirationTime(), new ToDeviceRpcRequestBody(src.getMethod(), src.getBody()));
242   - mainCtx.getDeviceRpcService().processRpcRequestToDevice(request, response -> {
  245 + mainCtx.getDeviceRpcService().forwardServerSideRPCRequestToDeviceActor(request, response -> {
243 246 if (src.isRestApiCall()) {
244   - mainCtx.getDeviceRpcService().processRestAPIRpcResponseFromRuleEngine(response);
  247 + ServerAddress requestOriginAddress;
  248 + if (!StringUtils.isEmpty(src.getOriginHost())) {
  249 + requestOriginAddress = new ServerAddress(src.getOriginHost(), src.getOriginPort(), ServerType.CORE);
  250 + } else {
  251 + requestOriginAddress = mainCtx.getRoutingService().getCurrentServer();
  252 + }
  253 + mainCtx.getDeviceRpcService().processResponseToServerSideRPCRequestFromRuleEngine(requestOriginAddress, response);
245 254 }
246 255 consumer.accept(RuleEngineDeviceRpcResponse.builder()
247 256 .deviceId(src.getDeviceId())
... ...
... ... @@ -35,5 +35,4 @@ public interface ActorService extends SessionMsgProcessor, RpcMsgListener, Disco
35 35
36 36 void onDeviceNameOrTypeUpdate(TenantId tenantId, DeviceId deviceId, String deviceName, String deviceType);
37 37
38   - void onMsg(ServiceToRuleEngineMsg serviceToRuleEngineMsg);
39 38 }
... ...
... ... @@ -42,7 +42,6 @@ import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
42 42 import org.thingsboard.server.common.msg.cluster.ServerAddress;
43 43 import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg;
44 44 import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
45   -import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg;
46 45 import org.thingsboard.server.gen.cluster.ClusterAPIProtos;
47 46 import org.thingsboard.server.service.cluster.discovery.DiscoveryService;
48 47 import org.thingsboard.server.service.cluster.discovery.ServerInstance;
... ... @@ -66,10 +65,7 @@ public class DefaultActorService implements ActorService {
66 65 public static final String APP_DISPATCHER_NAME = "app-dispatcher";
67 66 public static final String CORE_DISPATCHER_NAME = "core-dispatcher";
68 67 public static final String SYSTEM_RULE_DISPATCHER_NAME = "system-rule-dispatcher";
69   - public static final String SYSTEM_PLUGIN_DISPATCHER_NAME = "system-plugin-dispatcher";
70 68 public static final String TENANT_RULE_DISPATCHER_NAME = "rule-dispatcher";
71   - public static final String TENANT_PLUGIN_DISPATCHER_NAME = "plugin-dispatcher";
72   - public static final String SESSION_DISPATCHER_NAME = "session-dispatcher";
73 69 public static final String RPC_DISPATCHER_NAME = "rpc-dispatcher";
74 70
75 71 @Autowired
... ... @@ -162,11 +158,6 @@ public class DefaultActorService implements ActorService {
162 158 appActor.tell(new SendToClusterMsg(deviceId, msg), ActorRef.noSender());
163 159 }
164 160
165   - @Override
166   - public void onMsg(ServiceToRuleEngineMsg msg) {
167   - appActor.tell(msg, ActorRef.noSender());
168   - }
169   -
170 161 public void broadcast(ToAllNodesMsg msg) {
171 162 actorContext.getEncodingService().encode(msg);
172 163 rpcService.broadcast(new RpcBroadcastMsg(ClusterAPIProtos.ClusterMessage
... ... @@ -186,9 +177,9 @@ public class DefaultActorService implements ActorService {
186 177 @Override
187 178 public void onReceivedMsg(ServerAddress source, ClusterAPIProtos.ClusterMessage msg) {
188 179 ServerAddress serverAddress = new ServerAddress(source.getHost(), source.getPort(), source.getServerType());
189   - log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress);
190 180 if (log.isDebugEnabled()) {
191   - log.info("MSG: ", msg);
  181 + log.info("Received msg [{}] from [{}]", msg.getMessageType().name(), serverAddress);
  182 + log.info("MSG: {}", msg);
192 183 }
193 184 switch (msg.getMessageType()) {
194 185 case CLUSTER_ACTOR_MESSAGE:
... ... @@ -222,7 +213,7 @@ public class DefaultActorService implements ActorService {
222 213 actorContext.getTsSubService().onRemoteTsUpdate(serverAddress, msg.getPayload().toByteArray());
223 214 break;
224 215 case CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE:
225   - actorContext.getDeviceRpcService().processRemoteResponseFromDevice(serverAddress, msg.getPayload().toByteArray());
  216 + actorContext.getDeviceRpcService().processResponseToServerSideRPCRequestFromRemoteServer(serverAddress, msg.getPayload().toByteArray());
226 217 break;
227 218 case CLUSTER_DEVICE_STATE_SERVICE_MESSAGE:
228 219 actorContext.getDeviceStateService().onRemoteMsg(serverAddress, msg.getPayload().toByteArray());
... ...
... ... @@ -40,43 +40,31 @@ public abstract class AbstractContextAwareMsgProcessor {
40 40 this.logger = logger;
41 41 }
42 42
43   - protected ActorRef getAppActor() {
44   - return systemContext.getAppActor();
45   - }
46   -
47   - protected Scheduler getScheduler() {
  43 + private Scheduler getScheduler() {
48 44 return systemContext.getScheduler();
49 45 }
50 46
51   - protected ExecutionContextExecutor getSystemDispatcher() {
  47 + private ExecutionContextExecutor getSystemDispatcher() {
52 48 return systemContext.getActorSystem().dispatcher();
53 49 }
54 50
55 51 protected void schedulePeriodicMsgWithDelay(ActorContext ctx, Object msg, long delayInMs, long periodInMs) {
56   - schedulePeriodicMsgWithDelay(ctx, msg, delayInMs, periodInMs, ctx.self());
  52 + schedulePeriodicMsgWithDelay(msg, delayInMs, periodInMs, ctx.self());
57 53 }
58 54
59   - protected void schedulePeriodicMsgWithDelay(ActorContext ctx, Object msg, long delayInMs, long periodInMs, ActorRef target) {
  55 + private void schedulePeriodicMsgWithDelay(Object msg, long delayInMs, long periodInMs, ActorRef target) {
60 56 logger.debug("Scheduling periodic msg {} every {} ms with delay {} ms", msg, periodInMs, delayInMs);
61 57 getScheduler().schedule(Duration.create(delayInMs, TimeUnit.MILLISECONDS), Duration.create(periodInMs, TimeUnit.MILLISECONDS), target, msg, getSystemDispatcher(), null);
62 58 }
63 59
64   -
65 60 protected void scheduleMsgWithDelay(ActorContext ctx, Object msg, long delayInMs) {
66   - scheduleMsgWithDelay(ctx, msg, delayInMs, ctx.self());
  61 + scheduleMsgWithDelay(msg, delayInMs, ctx.self());
67 62 }
68 63
69   - protected void scheduleMsgWithDelay(ActorContext ctx, Object msg, long delayInMs, ActorRef target) {
  64 + private void scheduleMsgWithDelay(Object msg, long delayInMs, ActorRef target) {
70 65 logger.debug("Scheduling msg {} with delay {} ms", msg, delayInMs);
71 66 getScheduler().scheduleOnce(Duration.create(delayInMs, TimeUnit.MILLISECONDS), target, msg, getSystemDispatcher(), null);
72 67 }
73 68
74   - @Data
75   - @AllArgsConstructor
76   - private static class ComponentConfiguration {
77   - private final String clazz;
78   - private final String name;
79   - private final String configuration;
80   - }
81 69
82 70 }
... ...
... ... @@ -108,7 +108,7 @@ public class TenantActor extends RuleChainManagerActor {
108 108 @Override
109 109 protected void broadcast(Object msg) {
110 110 super.broadcast(msg);
111   - deviceActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender()));
  111 +// deviceActors.values().forEach(actorRef -> actorRef.tell(msg, ActorRef.noSender()));
112 112 }
113 113
114 114 private void onServiceToRuleEngineMsg(ServiceToRuleEngineMsg msg) {
... ... @@ -127,7 +127,6 @@ public class TenantActor extends RuleChainManagerActor {
127 127 ruleChainManager.getOrCreateActor(context(), msg.getRuleChainId()).tell(msg, self());
128 128 }
129 129
130   -
131 130 private void onToDeviceActorMsg(DeviceAwareMsg msg) {
132 131 getOrCreateDeviceActor(msg.getDeviceId()).tell(msg, ActorRef.noSender());
133 132 }
... ...
... ... @@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle;
48 48 import org.thingsboard.server.common.msg.TbMsg;
49 49 import org.thingsboard.server.common.msg.TbMsgDataType;
50 50 import org.thingsboard.server.common.msg.TbMsgMetaData;
  51 +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
51 52 import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg;
52 53 import org.thingsboard.server.dao.alarm.AlarmService;
53 54 import org.thingsboard.server.dao.asset.AssetService;
... ... @@ -673,7 +674,7 @@ public abstract class BaseController {
673 674 TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, entityId, metaData, TbMsgDataType.JSON
674 675 , json.writeValueAsString(entityNode)
675 676 , null, null, 0L);
676   - actorService.onMsg(new ServiceToRuleEngineMsg(user.getTenantId(), tbMsg));
  677 + actorService.onMsg(new SendToClusterMsg(entityId, new ServiceToRuleEngineMsg(user.getTenantId(), tbMsg)));
677 678 } catch (Exception e) {
678 679 log.warn("[{}] Failed to push entity action to rule engine: {}", entityId, actionType, e);
679 680 }
... ...
... ... @@ -29,7 +29,6 @@ import org.springframework.web.bind.annotation.RequestParam;
29 29 import org.springframework.web.bind.annotation.ResponseBody;
30 30 import org.springframework.web.bind.annotation.ResponseStatus;
31 31 import org.springframework.web.bind.annotation.RestController;
32   -import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg;
33 32 import org.thingsboard.server.common.data.Customer;
34 33 import org.thingsboard.server.common.data.DataConstants;
35 34 import org.thingsboard.server.common.data.EntitySubtype;
... ... @@ -39,7 +38,6 @@ import org.thingsboard.server.common.data.audit.ActionType;
39 38 import org.thingsboard.server.common.data.entityview.EntityViewSearchQuery;
40 39 import org.thingsboard.server.common.data.exception.ThingsboardException;
41 40 import org.thingsboard.server.common.data.id.CustomerId;
42   -import org.thingsboard.server.common.data.id.DeviceId;
43 41 import org.thingsboard.server.common.data.id.EntityId;
44 42 import org.thingsboard.server.common.data.id.EntityViewId;
45 43 import org.thingsboard.server.common.data.id.TenantId;
... ... @@ -47,7 +45,6 @@ import org.thingsboard.server.common.data.id.UUIDBased;
47 45 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
48 46 import org.thingsboard.server.common.data.page.TextPageData;
49 47 import org.thingsboard.server.common.data.page.TextPageLink;
50   -import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
51 48 import org.thingsboard.server.dao.exception.IncorrectParameterException;
52 49 import org.thingsboard.server.dao.model.ModelConstants;
53 50 import org.thingsboard.server.service.security.model.SecurityUser;
... ... @@ -174,7 +171,7 @@ public class EntityViewController extends BaseController {
174 171 EntityView entityView = checkEntityViewId(entityViewId);
175 172 entityViewService.deleteEntityView(entityViewId);
176 173 logEntityAction(entityViewId, entityView, entityView.getCustomerId(),
177   - ActionType.DELETED,null, strEntityViewId);
  174 + ActionType.DELETED, null, strEntityViewId);
178 175 } catch (Exception e) {
179 176 logEntityAction(emptyId(EntityType.ENTITY_VIEW),
180 177 null,
... ... @@ -185,10 +182,23 @@ public class EntityViewController extends BaseController {
185 182 }
186 183
187 184 @PreAuthorize("hasAuthority('TENANT_ADMIN')")
  185 + @RequestMapping(value = "/tenant/entityViews", params = {"entityViewName"}, method = RequestMethod.GET)
  186 + @ResponseBody
  187 + public EntityView getTenantEntityView(
  188 + @RequestParam String entityViewName) throws ThingsboardException {
  189 + try {
  190 + TenantId tenantId = getCurrentUser().getTenantId();
  191 + return checkNotNull(entityViewService.findEntityViewByTenantIdAndName(tenantId, entityViewName));
  192 + } catch (Exception e) {
  193 + throw handleException(e);
  194 + }
  195 + }
  196 +
  197 + @PreAuthorize("hasAuthority('TENANT_ADMIN')")
188 198 @RequestMapping(value = "/customer/{customerId}/entityView/{entityViewId}", method = RequestMethod.POST)
189 199 @ResponseBody
190 200 public EntityView assignEntityViewToCustomer(@PathVariable(CUSTOMER_ID) String strCustomerId,
191   - @PathVariable(ENTITY_VIEW_ID) String strEntityViewId) throws ThingsboardException {
  201 + @PathVariable(ENTITY_VIEW_ID) String strEntityViewId) throws ThingsboardException {
192 202 checkParameter(CUSTOMER_ID, strCustomerId);
193 203 checkParameter(ENTITY_VIEW_ID, strEntityViewId);
194 204 try {
... ...
... ... @@ -49,9 +49,11 @@ import org.thingsboard.server.common.data.kv.Aggregation;
49 49 import org.thingsboard.server.common.data.kv.AttributeKey;
50 50 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
51 51 import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
  52 +import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery;
52 53 import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
53 54 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
54 55 import org.thingsboard.server.common.data.kv.BooleanDataEntry;
  56 +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
55 57 import org.thingsboard.server.common.data.kv.DoubleDataEntry;
56 58 import org.thingsboard.server.common.data.kv.KvEntry;
57 59 import org.thingsboard.server.common.data.kv.LongDataEntry;
... ... @@ -60,12 +62,10 @@ import org.thingsboard.server.common.data.kv.StringDataEntry;
60 62 import org.thingsboard.server.common.data.kv.TsKvEntry;
61 63 import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
62 64 import org.thingsboard.server.common.transport.adaptor.JsonConverter;
63   -import org.thingsboard.server.dao.attributes.AttributesService;
64 65 import org.thingsboard.server.dao.timeseries.TimeseriesService;
65 66 import org.thingsboard.server.service.security.AccessValidator;
66 67 import org.thingsboard.server.service.security.model.SecurityUser;
67 68 import org.thingsboard.server.service.telemetry.AttributeData;
68   -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
69 69 import org.thingsboard.server.service.telemetry.TsData;
70 70 import org.thingsboard.server.service.telemetry.exception.InvalidParametersException;
71 71 import org.thingsboard.server.service.telemetry.exception.UncheckedApiException;
... ... @@ -250,6 +250,60 @@ public class TelemetryController extends BaseController {
250 250 }
251 251
252 252 @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
  253 + @RequestMapping(value = "/{entityType}/{entityId}/timeseries/delete", method = RequestMethod.DELETE)
  254 + @ResponseBody
  255 + public DeferredResult<ResponseEntity> deleteEntityTimeseries(@PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr,
  256 + @RequestParam(name = "keys") String keysStr,
  257 + @RequestParam(name = "deleteAllDataForKeys", defaultValue = "false") boolean deleteAllDataForKeys,
  258 + @RequestParam(name = "startTs", required = false) Long startTs,
  259 + @RequestParam(name = "endTs", required = false) Long endTs,
  260 + @RequestParam(name = "rewriteLatestIfDeleted", defaultValue = "false") boolean rewriteLatestIfDeleted) throws ThingsboardException {
  261 + EntityId entityId = EntityIdFactory.getByTypeAndId(entityType, entityIdStr);
  262 + return deleteTimeseries(entityId, keysStr, deleteAllDataForKeys, startTs, endTs, rewriteLatestIfDeleted);
  263 + }
  264 +
  265 + private DeferredResult<ResponseEntity> deleteTimeseries(EntityId entityIdStr, String keysStr, boolean deleteAllDataForKeys,
  266 + Long startTs, Long endTs, boolean rewriteLatestIfDeleted) throws ThingsboardException {
  267 + List<String> keys = toKeysList(keysStr);
  268 + if (keys.isEmpty()) {
  269 + return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST);
  270 + }
  271 + SecurityUser user = getCurrentUser();
  272 +
  273 + long deleteFromTs;
  274 + long deleteToTs;
  275 + if (deleteAllDataForKeys) {
  276 + deleteFromTs = 0L;
  277 + deleteToTs = System.currentTimeMillis();
  278 + } else {
  279 + deleteFromTs = startTs;
  280 + deleteToTs = endTs;
  281 + }
  282 +
  283 + return accessValidator.validateEntityAndCallback(user, entityIdStr, (result, entityId) -> {
  284 + List<DeleteTsKvQuery> deleteTsKvQueries = new ArrayList<>();
  285 + for (String key : keys) {
  286 + deleteTsKvQueries.add(new BaseDeleteTsKvQuery(key, deleteFromTs, deleteToTs, rewriteLatestIfDeleted));
  287 + }
  288 +
  289 + ListenableFuture<List<Void>> future = tsService.remove(entityId, deleteTsKvQueries);
  290 + Futures.addCallback(future, new FutureCallback<List<Void>>() {
  291 + @Override
  292 + public void onSuccess(@Nullable List<Void> tmp) {
  293 + logTimeseriesDeleted(user, entityId, keys, null);
  294 + result.setResult(new ResponseEntity<>(HttpStatus.OK));
  295 + }
  296 +
  297 + @Override
  298 + public void onFailure(Throwable t) {
  299 + logTimeseriesDeleted(user, entityId, keys, t);
  300 + result.setResult(new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR));
  301 + }
  302 + }, executor);
  303 + });
  304 + }
  305 +
  306 + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
253 307 @RequestMapping(value = "/{deviceId}/{scope}", method = RequestMethod.DELETE)
254 308 @ResponseBody
255 309 public DeferredResult<ResponseEntity> deleteEntityAttributes(@PathVariable("deviceId") String deviceIdStr,
... ... @@ -506,6 +560,15 @@ public class TelemetryController extends BaseController {
506 560 };
507 561 }
508 562
  563 + private void logTimeseriesDeleted(SecurityUser user, EntityId entityId, List<String> keys, Throwable e) {
  564 + try {
  565 + logEntityAction(user, (UUIDBased & EntityId) entityId, null, null, ActionType.TIMESERIES_DELETED, toException(e),
  566 + keys);
  567 + } catch (ThingsboardException te) {
  568 + log.warn("Failed to log timeseries delete", te);
  569 + }
  570 + }
  571 +
509 572 private void logAttributesDeleted(SecurityUser user, EntityId entityId, String scope, List<String> keys, Throwable e) {
510 573 try {
511 574 logEntityAction(user, (UUIDBased & EntityId) entityId, null, null, ActionType.ATTRIBUTES_DELETED, toException(e),
... ...
... ... @@ -101,6 +101,10 @@ public class ThingsboardInstallService {
101 101 log.info("Upgrading ThingsBoard from version 2.1.1 to 2.1.2 ...");
102 102
103 103 databaseUpgradeService.upgradeDatabase("2.1.1");
  104 + case "2.1.3":
  105 + log.info("Upgrading ThingsBoard from version 2.1.3 to 2.2.0 ...");
  106 +
  107 + databaseUpgradeService.upgradeDatabase("2.1.3");
104 108
105 109 log.info("Updating system data...");
106 110
... ...
... ... @@ -15,6 +15,8 @@
15 15 */
16 16 package org.thingsboard.server.service.cluster.rpc;
17 17
  18 +import io.grpc.Channel;
  19 +import io.grpc.ManagedChannel;
18 20 import io.grpc.stub.StreamObserver;
19 21 import lombok.Data;
20 22 import lombok.extern.slf4j.Slf4j;
... ... @@ -34,6 +36,7 @@ public final class GrpcSession implements Closeable {
34 36 private final UUID sessionId;
35 37 private final boolean client;
36 38 private final GrpcSessionListener listener;
  39 + private final ManagedChannel channel;
37 40 private StreamObserver<ClusterAPIProtos.ClusterMessage> inputStream;
38 41 private StreamObserver<ClusterAPIProtos.ClusterMessage> outputStream;
39 42
... ... @@ -41,10 +44,10 @@ public final class GrpcSession implements Closeable {
41 44 private ServerAddress remoteServer;
42 45
43 46 public GrpcSession(GrpcSessionListener listener) {
44   - this(null, listener);
  47 + this(null, listener, null);
45 48 }
46 49
47   - public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener) {
  50 + public GrpcSession(ServerAddress remoteServer, GrpcSessionListener listener, ManagedChannel channel) {
48 51 this.sessionId = UUID.randomUUID();
49 52 this.listener = listener;
50 53 if (remoteServer != null) {
... ... @@ -54,6 +57,7 @@ public final class GrpcSession implements Closeable {
54 57 } else {
55 58 this.client = false;
56 59 }
  60 + this.channel = channel;
57 61 }
58 62
59 63 public void initInputStream() {
... ... @@ -105,5 +109,8 @@ public final class GrpcSession implements Closeable {
105 109 } catch (IllegalStateException e) {
106 110 log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage());
107 111 }
  112 + if (channel != null) {
  113 + channel.shutdownNow();
  114 + }
108 115 }
109 116 }
... ...
... ... @@ -30,7 +30,6 @@ import org.thingsboard.rule.engine.api.NodeConfiguration;
30 30 import org.thingsboard.rule.engine.api.NodeDefinition;
31 31 import org.thingsboard.rule.engine.api.RuleNode;
32 32 import org.thingsboard.rule.engine.api.TbRelationTypes;
33   -import org.thingsboard.server.common.data.DataConstants;
34 33 import org.thingsboard.server.common.data.plugin.ComponentDescriptor;
35 34 import org.thingsboard.server.common.data.plugin.ComponentType;
36 35 import org.thingsboard.server.dao.component.ComponentDescriptorService;
... ... @@ -52,6 +51,7 @@ import java.util.Set;
52 51 @Slf4j
53 52 public class AnnotationComponentDiscoveryService implements ComponentDiscoveryService {
54 53
  54 + public static final int MAX_OPTIMISITC_RETRIES = 3;
55 55 @Value("${plugins.scan_packages}")
56 56 private String[] scanPackages;
57 57
... ... @@ -81,17 +81,32 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe
81 81 private void registerRuleNodeComponents() {
82 82 Set<BeanDefinition> ruleNodeBeanDefinitions = getBeanDefinitions(RuleNode.class);
83 83 for (BeanDefinition def : ruleNodeBeanDefinitions) {
84   - try {
85   - String clazzName = def.getBeanClassName();
86   - Class<?> clazz = Class.forName(clazzName);
87   - RuleNode ruleNodeAnnotation = clazz.getAnnotation(RuleNode.class);
88   - ComponentType type = ruleNodeAnnotation.type();
89   - ComponentDescriptor component = scanAndPersistComponent(def, type);
90   - components.put(component.getClazz(), component);
91   - componentsMap.computeIfAbsent(type, k -> new ArrayList<>()).add(component);
92   - } catch (Exception e) {
93   - log.error("Can't initialize component {}, due to {}", def.getBeanClassName(), e.getMessage(), e);
94   - throw new RuntimeException(e);
  84 + int retryCount = 0;
  85 + Exception cause = null;
  86 + while (retryCount < MAX_OPTIMISITC_RETRIES) {
  87 + try {
  88 + String clazzName = def.getBeanClassName();
  89 + Class<?> clazz = Class.forName(clazzName);
  90 + RuleNode ruleNodeAnnotation = clazz.getAnnotation(RuleNode.class);
  91 + ComponentType type = ruleNodeAnnotation.type();
  92 + ComponentDescriptor component = scanAndPersistComponent(def, type);
  93 + components.put(component.getClazz(), component);
  94 + componentsMap.computeIfAbsent(type, k -> new ArrayList<>()).add(component);
  95 + break;
  96 + } catch (Exception e) {
  97 + log.trace("Can't initialize component {}, due to {}", def.getBeanClassName(), e.getMessage(), e);
  98 + cause = e;
  99 + retryCount++;
  100 + try {
  101 + Thread.sleep(1000);
  102 + } catch (InterruptedException e1) {
  103 + throw new RuntimeException(e1);
  104 + }
  105 + }
  106 + }
  107 + if (cause != null && retryCount == MAX_OPTIMISITC_RETRIES) {
  108 + log.error("Can't initialize component {}, due to {}", def.getBeanClassName(), cause.getMessage(), cause);
  109 + throw new RuntimeException(cause);
95 110 }
96 111 }
97 112 }
... ...
... ... @@ -251,7 +251,8 @@ public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService {
251 251 log.info("Entity views restored.");
252 252
253 253 break;
254   -
  254 + case "2.1.3":
  255 + break;
255 256 default:
256 257 throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion);
257 258 }
... ...
... ... @@ -149,7 +149,14 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService {
149 149 log.info("Entity views restored.");
150 150 }
151 151 break;
152   -
  152 + case "2.1.3":
  153 + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
  154 + log.info("Updating schema ...");
  155 + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.2.0", SCHEMA_UPDATE_SQL);
  156 + loadSql(schemaUpdateFile, conn);
  157 + log.info("Schema updated.");
  158 + }
  159 + break;
153 160 default:
154 161 throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
155 162 }
... ...
... ... @@ -90,7 +90,7 @@ public class DefaultDeviceRpcService implements DeviceRpcService {
90 90
91 91 @Override
92 92 public void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer) {
93   - log.trace("[{}] Processing local rpc call to rule engine [{}]", request.getTenantId(), request.getDeviceId());
  93 + log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId());
94 94 UUID requestId = request.getId();
95 95 localToRuleEngineRpcRequests.put(requestId, responseConsumer);
96 96 sendRpcRequestToRuleEngine(request);
... ... @@ -98,31 +98,11 @@ public class DefaultDeviceRpcService implements DeviceRpcService {
98 98 }
99 99
100 100 @Override
101   - public void processRestAPIRpcResponseFromRuleEngine(FromDeviceRpcResponse response) {
102   - UUID requestId = response.getId();
103   - Consumer<FromDeviceRpcResponse> consumer = localToRuleEngineRpcRequests.remove(requestId);
104   - if (consumer != null) {
105   - consumer.accept(response);
106   - } else {
107   - log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response);
108   - }
109   - }
110   -
111   - @Override
112   - public void processRpcRequestToDevice(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer) {
113   - log.trace("[{}] Processing local rpc call to device [{}]", request.getTenantId(), request.getDeviceId());
114   - UUID requestId = request.getId();
115   - localToDeviceRpcRequests.put(requestId, responseConsumer);
116   - sendRpcRequestToDevice(request);
117   - scheduleTimeout(request, requestId, localToDeviceRpcRequests);
118   - }
119   -
120   - @Override
121   - public void processRpcResponseFromDevice(FromDeviceRpcResponse response) {
122   - log.trace("[{}] response to request: [{}]", this.hashCode(), response.getId());
123   - if (routingService.getCurrentServer().equals(response.getServerAddress())) {
  101 + public void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response) {
  102 + log.trace("[{}] Received response to server-side RPC request from rule engine: [{}]", response.getId(), requestOriginAddress);
  103 + if (routingService.getCurrentServer().equals(requestOriginAddress)) {
124 104 UUID requestId = response.getId();
125   - Consumer<FromDeviceRpcResponse> consumer = localToDeviceRpcRequests.remove(requestId);
  105 + Consumer<FromDeviceRpcResponse> consumer = localToRuleEngineRpcRequests.remove(requestId);
126 106 if (consumer != null) {
127 107 consumer.accept(response);
128 108 } else {
... ... @@ -138,12 +118,33 @@ public class DefaultDeviceRpcService implements DeviceRpcService {
138 118 } else {
139 119 builder.setError(-1);
140 120 }
141   - rpcService.tell(response.getServerAddress(), ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray());
  121 + rpcService.tell(requestOriginAddress, ClusterAPIProtos.MessageType.CLUSTER_RPC_FROM_DEVICE_RESPONSE_MESSAGE, builder.build().toByteArray());
  122 + }
  123 + }
  124 +
  125 + @Override
  126 + public void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer) {
  127 + log.trace("[{}][{}] Processing local rpc call to device actor [{}]", request.getTenantId(), request.getId(), request.getDeviceId());
  128 + UUID requestId = request.getId();
  129 + localToDeviceRpcRequests.put(requestId, responseConsumer);
  130 + sendRpcRequestToDevice(request);
  131 + scheduleTimeout(request, requestId, localToDeviceRpcRequests);
  132 + }
  133 +
  134 + @Override
  135 + public void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response) {
  136 + log.trace("[{}] Received response to server-side RPC request from device actor.", response.getId());
  137 + UUID requestId = response.getId();
  138 + Consumer<FromDeviceRpcResponse> consumer = localToDeviceRpcRequests.remove(requestId);
  139 + if (consumer != null) {
  140 + consumer.accept(response);
  141 + } else {
  142 + log.trace("[{}] Unknown or stale rpc response received [{}]", requestId, response);
142 143 }
143 144 }
144 145
145 146 @Override
146   - public void processRemoteResponseFromDevice(ServerAddress serverAddress, byte[] data) {
  147 + public void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data) {
147 148 ClusterAPIProtos.FromDeviceRPCResponseProto proto;
148 149 try {
149 150 proto = ClusterAPIProtos.FromDeviceRPCResponseProto.parseFrom(data);
... ... @@ -151,13 +152,12 @@ public class DefaultDeviceRpcService implements DeviceRpcService {
151 152 throw new RuntimeException(e);
152 153 }
153 154 RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null;
154   - FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), routingService.getCurrentServer(),
155   - proto.getResponse(), error);
156   - processRpcResponseFromDevice(response);
  155 + FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()), proto.getResponse(), error);
  156 + processResponseToServerSideRPCRequestFromRuleEngine(routingService.getCurrentServer(), response);
157 157 }
158 158
159 159 @Override
160   - public void sendRpcReplyToDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) {
  160 + public void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body) {
161 161 ToServerRpcResponseActorMsg rpcMsg = new ToServerRpcResponseActorMsg(tenantId, deviceId, new ToServerRpcResponseMsg(requestId, body));
162 162 forward(deviceId, rpcMsg);
163 163 }
... ... @@ -166,6 +166,8 @@ public class DefaultDeviceRpcService implements DeviceRpcService {
166 166 ObjectNode entityNode = json.createObjectNode();
167 167 TbMsgMetaData metaData = new TbMsgMetaData();
168 168 metaData.putValue("requestUUID", msg.getId().toString());
  169 + metaData.putValue("originHost", routingService.getCurrentServer().getHost());
  170 + metaData.putValue("originPort", Integer.toString(routingService.getCurrentServer().getPort()));
169 171 metaData.putValue("expirationTime", Long.toString(msg.getExpirationTime()));
170 172 metaData.putValue("oneway", Boolean.toString(msg.isOneway()));
171 173
... ... @@ -176,7 +178,7 @@ public class DefaultDeviceRpcService implements DeviceRpcService {
176 178 TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON
177 179 , json.writeValueAsString(entityNode)
178 180 , null, null, 0L);
179   - actorService.onMsg(new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg));
  181 + actorService.onMsg(new SendToClusterMsg(msg.getDeviceId(), new ServiceToRuleEngineMsg(msg.getTenantId(), tbMsg)));
180 182 } catch (JsonProcessingException e) {
181 183 throw new RuntimeException(e);
182 184 }
... ... @@ -199,7 +201,7 @@ public class DefaultDeviceRpcService implements DeviceRpcService {
199 201 log.trace("[{}] timeout the request: [{}]", this.hashCode(), requestId);
200 202 Consumer<FromDeviceRpcResponse> consumer = requestsMap.remove(requestId);
201 203 if (consumer != null) {
202   - consumer.accept(new FromDeviceRpcResponse(requestId, null, null, RpcError.TIMEOUT));
  204 + consumer.accept(new FromDeviceRpcResponse(requestId, null, RpcError.TIMEOUT));
203 205 }
204 206 }, timeout, TimeUnit.MILLISECONDS);
205 207 }
... ...
... ... @@ -29,13 +29,13 @@ public interface DeviceRpcService {
29 29
30 30 void processRestAPIRpcRequestToRuleEngine(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer);
31 31
32   - void processRestAPIRpcResponseFromRuleEngine(FromDeviceRpcResponse response);
  32 + void processResponseToServerSideRPCRequestFromRuleEngine(ServerAddress requestOriginAddress, FromDeviceRpcResponse response);
33 33
34   - void processRpcRequestToDevice(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer);
  34 + void forwardServerSideRPCRequestToDeviceActor(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer);
35 35
36   - void processRpcResponseFromDevice(FromDeviceRpcResponse response);
  36 + void processResponseToServerSideRPCRequestFromDeviceActor(FromDeviceRpcResponse response);
37 37
38   - void sendRpcReplyToDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body);
  38 + void processResponseToServerSideRPCRequestFromRemoteServer(ServerAddress serverAddress, byte[] data);
39 39
40   - void processRemoteResponseFromDevice(ServerAddress serverAddress, byte[] bytes);
  40 + void sendReplyToRpcCallFromDevice(TenantId tenantId, DeviceId deviceId, int requestId, String body);
41 41 }
... ...
... ... @@ -32,8 +32,6 @@ import java.util.UUID;
32 32 public class FromDeviceRpcResponse {
33 33 @Getter
34 34 private final UUID id;
35   - @Getter
36   - private final ServerAddress serverAddress;
37 35 private final String response;
38 36 private final RpcError error;
39 37
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.session;
  17 +
  18 +import lombok.extern.slf4j.Slf4j;
  19 +import org.springframework.cache.annotation.CachePut;
  20 +import org.springframework.cache.annotation.Cacheable;
  21 +import org.springframework.stereotype.Service;
  22 +import org.thingsboard.server.common.data.id.DeviceId;
  23 +import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry;
  24 +
  25 +import java.util.ArrayList;
  26 +import java.util.Collections;
  27 +
  28 +import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE;
  29 +
  30 +/**
  31 + * Created by ashvayka on 29.10.18.
  32 + */
  33 +@Service
  34 +@Slf4j
  35 +public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService {
  36 +
  37 + @Override
  38 + @Cacheable(cacheNames = SESSIONS_CACHE, key = "#deviceId")
  39 + public DeviceSessionsCacheEntry get(DeviceId deviceId) {
  40 + log.debug("[{}] Fetching session data from cache", deviceId);
  41 + return DeviceSessionsCacheEntry.newBuilder().addAllSessions(Collections.emptyList()).build();
  42 + }
  43 +
  44 + @Override
  45 + @CachePut(cacheNames = SESSIONS_CACHE, key = "#deviceId")
  46 + public DeviceSessionsCacheEntry put(DeviceId deviceId, DeviceSessionsCacheEntry sessions) {
  47 + log.debug("[{}] Pushing session data from cache: {}", deviceId, sessions);
  48 + return sessions;
  49 + }
  50 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.session;
  17 +
  18 +import org.thingsboard.server.common.data.id.DeviceId;
  19 +import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry;
  20 +
  21 +/**
  22 + * Created by ashvayka on 29.10.18.
  23 + */
  24 +public interface DeviceSessionCacheService {
  25 +
  26 + DeviceSessionsCacheEntry get(DeviceId deviceId);
  27 +
  28 + DeviceSessionsCacheEntry put(DeviceId deviceId, DeviceSessionsCacheEntry sessions);
  29 +
  30 +}
... ...
... ... @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
43 43 import org.thingsboard.server.common.msg.TbMsg;
44 44 import org.thingsboard.server.common.msg.TbMsgDataType;
45 45 import org.thingsboard.server.common.msg.TbMsgMetaData;
  46 +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
46 47 import org.thingsboard.server.common.msg.cluster.ServerAddress;
47 48 import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg;
48 49 import org.thingsboard.server.dao.attributes.AttributesService;
... ... @@ -457,7 +458,7 @@ public class DefaultDeviceStateService implements DeviceStateService {
457 458 TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON
458 459 , json.writeValueAsString(state)
459 460 , null, null, 0L);
460   - actorService.onMsg(new ServiceToRuleEngineMsg(stateData.getTenantId(), tbMsg));
  461 + actorService.onMsg(new SendToClusterMsg(stateData.getDeviceId(), new ServiceToRuleEngineMsg(stateData.getTenantId(), tbMsg)));
461 462 } catch (Exception e) {
462 463 log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e);
463 464 }
... ...
... ... @@ -166,7 +166,7 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
166 166
167 167 private SubscriptionState getUpdatedSubscriptionState(EntityId entityId, SubscriptionState sub, EntityView entityView) {
168 168 Map<String, Long> keyStates;
169   - if(sub.isAllKeys()) {
  169 + if (sub.isAllKeys()) {
170 170 keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L));
171 171 } else {
172 172 keyStates = sub.getKeyStates().entrySet()
... ... @@ -618,7 +618,9 @@ public class DefaultTelemetrySubscriptionService implements TelemetrySubscriptio
618 618 builder.setEntityId(sub.getEntityId().getId().toString());
619 619 builder.setType(sub.getType().name());
620 620 builder.setAllKeys(sub.isAllKeys());
621   - builder.setScope(sub.getScope());
  621 + if (sub.getScope() != null) {
  622 + builder.setScope(sub.getScope());
  623 + }
622 624 sub.getKeyStates().entrySet().forEach(e -> builder.addKeyStates(
623 625 ClusterAPIProtos.SubscriptionKetStateProto.newBuilder().setKey(e.getKey()).setTs(e.getValue()).build()));
624 626 rpcService.tell(address, ClusterAPIProtos.MessageType.CLUSTER_TELEMETRY_SUBSCRIPTION_CREATE_MESSAGE, builder.build().toByteArray());
... ...
... ... @@ -133,59 +133,48 @@ public class LocalTransportService extends AbstractTransportService implements R
133 133 }
134 134
135 135 @Override
136   - public void process(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback<Void> callback) {
137   - if (checkLimits(sessionInfo, callback)) {
138   - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSessionEvent(msg).build(), callback);
139   - }
  136 + protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback<Void> callback) {
  137 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSessionEvent(msg).build(), callback);
140 138 }
141 139
142 140 @Override
143   - public void process(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback<Void> callback) {
144   - if (checkLimits(sessionInfo, callback)) {
145   - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostTelemetry(msg).build(), callback);
146   - }
  141 + protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback<Void> callback) {
  142 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostTelemetry(msg).build(), callback);
147 143 }
148 144
149 145 @Override
150   - public void process(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback<Void> callback) {
151   - if (checkLimits(sessionInfo, callback)) {
152   - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostAttributes(msg).build(), callback);
153   - }
  146 + protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback<Void> callback) {
  147 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setPostAttributes(msg).build(), callback);
154 148 }
155 149
156 150 @Override
157   - public void process(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback) {
158   - if (checkLimits(sessionInfo, callback)) {
159   - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setGetAttributes(msg).build(), callback);
160   - }
  151 + protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback) {
  152 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setGetAttributes(msg).build(), callback);
161 153 }
162 154
163 155 @Override
164   - public void process(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback<Void> callback) {
165   - if (checkLimits(sessionInfo, callback)) {
166   - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), callback);
167   - }
  156 + public void process(SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback<Void> callback) {
  157 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscriptionInfo(msg).build(), callback);
168 158 }
169 159
170 160 @Override
171   - public void process(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback<Void> callback) {
172   - if (checkLimits(sessionInfo, callback)) {
173   - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), callback);
174   - }
  161 + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback<Void> callback) {
  162 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), callback);
175 163 }
176 164
177 165 @Override
178   - public void process(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback) {
179   - if (checkLimits(sessionInfo, callback)) {
180   - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), callback);
181   - }
  166 + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback<Void> callback) {
  167 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), callback);
182 168 }
183 169
184 170 @Override
185   - public void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback) {
186   - if (checkLimits(sessionInfo, callback)) {
187   - forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToServerRPCCallRequest(msg).build(), callback);
188   - }
  171 + protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback) {
  172 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), callback);
  173 + }
  174 +
  175 + @Override
  176 + protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback) {
  177 + forwardToDeviceActor(TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToServerRPCCallRequest(msg).build(), callback);
189 178 }
190 179
191 180 @Override
... ...
... ... @@ -17,6 +17,11 @@ package org.thingsboard.server.service.transport;
17 17
18 18 import akka.actor.ActorRef;
19 19 import com.fasterxml.jackson.databind.ObjectMapper;
  20 +import io.github.bucket4j.Bandwidth;
  21 +import io.github.bucket4j.BlockingBucket;
  22 +import io.github.bucket4j.Bucket4j;
  23 +import io.github.bucket4j.local.LocalBucket;
  24 +import io.github.bucket4j.local.LocalBucketBuilder;
20 25 import lombok.extern.slf4j.Slf4j;
21 26 import org.apache.kafka.clients.consumer.ConsumerRecords;
22 27 import org.apache.kafka.clients.producer.Callback;
... ... @@ -49,6 +54,7 @@ import java.util.Optional;
49 54 import java.util.UUID;
50 55 import java.util.concurrent.ExecutorService;
51 56 import java.util.concurrent.Executors;
  57 +import java.util.concurrent.TimeUnit;
52 58 import java.util.function.Consumer;
53 59
54 60 /**
... ... @@ -68,6 +74,13 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ
68 74 @Value("${transport.remote.rule_engine.auto_commit_interval}")
69 75 private int autoCommitInterval;
70 76
  77 + @Value("${transport.remote.rule_engine.poll_records_pack_size}")
  78 + private int pollRecordsPackSize;
  79 + @Value("${transport.remote.rule_engine.max_poll_records_per_second}")
  80 + private long pollRecordsPerSecond;
  81 + @Value("${transport.remote.rule_engine.max_poll_records_per_minute}")
  82 + private long pollRecordsPerMinute;
  83 +
71 84 @Autowired
72 85 private TbKafkaSettings kafkaSettings;
73 86
... ... @@ -109,15 +122,30 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ
109 122 ruleEngineConsumerBuilder.groupId("tb-node");
110 123 ruleEngineConsumerBuilder.autoCommit(true);
111 124 ruleEngineConsumerBuilder.autoCommitIntervalMs(autoCommitInterval);
  125 + ruleEngineConsumerBuilder.maxPollRecords(pollRecordsPackSize);
112 126 ruleEngineConsumerBuilder.decoder(new ToRuleEngineMsgDecoder());
113 127
114 128 ruleEngineConsumer = ruleEngineConsumerBuilder.build();
115 129 ruleEngineConsumer.subscribe();
116 130
  131 + LocalBucketBuilder builder = Bucket4j.builder();
  132 + builder.addLimit(Bandwidth.simple(pollRecordsPerSecond, Duration.ofSeconds(1)));
  133 + builder.addLimit(Bandwidth.simple(pollRecordsPerMinute, Duration.ofMinutes(1)));
  134 + LocalBucket pollRateBucket = builder.build();
  135 + BlockingBucket blockingPollRateBucket = pollRateBucket.asScheduler();
  136 +
117 137 mainConsumerExecutor.execute(() -> {
118 138 while (!stopped) {
119 139 try {
120 140 ConsumerRecords<String, byte[]> records = ruleEngineConsumer.poll(Duration.ofMillis(pollDuration));
  141 + int recordsCount = records.count();
  142 + if (recordsCount > 0) {
  143 + while (!blockingPollRateBucket.tryConsume(recordsCount, TimeUnit.SECONDS.toNanos(5))) {
  144 + log.info("Rule Engine consumer is busy. Required tokens: [{}]. Available tokens: [{}].", recordsCount, pollRateBucket.getAvailableTokens());
  145 + Thread.sleep(TimeUnit.SECONDS.toMillis(1));
  146 + }
  147 + log.trace("Processing {} records", recordsCount);
  148 + }
121 149 records.forEach(record -> {
122 150 try {
123 151 ToRuleEngineMsg toRuleEngineMsg = ruleEngineConsumer.decode(record);
... ...
... ... @@ -22,6 +22,7 @@ option java_outer_classname = "ClusterAPIProtos";
22 22 service ClusterRpcService {
23 23 rpc handleMsgs(stream ClusterMessage) returns (stream ClusterMessage) {}
24 24 }
  25 +
25 26 message ClusterMessage {
26 27 MessageType messageType = 1;
27 28 MessageMataInfo messageMetaInfo = 2;
... ... @@ -139,4 +140,4 @@ message DeviceStateServiceMsgProto {
139 140 bool added = 5;
140 141 bool updated = 6;
141 142 bool deleted = 7;
142   -}
\ No newline at end of file
  143 +}
... ...
... ... @@ -92,7 +92,7 @@ core-dispatcher {
92 92 throughput = 5
93 93 }
94 94
95   -# This dispatcher is used for system rule actors
  95 +# This dispatcher is used for system rule chains and rule node actors
96 96 system-rule-dispatcher {
97 97 type = Dispatcher
98 98 executor = "fork-join-executor"
... ... @@ -115,30 +115,7 @@ system-rule-dispatcher {
115 115 throughput = 5
116 116 }
117 117
118   -# This dispatcher is used for system plugin actors
119   -system-plugin-dispatcher {
120   - type = Dispatcher
121   - executor = "fork-join-executor"
122   - fork-join-executor {
123   - # Min number of threads to cap factor-based parallelism number to
124   - parallelism-min = 2
125   - # Max number of threads to cap factor-based parallelism number to
126   - parallelism-max = 12
127   -
128   - # The parallelism factor is used to determine thread pool size using the
129   - # following formula: ceil(available processors * factor). Resulting size
130   - # is then bounded by the parallelism-min and parallelism-max values.
131   - parallelism-factor = 0.25
132   - }
133   - # How long time the dispatcher will wait for new actors until it shuts down
134   - shutdown-timeout = 1s
135   -
136   - # Throughput defines the number of messages that are processed in a batch
137   - # before the thread is returned to the pool. Set to 1 for as fair as possible.
138   - throughput = 5
139   -}
140   -
141   -# This dispatcher is used for tenant rule actors
  118 +# This dispatcher is used for tenant rule chains and rule node actors
142 119 rule-dispatcher {
143 120 type = Dispatcher
144 121 executor = "fork-join-executor"
... ... @@ -160,50 +137,3 @@ rule-dispatcher {
160 137 # before the thread is returned to the pool. Set to 1 for as fair as possible.
161 138 throughput = 5
162 139 }
163   -
164   -# This dispatcher is used for tenant plugin actors
165   -plugin-dispatcher {
166   - type = Dispatcher
167   - executor = "fork-join-executor"
168   - fork-join-executor {
169   - # Min number of threads to cap factor-based parallelism number to
170   - parallelism-min = 2
171   - # Max number of threads to cap factor-based parallelism number to
172   - parallelism-max = 12
173   -
174   - # The parallelism factor is used to determine thread pool size using the
175   - # following formula: ceil(available processors * factor). Resulting size
176   - # is then bounded by the parallelism-min and parallelism-max values.
177   - parallelism-factor = 0.25
178   - }
179   - # How long time the dispatcher will wait for new actors until it shuts down
180   - shutdown-timeout = 1s
181   -
182   - # Throughput defines the number of messages that are processed in a batch
183   - # before the thread is returned to the pool. Set to 1 for as fair as possible.
184   - throughput = 5
185   -}
186   -
187   -
188   -# This dispatcher is used for rule actors
189   -session-dispatcher {
190   - type = Dispatcher
191   - executor = "fork-join-executor"
192   - fork-join-executor {
193   - # Min number of threads to cap factor-based parallelism number to
194   - parallelism-min = 2
195   - # Max number of threads to cap factor-based parallelism number to
196   - parallelism-max = 12
197   -
198   - # The parallelism factor is used to determine thread pool size using the
199   - # following formula: ceil(available processors * factor). Resulting size
200   - # is then bounded by the parallelism-min and parallelism-max values.
201   - parallelism-factor = 0.25
202   - }
203   - # How long time the dispatcher will wait for new actors until it shuts down
204   - shutdown-timeout = 1s
205   -
206   - # Throughput defines the number of messages that are processed in a batch
207   - # before the thread is returned to the pool. Set to 1 for as fair as possible.
208   - throughput = 5
209   -}
\ No newline at end of file
... ...
... ... @@ -140,7 +140,7 @@ cassandra:
140 140 buffer_size: "${CASSANDRA_QUERY_BUFFER_SIZE:200000}"
141 141 concurrent_limit: "${CASSANDRA_QUERY_CONCURRENT_LIMIT:1000}"
142 142 permit_max_wait_time: "${PERMIT_MAX_WAIT_TIME:120000}"
143   - rate_limit_print_interval_ms: "${CASSANDRA_QUERY_RATE_LIMIT_PRINT_MS:30000}"
  143 + rate_limit_print_interval_ms: "${CASSANDRA_QUERY_RATE_LIMIT_PRINT_MS:10000}"
144 144
145 145 # SQL configuration parameters
146 146 sql:
... ... @@ -202,6 +202,9 @@ caffeine:
202 202 devices:
203 203 timeToLiveInMinutes: 1440
204 204 maxSize: 100000
  205 + sessions:
  206 + timeToLiveInMinutes: 1440
  207 + maxSize: 100000
205 208 assets:
206 209 timeToLiveInMinutes: 1440
207 210 maxSize: 100000
... ... @@ -222,7 +225,7 @@ redis:
222 225 updates:
223 226 # Enable/disable updates checking.
224 227 enabled: "${UPDATES_ENABLED:true}"
225   -
  228 +
226 229 # spring CORS configuration
227 230 spring.mvc.cors:
228 231 mappings:
... ... @@ -322,8 +325,8 @@ audit_log:
322 325 password: "${AUDIT_LOG_SINK_PASSWORD:}"
323 326
324 327 state:
325   - defaultInactivityTimeoutInSec: 10
326   - defaultStateCheckIntervalInSec: 10
  328 + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:10}"
  329 + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:10}"
327 330
328 331 kafka:
329 332 enabled: true
... ... @@ -390,8 +393,14 @@ transport:
390 393 topic: "${TB_RULE_ENGINE_TOPIC:tb.rule-engine}"
391 394 poll_interval: "${TB_RULE_ENGINE_POLL_INTERVAL_MS:25}"
392 395 auto_commit_interval: "${TB_RULE_ENGINE_AUTO_COMMIT_INTERVAL_MS:100}"
  396 + poll_records_pack_size: "${TB_RULE_ENGINE_MAX_POLL_RECORDS:1000}"
  397 + max_poll_records_per_second: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_SECOND:10000}"
  398 + max_poll_records_per_minute: "${TB_RULE_ENGINE_MAX_POLL_RECORDS_PER_MINUTE:120000}"
393 399 notifications:
394 400 topic: "${TB_TRANSPORT_NOTIFICATIONS_TOPIC:tb.transport.notifications}"
  401 + sessions:
  402 + inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}"
  403 + report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}"
395 404 rate_limits:
396 405 enabled: "${TB_TRANSPORT_RATE_LIMITS_ENABLED:false}"
397 406 tenant: "${TB_TRANSPORT_RATE_LIMITS_TENANT:1000:1,20000:60}"
... ...
... ... @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.rule.RuleNode;
38 38 import org.thingsboard.server.common.data.security.Authority;
39 39 import org.thingsboard.server.common.msg.TbMsg;
40 40 import org.thingsboard.server.common.msg.TbMsgMetaData;
  41 +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
41 42 import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg;
42 43 import org.thingsboard.server.controller.AbstractRuleEngineControllerTest;
43 44 import org.thingsboard.server.dao.attributes.AttributesService;
... ... @@ -155,7 +156,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
155 156 device.getId(),
156 157 new TbMsgMetaData(),
157 158 "{}", null, null, 0L);
158   - actorService.onMsg(new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg));
  159 + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg)));
159 160
160 161 Thread.sleep(3000);
161 162
... ... @@ -270,7 +271,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
270 271 device.getId(),
271 272 new TbMsgMetaData(),
272 273 "{}", null, null, 0L);
273   - actorService.onMsg(new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg));
  274 + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg)));
274 275
275 276 Thread.sleep(3000);
276 277
... ...
... ... @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.rule.RuleNode;
39 39 import org.thingsboard.server.common.data.security.Authority;
40 40 import org.thingsboard.server.common.msg.TbMsg;
41 41 import org.thingsboard.server.common.msg.TbMsgMetaData;
  42 +import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
42 43 import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg;
43 44 import org.thingsboard.server.controller.AbstractRuleEngineControllerTest;
44 45 import org.thingsboard.server.dao.attributes.AttributesService;
... ... @@ -142,7 +143,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac
142 143 new TbMsgMetaData(),
143 144 "{}",
144 145 null, null, 0L);
145   - actorService.onMsg(new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg));
  146 + actorService.onMsg(new SendToClusterMsg(device.getId(), new ServiceToRuleEngineMsg(savedTenant.getId(), tbMsg)));
146 147
147 148 Thread.sleep(3000);
148 149
... ...
... ... @@ -19,6 +19,7 @@ public class CacheConstants {
19 19 public static final String DEVICE_CREDENTIALS_CACHE = "deviceCredentials";
20 20 public static final String RELATIONS_CACHE = "relations";
21 21 public static final String DEVICE_CACHE = "devices";
  22 + public static final String SESSIONS_CACHE = "sessions";
22 23 public static final String ASSET_CACHE = "assets";
23 24 public static final String ENTITY_VIEW_CACHE = "entityViews";
24 25 }
... ...
... ... @@ -24,6 +24,7 @@ public enum ActionType {
24 24 UPDATED(false), // log entity
25 25 ATTRIBUTES_UPDATED(false), // log attributes/values
26 26 ATTRIBUTES_DELETED(false), // log attributes
  27 + TIMESERIES_DELETED(false), // log timeseries
27 28 RPC_CALL(false), // log method and params
28 29 CREDENTIALS_UPDATED(false), // log new credentials
29 30 ASSIGNED_TO_CUSTOMER(false), // log customer name
... ... @@ -32,11 +33,11 @@ public enum ActionType {
32 33 SUSPENDED(false), // log string id
33 34 CREDENTIALS_READ(true), // log device id
34 35 ATTRIBUTES_READ(true), // log attributes
35   - RELATION_ADD_OR_UPDATE (false),
36   - RELATION_DELETED (false),
37   - RELATIONS_DELETED (false),
38   - ALARM_ACK (false),
39   - ALARM_CLEAR (false);
  36 + RELATION_ADD_OR_UPDATE(false),
  37 + RELATION_DELETED(false),
  38 + RELATIONS_DELETED(false),
  39 + ALARM_ACK(false),
  40 + ALARM_CLEAR(false);
40 41
41 42 private final boolean isRead;
42 43
... ...
... ... @@ -96,13 +96,8 @@ public enum MsgType {
96 96 */
97 97 DEVICE_ACTOR_TO_RULE_ENGINE_MSG,
98 98
99   - /**
100   - * Message that is sent from Rule Engine to the Device Actor when message is successfully pushed to queue.
101   - */
102   - ACTOR_SYSTEM_TO_DEVICE_SESSION_ACTOR_MSG,
103   - TRANSPORT_TO_DEVICE_SESSION_ACTOR_MSG,
104 99 SESSION_TIMEOUT_MSG,
105   - SESSION_CTRL_MSG,
  100 +
106 101 STATS_PERSIST_TICK_MSG,
107 102
108 103
... ...
... ... @@ -19,6 +19,5 @@ package org.thingsboard.server.common.msg.cluster;
19 19 * Created by ashvayka on 23.09.18.
20 20 */
21 21 public enum ServerType {
22   - //Should match content of enum in discovery.proto.
23   - CORE, JS_EVALUATOR
  22 + CORE
24 23 }
... ...
... ... @@ -21,11 +21,13 @@ import org.thingsboard.server.common.msg.MsgType;
21 21 import org.thingsboard.server.common.msg.TbActorMsg;
22 22 import org.thingsboard.server.common.msg.TbMsg;
23 23
  24 +import java.io.Serializable;
  25 +
24 26 /**
25 27 * Created by ashvayka on 15.03.18.
26 28 */
27 29 @Data
28   -public final class ServiceToRuleEngineMsg implements TbActorMsg {
  30 +public final class ServiceToRuleEngineMsg implements TbActorMsg, Serializable {
29 31
30 32 private final TenantId tenantId;
31 33 private final TbMsg tbMsg;
... ...
... ... @@ -46,7 +46,8 @@ public class TBKafkaConsumerTemplate<T> {
46 46 private TBKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder<T> decoder,
47 47 TbKafkaRequestIdExtractor<T> requestIdExtractor,
48 48 String clientId, String groupId, String topic,
49   - boolean autoCommit, int autoCommitIntervalMs) {
  49 + boolean autoCommit, int autoCommitIntervalMs,
  50 + int maxPollRecords) {
50 51 Properties props = settings.toProps();
51 52 props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId);
52 53 if (groupId != null) {
... ... @@ -56,6 +57,9 @@ public class TBKafkaConsumerTemplate<T> {
56 57 props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs);
57 58 props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
58 59 props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer");
  60 + if (maxPollRecords > 0) {
  61 + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
  62 + }
59 63 this.consumer = new KafkaConsumer<>(props);
60 64 this.decoder = decoder;
61 65 this.requestIdExtractor = requestIdExtractor;
... ...
... ... @@ -26,6 +26,7 @@ import org.apache.kafka.clients.producer.ProducerConfig;
26 26 import org.apache.kafka.clients.producer.ProducerRecord;
27 27 import org.apache.kafka.clients.producer.RecordMetadata;
28 28 import org.apache.kafka.common.PartitionInfo;
  29 +import org.apache.kafka.common.errors.TopicExistsException;
29 30 import org.apache.kafka.common.header.Header;
30 31
31 32 import java.util.List;
... ... @@ -75,7 +76,11 @@ public class TBKafkaProducerTemplate<T> {
75 76 CreateTopicsResult result = admin.createTopic(new NewTopic(defaultTopic, 100, (short) 1));
76 77 result.all().get();
77 78 } catch (Exception e) {
78   - log.trace("Failed to create topic: {}", e.getMessage(), e);
  79 + if ((e instanceof TopicExistsException) || (e.getCause() != null && e.getCause() instanceof TopicExistsException)) {
  80 + log.trace("[{}] Topic already exists: ", defaultTopic);
  81 + } else {
  82 + log.trace("[{}] Failed to create topic: {}", defaultTopic, e.getMessage(), e);
  83 + }
79 84 }
80 85 //Maybe this should not be cached, but we don't plan to change size of partitions
81 86 this.partitionInfoMap = new ConcurrentHashMap<>();
... ...
... ... @@ -43,10 +43,10 @@ import org.thingsboard.server.common.transport.TransportService;
43 43 import org.thingsboard.server.common.transport.TransportServiceCallback;
44 44 import org.thingsboard.server.common.transport.adaptor.AdaptorException;
45 45 import org.thingsboard.server.common.msg.EncryptionUtil;
  46 +import org.thingsboard.server.common.transport.service.AbstractTransportService;
46 47 import org.thingsboard.server.gen.transport.TransportProtos;
47 48 import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
48 49 import org.thingsboard.server.gen.transport.TransportProtos.SessionEvent;
49   -import org.thingsboard.server.gen.transport.TransportProtos.SessionEventMsg;
50 50 import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto;
51 51 import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg;
52 52 import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
... ... @@ -141,9 +141,12 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
141 141 processUnsubscribe(ctx, (MqttUnsubscribeMessage) msg);
142 142 break;
143 143 case PINGREQ:
144   - //TODO: should we push the notification to the rule engine?
145 144 if (checkConnected(ctx)) {
146 145 ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0)));
  146 + transportService.reportActivity(sessionInfo);
  147 + if (gatewaySessionHandler != null) {
  148 + gatewaySessionHandler.reportActivity();
  149 + }
147 150 }
148 151 break;
149 152 case DISCONNECT:
... ... @@ -394,7 +397,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
394 397 private void processDisconnect(ChannelHandlerContext ctx) {
395 398 ctx.close();
396 399 if (deviceSessionCtx.isConnected()) {
397   - transportService.process(sessionInfo, getSessionEventMsg(SessionEvent.CLOSED), null);
  400 + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null);
398 401 transportService.deregisterSession(sessionInfo);
399 402 if (gatewaySessionHandler != null) {
400 403 gatewaySessionHandler.onGatewayDisconnect();
... ... @@ -466,16 +469,10 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
466 469 }
467 470 }
468 471
469   - public static SessionEventMsg getSessionEventMsg(SessionEvent event) {
470   - return SessionEventMsg.newBuilder()
471   - .setSessionType(TransportProtos.SessionType.ASYNC)
472   - .setEvent(event).build();
473   - }
474   -
475 472 @Override
476 473 public void operationComplete(Future<? super Void> future) throws Exception {
477 474 if (deviceSessionCtx.isConnected()) {
478   - transportService.process(sessionInfo, getSessionEventMsg(SessionEvent.CLOSED), null);
  475 + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.CLOSED), null);
479 476 transportService.deregisterSession(sessionInfo);
480 477 }
481 478 }
... ... @@ -495,7 +492,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
495 492 .setTenantIdMSB(msg.getDeviceInfo().getTenantIdMSB())
496 493 .setTenantIdLSB(msg.getDeviceInfo().getTenantIdLSB())
497 494 .build();
498   - transportService.process(sessionInfo, getSessionEventMsg(SessionEvent.OPEN), null);
  495 + transportService.process(sessionInfo, AbstractTransportService.getSessionEventMsg(SessionEvent.OPEN), null);
499 496 transportService.registerAsyncSession(sessionInfo, this);
500 497 checkGatewaySession();
501 498 ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED));
... ...
... ... @@ -34,6 +34,7 @@ import org.thingsboard.server.common.transport.TransportService;
34 34 import org.thingsboard.server.common.transport.TransportServiceCallback;
35 35 import org.thingsboard.server.common.transport.adaptor.AdaptorException;
36 36 import org.thingsboard.server.common.transport.adaptor.JsonConverter;
  37 +import org.thingsboard.server.common.transport.service.AbstractTransportService;
37 38 import org.thingsboard.server.gen.transport.TransportProtos;
38 39 import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
39 40 import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg;
... ... @@ -118,7 +119,7 @@ public class GatewaySessionHandler {
118 119 GatewayDeviceSessionCtx deviceSessionCtx = new GatewayDeviceSessionCtx(GatewaySessionHandler.this, msg.getDeviceInfo(), mqttQoSMap);
119 120 if (devices.putIfAbsent(deviceName, deviceSessionCtx) == null) {
120 121 SessionInfoProto deviceSessionInfo = deviceSessionCtx.getSessionInfo();
121   - transportService.process(deviceSessionInfo, MqttTransportHandler.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null);
  122 + transportService.process(deviceSessionInfo, AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.OPEN), null);
122 123 transportService.process(deviceSessionInfo, TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), null);
123 124 transportService.process(deviceSessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), null);
124 125 transportService.registerAsyncSession(deviceSessionInfo, deviceSessionCtx);
... ... @@ -334,7 +335,7 @@ public class GatewaySessionHandler {
334 335
335 336 private void deregisterSession(String deviceName, GatewayDeviceSessionCtx deviceSessionCtx) {
336 337 transportService.deregisterSession(deviceSessionCtx.getSessionInfo());
337   - transportService.process(deviceSessionCtx.getSessionInfo(), MqttTransportHandler.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null);
  338 + transportService.process(deviceSessionCtx.getSessionInfo(), AbstractTransportService.getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null);
338 339 log.debug("[{}] Removed device [{}] from the gateway session", sessionId, deviceName);
339 340 }
340 341
... ... @@ -360,11 +361,15 @@ public class GatewaySessionHandler {
360 361 return context;
361 362 }
362 363
363   - public MqttTransportAdaptor getAdaptor() {
  364 + MqttTransportAdaptor getAdaptor() {
364 365 return context.getAdaptor();
365 366 }
366 367
367   - public int nextMsgId() {
  368 + int nextMsgId() {
368 369 return deviceSessionCtx.nextMsgId();
369 370 }
  371 +
  372 + public void reportActivity() {
  373 + devices.forEach((id, deviceCtx) -> transportService.reportActivity(deviceCtx.getSessionInfo()));
  374 + }
370 375 }
... ...
... ... @@ -61,10 +61,14 @@ public interface TransportService {
61 61
62 62 void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback);
63 63
  64 + void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback<Void> callback);
  65 +
64 66 void registerAsyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener);
65 67
66 68 void registerSyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout);
67 69
  70 + void reportActivity(SessionInfoProto sessionInfo);
  71 +
68 72 void deregisterSession(SessionInfoProto sessionInfo);
69 73
70 74 }
... ...
... ... @@ -47,9 +47,14 @@ public abstract class AbstractTransportService implements TransportService {
47 47 private String perTenantLimitsConf;
48 48 @Value("${transport.rate_limits.tenant}")
49 49 private String perDevicesLimitsConf;
  50 + @Value("${transport.sessions.inactivity_timeout}")
  51 + private long sessionInactivityTimeout;
  52 + @Value("${transport.sessions.report_timeout}")
  53 + private long sessionReportTimeout;
50 54
51 55 protected ScheduledExecutorService schedulerExecutor;
52 56 protected ExecutorService transportCallbackExecutor;
  57 +
53 58 private ConcurrentMap<UUID, SessionMetaData> sessions = new ConcurrentHashMap<>();
54 59
55 60 //TODO: Implement cleanup of this maps.
... ... @@ -59,7 +64,121 @@ public abstract class AbstractTransportService implements TransportService {
59 64 @Override
60 65 public void registerAsyncSession(TransportProtos.SessionInfoProto sessionInfo, SessionMsgListener listener) {
61 66 sessions.putIfAbsent(toId(sessionInfo), new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listener));
62   - //TODO: monitor sessions periodically: PING REQ/RESP, etc.
  67 + }
  68 +
  69 + @Override
  70 + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback<Void> callback) {
  71 + if (checkLimits(sessionInfo, callback)) {
  72 + reportActivityInternal(sessionInfo);
  73 + doProcess(sessionInfo, msg, callback);
  74 + }
  75 + }
  76 +
  77 + @Override
  78 + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback<Void> callback) {
  79 + if (checkLimits(sessionInfo, callback)) {
  80 + reportActivityInternal(sessionInfo);
  81 + doProcess(sessionInfo, msg, callback);
  82 + }
  83 + }
  84 +
  85 + @Override
  86 + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback<Void> callback) {
  87 + if (checkLimits(sessionInfo, callback)) {
  88 + reportActivityInternal(sessionInfo);
  89 + doProcess(sessionInfo, msg, callback);
  90 + }
  91 + }
  92 +
  93 + @Override
  94 + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback) {
  95 + if (checkLimits(sessionInfo, callback)) {
  96 + reportActivityInternal(sessionInfo);
  97 + doProcess(sessionInfo, msg, callback);
  98 + }
  99 + }
  100 +
  101 + @Override
  102 + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback<Void> callback) {
  103 + if (checkLimits(sessionInfo, callback)) {
  104 + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo);
  105 + sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe());
  106 + doProcess(sessionInfo, msg, callback);
  107 + }
  108 + }
  109 +
  110 + @Override
  111 + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback<Void> callback) {
  112 + if (checkLimits(sessionInfo, callback)) {
  113 + SessionMetaData sessionMetaData = reportActivityInternal(sessionInfo);
  114 + sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe());
  115 + doProcess(sessionInfo, msg, callback);
  116 + }
  117 + }
  118 +
  119 + @Override
  120 + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback) {
  121 + if (checkLimits(sessionInfo, callback)) {
  122 + reportActivityInternal(sessionInfo);
  123 + doProcess(sessionInfo, msg, callback);
  124 + }
  125 + }
  126 +
  127 + @Override
  128 + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback) {
  129 + if (checkLimits(sessionInfo, callback)) {
  130 + reportActivityInternal(sessionInfo);
  131 + doProcess(sessionInfo, msg, callback);
  132 + }
  133 + }
  134 +
  135 + @Override
  136 + public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) {
  137 + reportActivityInternal(sessionInfo);
  138 + }
  139 +
  140 + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback<Void> callback);
  141 +
  142 + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostTelemetryMsg msg, TransportServiceCallback<Void> callback);
  143 +
  144 + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TransportServiceCallback<Void> callback);
  145 +
  146 + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback);
  147 +
  148 + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback<Void> callback);
  149 +
  150 + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscribeToRPCMsg msg, TransportServiceCallback<Void> callback);
  151 +
  152 + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback);
  153 +
  154 + protected abstract void doProcess(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback);
  155 +
  156 + private SessionMetaData reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) {
  157 + UUID sessionId = toId(sessionInfo);
  158 + SessionMetaData sessionMetaData = sessions.get(sessionId);
  159 + if (sessionMetaData != null) {
  160 + sessionMetaData.updateLastActivityTime();
  161 + }
  162 + return sessionMetaData;
  163 + }
  164 +
  165 + private void checkInactivityAndReportActivity() {
  166 + long expTime = System.currentTimeMillis() - sessionInactivityTimeout;
  167 + sessions.forEach((uuid, sessionMD) -> {
  168 + if (sessionMD.getLastActivityTime() < expTime) {
  169 + if (log.isDebugEnabled()) {
  170 + log.debug("[{}] Session has expired due to last activity time: {}", toId(sessionMD.getSessionInfo()), sessionMD.getLastActivityTime());
  171 + }
  172 + process(sessionMD.getSessionInfo(), getSessionEventMsg(TransportProtos.SessionEvent.CLOSED), null);
  173 + sessions.remove(uuid);
  174 + sessionMD.getListener().onRemoteSessionCloseCommand(TransportProtos.SessionCloseNotificationProto.getDefaultInstance());
  175 + } else {
  176 + process(sessionMD.getSessionInfo(), TransportProtos.SubscriptionInfoProto.newBuilder()
  177 + .setAttributeSubscription(sessionMD.isSubscribedToAttributes())
  178 + .setRpcSubscription(sessionMD.isSubscribedToRPC())
  179 + .setLastActivityTime(sessionMD.getLastActivityTime()).build(), null);
  180 + }
  181 + });
63 182 }
64 183
65 184 @Override
... ... @@ -131,7 +250,7 @@ public abstract class AbstractTransportService implements TransportService {
131 250 }
132 251 }
133 252
134   - protected UUID toId(TransportProtos.SessionInfoProto sessionInfo) {
  253 + private UUID toId(TransportProtos.SessionInfoProto sessionInfo) {
135 254 return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB());
136 255 }
137 256
... ... @@ -147,6 +266,7 @@ public abstract class AbstractTransportService implements TransportService {
147 266 }
148 267 this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor();
149 268 this.transportCallbackExecutor = new ThreadPoolExecutor(0, 20, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
  269 + this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, sessionReportTimeout, sessionReportTimeout, TimeUnit.MILLISECONDS);
150 270 }
151 271
152 272 public void destroy() {
... ... @@ -161,4 +281,10 @@ public abstract class AbstractTransportService implements TransportService {
161 281 transportCallbackExecutor.shutdownNow();
162 282 }
163 283 }
  284 +
  285 + public static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) {
  286 + return TransportProtos.SessionEventMsg.newBuilder()
  287 + .setSessionType(TransportProtos.SessionType.ASYNC)
  288 + .setEvent(event).build();
  289 + }
164 290 }
... ...
... ... @@ -217,91 +217,84 @@ public class RemoteTransportService extends AbstractTransportService {
217 217 }
218 218
219 219 @Override
220   - public void process(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback<Void> callback) {
221   - if (checkLimits(sessionInfo, callback)) {
222   - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
223   - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
224   - .setSessionEvent(msg).build()
225   - ).build();
226   - send(sessionInfo, toRuleEngineMsg, callback);
227   - }
  220 + public void process(SessionInfoProto sessionInfo, SubscriptionInfoProto msg, TransportServiceCallback<Void> callback) {
  221 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  222 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  223 + .setSubscriptionInfo(msg).build()
  224 + ).build();
  225 + send(sessionInfo, toRuleEngineMsg, callback);
228 226 }
229 227
230 228 @Override
231   - public void process(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback<Void> callback) {
232   - if (checkLimits(sessionInfo, callback)) {
233   - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
234   - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
235   - .setPostTelemetry(msg).build()
236   - ).build();
237   - send(sessionInfo, toRuleEngineMsg, callback);
238   - }
  229 + protected void doProcess(SessionInfoProto sessionInfo, SessionEventMsg msg, TransportServiceCallback<Void> callback) {
  230 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  231 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  232 + .setSessionEvent(msg).build()
  233 + ).build();
  234 + send(sessionInfo, toRuleEngineMsg, callback);
239 235 }
240 236
241 237 @Override
242   - public void process(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback<Void> callback) {
243   - if (checkLimits(sessionInfo, callback)) {
244   - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
245   - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
246   - .setPostAttributes(msg).build()
247   - ).build();
248   - send(sessionInfo, toRuleEngineMsg, callback);
249   - }
  238 + protected void doProcess(SessionInfoProto sessionInfo, PostTelemetryMsg msg, TransportServiceCallback<Void> callback) {
  239 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  240 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  241 + .setPostTelemetry(msg).build()
  242 + ).build();
  243 + send(sessionInfo, toRuleEngineMsg, callback);
250 244 }
251 245
252 246 @Override
253   - public void process(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback) {
254   - if (checkLimits(sessionInfo, callback)) {
255   - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
256   - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
257   - .setGetAttributes(msg).build()
258   - ).build();
259   - send(sessionInfo, toRuleEngineMsg, callback);
260   - }
  247 + protected void doProcess(SessionInfoProto sessionInfo, PostAttributeMsg msg, TransportServiceCallback<Void> callback) {
  248 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  249 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  250 + .setPostAttributes(msg).build()
  251 + ).build();
  252 + send(sessionInfo, toRuleEngineMsg, callback);
261 253 }
262 254
263 255 @Override
264   - public void process(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback<Void> callback) {
265   - if (checkLimits(sessionInfo, callback)) {
266   - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
267   - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
268   - .setSubscribeToAttributes(msg).build()
269   - ).build();
270   - send(sessionInfo, toRuleEngineMsg, callback);
271   - }
  256 + protected void doProcess(SessionInfoProto sessionInfo, GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback) {
  257 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  258 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  259 + .setGetAttributes(msg).build()
  260 + ).build();
  261 + send(sessionInfo, toRuleEngineMsg, callback);
272 262 }
273 263
274 264 @Override
275   - public void process(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback<Void> callback) {
276   - if (checkLimits(sessionInfo, callback)) {
277   - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
278   - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
279   - .setSubscribeToRPC(msg).build()
280   - ).build();
281   - send(sessionInfo, toRuleEngineMsg, callback);
282   - }
  265 + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToAttributeUpdatesMsg msg, TransportServiceCallback<Void> callback) {
  266 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  267 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  268 + .setSubscribeToAttributes(msg).build()
  269 + ).build();
  270 + send(sessionInfo, toRuleEngineMsg, callback);
283 271 }
284 272
285 273 @Override
286   - public void process(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback) {
287   - if (checkLimits(sessionInfo, callback)) {
288   - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
289   - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
290   - .setToDeviceRPCCallResponse(msg).build()
291   - ).build();
292   - send(sessionInfo, toRuleEngineMsg, callback);
293   - }
  274 + protected void doProcess(SessionInfoProto sessionInfo, SubscribeToRPCMsg msg, TransportServiceCallback<Void> callback) {
  275 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  276 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  277 + .setSubscribeToRPC(msg).build()
  278 + ).build();
  279 + send(sessionInfo, toRuleEngineMsg, callback);
294 280 }
295 281
296 282 @Override
297   - public void process(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback) {
298   - if (checkLimits(sessionInfo, callback)) {
299   - ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
300   - TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
301   - .setToServerRPCCallRequest(msg).build()
302   - ).build();
303   - send(sessionInfo, toRuleEngineMsg, callback);
304   - }
  283 + protected void doProcess(SessionInfoProto sessionInfo, ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback) {
  284 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  285 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  286 + .setToDeviceRPCCallResponse(msg).build()
  287 + ).build();
  288 + send(sessionInfo, toRuleEngineMsg, callback);
  289 + }
  290 +
  291 + @Override
  292 + protected void doProcess(SessionInfoProto sessionInfo, ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback) {
  293 + ToRuleEngineMsg toRuleEngineMsg = ToRuleEngineMsg.newBuilder().setToDeviceActorMsg(
  294 + TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
  295 + .setToServerRPCCallRequest(msg).build()
  296 + ).build();
  297 + send(sessionInfo, toRuleEngineMsg, callback);
305 298 }
306 299
307 300 private static class TransportCallbackAdaptor implements Callback {
... ...
... ... @@ -23,10 +23,25 @@ import org.thingsboard.server.gen.transport.TransportProtos;
23 23 * Created by ashvayka on 15.10.18.
24 24 */
25 25 @Data
26   -public class SessionMetaData {
  26 +class SessionMetaData {
27 27
28 28 private final TransportProtos.SessionInfoProto sessionInfo;
29 29 private final TransportProtos.SessionType sessionType;
30 30 private final SessionMsgListener listener;
31 31
  32 + private volatile long lastActivityTime;
  33 + private volatile boolean subscribedToAttributes;
  34 + private volatile boolean subscribedToRPC;
  35 +
  36 + SessionMetaData(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionType sessionType, SessionMsgListener listener) {
  37 + this.sessionInfo = sessionInfo;
  38 + this.sessionType = sessionType;
  39 + this.listener = listener;
  40 + this.lastActivityTime = System.currentTimeMillis();
  41 + }
  42 +
  43 + void updateLastActivityTime() {
  44 + this.lastActivityTime = System.currentTimeMillis();
  45 + }
  46 +
32 47 }
... ...
... ... @@ -172,6 +172,22 @@ message ToServerRpcResponseMsg {
172 172 string error = 3;
173 173 }
174 174
  175 +//Used to report session state to tb-node and persist this state in the cache on the tb-node level.
  176 +message SubscriptionInfoProto {
  177 + int64 lastActivityTime = 1;
  178 + bool attributeSubscription = 2;
  179 + bool rpcSubscription = 3;
  180 +}
  181 +
  182 +message SessionSubscriptionInfoProto {
  183 + SessionInfoProto sessionInfo = 1;
  184 + SubscriptionInfoProto subscriptionInfo = 2;
  185 +}
  186 +
  187 +message DeviceSessionsCacheEntry {
  188 + repeated SessionSubscriptionInfoProto sessions = 1;
  189 +}
  190 +
175 191 message TransportToDeviceActorMsg {
176 192 SessionInfoProto sessionInfo = 1;
177 193 SessionEventMsg sessionEvent = 2;
... ... @@ -182,6 +198,7 @@ message TransportToDeviceActorMsg {
182 198 SubscribeToRPCMsg subscribeToRPC = 7;
183 199 ToDeviceRpcResponseMsg toDeviceRPCCallResponse = 8;
184 200 ToServerRpcRequestMsg toServerRPCCallRequest = 9;
  201 + SubscriptionInfoProto subscriptionInfo = 10;
185 202 }
186 203
187 204 message DeviceActorToTransportMsg {
... ... @@ -214,4 +231,4 @@ message TransportApiRequestMsg {
214 231 message TransportApiResponseMsg {
215 232 ValidateDeviceCredentialsResponseMsg validateTokenResponseMsg = 1;
216 233 GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2;
217   -}
\ No newline at end of file
  234 +}
... ...
... ... @@ -43,6 +43,8 @@ public interface EntityViewService {
43 43
44 44 EntityView findEntityViewById(EntityViewId entityViewId);
45 45
  46 + EntityView findEntityViewByTenantIdAndName(TenantId tenantId, String name);
  47 +
46 48 TextPageData<EntityView> findEntityViewByTenantId(TenantId tenantId, TextPageLink pageLink);
47 49
48 50 TextPageData<EntityView> findEntityViewByTenantIdAndType(TenantId tenantId, TextPageLink pageLink, String type);
... ...
... ... @@ -29,8 +29,6 @@ import org.springframework.cache.annotation.Cacheable;
29 29 import org.springframework.cache.annotation.Caching;
30 30 import org.springframework.stereotype.Service;
31 31 import org.thingsboard.server.common.data.Customer;
32   -import org.thingsboard.server.common.data.DataConstants;
33   -import org.thingsboard.server.common.data.Device;
34 32 import org.thingsboard.server.common.data.EntitySubtype;
35 33 import org.thingsboard.server.common.data.EntityType;
36 34 import org.thingsboard.server.common.data.EntityView;
... ... @@ -40,12 +38,10 @@ import org.thingsboard.server.common.data.id.CustomerId;
40 38 import org.thingsboard.server.common.data.id.EntityId;
41 39 import org.thingsboard.server.common.data.id.EntityViewId;
42 40 import org.thingsboard.server.common.data.id.TenantId;
43   -import org.thingsboard.server.common.data.kv.AttributeKvEntry;
44 41 import org.thingsboard.server.common.data.page.TextPageData;
45 42 import org.thingsboard.server.common.data.page.TextPageLink;
46 43 import org.thingsboard.server.common.data.relation.EntityRelation;
47 44 import org.thingsboard.server.common.data.relation.EntitySearchDirection;
48   -import org.thingsboard.server.dao.attributes.AttributesService;
49 45 import org.thingsboard.server.dao.customer.CustomerDao;
50 46 import org.thingsboard.server.dao.entity.AbstractEntityService;
51 47 import org.thingsboard.server.dao.exception.DataValidationException;
... ... @@ -56,15 +52,13 @@ import org.thingsboard.server.dao.tenant.TenantDao;
56 52 import javax.annotation.Nullable;
57 53 import java.util.ArrayList;
58 54 import java.util.Arrays;
59   -import java.util.Collection;
60 55 import java.util.Collections;
61 56 import java.util.Comparator;
62 57 import java.util.List;
63   -import java.util.concurrent.ExecutionException;
  58 +import java.util.Optional;
64 59 import java.util.stream.Collectors;
65 60
66 61 import static org.thingsboard.server.common.data.CacheConstants.ENTITY_VIEW_CACHE;
67   -import static org.thingsboard.server.common.data.CacheConstants.RELATIONS_CACHE;
68 62 import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
69 63 import static org.thingsboard.server.dao.service.Validator.validateId;
70 64 import static org.thingsboard.server.dao.service.Validator.validatePageLink;
... ... @@ -96,6 +90,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
96 90
97 91 @Caching(evict = {
98 92 @CacheEvict(cacheNames = ENTITY_VIEW_CACHE, key = "{#entityView.tenantId, #entityView.entityId}"),
  93 + @CacheEvict(cacheNames = ENTITY_VIEW_CACHE, key = "{#entityView.tenantId, #entityView.name}"),
99 94 @CacheEvict(cacheNames = ENTITY_VIEW_CACHE, key = "{#entityView.id}")})
100 95 @Override
101 96 public EntityView saveEntityView(EntityView entityView) {
... ... @@ -137,6 +132,15 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
137 132 return entityViewDao.findById(entityViewId.getId());
138 133 }
139 134
  135 + @Cacheable(cacheNames = ENTITY_VIEW_CACHE, key = "{#tenantId, #name}")
  136 + @Override
  137 + public EntityView findEntityViewByTenantIdAndName(TenantId tenantId, String name) {
  138 + log.trace("Executing findEntityViewByTenantIdAndName [{}][{}]", tenantId, name);
  139 + validateId(tenantId, INCORRECT_TENANT_ID + tenantId);
  140 + Optional<EntityView> entityViewOpt = entityViewDao.findEntityViewByTenantIdAndName(tenantId.getId(), name);
  141 + return entityViewOpt.orElse(null);
  142 + }
  143 +
140 144 @Override
141 145 public TextPageData<EntityView> findEntityViewByTenantId(TenantId tenantId, TextPageLink pageLink) {
142 146 log.trace("Executing findEntityViewsByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink);
... ... @@ -255,6 +259,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
255 259 deleteEntityRelations(entityViewId);
256 260 EntityView entityView = entityViewDao.findById(entityViewId.getId());
257 261 cacheManager.getCache(ENTITY_VIEW_CACHE).evict(Arrays.asList(entityView.getTenantId(), entityView.getEntityId()));
  262 + cacheManager.getCache(ENTITY_VIEW_CACHE).evict(Arrays.asList(entityView.getTenantId(), entityView.getName()));
258 263 entityViewDao.removeById(entityViewId.getId());
259 264 }
260 265
... ...
... ... @@ -34,6 +34,7 @@ import javax.persistence.Entity;
34 34 import javax.persistence.EnumType;
35 35 import javax.persistence.Enumerated;
36 36 import javax.persistence.Table;
  37 +import javax.persistence.UniqueConstraint;
37 38
38 39 @Data
39 40 @EqualsAndHashCode(callSuper = true)
... ... @@ -53,7 +54,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity<ComponentDescriptor
53 54 @Column(name = ModelConstants.COMPONENT_DESCRIPTOR_NAME_PROPERTY)
54 55 private String name;
55 56
56   - @Column(name = ModelConstants.COMPONENT_DESCRIPTOR_CLASS_PROPERTY)
  57 + @Column(name = ModelConstants.COMPONENT_DESCRIPTOR_CLASS_PROPERTY, unique = true)
57 58 private String clazz;
58 59
59 60 @Type(type = "json")
... ...
... ... @@ -35,7 +35,6 @@ import org.thingsboard.server.dao.model.type.ComponentTypeCodec;
35 35 import org.thingsboard.server.dao.model.type.DeviceCredentialsTypeCodec;
36 36 import org.thingsboard.server.dao.model.type.EntityTypeCodec;
37 37 import org.thingsboard.server.dao.model.type.JsonCodec;
38   -import org.thingsboard.server.dao.util.BufferedRateLimiter;
39 38
40 39 import java.util.concurrent.ConcurrentHashMap;
41 40 import java.util.concurrent.ConcurrentMap;
... ... @@ -49,7 +48,7 @@ public abstract class CassandraAbstractDao {
49 48 private ConcurrentMap<String, PreparedStatement> preparedStatementMap = new ConcurrentHashMap<>();
50 49
51 50 @Autowired
52   - private BufferedRateLimiter rateLimiter;
  51 + private CassandraBufferedRateExecutor rateLimiter;
53 52
54 53 private Session session;
55 54
... ... @@ -115,12 +114,12 @@ public abstract class CassandraAbstractDao {
115 114 if (statement.getConsistencyLevel() == null) {
116 115 statement.setConsistencyLevel(level);
117 116 }
118   - return new RateLimitedResultSetFuture(getSession(), rateLimiter, statement);
  117 + return rateLimiter.submit(new CassandraStatementTask(getSession(), statement));
119 118 }
120 119
121 120 private static String statementToString(Statement statement) {
122 121 if (statement instanceof BoundStatement) {
123   - return ((BoundStatement)statement).preparedStatement().getQueryString();
  122 + return ((BoundStatement) statement).preparedStatement().getQueryString();
124 123 } else {
125 124 return statement.toString();
126 125 }
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.nosql;
  17 +
  18 +import com.datastax.driver.core.ResultSet;
  19 +import com.datastax.driver.core.ResultSetFuture;
  20 +import com.google.common.util.concurrent.SettableFuture;
  21 +import lombok.extern.slf4j.Slf4j;
  22 +import org.springframework.beans.factory.annotation.Value;
  23 +import org.springframework.scheduling.annotation.Scheduled;
  24 +import org.springframework.stereotype.Component;
  25 +import org.thingsboard.server.dao.util.AbstractBufferedRateExecutor;
  26 +import org.thingsboard.server.dao.util.AsyncTaskContext;
  27 +import org.thingsboard.server.dao.util.NoSqlAnyDao;
  28 +
  29 +import javax.annotation.PreDestroy;
  30 +
  31 +/**
  32 + * Created by ashvayka on 24.10.18.
  33 + */
  34 +@Component
  35 +@Slf4j
  36 +@NoSqlAnyDao
  37 +public class CassandraBufferedRateExecutor extends AbstractBufferedRateExecutor<CassandraStatementTask, ResultSetFuture, ResultSet> {
  38 +
  39 + public CassandraBufferedRateExecutor(
  40 + @Value("${cassandra.query.buffer_size}") int queueLimit,
  41 + @Value("${cassandra.query.concurrent_limit}") int concurrencyLimit,
  42 + @Value("${cassandra.query.permit_max_wait_time}") long maxWaitTime,
  43 + @Value("${cassandra.query.dispatcher_threads:2}") int dispatcherThreads,
  44 + @Value("${cassandra.query.callback_threads:2}") int callbackThreads,
  45 + @Value("${cassandra.query.poll_ms:50}") long pollMs) {
  46 + super(queueLimit, concurrencyLimit, maxWaitTime, dispatcherThreads, callbackThreads, pollMs);
  47 + }
  48 +
  49 + @Scheduled(fixedDelayString = "${cassandra.query.rate_limit_print_interval_ms}")
  50 + public void printStats() {
  51 + log.info("Permits queueSize [{}] totalAdded [{}] totalLaunched [{}] totalReleased [{}] totalFailed [{}] totalExpired [{}] totalRejected [{}] currBuffer [{}] ",
  52 + getQueueSize(),
  53 + totalAdded.getAndSet(0), totalLaunched.getAndSet(0), totalReleased.getAndSet(0),
  54 + totalFailed.getAndSet(0), totalExpired.getAndSet(0), totalRejected.getAndSet(0),
  55 + concurrencyLevel.get());
  56 + }
  57 +
  58 + @PreDestroy
  59 + public void stop() {
  60 + super.stop();
  61 + }
  62 +
  63 + @Override
  64 + protected SettableFuture<ResultSet> create() {
  65 + return SettableFuture.create();
  66 + }
  67 +
  68 + @Override
  69 + protected ResultSetFuture wrap(CassandraStatementTask task, SettableFuture<ResultSet> future) {
  70 + return new TbResultSetFuture(future);
  71 + }
  72 +
  73 + @Override
  74 + protected ResultSetFuture execute(AsyncTaskContext<CassandraStatementTask, ResultSet> taskCtx) {
  75 + CassandraStatementTask task = taskCtx.getTask();
  76 + return task.getSession().executeAsync(task.getStatement());
  77 + }
  78 +
  79 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.nosql;
  17 +
  18 +import com.datastax.driver.core.Session;
  19 +import com.datastax.driver.core.Statement;
  20 +import lombok.Data;
  21 +import org.thingsboard.server.dao.util.AsyncTask;
  22 +
  23 +/**
  24 + * Created by ashvayka on 24.10.18.
  25 + */
  26 +@Data
  27 +public class CassandraStatementTask implements AsyncTask {
  28 +
  29 + private final Session session;
  30 + private final Statement statement;
  31 +
  32 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.nosql;
  17 +
  18 +import com.datastax.driver.core.ResultSet;
  19 +import com.datastax.driver.core.ResultSetFuture;
  20 +import com.google.common.util.concurrent.SettableFuture;
  21 +
  22 +import java.util.concurrent.ExecutionException;
  23 +import java.util.concurrent.Executor;
  24 +import java.util.concurrent.TimeUnit;
  25 +import java.util.concurrent.TimeoutException;
  26 +
  27 +/**
  28 + * Created by ashvayka on 24.10.18.
  29 + */
  30 +public class TbResultSetFuture implements ResultSetFuture {
  31 +
  32 + private final SettableFuture<ResultSet> mainFuture;
  33 +
  34 + public TbResultSetFuture(SettableFuture<ResultSet> mainFuture) {
  35 + this.mainFuture = mainFuture;
  36 + }
  37 +
  38 + @Override
  39 + public ResultSet getUninterruptibly() {
  40 + return getSafe();
  41 + }
  42 +
  43 + @Override
  44 + public ResultSet getUninterruptibly(long timeout, TimeUnit unit) throws TimeoutException {
  45 + return getSafe(timeout, unit);
  46 + }
  47 +
  48 + @Override
  49 + public boolean cancel(boolean mayInterruptIfRunning) {
  50 + return mainFuture.cancel(mayInterruptIfRunning);
  51 + }
  52 +
  53 + @Override
  54 + public boolean isCancelled() {
  55 + return mainFuture.isCancelled();
  56 + }
  57 +
  58 + @Override
  59 + public boolean isDone() {
  60 + return mainFuture.isDone();
  61 + }
  62 +
  63 + @Override
  64 + public ResultSet get() throws InterruptedException, ExecutionException {
  65 + return mainFuture.get();
  66 + }
  67 +
  68 + @Override
  69 + public ResultSet get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
  70 + return mainFuture.get(timeout, unit);
  71 + }
  72 +
  73 + @Override
  74 + public void addListener(Runnable listener, Executor executor) {
  75 + mainFuture.addListener(listener, executor);
  76 + }
  77 +
  78 + private ResultSet getSafe() {
  79 + try {
  80 + return mainFuture.get();
  81 + } catch (InterruptedException | ExecutionException e) {
  82 + throw new IllegalStateException(e);
  83 + }
  84 + }
  85 +
  86 + private ResultSet getSafe(long timeout, TimeUnit unit) throws TimeoutException {
  87 + try {
  88 + return mainFuture.get(timeout, unit);
  89 + } catch (InterruptedException | ExecutionException e) {
  90 + throw new IllegalStateException(e);
  91 + }
  92 + }
  93 +
  94 +}
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.timeseries;
17 17
18 18 import com.google.common.base.Function;
19 19 import com.google.common.collect.Lists;
  20 +import com.google.common.util.concurrent.FutureCallback;
20 21 import com.google.common.util.concurrent.Futures;
21 22 import com.google.common.util.concurrent.ListenableFuture;
22 23 import com.google.common.util.concurrent.ListeningExecutorService;
... ... @@ -31,6 +32,7 @@ import org.springframework.stereotype.Component;
31 32 import org.thingsboard.server.common.data.UUIDConverter;
32 33 import org.thingsboard.server.common.data.id.EntityId;
33 34 import org.thingsboard.server.common.data.kv.Aggregation;
  35 +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
34 36 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
35 37 import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
36 38 import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
... ... @@ -41,9 +43,9 @@ import org.thingsboard.server.dao.model.sql.TsKvEntity;
41 43 import org.thingsboard.server.dao.model.sql.TsKvLatestCompositeKey;
42 44 import org.thingsboard.server.dao.model.sql.TsKvLatestEntity;
43 45 import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
  46 +import org.thingsboard.server.dao.timeseries.SimpleListenableFuture;
44 47 import org.thingsboard.server.dao.timeseries.TimeseriesDao;
45 48 import org.thingsboard.server.dao.timeseries.TsInsertExecutorType;
46   -import org.thingsboard.server.dao.util.SqlDao;
47 49 import org.thingsboard.server.dao.util.SqlTsDao;
48 50
49 51 import javax.annotation.Nullable;
... ... @@ -53,6 +55,7 @@ import java.util.ArrayList;
53 55 import java.util.List;
54 56 import java.util.Optional;
55 57 import java.util.concurrent.CompletableFuture;
  58 +import java.util.concurrent.ExecutionException;
56 59 import java.util.concurrent.Executors;
57 60 import java.util.stream.Collectors;
58 61
... ... @@ -64,6 +67,8 @@ import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
64 67 @SqlTsDao
65 68 public class JpaTimeseriesDao extends JpaAbstractDaoListeningExecutorService implements TimeseriesDao {
66 69
  70 + private static final String DESC_ORDER = "DESC";
  71 +
67 72 @Value("${sql.ts_inserts_executor_type}")
68 73 private String insertExecutorType;
69 74
... ... @@ -326,14 +331,72 @@ public class JpaTimeseriesDao extends JpaAbstractDaoListeningExecutorService imp
326 331
327 332 @Override
328 333 public ListenableFuture<Void> removeLatest(EntityId entityId, DeleteTsKvQuery query) {
329   - TsKvLatestEntity latestEntity = new TsKvLatestEntity();
330   - latestEntity.setEntityType(entityId.getEntityType());
331   - latestEntity.setEntityId(fromTimeUUID(entityId.getId()));
332   - latestEntity.setKey(query.getKey());
333   - return service.submit(() -> {
334   - tsKvLatestRepository.delete(latestEntity);
335   - return null;
  334 + ListenableFuture<TsKvEntry> latestFuture = findLatest(entityId, query.getKey());
  335 +
  336 + ListenableFuture<Boolean> booleanFuture = Futures.transform(latestFuture, tsKvEntry -> {
  337 + long ts = tsKvEntry.getTs();
  338 + return ts > query.getStartTs() && ts <= query.getEndTs();
  339 + }, service);
  340 +
  341 + ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
  342 + if (isRemove) {
  343 + TsKvLatestEntity latestEntity = new TsKvLatestEntity();
  344 + latestEntity.setEntityType(entityId.getEntityType());
  345 + latestEntity.setEntityId(fromTimeUUID(entityId.getId()));
  346 + latestEntity.setKey(query.getKey());
  347 + return service.submit(() -> {
  348 + tsKvLatestRepository.delete(latestEntity);
  349 + return null;
  350 + });
  351 + }
  352 + return Futures.immediateFuture(null);
  353 + }, service);
  354 +
  355 + final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>();
  356 + Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() {
  357 + @Override
  358 + public void onSuccess(@Nullable Void result) {
  359 + if (query.getRewriteLatestIfDeleted()) {
  360 + ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
  361 + if (isRemove) {
  362 + return getNewLatestEntryFuture(entityId, query);
  363 + }
  364 + return Futures.immediateFuture(null);
  365 + }, service);
  366 +
  367 + try {
  368 + resultFuture.set(savedLatestFuture.get());
  369 + } catch (InterruptedException | ExecutionException e) {
  370 + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e);
  371 + }
  372 + } else {
  373 + resultFuture.set(null);
  374 + }
  375 + }
  376 +
  377 + @Override
  378 + public void onFailure(Throwable t) {
  379 + log.warn("[{}] Failed to process remove of the latest value", entityId, t);
  380 + }
336 381 });
  382 + return resultFuture;
  383 + }
  384 +
  385 + private ListenableFuture<Void> getNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) {
  386 + long startTs = 0;
  387 + long endTs = query.getStartTs() - 1;
  388 + ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1,
  389 + Aggregation.NONE, DESC_ORDER);
  390 + ListenableFuture<List<TsKvEntry>> future = findAllAsync(entityId, findNewLatestQuery);
  391 +
  392 + return Futures.transformAsync(future, entryList -> {
  393 + if (entryList.size() == 1) {
  394 + return saveLatest(entityId, entryList.get(0));
  395 + } else {
  396 + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey());
  397 + }
  398 + return Futures.immediateFuture(null);
  399 + }, service);
337 400 }
338 401
339 402 @Override
... ...
... ... @@ -47,7 +47,7 @@ public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvComposite
47 47 @Modifying
48 48 @Query("DELETE FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " +
49 49 "AND tskv.entityType = :entityType AND tskv.key = :entityKey " +
50   - "AND tskv.ts > :startTs AND tskv.ts < :endTs")
  50 + "AND tskv.ts > :startTs AND tskv.ts <= :endTs")
51 51 void delete(@Param("entityId") String entityId,
52 52 @Param("entityType") EntityType entityType,
53 53 @Param("entityKey") String key,
... ...
... ... @@ -48,7 +48,6 @@ import org.thingsboard.server.common.data.kv.StringDataEntry;
48 48 import org.thingsboard.server.common.data.kv.TsKvEntry;
49 49 import org.thingsboard.server.dao.model.ModelConstants;
50 50 import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao;
51   -import org.thingsboard.server.dao.util.NoSqlDao;
52 51 import org.thingsboard.server.dao.util.NoSqlTsDao;
53 52
54 53 import javax.annotation.Nullable;
... ... @@ -62,6 +61,7 @@ import java.util.Arrays;
62 61 import java.util.Collections;
63 62 import java.util.List;
64 63 import java.util.Optional;
  64 +import java.util.concurrent.ExecutionException;
65 65 import java.util.stream.Collectors;
66 66
67 67 import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
... ... @@ -434,14 +434,14 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
434 434 public ListenableFuture<Void> removeLatest(EntityId entityId, DeleteTsKvQuery query) {
435 435 ListenableFuture<TsKvEntry> latestEntryFuture = findLatest(entityId, query.getKey());
436 436
437   - ListenableFuture<Boolean> booleanFuture = Futures.transformAsync(latestEntryFuture, latestEntry -> {
  437 + ListenableFuture<Boolean> booleanFuture = Futures.transform(latestEntryFuture, latestEntry -> {
438 438 long ts = latestEntry.getTs();
439   - if (ts >= query.getStartTs() && ts <= query.getEndTs()) {
440   - return Futures.immediateFuture(true);
  439 + if (ts > query.getStartTs() && ts <= query.getEndTs()) {
  440 + return true;
441 441 } else {
442 442 log.trace("Won't be deleted latest value for [{}], key - {}", entityId, query.getKey());
443 443 }
444   - return Futures.immediateFuture(false);
  444 + return false;
445 445 }, readResultsProcessingExecutor);
446 446
447 447 ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
... ... @@ -451,18 +451,34 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
451 451 return Futures.immediateFuture(null);
452 452 }, readResultsProcessingExecutor);
453 453
454   - if (query.getRewriteLatestIfDeleted()) {
455   - ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
456   - if (isRemove) {
457   - return getNewLatestEntryFuture(entityId, query);
  454 + final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>();
  455 + Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() {
  456 + @Override
  457 + public void onSuccess(@Nullable Void result) {
  458 + if (query.getRewriteLatestIfDeleted()) {
  459 + ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
  460 + if (isRemove) {
  461 + return getNewLatestEntryFuture(entityId, query);
  462 + }
  463 + return Futures.immediateFuture(null);
  464 + }, readResultsProcessingExecutor);
  465 +
  466 + try {
  467 + resultFuture.set(savedLatestFuture.get());
  468 + } catch (InterruptedException | ExecutionException e) {
  469 + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e);
  470 + }
  471 + } else {
  472 + resultFuture.set(null);
458 473 }
459   - return Futures.immediateFuture(null);
460   - }, readResultsProcessingExecutor);
  474 + }
461 475
462   - return Futures.transformAsync(Futures.allAsList(Arrays.asList(savedLatestFuture, removedLatestFuture)),
463   - list -> Futures.immediateFuture(null), readResultsProcessingExecutor);
464   - }
465   - return removedLatestFuture;
  476 + @Override
  477 + public void onFailure(Throwable t) {
  478 + log.warn("[{}] Failed to process remove of the latest value", entityId, t);
  479 + }
  480 + });
  481 + return resultFuture;
466 482 }
467 483
468 484 private ListenableFuture<Void> getNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) {
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.util;
  17 +
  18 +import com.google.common.util.concurrent.FutureCallback;
  19 +import com.google.common.util.concurrent.Futures;
  20 +import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.SettableFuture;
  22 +import lombok.extern.slf4j.Slf4j;
  23 +
  24 +import javax.annotation.Nullable;
  25 +import java.util.UUID;
  26 +import java.util.concurrent.BlockingQueue;
  27 +import java.util.concurrent.ExecutorService;
  28 +import java.util.concurrent.Executors;
  29 +import java.util.concurrent.LinkedBlockingDeque;
  30 +import java.util.concurrent.ScheduledExecutorService;
  31 +import java.util.concurrent.TimeUnit;
  32 +import java.util.concurrent.TimeoutException;
  33 +import java.util.concurrent.atomic.AtomicInteger;
  34 +
  35 +/**
  36 + * Created by ashvayka on 24.10.18.
  37 + */
  38 +@Slf4j
  39 +public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extends ListenableFuture<V>, V> implements BufferedRateExecutor<T, F> {
  40 +
  41 + private final long maxWaitTime;
  42 + private final long pollMs;
  43 + private final BlockingQueue<AsyncTaskContext<T, V>> queue;
  44 + private final ExecutorService dispatcherExecutor;
  45 + private final ExecutorService callbackExecutor;
  46 + private final ScheduledExecutorService timeoutExecutor;
  47 + private final int concurrencyLimit;
  48 +
  49 + protected final AtomicInteger concurrencyLevel = new AtomicInteger();
  50 + protected final AtomicInteger totalAdded = new AtomicInteger();
  51 + protected final AtomicInteger totalLaunched = new AtomicInteger();
  52 + protected final AtomicInteger totalReleased = new AtomicInteger();
  53 + protected final AtomicInteger totalFailed = new AtomicInteger();
  54 + protected final AtomicInteger totalExpired = new AtomicInteger();
  55 + protected final AtomicInteger totalRejected = new AtomicInteger();
  56 +
  57 + public AbstractBufferedRateExecutor(int queueLimit, int concurrencyLimit, long maxWaitTime, int dispatcherThreads, int callbackThreads, long pollMs) {
  58 + this.maxWaitTime = maxWaitTime;
  59 + this.pollMs = pollMs;
  60 + this.concurrencyLimit = concurrencyLimit;
  61 + this.queue = new LinkedBlockingDeque<>(queueLimit);
  62 + this.dispatcherExecutor = Executors.newFixedThreadPool(dispatcherThreads);
  63 + this.callbackExecutor = Executors.newFixedThreadPool(callbackThreads);
  64 + this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor();
  65 + for (int i = 0; i < dispatcherThreads; i++) {
  66 + dispatcherExecutor.submit(this::dispatch);
  67 + }
  68 + }
  69 +
  70 + @Override
  71 + public F submit(T task) {
  72 + SettableFuture<V> settableFuture = create();
  73 + F result = wrap(task, settableFuture);
  74 + try {
  75 + totalAdded.incrementAndGet();
  76 + queue.add(new AsyncTaskContext<>(UUID.randomUUID(), task, settableFuture, System.currentTimeMillis()));
  77 + } catch (IllegalStateException e) {
  78 + totalRejected.incrementAndGet();
  79 + settableFuture.setException(e);
  80 + }
  81 + return result;
  82 + }
  83 +
  84 + public void stop() {
  85 + if (dispatcherExecutor != null) {
  86 + dispatcherExecutor.shutdownNow();
  87 + }
  88 + if (callbackExecutor != null) {
  89 + callbackExecutor.shutdownNow();
  90 + }
  91 + if (timeoutExecutor != null) {
  92 + timeoutExecutor.shutdownNow();
  93 + }
  94 + }
  95 +
  96 + protected abstract SettableFuture<V> create();
  97 +
  98 + protected abstract F wrap(T task, SettableFuture<V> future);
  99 +
  100 + protected abstract ListenableFuture<V> execute(AsyncTaskContext<T, V> taskCtx);
  101 +
  102 + private void dispatch() {
  103 + log.info("Buffered rate executor thread started");
  104 + while (!Thread.interrupted()) {
  105 + int curLvl = concurrencyLevel.get();
  106 + AsyncTaskContext<T, V> taskCtx = null;
  107 + try {
  108 + if (curLvl <= concurrencyLimit) {
  109 + taskCtx = queue.take();
  110 + final AsyncTaskContext<T, V> finalTaskCtx = taskCtx;
  111 + logTask("Processing", finalTaskCtx);
  112 + concurrencyLevel.incrementAndGet();
  113 + long timeout = finalTaskCtx.getCreateTime() + maxWaitTime - System.currentTimeMillis();
  114 + if (timeout > 0) {
  115 + totalLaunched.incrementAndGet();
  116 + ListenableFuture<V> result = execute(finalTaskCtx);
  117 + result = Futures.withTimeout(result, timeout, TimeUnit.MILLISECONDS, timeoutExecutor);
  118 + Futures.addCallback(result, new FutureCallback<V>() {
  119 + @Override
  120 + public void onSuccess(@Nullable V result) {
  121 + logTask("Releasing", finalTaskCtx);
  122 + totalReleased.incrementAndGet();
  123 + concurrencyLevel.decrementAndGet();
  124 + finalTaskCtx.getFuture().set(result);
  125 + }
  126 +
  127 + @Override
  128 + public void onFailure(Throwable t) {
  129 + if (t instanceof TimeoutException) {
  130 + logTask("Expired During Execution", finalTaskCtx);
  131 + } else {
  132 + logTask("Failed", finalTaskCtx);
  133 + }
  134 + totalFailed.incrementAndGet();
  135 + concurrencyLevel.decrementAndGet();
  136 + finalTaskCtx.getFuture().setException(t);
  137 + log.debug("[{}] Failed to execute task: {}", finalTaskCtx.getId(), finalTaskCtx.getTask(), t);
  138 + }
  139 + }, callbackExecutor);
  140 + } else {
  141 + logTask("Expired Before Execution", finalTaskCtx);
  142 + totalExpired.incrementAndGet();
  143 + concurrencyLevel.decrementAndGet();
  144 + taskCtx.getFuture().setException(new TimeoutException());
  145 + }
  146 + } else {
  147 + Thread.sleep(pollMs);
  148 + }
  149 + } catch (InterruptedException e) {
  150 + break;
  151 + } catch (Throwable e) {
  152 + if (taskCtx != null) {
  153 + log.debug("[{}] Failed to execute task: {}", taskCtx.getId(), taskCtx, e);
  154 + totalFailed.incrementAndGet();
  155 + concurrencyLevel.decrementAndGet();
  156 + } else {
  157 + log.debug("Failed to queue task:", e);
  158 + }
  159 + }
  160 + }
  161 + log.info("Buffered rate executor thread stopped");
  162 + }
  163 +
  164 + private void logTask(String action, AsyncTaskContext<T, V> taskCtx) {
  165 + if (log.isTraceEnabled()) {
  166 + log.trace("[{}] {} task: {}", taskCtx.getId(), action, taskCtx);
  167 + } else {
  168 + log.debug("[{}] {} task", taskCtx.getId(), action);
  169 + }
  170 + }
  171 +
  172 + protected int getQueueSize() {
  173 + return queue.size();
  174 + }
  175 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.util;
  17 +
  18 +/**
  19 + * Created by ashvayka on 24.10.18.
  20 + */
  21 +public interface AsyncTask {
  22 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.util;
  17 +
  18 +import com.google.common.util.concurrent.SettableFuture;
  19 +import lombok.Data;
  20 +
  21 +import java.util.UUID;
  22 +
  23 +/**
  24 + * Created by ashvayka on 24.10.18.
  25 + */
  26 +@Data
  27 +public class AsyncTaskContext<T extends AsyncTask, V> {
  28 +
  29 + private final UUID id;
  30 + private final T task;
  31 + private final SettableFuture<V> future;
  32 + private final long createTime;
  33 +
  34 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2018 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.util;
  17 +
  18 +import com.google.common.util.concurrent.ListenableFuture;
  19 +
  20 +/**
  21 + * Created by ashvayka on 24.10.18.
  22 + */
  23 +public interface BufferedRateExecutor<T extends AsyncTask, F extends ListenableFuture> {
  24 +
  25 + F submit(T task);
  26 +
  27 +}
... ...
1   -/**
2   - * Copyright © 2016-2018 The Thingsboard Authors
3   - *
4   - * Licensed under the Apache License, Version 2.0 (the "License");
5   - * you may not use this file except in compliance with the License.
6   - * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
10   - * Unless required by applicable law or agreed to in writing, software
11   - * distributed under the License is distributed on an "AS IS" BASIS,
12   - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   - * See the License for the specific language governing permissions and
14   - * limitations under the License.
15   - */
16   -package org.thingsboard.server.dao.util;
17   -
18   -import com.google.common.util.concurrent.Futures;
19   -import com.google.common.util.concurrent.ListenableFuture;
20   -import com.google.common.util.concurrent.ListeningExecutorService;
21   -import com.google.common.util.concurrent.MoreExecutors;
22   -import lombok.extern.slf4j.Slf4j;
23   -import org.springframework.beans.factory.annotation.Value;
24   -import org.springframework.scheduling.annotation.Scheduled;
25   -import org.springframework.stereotype.Component;
26   -import org.thingsboard.server.dao.exception.BufferLimitException;
27   -
28   -import java.util.concurrent.BlockingQueue;
29   -import java.util.concurrent.CountDownLatch;
30   -import java.util.concurrent.Executors;
31   -import java.util.concurrent.LinkedBlockingQueue;
32   -import java.util.concurrent.TimeUnit;
33   -import java.util.concurrent.atomic.AtomicInteger;
34   -
35   -@Component
36   -@Slf4j
37   -@NoSqlAnyDao
38   -public class BufferedRateLimiter implements AsyncRateLimiter {
39   -
40   - private final ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(10));
41   -
42   - private final int permitsLimit;
43   - private final int maxPermitWaitTime;
44   - private final AtomicInteger permits;
45   - private final BlockingQueue<LockedFuture> queue;
46   -
47   - private final AtomicInteger maxQueueSize = new AtomicInteger();
48   - private final AtomicInteger maxGrantedPermissions = new AtomicInteger();
49   - private final AtomicInteger totalGranted = new AtomicInteger();
50   - private final AtomicInteger totalReleased = new AtomicInteger();
51   - private final AtomicInteger totalRequested = new AtomicInteger();
52   -
53   - public BufferedRateLimiter(@Value("${cassandra.query.buffer_size}") int queueLimit,
54   - @Value("${cassandra.query.concurrent_limit}") int permitsLimit,
55   - @Value("${cassandra.query.permit_max_wait_time}") int maxPermitWaitTime) {
56   - this.permitsLimit = permitsLimit;
57   - this.maxPermitWaitTime = maxPermitWaitTime;
58   - this.permits = new AtomicInteger();
59   - this.queue = new LinkedBlockingQueue<>(queueLimit);
60   - }
61   -
62   - @Override
63   - public ListenableFuture<Void> acquireAsync() {
64   - totalRequested.incrementAndGet();
65   - if (queue.isEmpty()) {
66   - if (permits.incrementAndGet() <= permitsLimit) {
67   - if (permits.get() > maxGrantedPermissions.get()) {
68   - maxGrantedPermissions.set(permits.get());
69   - }
70   - totalGranted.incrementAndGet();
71   - return Futures.immediateFuture(null);
72   - }
73   - permits.decrementAndGet();
74   - }
75   -
76   - return putInQueue();
77   - }
78   -
79   - @Override
80   - public void release() {
81   - permits.decrementAndGet();
82   - totalReleased.incrementAndGet();
83   - reprocessQueue();
84   - }
85   -
86   - private void reprocessQueue() {
87   - while (permits.get() < permitsLimit) {
88   - if (permits.incrementAndGet() <= permitsLimit) {
89   - if (permits.get() > maxGrantedPermissions.get()) {
90   - maxGrantedPermissions.set(permits.get());
91   - }
92   - LockedFuture lockedFuture = queue.poll();
93   - if (lockedFuture != null) {
94   - totalGranted.incrementAndGet();
95   - lockedFuture.latch.countDown();
96   - } else {
97   - permits.decrementAndGet();
98   - break;
99   - }
100   - } else {
101   - permits.decrementAndGet();
102   - }
103   - }
104   - }
105   -
106   - private LockedFuture createLockedFuture() {
107   - CountDownLatch latch = new CountDownLatch(1);
108   - ListenableFuture<Void> future = pool.submit(() -> {
109   - latch.await();
110   - return null;
111   - });
112   - return new LockedFuture(latch, future, System.currentTimeMillis());
113   - }
114   -
115   - private ListenableFuture<Void> putInQueue() {
116   -
117   - int size = queue.size();
118   - if (size > maxQueueSize.get()) {
119   - maxQueueSize.set(size);
120   - }
121   -
122   - if (queue.remainingCapacity() > 0) {
123   - try {
124   - LockedFuture lockedFuture = createLockedFuture();
125   - if (!queue.offer(lockedFuture, 1, TimeUnit.SECONDS)) {
126   - lockedFuture.cancelFuture();
127   - return Futures.immediateFailedFuture(new BufferLimitException());
128   - }
129   - if(permits.get() < permitsLimit) {
130   - reprocessQueue();
131   - }
132   - if(permits.get() < permitsLimit) {
133   - reprocessQueue();
134   - }
135   - return lockedFuture.future;
136   - } catch (InterruptedException e) {
137   - return Futures.immediateFailedFuture(new BufferLimitException());
138   - }
139   - }
140   - return Futures.immediateFailedFuture(new BufferLimitException());
141   - }
142   -
143   - @Scheduled(fixedDelayString = "${cassandra.query.rate_limit_print_interval_ms}")
144   - public void printStats() {
145   - int expiredCount = 0;
146   - for (LockedFuture lockedFuture : queue) {
147   - if (lockedFuture.isExpired()) {
148   - lockedFuture.cancelFuture();
149   - expiredCount++;
150   - }
151   - }
152   - log.info("Permits maxBuffer [{}] maxPermits [{}] expired [{}] currPermits [{}] currBuffer [{}] " +
153   - "totalPermits [{}] totalRequests [{}] totalReleased [{}]",
154   - maxQueueSize.getAndSet(0), maxGrantedPermissions.getAndSet(0), expiredCount,
155   - permits.get(), queue.size(),
156   - totalGranted.getAndSet(0), totalRequested.getAndSet(0), totalReleased.getAndSet(0));
157   - }
158   -
159   - private class LockedFuture {
160   - final CountDownLatch latch;
161   - final ListenableFuture<Void> future;
162   - final long createTime;
163   -
164   - public LockedFuture(CountDownLatch latch, ListenableFuture<Void> future, long createTime) {
165   - this.latch = latch;
166   - this.future = future;
167   - this.createTime = createTime;
168   - }
169   -
170   - void cancelFuture() {
171   - future.cancel(false);
172   - latch.countDown();
173   - }
174   -
175   - boolean isExpired() {
176   - return (System.currentTimeMillis() - createTime) > maxPermitWaitTime;
177   - }
178   -
179   - }
180   -
181   -
182   -}
... ... @@ -78,7 +78,7 @@ CREATE TABLE IF NOT EXISTS attribute_kv (
78 78 CREATE TABLE IF NOT EXISTS component_descriptor (
79 79 id varchar(31) NOT NULL CONSTRAINT component_descriptor_pkey PRIMARY KEY,
80 80 actions varchar(255),
81   - clazz varchar,
  81 + clazz varchar UNIQUE,
82 82 configuration_descriptor varchar,
83 83 name varchar(255),
84 84 scope varchar(255),
... ...
... ... @@ -152,7 +152,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
152 152 }
153 153
154 154 @Test
155   - public void testDeleteDeviceTsData() throws Exception {
  155 + public void testDeleteDeviceTsDataWithoutOverwritingLatest() throws Exception {
156 156 DeviceId deviceId = new DeviceId(UUIDs.timeBased());
157 157
158 158 saveEntries(deviceId, 10000);
... ... @@ -172,6 +172,26 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
172 172 }
173 173
174 174 @Test
  175 + public void testDeleteDeviceTsDataWithOverwritingLatest() throws Exception {
  176 + DeviceId deviceId = new DeviceId(UUIDs.timeBased());
  177 +
  178 + saveEntries(deviceId, 10000);
  179 + saveEntries(deviceId, 20000);
  180 + saveEntries(deviceId, 30000);
  181 + saveEntries(deviceId, 40000);
  182 +
  183 + tsService.remove(deviceId, Collections.singletonList(
  184 + new BaseDeleteTsKvQuery(STRING_KEY, 25000, 45000, true))).get();
  185 +
  186 + List<TsKvEntry> list = tsService.findAll(deviceId, Collections.singletonList(
  187 + new BaseReadTsKvQuery(STRING_KEY, 5000, 45000, 10000, 10, Aggregation.NONE))).get();
  188 + Assert.assertEquals(2, list.size());
  189 +
  190 + List<TsKvEntry> latest = tsService.findLatest(deviceId, Collections.singletonList(STRING_KEY)).get();
  191 + Assert.assertEquals(20000, latest.get(0).getTs());
  192 + }
  193 +
  194 + @Test
175 195 public void testFindDeviceTsData() throws Exception {
176 196 DeviceId deviceId = new DeviceId(UUIDs.timeBased());
177 197 List<TsKvEntry> entries = new ArrayList<>();
... ...
1   -/**
2   - * Copyright © 2016-2018 The Thingsboard Authors
3   - *
4   - * Licensed under the Apache License, Version 2.0 (the "License");
5   - * you may not use this file except in compliance with the License.
6   - * You may obtain a copy of the License at
7   - *
8   - * http://www.apache.org/licenses/LICENSE-2.0
9   - *
10   - * Unless required by applicable law or agreed to in writing, software
11   - * distributed under the License is distributed on an "AS IS" BASIS,
12   - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   - * See the License for the specific language governing permissions and
14   - * limitations under the License.
15   - */
16   -package org.thingsboard.server.dao.util;
17   -
18   -import com.google.common.util.concurrent.FutureCallback;
19   -import com.google.common.util.concurrent.Futures;
20   -import com.google.common.util.concurrent.ListenableFuture;
21   -import com.google.common.util.concurrent.ListeningExecutorService;
22   -import com.google.common.util.concurrent.MoreExecutors;
23   -import org.junit.Test;
24   -import org.thingsboard.server.dao.exception.BufferLimitException;
25   -
26   -import javax.annotation.Nullable;
27   -import java.util.concurrent.ExecutionException;
28   -import java.util.concurrent.Executors;
29   -import java.util.concurrent.TimeUnit;
30   -import java.util.concurrent.atomic.AtomicInteger;
31   -
32   -import static org.junit.Assert.assertEquals;
33   -import static org.junit.Assert.assertFalse;
34   -import static org.junit.Assert.assertTrue;
35   -import static org.junit.Assert.fail;
36   -
37   -
38   -public class BufferedRateLimiterTest {
39   -
40   - @Test
41   - public void finishedFutureReturnedIfPermitsAreGranted() {
42   - BufferedRateLimiter limiter = new BufferedRateLimiter(10, 10, 100);
43   - ListenableFuture<Void> actual = limiter.acquireAsync();
44   - assertTrue(actual.isDone());
45   - }
46   -
47   - @Test
48   - public void notFinishedFutureReturnedIfPermitsAreNotGranted() {
49   - BufferedRateLimiter limiter = new BufferedRateLimiter(10, 1, 100);
50   - ListenableFuture<Void> actual1 = limiter.acquireAsync();
51   - ListenableFuture<Void> actual2 = limiter.acquireAsync();
52   - assertTrue(actual1.isDone());
53   - assertFalse(actual2.isDone());
54   - }
55   -
56   - @Test
57   - public void failedFutureReturnedIfQueueIsfull() {
58   - BufferedRateLimiter limiter = new BufferedRateLimiter(1, 1, 100);
59   - ListenableFuture<Void> actual1 = limiter.acquireAsync();
60   - ListenableFuture<Void> actual2 = limiter.acquireAsync();
61   - ListenableFuture<Void> actual3 = limiter.acquireAsync();
62   -
63   - assertTrue(actual1.isDone());
64   - assertFalse(actual2.isDone());
65   - assertTrue(actual3.isDone());
66   - try {
67   - actual3.get();
68   - fail();
69   - } catch (Exception e) {
70   - assertTrue(e instanceof ExecutionException);
71   - Throwable actualCause = e.getCause();
72   - assertTrue(actualCause instanceof BufferLimitException);
73   - assertEquals("Rate Limit Buffer is full", actualCause.getMessage());
74   - }
75   - }
76   -
77   - @Test
78   - public void releasedPermitTriggerTasksFromQueue() throws InterruptedException {
79   - BufferedRateLimiter limiter = new BufferedRateLimiter(10, 2, 100);
80   - ListenableFuture<Void> actual1 = limiter.acquireAsync();
81   - ListenableFuture<Void> actual2 = limiter.acquireAsync();
82   - ListenableFuture<Void> actual3 = limiter.acquireAsync();
83   - ListenableFuture<Void> actual4 = limiter.acquireAsync();
84   - assertTrue(actual1.isDone());
85   - assertTrue(actual2.isDone());
86   - assertFalse(actual3.isDone());
87   - assertFalse(actual4.isDone());
88   - limiter.release();
89   - TimeUnit.MILLISECONDS.sleep(100L);
90   - assertTrue(actual3.isDone());
91   - assertFalse(actual4.isDone());
92   - limiter.release();
93   - TimeUnit.MILLISECONDS.sleep(100L);
94   - assertTrue(actual4.isDone());
95   - }
96   -
97   - @Test
98   - public void permitsReleasedInConcurrentMode() throws InterruptedException {
99   - BufferedRateLimiter limiter = new BufferedRateLimiter(10, 2, 100);
100   - AtomicInteger actualReleased = new AtomicInteger();
101   - AtomicInteger actualRejected = new AtomicInteger();
102   - ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(5));
103   - for (int i = 0; i < 100; i++) {
104   - ListenableFuture<ListenableFuture<Void>> submit = pool.submit(limiter::acquireAsync);
105   - Futures.addCallback(submit, new FutureCallback<ListenableFuture<Void>>() {
106   - @Override
107   - public void onSuccess(@Nullable ListenableFuture<Void> result) {
108   - Futures.addCallback(result, new FutureCallback<Void>() {
109   - @Override
110   - public void onSuccess(@Nullable Void result) {
111   - try {
112   - TimeUnit.MILLISECONDS.sleep(100);
113   - } catch (InterruptedException e) {
114   - e.printStackTrace();
115   - }
116   - limiter.release();
117   - actualReleased.incrementAndGet();
118   - }
119   -
120   - @Override
121   - public void onFailure(Throwable t) {
122   - actualRejected.incrementAndGet();
123   - }
124   - });
125   - }
126   -
127   - @Override
128   - public void onFailure(Throwable t) {
129   - }
130   - });
131   - }
132   -
133   - TimeUnit.SECONDS.sleep(2);
134   - assertTrue("Unexpected released count " + actualReleased.get(),
135   - actualReleased.get() > 10 && actualReleased.get() < 20);
136   - assertTrue("Unexpected rejected count " + actualRejected.get(),
137   - actualRejected.get() > 80 && actualRejected.get() < 90);
138   -
139   - }
140   -
141   -
142   -}
\ No newline at end of file
1   -# cassandra environment variables
2   -CASSANDRA_DATA_DIR=/home/docker/cassandra_volume
3 1
4   -# postgres environment variables
5   -POSTGRES_DATA_DIR=/home/docker/postgres_volume
6   -POSTGRES_DB=thingsboard
  2 +DOCKER_REPO=thingsboard
7 3
8   -# hsqldb environment variables
9   -HSQLDB_DATA_DIR=/home/docker/hsqldb_volume
  4 +JS_EXECUTOR_DOCKER_NAME=tb-js-executor
  5 +TB_NODE_DOCKER_NAME=tb-node
  6 +WEB_UI_DOCKER_NAME=tb-web-ui
  7 +MQTT_TRANSPORT_DOCKER_NAME=tb-mqtt-transport
  8 +HTTP_TRANSPORT_DOCKER_NAME=tb-http-transport
  9 +COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport
10 10
11   -# environment variables for schema init and insert system and demo data
12   -ADD_SCHEMA_AND_SYSTEM_DATA=false
13   -ADD_DEMO_DATA=false
\ No newline at end of file
  11 +TB_VERSION=latest
  12 +
  13 +# Database used by ThingsBoard, can be either postgres (PostgreSQL) or cassandra (Cassandra).
  14 +# According to the database type corresponding docker service will be deployed (see docker-compose.postgres.yml, docker-compose.cassandra.yml for details).
  15 +
  16 +DATABASE=postgres
  17 +
  18 +KAFKA_TOPICS="js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1"
... ...
docker/.gitignore renamed from msa/docker/.gitignore
... ... @@ -2,4 +2,6 @@ haproxy/certs.d/**
2 2 haproxy/letsencrypt/**
3 3 tb-node/log/**
4 4 tb-node/db/**
  5 +tb-node/postgres/**
  6 +tb-node/cassandra/**
5 7 !.env
... ...
  1 +# Docker configuration for ThingsBoard Microservices
  2 +
  3 +This folder containing scripts and Docker Compose configurations to run ThingsBoard in Microservices mode.
  4 +
  5 +## Prerequisites
  6 +
  7 +ThingsBoard Microservices are running in dockerized environment.
  8 +Before starting please make sure [Docker CE](https://docs.docker.com/install/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed in your system.
  9 +
  10 +## Installation
  11 +
  12 +Before performing initial installation you can configure the type of database to be used with ThinsBoard.
  13 +In order to set database type change the value of `DATABASE` variable in `.env` file to one of the following:
  14 +
  15 +- `postgres` - use PostgreSQL database;
  16 +- `cassandra` - use Cassandra database;
  17 +
  18 +**NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.cassandra.yml` for details).
  19 +
  20 +Execute the following command to run installation:
  21 +
  22 +`
  23 +$ ./docker-install-tb.sh --loadDemo
  24 +`
  25 +
  26 +Where:
  27 +
  28 +- `--loadDemo` - optional argument. Whether to load additional demo data.
  29 +
  30 +## Running
  31 +
  32 +Execute the following command to start services:
  33 +
  34 +`
  35 +$ ./docker-start-services.sh
  36 +`
  37 +
  38 +After a while when all services will be successfully started you can open `http://{your-host-ip}` in you browser (for ex. `http://localhost`).
  39 +You should see ThingsBoard login page.
  40 +
  41 +Use the following default credentials:
  42 +
  43 +- **Systen Administrator**: sysadmin@thingsboard.org / sysadmin
  44 +
  45 +If you installed DataBase with demo data (using `--loadDemo` flag) you can also use the following credentials:
  46 +
  47 +- **Tenant Administrator**: tenant@thingsboard.org / tenant
  48 +- **Customer User**: customer@thingsboard.org / customer
  49 +
  50 +In case of any issues you can examine service logs for errors.
  51 +For example to see ThingsBoard node logs execute the following command:
  52 +
  53 +`
  54 +$ docker-compose logs -f tb1
  55 +`
  56 +
  57 +Or use `docker-compose ps` to see the state of all the containers.
  58 +Use `docker-compose logs --f` to inspect the logs of all running services.
  59 +See [docker-compose logs](https://docs.docker.com/compose/reference/logs/) command reference for details.
  60 +
  61 +Execute the following command to stop services:
  62 +
  63 +`
  64 +$ ./docker-stop-services.sh
  65 +`
  66 +
  67 +Execute the following command to stop and completely remove deployed docker containers:
  68 +
  69 +`
  70 +$ ./docker-remove-services.sh
  71 +`
  72 +
  73 +Execute the following command to update particular or all services (pull newer docker image and rebuild container):
  74 +
  75 +`
  76 +$ ./docker-update-service.sh [SERVICE...]
  77 +`
  78 +
  79 +Where:
  80 +
  81 +- `[SERVICE...]` - list of services to update (defined in docker-compose configurations). If not specified all services will be updated.
  82 +
  83 +## Upgrading
  84 +
  85 +In case when database upgrade is needed, execute the following commands:
  86 +
  87 +```
  88 +$ ./docker-stop-services.sh
  89 +$ ./docker-upgrade-tb.sh --fromVersion=[FROM_VERSION]
  90 +$ ./docker-start-services.sh
  91 +```
  92 +
  93 +Where:
  94 +
  95 +- `FROM_VERSION` - from which version upgrade should be started. See [Upgrade Instructions](https://thingsboard.io/docs/user-guide/install/upgrade-instructions) for valid `fromVersion` values.
... ...
1   -VERSION=2.1.0
2   -PROJECT=thingsboard
3   -APP=cassandra-setup
4   -
5   -build:
6   - cp ../../application/target/thingsboard.deb .
7   - docker build --pull -t ${PROJECT}/${APP}:${VERSION} -t ${PROJECT}/${APP}:latest .
8   - rm thingsboard.deb
9   -
10   -push: build
11   - docker push ${PROJECT}/${APP}:${VERSION}
12   - docker push ${PROJECT}/${APP}:latest
1   -VERSION=2.1.0
2   -PROJECT=thingsboard
3   -APP=cassandra-upgrade
4   -
5   -build:
6   - cp ../../application/target/thingsboard.deb .
7   - docker build --pull -t ${PROJECT}/${APP}:${VERSION} -t ${PROJECT}/${APP}:latest .
8   - rm thingsboard.deb
9   -
10   -push: build
11   - docker push ${PROJECT}/${APP}:${VERSION}
12   - docker push ${PROJECT}/${APP}:latest
1   -VERSION=2.1.0
2   -PROJECT=thingsboard
3   -APP=cassandra
4   -
5   -build:
6   - docker build --pull -t ${PROJECT}/${APP}:${VERSION} -t ${PROJECT}/${APP}:latest .
7   -
8   -push: build
9   - docker push ${PROJECT}/${APP}:${VERSION}
10   - docker push ${PROJECT}/${APP}:latest
docker/check-dirs.sh renamed from msa/docker/check-dirs.sh
... ... @@ -15,7 +15,7 @@
15 15 # limitations under the License.
16 16 #
17 17
18   -dirsArray=("./haproxy/certs.d" "./haproxy/letsencrypt" "./tb-node/db" "./tb-node/log")
  18 +dirsArray=("./haproxy/certs.d" "./haproxy/letsencrypt" "./tb-node/postgres" "./tb-node/cassandra" "./tb-node/log/tb1" "./tb-node/log/tb2")
19 19
20 20 for dir in ${dirsArray[@]}
21 21 do
... ...
1   -#
2   -# Copyright © 2016-2018 The Thingsboard Authors
3   -#
4   -# Licensed under the Apache License, Version 2.0 (the "License");
5   -# you may not use this file except in compliance with the License.
6   -# You may obtain a copy of the License at
7   -#
8   -# http://www.apache.org/licenses/LICENSE-2.0
9   -#
10   -# Unless required by applicable law or agreed to in writing, software
11   -# distributed under the License is distributed on an "AS IS" BASIS,
12   -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   -# See the License for the specific language governing permissions and
14   -# limitations under the License.
15   -#
16   -
17   -version: '3.3'
18   -services:
19   - zookeeper:
20   - image: wurstmeister/zookeeper
21   - networks:
22   - - core
23   - ports:
24   - - "2181:2181"
25   -
26   - cassandra:
27   - image: cassandra:3.11.2
28   - networks:
29   - - core
30   - ports:
31   - - "7199:7199"
32   - - "9160:9160"
33   - - "9042:9042"
34   -
35   - redis:
36   - image: redis:4.0
37   - networks:
38   - - core
39   - command: redis-server --maxclients 2000
40   - ports:
41   - - "6379:6379"
42   -
43   -networks:
44   - core:
45   -
  1 +#!/bin/bash
  2 +#
  3 +# Copyright © 2016-2018 The Thingsboard Authors
  4 +#
  5 +# Licensed under the Apache License, Version 2.0 (the "License");
  6 +# you may not use this file except in compliance with the License.
  7 +# You may obtain a copy of the License at
  8 +#
  9 +# http://www.apache.org/licenses/LICENSE-2.0
  10 +#
  11 +# Unless required by applicable law or agreed to in writing, software
  12 +# distributed under the License is distributed on an "AS IS" BASIS,
  13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 +# See the License for the specific language governing permissions and
  15 +# limitations under the License.
  16 +#
  17 +
  18 +function additionalComposeArgs() {
  19 + source .env
  20 + ADDITIONAL_COMPOSE_ARGS=""
  21 + case $DATABASE in
  22 + postgres)
  23 + ADDITIONAL_COMPOSE_ARGS="-f docker-compose.postgres.yml"
  24 + ;;
  25 + cassandra)
  26 + ADDITIONAL_COMPOSE_ARGS="-f docker-compose.cassandra.yml"
  27 + ;;
  28 + *)
  29 + echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2
  30 + exit 1
  31 + esac
  32 + echo $ADDITIONAL_COMPOSE_ARGS
  33 +}
  34 +
  35 +function additionalStartupServices() {
  36 + source .env
  37 + ADDITIONAL_STARTUP_SERVICES=""
  38 + case $DATABASE in
  39 + postgres)
  40 + ADDITIONAL_STARTUP_SERVICES=postgres
  41 + ;;
  42 + cassandra)
  43 + ADDITIONAL_STARTUP_SERVICES=cassandra
  44 + ;;
  45 + *)
  46 + echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or cassandra." >&2
  47 + exit 1
  48 + esac
  49 + echo $ADDITIONAL_STARTUP_SERVICES
  50 +}
... ...
docker/docker-compose.cassandra.yml renamed from docker/docker-compose.static.yml
... ... @@ -14,16 +14,27 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -version: '2'
  17 +version: '2.2'
18 18
19 19 services:
20 20 cassandra:
  21 + restart: always
  22 + image: "cassandra:3.11.3"
21 23 ports:
22   - - "9042:9042"
23   - - "9160:9160"
24   - zk:
25   - ports:
26   - - "2181:2181"
27   - postgres:
28   - ports:
29   - - "5432:5432"
\ No newline at end of file
  24 + - "9042"
  25 + volumes:
  26 + - ./tb-node/cassandra:/var/lib/cassandra
  27 + tb1:
  28 + env_file:
  29 + - tb-node.cassandra.env
  30 + depends_on:
  31 + - kafka
  32 + - redis
  33 + - cassandra
  34 + tb2:
  35 + env_file:
  36 + - tb-node.cassandra.env
  37 + depends_on:
  38 + - kafka
  39 + - redis
  40 + - cassandra
... ...
docker/docker-compose.postgres.yml renamed from docker/docker-compose-tests.yml
... ... @@ -14,14 +14,29 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -version: '3.3'
  17 +version: '2.2'
  18 +
18 19 services:
19   - redis:
20   - image: redis:4.0
21   - networks:
22   - - core
  20 + postgres:
  21 + restart: always
  22 + image: "postgres:9.6"
23 23 ports:
24   - - "6379:6379"
25   -
26   -networks:
27   - core:
  24 + - "5432"
  25 + environment:
  26 + POSTGRES_DB: thingsboard
  27 + volumes:
  28 + - ./tb-node/postgres:/var/lib/postgresql/data
  29 + tb1:
  30 + env_file:
  31 + - tb-node.postgres.env
  32 + depends_on:
  33 + - kafka
  34 + - redis
  35 + - postgres
  36 + tb2:
  37 + env_file:
  38 + - tb-node.postgres.env
  39 + depends_on:
  40 + - kafka
  41 + - redis
  42 + - postgres
... ...
... ... @@ -14,40 +14,163 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -version: '2'
  17 +
  18 +version: '2.2'
18 19
19 20 services:
20   - tb:
21   - image: "thingsboard/application:2.1.0"
  21 + zookeeper:
  22 + restart: always
  23 + image: "zookeeper:3.5"
22 24 ports:
23   - - "8080:8080"
24   - - "1883:1883"
25   - - "5683:5683/udp"
  25 + - "2181"
  26 + kafka:
  27 + restart: always
  28 + image: "wurstmeister/kafka"
  29 + ports:
  30 + - "9092:9092"
26 31 env_file:
27   - - tb.env
  32 + - kafka.env
  33 + depends_on:
  34 + - zookeeper
  35 + redis:
  36 + image: redis:4.0
  37 + ports:
  38 + - "6379"
  39 + tb-js-executor:
  40 + restart: always
  41 + image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}"
  42 + scale: 20
  43 + env_file:
  44 + - tb-js-executor.env
  45 + depends_on:
  46 + - kafka
  47 + tb1:
  48 + restart: always
  49 + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}"
  50 + ports:
  51 + - "8080"
  52 + logging:
  53 + driver: "json-file"
  54 + options:
  55 + max-size: "200m"
  56 + max-file: "30"
28 57 environment:
29   - - ADD_SCHEMA_AND_SYSTEM_DATA=${ADD_SCHEMA_AND_SYSTEM_DATA}
30   - - ADD_DEMO_DATA=${ADD_DEMO_DATA}
  58 + TB_HOST: tb1
  59 + env_file:
  60 + - tb-node.env
31 61 volumes:
32   - - "${HSQLDB_DATA_DIR}:/usr/share/thingsboard/data/sql"
33   - entrypoint: /run-application.sh
34   - cassandra:
35   - image: "cassandra:3.11.2"
  62 + - ./tb-node/conf:/config
  63 + - ./tb-node/log:/var/log/thingsboard
  64 + depends_on:
  65 + - kafka
  66 + - redis
  67 + tb2:
  68 + restart: always
  69 + image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}"
36 70 ports:
37   - - "9042"
38   - - "9160"
  71 + - "8080"
  72 + logging:
  73 + driver: "json-file"
  74 + options:
  75 + max-size: "200m"
  76 + max-file: "30"
  77 + environment:
  78 + TB_HOST: tb2
  79 + env_file:
  80 + - tb-node.env
39 81 volumes:
40   - - "${CASSANDRA_DATA_DIR}:/var/lib/cassandra"
41   - zk:
42   - image: "zookeeper:3.4.10"
  82 + - ./tb-node/conf:/config
  83 + - ./tb-node/log:/var/log/thingsboard
  84 + depends_on:
  85 + - kafka
  86 + - redis
  87 + tb-mqtt-transport1:
  88 + restart: always
  89 + image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
43 90 ports:
44   - - "2181"
  91 + - "1883"
  92 + env_file:
  93 + - tb-mqtt-transport.env
  94 + depends_on:
  95 + - kafka
  96 + tb-mqtt-transport2:
45 97 restart: always
46   - postgres:
47   - image: "postgres:9.6"
  98 + image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
48 99 ports:
49   - - "5432"
50   - environment:
51   - - POSTGRES_DB=${POSTGRES_DB}
  100 + - "1883"
  101 + env_file:
  102 + - tb-mqtt-transport.env
  103 + depends_on:
  104 + - kafka
  105 + tb-http-transport1:
  106 + restart: always
  107 + image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
  108 + ports:
  109 + - "8081"
  110 + env_file:
  111 + - tb-http-transport.env
  112 + depends_on:
  113 + - kafka
  114 + tb-http-transport2:
  115 + restart: always
  116 + image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
  117 + ports:
  118 + - "8081"
  119 + env_file:
  120 + - tb-http-transport.env
  121 + depends_on:
  122 + - kafka
  123 + tb-coap-transport:
  124 + restart: always
  125 + image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
  126 + ports:
  127 + - "5683:5683/udp"
  128 + env_file:
  129 + - tb-coap-transport.env
  130 + depends_on:
  131 + - kafka
  132 + tb-web-ui1:
  133 + restart: always
  134 + image: "${DOCKER_REPO}/${WEB_UI_DOCKER_NAME}:${TB_VERSION}"
  135 + ports:
  136 + - "8080"
  137 + env_file:
  138 + - tb-web-ui.env
  139 + tb-web-ui2:
  140 + restart: always
  141 + image: "${DOCKER_REPO}/${WEB_UI_DOCKER_NAME}:${TB_VERSION}"
  142 + ports:
  143 + - "8080"
  144 + env_file:
  145 + - tb-web-ui.env
  146 + haproxy:
  147 + restart: always
  148 + container_name: haproxy-certbot
  149 + image: xalauc/haproxy-certbot:1.7.9
52 150 volumes:
53   - - "${POSTGRES_DATA_DIR}:/var/lib/postgresql/data"
  151 + - ./haproxy/config:/config
  152 + - ./haproxy/letsencrypt:/etc/letsencrypt
  153 + - ./haproxy/certs.d:/usr/local/etc/haproxy/certs.d
  154 + ports:
  155 + - "80:80"
  156 + - "8080"
  157 + - "443:443"
  158 + - "1883:1883"
  159 + - "9999:9999"
  160 + cap_add:
  161 + - NET_ADMIN
  162 + environment:
  163 + HTTP_PORT: 80
  164 + HTTPS_PORT: 443
  165 + MQTT_PORT: 1883
  166 + TB_API_PORT: 8080
  167 + FORCE_HTTPS_REDIRECT: "false"
  168 + links:
  169 + - tb1
  170 + - tb2
  171 + - tb-web-ui1
  172 + - tb-web-ui2
  173 + - tb-mqtt-transport1
  174 + - tb-mqtt-transport2
  175 + - tb-http-transport1
  176 + - tb-http-transport2
... ...
docker/docker-install-tb.sh renamed from msa/docker/docker-install-tb.sh
... ... @@ -39,6 +39,18 @@ else
39 39 loadDemo=false
40 40 fi
41 41
42   -docker-compose run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb
  42 +set -e
  43 +
  44 +source compose-utils.sh
  45 +
  46 +ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
  47 +
  48 +ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
  49 +
  50 +if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
  51 + docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
  52 +fi
  53 +
  54 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb1
43 55
44 56
... ...
docker/docker-remove-services.sh renamed from docker/cassandra-setup/Dockerfile 100644 → 100755
  1 +#!/bin/bash
1 2 #
2 3 # Copyright © 2016-2018 The Thingsboard Authors
3 4 #
... ... @@ -14,11 +15,10 @@
14 15 # limitations under the License.
15 16 #
16 17
17   -FROM openjdk:8-jre
  18 +set -e
18 19
19   -ADD install.sh /install.sh
20   -ADD thingsboard.deb /thingsboard.deb
  20 +source compose-utils.sh
21 21
22   -RUN apt-get update \
23   - && apt-get install -y nmap \
24   - && chmod +x /install.sh
  22 +ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
  23 +
  24 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS down -v
... ...
docker/docker-start-services.sh renamed from docker/cassandra-upgrade/upgrade.sh
... ... @@ -15,14 +15,12 @@
15 15 # limitations under the License.
16 16 #
17 17
  18 +./check-dirs.sh
18 19
19   -dpkg -i /thingsboard.deb
  20 +set -e
20 21
21   -until nmap $CASSANDRA_HOST -p $CASSANDRA_PORT | grep "$CASSANDRA_PORT/tcp open"
22   -do
23   - echo "Wait for cassandra db to start..."
24   - sleep 10
25   -done
  22 +source compose-utils.sh
26 23
27   -echo "Upgrading 'Thingsboard' schema..."
28   -/usr/share/thingsboard/bin/install/upgrade.sh --fromVersion=$UPGRADE_FROM_VERSION
  24 +ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
  25 +
  26 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d
... ...
docker/docker-stop-services.sh renamed from docker/cassandra-upgrade/Dockerfile 100644 → 100755
  1 +#!/bin/bash
1 2 #
2 3 # Copyright © 2016-2018 The Thingsboard Authors
3 4 #
... ... @@ -14,11 +15,10 @@
14 15 # limitations under the License.
15 16 #
16 17
17   -FROM openjdk:8-jre
  18 +set -e
18 19
19   -ADD upgrade.sh /upgrade.sh
20   -ADD thingsboard.deb /thingsboard.deb
  20 +source compose-utils.sh
21 21
22   -RUN apt-get update \
23   - && apt-get install -y nmap \
24   - && chmod +x /upgrade.sh
  22 +ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
  23 +
  24 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS stop
... ...
docker/docker-update-service.sh renamed from docker/tb/Dockerfile 100644 → 100755
  1 +#!/bin/bash
1 2 #
2 3 # Copyright © 2016-2018 The Thingsboard Authors
3 4 #
... ... @@ -14,13 +15,11 @@
14 15 # limitations under the License.
15 16 #
16 17
17   -FROM openjdk:8-jre
  18 +set -e
18 19
19   -ADD run-application.sh /run-application.sh
20   -ADD thingsboard.deb /thingsboard.deb
  20 +source compose-utils.sh
21 21
22   -RUN apt-get update \
23   - && apt-get install --no-install-recommends -y nmap \
24   - && apt-get clean \
25   - && rm -r /var/lib/apt/lists/* \
26   - && chmod +x /run-application.sh
  22 +ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
  23 +
  24 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull $@
  25 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d --no-deps --build $@
... ...
docker/docker-upgrade-tb.sh renamed from msa/docker/docker-upgrade-tb.sh
... ... @@ -38,4 +38,18 @@ else
38 38 fromVersion="${FROM_VERSION// }"
39 39 fi
40 40
41   -docker-compose run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb
  41 +set -e
  42 +
  43 +source compose-utils.sh
  44 +
  45 +ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
  46 +
  47 +ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
  48 +
  49 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb1
  50 +
  51 +if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
  52 + docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
  53 +fi
  54 +
  55 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb1
... ...
docker/haproxy/config/haproxy.cfg renamed from msa/docker/haproxy/config/haproxy.cfg
... ... @@ -23,6 +23,13 @@ defaults
23 23 timeout connect 5000ms
24 24 timeout client 50000ms
25 25 timeout server 50000ms
  26 + timeout tunnel 1h # timeout to use with WebSocket and CONNECT
  27 +
  28 + default-server init-addr none
  29 +
  30 +#enable resolving throught docker dns and avoid crashing if service is down while proxy is starting
  31 +resolvers docker_resolver
  32 + nameserver dns 127.0.0.11:53
26 33
27 34 listen stats
28 35 bind *:9999
... ... @@ -39,8 +46,8 @@ listen mqtt-in
39 46 timeout server 3h
40 47 option tcplog
41 48 balance leastconn
42   - server tbMqtt1 tb-mqtt-transport1:1883 check
43   - server tbMqtt2 tb-mqtt-transport2:1883 check
  49 + server tbMqtt1 tb-mqtt-transport1:1883 check inter 5s resolvers docker_resolver resolve-prefer ipv4
  50 + server tbMqtt2 tb-mqtt-transport2:1883 check inter 5s resolvers docker_resolver resolve-prefer ipv4
44 51
45 52 frontend http-in
46 53 bind *:${HTTP_PORT}
... ... @@ -51,7 +58,7 @@ frontend http-in
51 58
52 59 acl transport_http_acl path_beg /api/v1/
53 60 acl letsencrypt_http_acl path_beg /.well-known/acme-challenge/
54   - redirect scheme https if !letsencrypt_http_acl !transport_http_acl
  61 + redirect scheme https if !letsencrypt_http_acl !transport_http_acl { env(FORCE_HTTPS_REDIRECT) -m str true }
55 62 use_backend letsencrypt_http if letsencrypt_http_acl
56 63 use_backend tb-http-backend if transport_http_acl
57 64
... ... @@ -69,6 +76,11 @@ frontend https_in
69 76
70 77 default_backend tb-web-backend
71 78
  79 +frontend http-api-in
  80 + bind *:${TB_API_PORT}
  81 +
  82 + default_backend tb-api-backend
  83 +
72 84 backend letsencrypt_http
73 85 server letsencrypt_http_srv 127.0.0.1:8080
74 86
... ... @@ -76,13 +88,20 @@ backend tb-web-backend
76 88 balance leastconn
77 89 option tcp-check
78 90 option log-health-checks
79   - server tbWeb1 tb-web-ui1:8080 check
80   - server tbWeb2 tb-web-ui2:8080 check
  91 + server tbWeb1 tb-web-ui1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4
  92 + server tbWeb2 tb-web-ui2:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4
81 93 http-request set-header X-Forwarded-Port %[dst_port]
82 94
83 95 backend tb-http-backend
84 96 balance leastconn
85 97 option tcp-check
86 98 option log-health-checks
87   - server tbHttp1 tb-http-transport1:8081 check
88   - server tbHttp2 tb-http-transport2:8081 check
  99 + server tbHttp1 tb-http-transport1:8081 check inter 5s resolvers docker_resolver resolve-prefer ipv4
  100 + server tbHttp2 tb-http-transport2:8081 check inter 5s resolvers docker_resolver resolve-prefer ipv4
  101 +
  102 +backend tb-api-backend
  103 + balance leastconn
  104 + option tcp-check
  105 + option log-health-checks
  106 + server tbApi1 tb1:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4
  107 + server tbApi2 tb2:8080 check inter 5s resolvers docker_resolver resolve-prefer ipv4
... ...
1   -#
2   -# Copyright © 2016-2018 The Thingsboard Authors
3   -#
4   -# Licensed under the Apache License, Version 2.0 (the "License");
5   -# you may not use this file except in compliance with the License.
6   -# You may obtain a copy of the License at
7   -#
8   -# http://www.apache.org/licenses/LICENSE-2.0
9   -#
10   -# Unless required by applicable law or agreed to in writing, software
11   -# distributed under the License is distributed on an "AS IS" BASIS,
12   -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   -# See the License for the specific language governing permissions and
14   -# limitations under the License.
15   -#
16   -
17   -apiVersion: v1
18   -kind: Pod
19   -metadata:
20   - name: cassandra-setup
21   -spec:
22   - containers:
23   - - name: cassandra-setup
24   - imagePullPolicy: Always
25   - image: thingsboard/cassandra-setup:2.1.0
26   - env:
27   - - name: ADD_DEMO_DATA
28   - value: "true"
29   - - name : CASSANDRA_HOST
30   - value: "cassandra-headless"
31   - - name : CASSANDRA_PORT
32   - value: "9042"
33   - - name : DATABASE_ENTITIES_TYPE
34   - value: "cassandra"
35   - - name : DATABASE_TS_TYPE
36   - value: "cassandra"
37   - - name : CASSANDRA_URL
38   - value: "cassandra-headless:9042"
39   - command:
40   - - sh
41   - - -c
42   - - /install.sh
43   - restartPolicy: Never
1   -#
2   -# Copyright © 2016-2018 The Thingsboard Authors
3   -#
4   -# Licensed under the Apache License, Version 2.0 (the "License");
5   -# you may not use this file except in compliance with the License.
6   -# You may obtain a copy of the License at
7   -#
8   -# http://www.apache.org/licenses/LICENSE-2.0
9   -#
10   -# Unless required by applicable law or agreed to in writing, software
11   -# distributed under the License is distributed on an "AS IS" BASIS,
12   -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   -# See the License for the specific language governing permissions and
14   -# limitations under the License.
15   -#
16   -
17   -apiVersion: v1
18   -kind: Pod
19   -metadata:
20   - name: cassandra-upgrade
21   -spec:
22   - containers:
23   - - name: cassandra-upgrade
24   - imagePullPolicy: Always
25   - image: thingsboard/cassandra-upgrade:2.1.0
26   - env:
27   - - name: ADD_DEMO_DATA
28   - value: "true"
29   - - name : CASSANDRA_HOST
30   - value: "cassandra-headless"
31   - - name : CASSANDRA_PORT
32   - value: "9042"
33   - - name : DATABASE_ENTITIES_TYPE
34   - value: "cassandra"
35   - - name : DATABASE_TS_TYPE
36   - value: "cassandra"
37   - - name : CASSANDRA_URL
38   - value: "cassandra-headless:9042"
39   - - name : UPGRADE_FROM_VERSION
40   - value: "1.4.0"
41   - command:
42   - - sh
43   - - -c
44   - - /upgrade.sh
45   - restartPolicy: Never
1   -#
2   -# Copyright © 2016-2018 The Thingsboard Authors
3   -#
4   -# Licensed under the Apache License, Version 2.0 (the "License");
5   -# you may not use this file except in compliance with the License.
6   -# You may obtain a copy of the License at
7   -#
8   -# http://www.apache.org/licenses/LICENSE-2.0
9   -#
10   -# Unless required by applicable law or agreed to in writing, software
11   -# distributed under the License is distributed on an "AS IS" BASIS,
12   -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   -# See the License for the specific language governing permissions and
14   -# limitations under the License.
15   -#
16   -
17   -apiVersion: v1
18   -kind: Service
19   -metadata:
20   - name: cassandra-headless
21   - labels:
22   - app: cassandra-headless
23   -spec:
24   - ports:
25   - - port: 9042
26   - name: cql
27   - clusterIP: None
28   - selector:
29   - app: cassandra
30   ----
31   -apiVersion: "apps/v1beta1"
32   -kind: StatefulSet
33   -metadata:
34   - name: cassandra
35   -spec:
36   - serviceName: cassandra-headless
37   - replicas: 2
38   - template:
39   - metadata:
40   - labels:
41   - app: cassandra
42   - spec:
43   - nodeSelector:
44   - machinetype: other
45   - affinity:
46   - podAntiAffinity:
47   - requiredDuringSchedulingIgnoredDuringExecution:
48   - - labelSelector:
49   - matchExpressions:
50   - - key: "app"
51   - operator: In
52   - values:
53   - - cassandra-headless
54   - topologyKey: "kubernetes.io/hostname"
55   - containers:
56   - - name: cassandra
57   - image: thingsboard/cassandra:2.1.0
58   - imagePullPolicy: Always
59   - ports:
60   - - containerPort: 7000
61   - name: intra-node
62   - - containerPort: 7001
63   - name: tls-intra-node
64   - - containerPort: 7199
65   - name: jmx
66   - - containerPort: 9042
67   - name: cql
68   - - containerPort: 9160
69   - name: thrift
70   - securityContext:
71   - capabilities:
72   - add:
73   - - IPC_LOCK
74   - lifecycle:
75   - preStop:
76   - exec:
77   - command: ["/bin/sh", "-c", "PID=$(pidof java) && kill $PID && while ps -p $PID > /dev/null; do sleep 1; done"]
78   - env:
79   - - name: MAX_HEAP_SIZE
80   - value: 2048M
81   - - name: HEAP_NEWSIZE
82   - value: 100M
83   - - name: CASSANDRA_SEEDS
84   - value: "cassandra-0.cassandra-headless.default.svc.cluster.local"
85   - - name: CASSANDRA_CLUSTER_NAME
86   - value: "Thingsboard-Cluster"
87   - - name: CASSANDRA_DC
88   - value: "DC1-Thingsboard-Cluster"
89   - - name: CASSANDRA_RACK
90   - value: "Rack-Thingsboard-Cluster"
91   - - name: CASSANDRA_AUTO_BOOTSTRAP
92   - value: "false"
93   - - name: POD_IP
94   - valueFrom:
95   - fieldRef:
96   - fieldPath: status.podIP
97   - - name: POD_NAMESPACE
98   - valueFrom:
99   - fieldRef:
100   - fieldPath: metadata.namespace
101   - readinessProbe:
102   - exec:
103   - command:
104   - - /bin/bash
105   - - -c
106   - - /ready-probe.sh
107   - initialDelaySeconds: 15
108   - timeoutSeconds: 5
109   - volumeMounts:
110   - - name: cassandra-data
111   - mountPath: /var/lib/cassandra/data
112   - - name: cassandra-commitlog
113   - mountPath: /var/lib/cassandra/commitlog
114   - volumeClaimTemplates:
115   - - metadata:
116   - name: cassandra-data
117   - annotations:
118   - volume.beta.kubernetes.io/storage-class: fast
119   - spec:
120   - accessModes: [ "ReadWriteOnce" ]
121   - resources:
122   - requests:
123   - storage: 3Gi
124   - - metadata:
125   - name: cassandra-commitlog
126   - annotations:
127   - volume.beta.kubernetes.io/storage-class: fast
128   - spec:
129   - accessModes: [ "ReadWriteOnce" ]
130   - resources:
131   - requests:
132   - storage: 2Gi
\ No newline at end of file
1   -#
2   -# Copyright © 2016-2018 The Thingsboard Authors
3   -#
4   -# Licensed under the Apache License, Version 2.0 (the "License");
5   -# you may not use this file except in compliance with the License.
6   -# You may obtain a copy of the License at
7   -#
8   -# http://www.apache.org/licenses/LICENSE-2.0
9   -#
10   -# Unless required by applicable law or agreed to in writing, software
11   -# distributed under the License is distributed on an "AS IS" BASIS,
12   -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   -# See the License for the specific language governing permissions and
14   -# limitations under the License.
15   -#
16   -
17   ----
18   -apiVersion: storage.k8s.io/v1beta1
19   -kind: StorageClass
20   -metadata:
21   - name: slow
22   -provisioner: kubernetes.io/gce-pd
23   -parameters:
24   - type: pd-standard
25   ----
26   -apiVersion: storage.k8s.io/v1beta1
27   -kind: StorageClass
28   -metadata:
29   - name: fast
30   -provisioner: kubernetes.io/gce-pd
31   -parameters:
32   - type: pd-ssd
33   ----
\ No newline at end of file
1   -#
2   -# Copyright © 2016-2018 The Thingsboard Authors
3   -#
4   -# Licensed under the Apache License, Version 2.0 (the "License");
5   -# you may not use this file except in compliance with the License.
6   -# You may obtain a copy of the License at
7   -#
8   -# http://www.apache.org/licenses/LICENSE-2.0
9   -#
10   -# Unless required by applicable law or agreed to in writing, software
11   -# distributed under the License is distributed on an "AS IS" BASIS,
12   -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   -# See the License for the specific language governing permissions and
14   -# limitations under the License.
15   -#
16   -
17   ----
18   -apiVersion: v1
19   -kind: Service
20   -metadata:
21   - labels:
22   - name: redis-service
23   - name: redis-service
24   -spec:
25   - ports:
26   - - name: redis-service
27   - protocol: TCP
28   - port: 6379
29   - targetPort: 6379
30   - selector:
31   - app: redis
32   ----
33   -apiVersion: v1
34   -kind: ConfigMap
35   -metadata:
36   - name: redis-conf
37   -data:
38   - redis.conf: |
39   - appendonly yes
40   - protected-mode no
41   - bind 0.0.0.0
42   - port 6379
43   - dir /var/lib/redis
44   ----
45   -apiVersion: apps/v1beta1
46   -kind: StatefulSet
47   -metadata:
48   - name: redis
49   -spec:
50   - serviceName: redis-service
51   - replicas: 1
52   - template:
53   - metadata:
54   - labels:
55   - app: redis
56   - spec:
57   - terminationGracePeriodSeconds: 10
58   - containers:
59   - - name: redis
60   - image: redis:4.0.9
61   - command:
62   - - redis-server
63   - args:
64   - - /etc/redis/redis.conf
65   - resources:
66   - requests:
67   - cpu: 100m
68   - memory: 100Mi
69   - ports:
70   - - containerPort: 6379
71   - name: redis
72   - volumeMounts:
73   - - name: redis-data
74   - mountPath: /var/lib/redis
75   - - name: redis-conf
76   - mountPath: /etc/redis
77   - volumes:
78   - - name: redis-conf
79   - configMap:
80   - name: redis-conf
81   - items:
82   - - key: redis.conf
83   - path: redis.conf
84   - volumeClaimTemplates:
85   - - metadata:
86   - name: redis-data
87   - annotations:
88   - volume.beta.kubernetes.io/storage-class: fast
89   - spec:
90   - accessModes: [ "ReadWriteOnce" ]
91   - resources:
92   - requests:
93   - storage: 1Gi
\ No newline at end of file
1   -#
2   -# Copyright © 2016-2018 The Thingsboard Authors
3   -#
4   -# Licensed under the Apache License, Version 2.0 (the "License");
5   -# you may not use this file except in compliance with the License.
6   -# You may obtain a copy of the License at
7   -#
8   -# http://www.apache.org/licenses/LICENSE-2.0
9   -#
10   -# Unless required by applicable law or agreed to in writing, software
11   -# distributed under the License is distributed on an "AS IS" BASIS,
12   -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   -# See the License for the specific language governing permissions and
14   -# limitations under the License.
15   -#
16   -
17   ----
18   -apiVersion: v1
19   -kind: Service
20   -metadata:
21   - name: tb-service
22   - labels:
23   - app: tb-service
24   -spec:
25   - ports:
26   - - port: 8080
27   - name: ui
28   - - port: 1883
29   - name: mqtt
30   - - port: 5683
31   - name: coap
32   - selector:
33   - app: tb
34   - type: LoadBalancer
35   ----
36   -apiVersion: policy/v1beta1
37   -kind: PodDisruptionBudget
38   -metadata:
39   - name: tb-budget
40   -spec:
41   - selector:
42   - matchLabels:
43   - app: tb
44   - minAvailable: 3
45   ----
46   -apiVersion: v1
47   -kind: ConfigMap
48   -metadata:
49   - name: tb-config
50   -data:
51   - zookeeper.enabled: "true"
52   - zookeeper.url: "zk-headless"
53   - cassandra.url: "cassandra-headless:9042"
54   - cassandra.host: "cassandra-headless"
55   - cassandra.port: "9042"
56   - database.type: "cassandra"
57   - cache.type: "redis"
58   - redis.host: "redis-service"
59   ----
60   -apiVersion: apps/v1beta1
61   -kind: StatefulSet
62   -metadata:
63   - name: tb
64   -spec:
65   - serviceName: "tb-service"
66   - replicas: 3
67   - template:
68   - metadata:
69   - labels:
70   - app: tb
71   - spec:
72   - nodeSelector:
73   - machinetype: tb
74   - affinity:
75   - podAntiAffinity:
76   - requiredDuringSchedulingIgnoredDuringExecution:
77   - - labelSelector:
78   - matchExpressions:
79   - - key: "app"
80   - operator: In
81   - values:
82   - - tb-service
83   - topologyKey: "kubernetes.io/hostname"
84   - containers:
85   - - name: tb
86   - imagePullPolicy: Always
87   - image: thingsboard/application:2.1.0
88   - ports:
89   - - containerPort: 8080
90   - name: ui
91   - - containerPort: 1883
92   - name: mqtt
93   - - containerPort: 5683
94   - name: coap
95   - - containerPort: 9001
96   - name: rpc
97   - env:
98   - - name: ZOOKEEPER_ENABLED
99   - valueFrom:
100   - configMapKeyRef:
101   - name: tb-config
102   - key: zookeeper.enabled
103   - - name: ZOOKEEPER_URL
104   - valueFrom:
105   - configMapKeyRef:
106   - name: tb-config
107   - key: zookeeper.url
108   - - name : CASSANDRA_HOST
109   - valueFrom:
110   - configMapKeyRef:
111   - name: tb-config
112   - key: cassandra.host
113   - - name : CASSANDRA_PORT
114   - valueFrom:
115   - configMapKeyRef:
116   - name: tb-config
117   - key: cassandra.port
118   - - name : CASSANDRA_URL
119   - valueFrom:
120   - configMapKeyRef:
121   - name: tb-config
122   - key: cassandra.url
123   - - name: DATABASE_ENTITIES_TYPE
124   - valueFrom:
125   - configMapKeyRef:
126   - name: tb-config
127   - key: database.type
128   - - name: DATABASE_TS_TYPE
129   - valueFrom:
130   - configMapKeyRef:
131   - name: tb-config
132   - key: database.type
133   - - name : RPC_HOST
134   - valueFrom:
135   - fieldRef:
136   - fieldPath: status.podIP
137   - - name: CACHE_TYPE
138   - valueFrom:
139   - configMapKeyRef:
140   - name: tb-config
141   - key: cache.type
142   - - name: REDIS_HOST
143   - valueFrom:
144   - configMapKeyRef:
145   - name: tb-config
146   - key: redis.host
147   - command:
148   - - sh
149   - - -c
150   - - /run-application.sh
151   - livenessProbe:
152   - httpGet:
153   - path: /login
154   - port: ui-port
155   - initialDelaySeconds: 120
156   - timeoutSeconds: 10
\ No newline at end of file