简柏林 / thingskit · Commits

Commit c3ee6785aa381ca130a9423c8f928df983d36227

Authored by Viacheslav Kukhtyn
2 parents 0fc43c3b a2eea121

Merge branch 'master' into feature/log-telemetry-updated

Inline Side-by-side
Showing 100 changed files with 2069 additions and 636 deletions

Too many changes to show.
Reload with full diff Plain diff Email patch

To preserve performance only 100 of 243 files are displayed.

@@ -391,16 +391,16 @@ @@ -391,16 +391,16 @@
391 }, 391 },
392 { 392 {
393 "alias": "web_camera_input", 393 "alias": "web_camera_input",
394 - "name": "Web Camera Input", 394 + "name": "Photo camera input",
395 "descriptor": { 395 "descriptor": {
396 "type": "latest", 396 "type": "latest",
397 "sizeX": 7.5, 397 "sizeX": 7.5,
398 "sizeY": 3, 398 "sizeY": 3,
399 "resources": [], 399 "resources": [],
400 - "templateHtml": "<tb-web-camera-widget \n [ctx]=\"ctx\">\n</tb-web-camera-widget>", 400 + "templateHtml": "<tb-photo-camera-widget \n [ctx]=\"ctx\">\n</tb-photo-camera-widget>",
401 "templateCss": "", 401 "templateCss": "",
402 - "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.webCameraInputWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true\n }\n}\n\nself.onDestroy = function() {\n}\n",  
403 - "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Web Camera\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"imageFormat\": {\n \"title\": \"Image Format\",\n \"type\": \"string\",\n \"default\": \"image/png\"\n },\n \"imageQuality\":{\n \"title\":\"Image quality that use lossy compression such as jpeg and webp\",\n \"type\":\"number\",\n \"default\": 0.92,\n \"min\": 0,\n \"max\": 1\n },\n \"maxWidth\": {\n \"title\": \"The maximal image width\",\n \"type\": \"number\",\n \"default\": 640\n }, \n \"maxHeight\": {\n \"title\": \"The maximal image heigth\",\n \"type\": \"number\",\n \"default\": 480\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n {\n \"key\": \"imageFormat\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"image/jpeg\",\n \"label\": \"JPEG\"\n },\n {\n \"value\": \"image/png\",\n \"label\": \"PNG\"\n },\n {\n \"value\": \"image/webp\",\n \"label\": \"WEBP\"\n }\n ]\n },\n \"imageQuality\",\n \"maxWidth\",\n \"maxHeight\"\n ]\n}", 402 + "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.photoCameraInputWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true\n }\n}\n\nself.onDestroy = function() {\n}\n",
  403 + "settingsSchema": "{\n \"schema\": {\n \"type\": \"object\",\n \"title\": \"Photo Camera\",\n \"properties\": {\n \"widgetTitle\": {\n \"title\": \"Widget title\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n \"imageFormat\": {\n \"title\": \"Image Format\",\n \"type\": \"string\",\n \"default\": \"image/png\"\n },\n \"imageQuality\":{\n \"title\":\"Image quality that use lossy compression such as jpeg and webp\",\n \"type\":\"number\",\n \"default\": 0.92,\n \"min\": 0,\n \"max\": 1\n },\n \"maxWidth\": {\n \"title\": \"The maximal image width\",\n \"type\": \"number\",\n \"default\": 640\n }, \n \"maxHeight\": {\n \"title\": \"The maximal image heigth\",\n \"type\": \"number\",\n \"default\": 480\n }\n },\n \"required\": []\n },\n \"form\": [\n \"widgetTitle\",\n {\n \"key\": \"imageFormat\",\n \"type\": \"rc-select\",\n \"multiple\": false,\n \"items\": [\n {\n \"value\": \"image/jpeg\",\n \"label\": \"JPEG\"\n },\n {\n \"value\": \"image/png\",\n \"label\": \"PNG\"\n },\n {\n \"value\": \"image/webp\",\n \"label\": \"WEBP\"\n }\n ]\n },\n \"imageQuality\",\n \"maxWidth\",\n \"maxHeight\"\n ]\n}",
404 "dataKeySettingsSchema": "{}\n", 404 "dataKeySettingsSchema": "{}\n",
405 "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Web Camera Input\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":false,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}" 405 "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Web Camera Input\",\"showTitleIcon\":false,\"titleIcon\":\"more_horiz\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"titleTooltip\":\"\",\"dropShadow\":true,\"enableFullscreen\":false,\"widgetStyle\":{},\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"useDashboardTimewindow\":true,\"displayTimewindow\":true,\"showLegend\":false,\"actions\":{}}"
406 } 406 }
@@ -96,6 +96,7 @@ CREATE TABLE IF NOT EXISTS device_profile ( @@ -96,6 +96,7 @@ CREATE TABLE IF NOT EXISTS device_profile (
96 is_default boolean, 96 is_default boolean,
97 tenant_id uuid, 97 tenant_id uuid,
98 default_rule_chain_id uuid, 98 default_rule_chain_id uuid,
  99 + default_queue_name varchar(255),
99 provision_device_key varchar, 100 provision_device_key varchar,
100 CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), 101 CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name),
101 CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), 102 CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key),
@@ -32,7 +32,6 @@ import org.springframework.data.redis.core.RedisTemplate; @@ -32,7 +32,6 @@ import org.springframework.data.redis.core.RedisTemplate;
32 import org.springframework.scheduling.annotation.Scheduled; 32 import org.springframework.scheduling.annotation.Scheduled;
33 import org.springframework.stereotype.Component; 33 import org.springframework.stereotype.Component;
34 import org.thingsboard.rule.engine.api.MailService; 34 import org.thingsboard.rule.engine.api.MailService;
35 -import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;  
36 import org.thingsboard.server.actors.service.ActorService; 35 import org.thingsboard.server.actors.service.ActorService;
37 import org.thingsboard.server.actors.tenant.DebugTbRateLimits; 36 import org.thingsboard.server.actors.tenant.DebugTbRateLimits;
38 import org.thingsboard.server.common.data.DataConstants; 37 import org.thingsboard.server.common.data.DataConstants;
@@ -65,6 +64,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -65,6 +64,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService;
65 import org.thingsboard.server.dao.user.UserService; 64 import org.thingsboard.server.dao.user.UserService;
66 import org.thingsboard.server.queue.discovery.PartitionService; 65 import org.thingsboard.server.queue.discovery.PartitionService;
67 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; 66 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
  67 +import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
  68 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
68 import org.thingsboard.server.service.component.ComponentDiscoveryService; 69 import org.thingsboard.server.service.component.ComponentDiscoveryService;
69 import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; 70 import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
70 import org.thingsboard.server.service.executors.DbCallbackExecutorService; 71 import org.thingsboard.server.service.executors.DbCallbackExecutorService;
@@ -72,7 +73,7 @@ import org.thingsboard.server.service.executors.ExternalCallExecutorService; @@ -72,7 +73,7 @@ import org.thingsboard.server.service.executors.ExternalCallExecutorService;
72 import org.thingsboard.server.service.executors.SharedEventLoopGroupService; 73 import org.thingsboard.server.service.executors.SharedEventLoopGroupService;
73 import org.thingsboard.server.service.mail.MailExecutorService; 74 import org.thingsboard.server.service.mail.MailExecutorService;
74 import org.thingsboard.server.service.profile.TbDeviceProfileCache; 75 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
75 -import org.thingsboard.server.service.profile.TbTenantProfileCache; 76 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
76 import org.thingsboard.server.service.queue.TbClusterService; 77 import org.thingsboard.server.service.queue.TbClusterService;
77 import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; 78 import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
78 import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; 79 import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
@@ -108,6 +109,14 @@ public class ActorSystemContext { @@ -108,6 +109,14 @@ public class ActorSystemContext {
108 109
109 @Autowired 110 @Autowired
110 @Getter 111 @Getter
  112 + private TbApiUsageStateService apiUsageStateService;
  113 +
  114 + @Autowired
  115 + @Getter
  116 + private TbApiUsageClient apiUsageClient;
  117 +
  118 + @Autowired
  119 + @Getter
111 @Setter 120 @Setter
112 private TbServiceInfoProvider serviceInfoProvider; 121 private TbServiceInfoProvider serviceInfoProvider;
113 122
@@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.Tenant; @@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.Tenant;
30 import org.thingsboard.server.common.data.TenantProfile; 30 import org.thingsboard.server.common.data.TenantProfile;
31 import org.thingsboard.server.common.data.id.EntityId; 31 import org.thingsboard.server.common.data.id.EntityId;
32 import org.thingsboard.server.common.data.id.TenantId; 32 import org.thingsboard.server.common.data.id.TenantId;
33 -import org.thingsboard.server.common.data.id.TenantProfileId;  
34 import org.thingsboard.server.common.data.page.PageDataIterable; 33 import org.thingsboard.server.common.data.page.PageDataIterable;
35 import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; 34 import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
36 import org.thingsboard.server.common.msg.MsgType; 35 import org.thingsboard.server.common.msg.MsgType;
@@ -40,10 +39,8 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; @@ -40,10 +39,8 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
40 import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; 39 import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
41 import org.thingsboard.server.common.msg.queue.RuleEngineException; 40 import org.thingsboard.server.common.msg.queue.RuleEngineException;
42 import org.thingsboard.server.common.msg.queue.ServiceType; 41 import org.thingsboard.server.common.msg.queue.ServiceType;
43 -import org.thingsboard.server.dao.model.ModelConstants;  
44 -import org.thingsboard.server.dao.tenant.TenantProfileService;  
45 import org.thingsboard.server.dao.tenant.TenantService; 42 import org.thingsboard.server.dao.tenant.TenantService;
46 -import org.thingsboard.server.service.profile.TbTenantProfileCache; 43 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
47 import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; 44 import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
48 45
49 import java.util.HashSet; 46 import java.util.HashSet;
@@ -150,15 +147,12 @@ public class AppActor extends ContextAwareActor { @@ -150,15 +147,12 @@ public class AppActor extends ContextAwareActor {
150 private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { 147 private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) {
151 TbActorRef target = null; 148 TbActorRef target = null;
152 if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { 149 if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) {
153 - if (msg.getEntityId().getEntityType() == EntityType.TENANT_PROFILE) {  
154 - tenantProfileCache.evict(new TenantProfileId(msg.getEntityId().getId()));  
155 - } else { 150 + if (!EntityType.TENANT_PROFILE.equals(msg.getEntityId().getEntityType())) {
156 log.warn("Message has system tenant id: {}", msg); 151 log.warn("Message has system tenant id: {}", msg);
157 } 152 }
158 } else { 153 } else {
159 - if (msg.getEntityId().getEntityType() == EntityType.TENANT) { 154 + if (EntityType.TENANT.equals(msg.getEntityId().getEntityType())) {
160 TenantId tenantId = new TenantId(msg.getEntityId().getId()); 155 TenantId tenantId = new TenantId(msg.getEntityId().getId());
161 - tenantProfileCache.evict(tenantId);  
162 if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { 156 if (msg.getEvent() == ComponentLifecycleEvent.DELETED) {
163 log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); 157 log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg);
164 deletedTenants.add(tenantId); 158 deletedTenants.add(tenantId);
@@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.Device; @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.Device;
38 import org.thingsboard.server.common.data.DeviceProfile; 38 import org.thingsboard.server.common.data.DeviceProfile;
39 import org.thingsboard.server.common.data.alarm.Alarm; 39 import org.thingsboard.server.common.data.alarm.Alarm;
40 import org.thingsboard.server.common.data.asset.Asset; 40 import org.thingsboard.server.common.data.asset.Asset;
  41 +import org.thingsboard.server.common.data.id.DeviceId;
41 import org.thingsboard.server.common.data.id.EntityId; 42 import org.thingsboard.server.common.data.id.EntityId;
42 import org.thingsboard.server.common.data.id.RuleChainId; 43 import org.thingsboard.server.common.data.id.RuleChainId;
43 import org.thingsboard.server.common.data.id.RuleNodeId; 44 import org.thingsboard.server.common.data.id.RuleNodeId;
@@ -73,6 +74,7 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; @@ -73,6 +74,7 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine;
73 74
74 import java.util.Collections; 75 import java.util.Collections;
75 import java.util.Set; 76 import java.util.Set;
  77 +import java.util.function.BiConsumer;
76 import java.util.function.Consumer; 78 import java.util.function.Consumer;
77 79
78 /** 80 /**
@@ -257,26 +259,26 @@ class DefaultTbContext implements TbContext { @@ -257,26 +259,26 @@ class DefaultTbContext implements TbContext {
257 } 259 }
258 260
259 public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) { 261 public TbMsg customerCreatedMsg(Customer customer, RuleNodeId ruleNodeId) {
260 - return entityCreatedMsg(customer, customer.getId(), ruleNodeId); 262 + return entityActionMsg(customer, customer.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
261 } 263 }
262 264
263 public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { 265 public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) {
264 - return entityCreatedMsg(device, device.getId(), ruleNodeId); 266 + return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
265 } 267 }
266 268
267 public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { 269 public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) {
268 - return entityCreatedMsg(asset, asset.getId(), ruleNodeId); 270 + return entityActionMsg(asset, asset.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
269 } 271 }
270 272
271 - public TbMsg alarmCreatedMsg(Alarm alarm, RuleNodeId ruleNodeId) {  
272 - return entityCreatedMsg(alarm, alarm.getId(), ruleNodeId); 273 + public TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action) {
  274 + return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action);
273 } 275 }
274 276
275 - public <E, I extends EntityId> TbMsg entityCreatedMsg(E entity, I id, RuleNodeId ruleNodeId) { 277 + public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action) {
276 try { 278 try {
277 - return TbMsg.newMsg(DataConstants.ENTITY_CREATED, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); 279 + return TbMsg.newMsg(action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)));
278 } catch (JsonProcessingException | IllegalArgumentException e) { 280 } catch (JsonProcessingException | IllegalArgumentException e) {
279 - throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " created msg: " + e); 281 + throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " " + action + " msg: " + e);
280 } 282 }
281 } 283 }
282 284
@@ -312,7 +314,7 @@ class DefaultTbContext implements TbContext { @@ -312,7 +314,7 @@ class DefaultTbContext implements TbContext {
312 314
313 @Override 315 @Override
314 public ScriptEngine createJsScriptEngine(String script, String... argNames) { 316 public ScriptEngine createJsScriptEngine(String script, String... argNames) {
315 - return new RuleNodeJsScriptEngine(mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames); 317 + return new RuleNodeJsScriptEngine(getTenantId(), mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames);
316 } 318 }
317 319
318 @Override 320 @Override
@@ -479,8 +481,16 @@ class DefaultTbContext implements TbContext { @@ -479,8 +481,16 @@ class DefaultTbContext implements TbContext {
479 } 481 }
480 482
481 @Override 483 @Override
482 - public void addProfileListener(Consumer<DeviceProfile> listener) {  
483 - mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), listener); 484 + public void removeRuleNodeStateForEntity(EntityId entityId) {
  485 + if (log.isDebugEnabled()) {
  486 + log.debug("[{}][{}][{}] Remove Rule Node State for entity.", getTenantId(), getSelfId(), entityId);
  487 + }
  488 + mainCtx.getRuleNodeStateService().removeByRuleNodeIdAndEntityId(getTenantId(), getSelfId(), entityId);
  489 + }
  490 +
  491 + @Override
  492 + public void addDeviceProfileListeners(Consumer<DeviceProfile> profileListener, BiConsumer<DeviceId, DeviceProfile> deviceListener) {
  493 + mainCtx.getDeviceProfileCache().addListener(getTenantId(), getSelfId(), profileListener, deviceListener);
484 } 494 }
485 495
486 @Override 496 @Override
@@ -23,6 +23,7 @@ import org.thingsboard.server.actors.TbActorRef; @@ -23,6 +23,7 @@ import org.thingsboard.server.actors.TbActorRef;
23 import org.thingsboard.server.actors.TbEntityActorId; 23 import org.thingsboard.server.actors.TbEntityActorId;
24 import org.thingsboard.server.actors.service.DefaultActorService; 24 import org.thingsboard.server.actors.service.DefaultActorService;
25 import org.thingsboard.server.actors.shared.ComponentMsgProcessor; 25 import org.thingsboard.server.actors.shared.ComponentMsgProcessor;
  26 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
26 import org.thingsboard.server.common.data.EntityType; 27 import org.thingsboard.server.common.data.EntityType;
27 import org.thingsboard.server.common.data.id.EntityId; 28 import org.thingsboard.server.common.data.id.EntityId;
28 import org.thingsboard.server.common.data.id.RuleChainId; 29 import org.thingsboard.server.common.data.id.RuleChainId;
@@ -46,6 +47,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -46,6 +47,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
46 import org.thingsboard.server.queue.TbQueueCallback; 47 import org.thingsboard.server.queue.TbQueueCallback;
47 import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper; 48 import org.thingsboard.server.queue.common.MultipleTbQueueTbMsgCallbackWrapper;
48 import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper; 49 import org.thingsboard.server.queue.common.TbQueueTbMsgCallbackWrapper;
  50 +import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
49 import org.thingsboard.server.service.queue.TbClusterService; 51 import org.thingsboard.server.service.queue.TbClusterService;
50 52
51 import java.util.ArrayList; 53 import java.util.ArrayList;
@@ -68,15 +70,16 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh @@ -68,15 +70,16 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
68 private final Map<RuleNodeId, List<RuleNodeRelation>> nodeRoutes; 70 private final Map<RuleNodeId, List<RuleNodeRelation>> nodeRoutes;
69 private final RuleChainService service; 71 private final RuleChainService service;
70 private final TbClusterService clusterService; 72 private final TbClusterService clusterService;
  73 + private final TbApiUsageClient apiUsageClient;
71 private String ruleChainName; 74 private String ruleChainName;
72 75
73 private RuleNodeId firstId; 76 private RuleNodeId firstId;
74 private RuleNodeCtx firstNode; 77 private RuleNodeCtx firstNode;
75 private boolean started; 78 private boolean started;
76 79
77 - RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext  
78 - , TbActorRef parent, TbActorRef self) { 80 + RuleChainActorMessageProcessor(TenantId tenantId, RuleChain ruleChain, ActorSystemContext systemContext, TbActorRef parent, TbActorRef self) {
79 super(systemContext, tenantId, ruleChain.getId()); 81 super(systemContext, tenantId, ruleChain.getId());
  82 + this.apiUsageClient = systemContext.getApiUsageClient();
80 this.ruleChainName = ruleChain.getName(); 83 this.ruleChainName = ruleChain.getName();
81 this.parent = parent; 84 this.parent = parent;
82 this.self = self; 85 this.self = self;
@@ -64,6 +64,12 @@ public abstract class RuleChainManagerActor extends ContextAwareActor { @@ -64,6 +64,12 @@ public abstract class RuleChainManagerActor extends ContextAwareActor {
64 } 64 }
65 } 65 }
66 66
  67 + protected void destroyRuleChains() {
  68 + for (RuleChain ruleChain : new PageDataIterable<>(link -> ruleChainService.findTenantRuleChains(tenantId, link), ContextAwareActor.ENTITY_PACK_LIMIT)) {
  69 + ctx.stop(new TbEntityActorId(ruleChain.getId()));
  70 + }
  71 + }
  72 +
67 protected void visit(RuleChain entity, TbActorRef actorRef) { 73 protected void visit(RuleChain entity, TbActorRef actorRef) {
68 if (entity != null && entity.isRoot()) { 74 if (entity != null && entity.isRoot()) {
69 rootChain = entity; 75 rootChain = entity;
@@ -21,13 +21,18 @@ import org.thingsboard.server.actors.ActorSystemContext; @@ -21,13 +21,18 @@ import org.thingsboard.server.actors.ActorSystemContext;
21 import org.thingsboard.server.actors.TbActorCtx; 21 import org.thingsboard.server.actors.TbActorCtx;
22 import org.thingsboard.server.actors.TbActorRef; 22 import org.thingsboard.server.actors.TbActorRef;
23 import org.thingsboard.server.actors.shared.ComponentMsgProcessor; 23 import org.thingsboard.server.actors.shared.ComponentMsgProcessor;
  24 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
  25 +import org.thingsboard.server.common.data.TenantProfile;
24 import org.thingsboard.server.common.data.id.RuleNodeId; 26 import org.thingsboard.server.common.data.id.RuleNodeId;
25 import org.thingsboard.server.common.data.id.TenantId; 27 import org.thingsboard.server.common.data.id.TenantId;
26 import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; 28 import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
27 import org.thingsboard.server.common.data.rule.RuleNode; 29 import org.thingsboard.server.common.data.rule.RuleNode;
  30 +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
  31 +import org.thingsboard.server.common.msg.TbMsg;
28 import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; 32 import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
29 import org.thingsboard.server.common.msg.queue.RuleNodeException; 33 import org.thingsboard.server.common.msg.queue.RuleNodeException;
30 import org.thingsboard.server.common.msg.queue.RuleNodeInfo; 34 import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
  35 +import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
31 36
32 /** 37 /**
33 * @author Andrew Shvayka 38 * @author Andrew Shvayka
@@ -36,6 +41,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod @@ -36,6 +41,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
36 41
37 private final String ruleChainName; 42 private final String ruleChainName;
38 private final TbActorRef self; 43 private final TbActorRef self;
  44 + private final TbApiUsageClient apiUsageClient;
39 private RuleNode ruleNode; 45 private RuleNode ruleNode;
40 private TbNode tbNode; 46 private TbNode tbNode;
41 private DefaultTbContext defaultCtx; 47 private DefaultTbContext defaultCtx;
@@ -44,6 +50,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod @@ -44,6 +50,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
44 RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext 50 RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext
45 , TbActorRef parent, TbActorRef self) { 51 , TbActorRef parent, TbActorRef self) {
46 super(systemContext, tenantId, ruleNodeId); 52 super(systemContext, tenantId, ruleNodeId);
  53 + this.apiUsageClient = systemContext.getApiUsageClient();
47 this.ruleChainName = ruleChainName; 54 this.ruleChainName = ruleChainName;
48 this.self = self; 55 this.self = self;
49 this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId); 56 this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId);
@@ -92,26 +99,42 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod @@ -92,26 +99,42 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
92 99
93 public void onRuleToSelfMsg(RuleNodeToSelfMsg msg) throws Exception { 100 public void onRuleToSelfMsg(RuleNodeToSelfMsg msg) throws Exception {
94 checkActive(msg.getMsg()); 101 checkActive(msg.getMsg());
95 - if (ruleNode.isDebugMode()) {  
96 - systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self");  
97 - }  
98 - try {  
99 - tbNode.onMsg(defaultCtx, msg.getMsg());  
100 - } catch (Exception e) {  
101 - defaultCtx.tellFailure(msg.getMsg(), e); 102 + TbMsg tbMsg = msg.getMsg();
  103 + int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
  104 + int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
  105 + if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
  106 + apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
  107 + if (ruleNode.isDebugMode()) {
  108 + systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self");
  109 + }
  110 + try {
  111 + tbNode.onMsg(defaultCtx, msg.getMsg());
  112 + } catch (Exception e) {
  113 + defaultCtx.tellFailure(msg.getMsg(), e);
  114 + }
  115 + } else {
  116 + tbMsg.getCallback().onFailure(new RuleNodeException("Message is processed by more then " + maxRuleNodeExecutionsPerMessage + " rule nodes!", ruleChainName, ruleNode));
102 } 117 }
103 } 118 }
104 119
105 void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception { 120 void onRuleChainToRuleNodeMsg(RuleChainToRuleNodeMsg msg) throws Exception {
106 msg.getMsg().getCallback().onProcessingStart(info); 121 msg.getMsg().getCallback().onProcessingStart(info);
107 checkActive(msg.getMsg()); 122 checkActive(msg.getMsg());
108 - if (ruleNode.isDebugMode()) {  
109 - systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());  
110 - }  
111 - try {  
112 - tbNode.onMsg(msg.getCtx(), msg.getMsg());  
113 - } catch (Exception e) {  
114 - msg.getCtx().tellFailure(msg.getMsg(), e); 123 + TbMsg tbMsg = msg.getMsg();
  124 + int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
  125 + int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
  126 + if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
  127 + apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
  128 + if (ruleNode.isDebugMode()) {
  129 + systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());
  130 + }
  131 + try {
  132 + tbNode.onMsg(msg.getCtx(), msg.getMsg());
  133 + } catch (Exception e) {
  134 + msg.getCtx().tellFailure(msg.getMsg(), e);
  135 + }
  136 + } else {
  137 + tbMsg.getCallback().onFailure(new RuleNodeException("Message is processed by more then " + maxRuleNodeExecutionsPerMessage + " rule nodes!", ruleChainName, ruleNode));
115 } 138 }
116 } 139 }
117 140
@@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired;
20 import org.springframework.beans.factory.annotation.Value; 20 import org.springframework.beans.factory.annotation.Value;
21 import org.springframework.boot.context.event.ApplicationReadyEvent; 21 import org.springframework.boot.context.event.ApplicationReadyEvent;
22 import org.springframework.context.event.EventListener; 22 import org.springframework.context.event.EventListener;
  23 +import org.springframework.core.annotation.Order;
23 import org.springframework.stereotype.Service; 24 import org.springframework.stereotype.Service;
24 import org.thingsboard.common.util.ThingsBoardThreadFactory; 25 import org.thingsboard.common.util.ThingsBoardThreadFactory;
25 import org.thingsboard.server.actors.ActorSystemContext; 26 import org.thingsboard.server.actors.ActorSystemContext;
@@ -113,6 +114,7 @@ public class DefaultActorService implements ActorService { @@ -113,6 +114,7 @@ public class DefaultActorService implements ActorService {
113 } 114 }
114 115
115 @EventListener(ApplicationReadyEvent.class) 116 @EventListener(ApplicationReadyEvent.class)
  117 + @Order(value = 2)
116 public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { 118 public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
117 log.info("Received application ready event. Sending application init message to actor system"); 119 log.info("Received application ready event. Sending application init message to actor system");
118 appActor.tellWithHighPriority(new AppInitMsg()); 120 appActor.tellWithHighPriority(new AppInitMsg());
@@ -19,9 +19,11 @@ import lombok.extern.slf4j.Slf4j; @@ -19,9 +19,11 @@ import lombok.extern.slf4j.Slf4j;
19 import org.thingsboard.server.actors.ActorSystemContext; 19 import org.thingsboard.server.actors.ActorSystemContext;
20 import org.thingsboard.server.actors.TbActorCtx; 20 import org.thingsboard.server.actors.TbActorCtx;
21 import org.thingsboard.server.actors.stats.StatsPersistTick; 21 import org.thingsboard.server.actors.stats.StatsPersistTick;
  22 +import org.thingsboard.server.common.data.TenantProfile;
22 import org.thingsboard.server.common.data.id.EntityId; 23 import org.thingsboard.server.common.data.id.EntityId;
23 import org.thingsboard.server.common.data.id.TenantId; 24 import org.thingsboard.server.common.data.id.TenantId;
24 import org.thingsboard.server.common.data.plugin.ComponentLifecycleState; 25 import org.thingsboard.server.common.data.plugin.ComponentLifecycleState;
  26 +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
25 import org.thingsboard.server.common.msg.TbMsg; 27 import org.thingsboard.server.common.msg.TbMsg;
26 import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; 28 import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
27 import org.thingsboard.server.common.msg.queue.RuleNodeException; 29 import org.thingsboard.server.common.msg.queue.RuleNodeException;
@@ -39,6 +41,10 @@ public abstract class ComponentMsgProcessor<T extends EntityId> extends Abstract @@ -39,6 +41,10 @@ public abstract class ComponentMsgProcessor<T extends EntityId> extends Abstract
39 this.entityId = id; 41 this.entityId = id;
40 } 42 }
41 43
  44 + protected TenantProfileConfiguration getTenantProfileConfiguration() {
  45 + return systemContext.getTenantProfileCache().get(tenantId).getProfileData().getConfiguration();
  46 + }
  47 +
42 public abstract String getComponentName(); 48 public abstract String getComponentName();
43 49
44 public abstract void start(TbActorCtx context) throws Exception; 50 public abstract void start(TbActorCtx context) throws Exception;
@@ -29,6 +29,7 @@ import org.thingsboard.server.actors.device.DeviceActorCreator; @@ -29,6 +29,7 @@ import org.thingsboard.server.actors.device.DeviceActorCreator;
29 import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor; 29 import org.thingsboard.server.actors.ruleChain.RuleChainManagerActor;
30 import org.thingsboard.server.actors.service.ContextBasedCreator; 30 import org.thingsboard.server.actors.service.ContextBasedCreator;
31 import org.thingsboard.server.actors.service.DefaultActorService; 31 import org.thingsboard.server.actors.service.DefaultActorService;
  32 +import org.thingsboard.server.common.data.ApiUsageState;
32 import org.thingsboard.server.common.data.EntityType; 33 import org.thingsboard.server.common.data.EntityType;
33 import org.thingsboard.server.common.data.Tenant; 34 import org.thingsboard.server.common.data.Tenant;
34 import org.thingsboard.server.common.data.TenantProfile; 35 import org.thingsboard.server.common.data.TenantProfile;
@@ -57,6 +58,7 @@ public class TenantActor extends RuleChainManagerActor { @@ -57,6 +58,7 @@ public class TenantActor extends RuleChainManagerActor {
57 58
58 private boolean isRuleEngineForCurrentTenant; 59 private boolean isRuleEngineForCurrentTenant;
59 private boolean isCore; 60 private boolean isCore;
  61 + private ApiUsageState apiUsageState;
60 62
61 private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { 63 private TenantActor(ActorSystemContext systemContext, TenantId tenantId) {
62 super(systemContext, tenantId); 64 super(systemContext, tenantId);
@@ -74,19 +76,24 @@ public class TenantActor extends RuleChainManagerActor { @@ -74,19 +76,24 @@ public class TenantActor extends RuleChainManagerActor {
74 cantFindTenant = true; 76 cantFindTenant = true;
75 log.info("[{}] Started tenant actor for missing tenant.", tenantId); 77 log.info("[{}] Started tenant actor for missing tenant.", tenantId);
76 } else { 78 } else {
  79 + apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenant.getId()));
  80 +
77 // This Service may be started for specific tenant only. 81 // This Service may be started for specific tenant only.
78 Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant(); 82 Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant();
79 83
80 TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); 84 TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId());
81 85
82 - isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);  
83 isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); 86 isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE);
84 - 87 + isRuleEngineForCurrentTenant = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
85 if (isRuleEngineForCurrentTenant) { 88 if (isRuleEngineForCurrentTenant) {
86 try { 89 try {
87 if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) { 90 if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) {
88 - log.info("[{}] Going to init rule chains", tenantId);  
89 - initRuleChains(); 91 + if (apiUsageState.isReExecEnabled()) {
  92 + log.info("[{}] Going to init rule chains", tenantId);
  93 + initRuleChains();
  94 + } else {
  95 + log.info("[{}] Skip init of the rule chains due to API limits", tenantId);
  96 + }
90 } else { 97 } else {
91 isRuleEngineForCurrentTenant = false; 98 isRuleEngineForCurrentTenant = false;
92 } 99 }
@@ -98,8 +105,6 @@ public class TenantActor extends RuleChainManagerActor { @@ -98,8 +105,6 @@ public class TenantActor extends RuleChainManagerActor {
98 } 105 }
99 } catch (Exception e) { 106 } catch (Exception e) {
100 log.warn("[{}] Unknown failure", tenantId, e); 107 log.warn("[{}] Unknown failure", tenantId, e);
101 -// TODO: throw this in 3.1?  
102 -// throw new TbActorException("Failed to init actor", e);  
103 } 108 }
104 } 109 }
105 110
@@ -115,7 +120,7 @@ public class TenantActor extends RuleChainManagerActor { @@ -115,7 +120,7 @@ public class TenantActor extends RuleChainManagerActor {
115 if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { 120 if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) {
116 QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; 121 QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg;
117 queueMsg.getTbMsg().getCallback().onSuccess(); 122 queueMsg.getTbMsg().getCallback().onSuccess();
118 - } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)){ 123 + } else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)) {
119 TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg; 124 TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg;
120 transportMsg.getCallback().onSuccess(); 125 transportMsg.getCallback().onSuccess();
121 } 126 }
@@ -173,26 +178,33 @@ public class TenantActor extends RuleChainManagerActor { @@ -173,26 +178,33 @@ public class TenantActor extends RuleChainManagerActor {
173 return; 178 return;
174 } 179 }
175 TbMsg tbMsg = msg.getTbMsg(); 180 TbMsg tbMsg = msg.getTbMsg();
176 - if (tbMsg.getRuleChainId() == null) {  
177 - if (getRootChainActor() != null) {  
178 - getRootChainActor().tell(msg); 181 + if (apiUsageState.isReExecEnabled()) {
  182 + if (tbMsg.getRuleChainId() == null) {
  183 + if (getRootChainActor() != null) {
  184 + getRootChainActor().tell(msg);
  185 + } else {
  186 + tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!"));
  187 + log.info("[{}] No Root Chain: {}", tenantId, msg);
  188 + }
179 } else { 189 } else {
180 - tbMsg.getCallback().onFailure(new RuleEngineException("No Root Rule Chain available!"));  
181 - log.info("[{}] No Root Chain: {}", tenantId, msg); 190 + try {
  191 + ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg);
  192 + } catch (TbActorNotRegisteredException ex) {
  193 + log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId());
  194 + //TODO: 3.1 Log it to dead letters queue;
  195 + tbMsg.getCallback().onSuccess();
  196 + }
182 } 197 }
183 } else { 198 } else {
184 - try {  
185 - ctx.tell(new TbEntityActorId(tbMsg.getRuleChainId()), msg);  
186 - } catch (TbActorNotRegisteredException ex) {  
187 - log.trace("Received message for non-existing rule chain: [{}]", tbMsg.getRuleChainId());  
188 - //TODO: 3.1 Log it to dead letters queue;  
189 - tbMsg.getCallback().onSuccess();  
190 - } 199 + log.trace("[{}] Ack message because Rule Engine is disabled", tenantId);
  200 + tbMsg.getCallback().onSuccess();
191 } 201 }
192 } 202 }
193 203
194 private void onRuleChainMsg(RuleChainAwareMsg msg) { 204 private void onRuleChainMsg(RuleChainAwareMsg msg) {
195 - getOrCreateActor(msg.getRuleChainId()).tell(msg); 205 + if (apiUsageState.isReExecEnabled()) {
  206 + getOrCreateActor(msg.getRuleChainId()).tell(msg);
  207 + }
196 } 208 }
197 209
198 private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) { 210 private void onToDeviceActorMsg(DeviceAwareMsg msg, boolean priority) {
@@ -208,6 +220,17 @@ public class TenantActor extends RuleChainManagerActor { @@ -208,6 +220,17 @@ public class TenantActor extends RuleChainManagerActor {
208 } 220 }
209 221
210 private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) { 222 private void onComponentLifecycleMsg(ComponentLifecycleMsg msg) {
  223 + if (msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) {
  224 + ApiUsageState old = apiUsageState;
  225 + apiUsageState = new ApiUsageState(systemContext.getApiUsageStateService().getApiUsageState(tenantId));
  226 + if (old.isReExecEnabled() && !apiUsageState.isReExecEnabled()) {
  227 + log.info("[{}] Received API state update. Going to DISABLE Rule Engine execution.", tenantId);
  228 + destroyRuleChains();
  229 + } else if (!old.isReExecEnabled() && apiUsageState.isReExecEnabled()) {
  230 + log.info("[{}] Received API state update. Going to ENABLE Rule Engine execution.", tenantId);
  231 + initRuleChains();
  232 + }
  233 + }
211 if (isRuleEngineForCurrentTenant) { 234 if (isRuleEngineForCurrentTenant) {
212 TbActorRef target = getEntityActorRef(msg.getEntityId()); 235 TbActorRef target = getEntityActorRef(msg.getEntityId());
213 if (target != null) { 236 if (target != null) {
@@ -126,6 +126,7 @@ public class AlarmController extends BaseController { @@ -126,6 +126,7 @@ public class AlarmController extends BaseController {
126 long ackTs = System.currentTimeMillis(); 126 long ackTs = System.currentTimeMillis();
127 alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get(); 127 alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get();
128 alarm.setAckTs(ackTs); 128 alarm.setAckTs(ackTs);
  129 + alarm.setStatus(alarm.getStatus().isCleared() ? AlarmStatus.CLEARED_ACK : AlarmStatus.ACTIVE_ACK);
129 logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null); 130 logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null);
130 } catch (Exception e) { 131 } catch (Exception e) {
131 throw handleException(e); 132 throw handleException(e);
@@ -143,6 +144,7 @@ public class AlarmController extends BaseController { @@ -143,6 +144,7 @@ public class AlarmController extends BaseController {
143 long clearTs = System.currentTimeMillis(); 144 long clearTs = System.currentTimeMillis();
144 alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get(); 145 alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get();
145 alarm.setClearTs(clearTs); 146 alarm.setClearTs(clearTs);
  147 + alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK);
146 logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null); 148 logEntityAction(alarm.getOriginator(), alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null);
147 } catch (Exception e) { 149 } catch (Exception e) {
148 throw handleException(e); 150 throw handleException(e);
@@ -166,7 +166,8 @@ public class AuthController extends BaseController { @@ -166,7 +166,8 @@ public class AuthController extends BaseController {
166 try { 166 try {
167 String email = resetPasswordByEmailRequest.get("email").asText(); 167 String email = resetPasswordByEmailRequest.get("email").asText();
168 UserCredentials userCredentials = userService.requestPasswordReset(TenantId.SYS_TENANT_ID, email); 168 UserCredentials userCredentials = userService.requestPasswordReset(TenantId.SYS_TENANT_ID, email);
169 - String baseUrl = MiscUtils.constructBaseUrl(request); 169 + User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId());
  170 + String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
170 String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl, 171 String resetUrl = String.format("%s/api/noauth/resetPassword?resetToken=%s", baseUrl,
171 userCredentials.getResetToken()); 172 userCredentials.getResetToken());
172 173
@@ -214,7 +215,7 @@ public class AuthController extends BaseController { @@ -214,7 +215,7 @@ public class AuthController extends BaseController {
214 User user = userService.findUserById(TenantId.SYS_TENANT_ID, credentials.getUserId()); 215 User user = userService.findUserById(TenantId.SYS_TENANT_ID, credentials.getUserId());
215 UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); 216 UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail());
216 SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal); 217 SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal);
217 - String baseUrl = MiscUtils.constructBaseUrl(request); 218 + String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
218 String loginUrl = String.format("%s/login", baseUrl); 219 String loginUrl = String.format("%s/login", baseUrl);
219 String email = user.getEmail(); 220 String email = user.getEmail();
220 221
@@ -261,7 +262,7 @@ public class AuthController extends BaseController { @@ -261,7 +262,7 @@ public class AuthController extends BaseController {
261 User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId()); 262 User user = userService.findUserById(TenantId.SYS_TENANT_ID, userCredentials.getUserId());
262 UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()); 263 UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail());
263 SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), principal); 264 SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), principal);
264 - String baseUrl = MiscUtils.constructBaseUrl(request); 265 + String baseUrl = systemSecurityService.getBaseUrl(user.getTenantId(), user.getCustomerId(), request);
265 String loginUrl = String.format("%s/login", baseUrl); 266 String loginUrl = String.format("%s/login", baseUrl);
266 String email = user.getEmail(); 267 String email = user.getEmail();
267 mailService.sendPasswordWasResetEmail(loginUrl, email); 268 mailService.sendPasswordWasResetEmail(loginUrl, email);
@@ -95,7 +95,7 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider; @@ -95,7 +95,7 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
95 import org.thingsboard.server.queue.util.TbCoreComponent; 95 import org.thingsboard.server.queue.util.TbCoreComponent;
96 import org.thingsboard.server.service.component.ComponentDiscoveryService; 96 import org.thingsboard.server.service.component.ComponentDiscoveryService;
97 import org.thingsboard.server.service.profile.TbDeviceProfileCache; 97 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
98 -import org.thingsboard.server.service.profile.TbTenantProfileCache; 98 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
99 import org.thingsboard.server.service.queue.TbClusterService; 99 import org.thingsboard.server.service.queue.TbClusterService;
100 import org.thingsboard.server.service.security.model.SecurityUser; 100 import org.thingsboard.server.service.security.model.SecurityUser;
101 import org.thingsboard.server.service.security.permission.AccessControlService; 101 import org.thingsboard.server.service.security.permission.AccessControlService;
@@ -51,6 +51,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -51,6 +51,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
51 import org.thingsboard.server.common.data.id.TenantId; 51 import org.thingsboard.server.common.data.id.TenantId;
52 import org.thingsboard.server.common.data.page.PageData; 52 import org.thingsboard.server.common.data.page.PageData;
53 import org.thingsboard.server.common.data.page.PageLink; 53 import org.thingsboard.server.common.data.page.PageLink;
  54 +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
54 import org.thingsboard.server.common.data.security.DeviceCredentials; 55 import org.thingsboard.server.common.data.security.DeviceCredentials;
55 import org.thingsboard.server.common.msg.TbMsg; 56 import org.thingsboard.server.common.msg.TbMsg;
56 import org.thingsboard.server.common.msg.TbMsgDataType; 57 import org.thingsboard.server.common.msg.TbMsgDataType;
@@ -119,6 +120,8 @@ public class DeviceController extends BaseController { @@ -119,6 +120,8 @@ public class DeviceController extends BaseController {
119 120
120 tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(), 121 tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(),
121 savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null); 122 savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null);
  123 + tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(),
  124 + device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
122 125
123 logEntityAction(savedDevice.getId(), savedDevice, 126 logEntityAction(savedDevice.getId(), savedDevice,
124 savedDevice.getCustomerId(), 127 savedDevice.getCustomerId(),
@@ -160,7 +160,7 @@ public class RuleChainController extends BaseController { @@ -160,7 +160,7 @@ public class RuleChainController extends BaseController {
160 public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException { 160 public RuleChain saveRuleChain(@RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException {
161 try { 161 try {
162 checkNotNull(request); 162 checkNotNull(request);
163 - checkNotNull(request.getName()); 163 + checkParameter(request.getName(), "name");
164 164
165 RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); 165 RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName());
166 166
@@ -347,7 +347,7 @@ public class RuleChainController extends BaseController { @@ -347,7 +347,7 @@ public class RuleChainController extends BaseController {
347 String errorText = ""; 347 String errorText = "";
348 ScriptEngine engine = null; 348 ScriptEngine engine = null;
349 try { 349 try {
350 - engine = new RuleNodeJsScriptEngine(jsInvokeService, getCurrentUser().getId(), script, argNames); 350 + engine = new RuleNodeJsScriptEngine(getTenantId(), jsInvokeService, getCurrentUser().getId(), script, argNames);
351 TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data); 351 TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data);
352 switch (scriptType) { 352 switch (scriptType) {
353 case "update": 353 case "update":
@@ -93,6 +93,8 @@ public class TenantController extends BaseController { @@ -93,6 +93,8 @@ public class TenantController extends BaseController {
93 } 93 }
94 tenantProfileCache.evict(tenant.getId()); 94 tenantProfileCache.evict(tenant.getId());
95 tbClusterService.onTenantChange(tenant, null); 95 tbClusterService.onTenantChange(tenant, null);
  96 + tbClusterService.onEntityStateChange(tenant.getId(), tenant.getId(),
  97 + newTenant ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
96 return tenant; 98 return tenant;
97 } catch (Exception e) { 99 } catch (Exception e) {
98 throw handleException(e); 100 throw handleException(e);
@@ -52,6 +52,7 @@ import org.thingsboard.server.service.security.model.token.JwtToken; @@ -52,6 +52,7 @@ import org.thingsboard.server.service.security.model.token.JwtToken;
52 import org.thingsboard.server.service.security.model.token.JwtTokenFactory; 52 import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
53 import org.thingsboard.server.service.security.permission.Operation; 53 import org.thingsboard.server.service.security.permission.Operation;
54 import org.thingsboard.server.service.security.permission.Resource; 54 import org.thingsboard.server.service.security.permission.Resource;
  55 +import org.thingsboard.server.service.security.system.SystemSecurityService;
55 import org.thingsboard.server.utils.MiscUtils; 56 import org.thingsboard.server.utils.MiscUtils;
56 57
57 import javax.servlet.http.HttpServletRequest; 58 import javax.servlet.http.HttpServletRequest;
@@ -78,6 +79,9 @@ public class UserController extends BaseController { @@ -78,6 +79,9 @@ public class UserController extends BaseController {
78 @Autowired 79 @Autowired
79 private RefreshTokenRepository refreshTokenRepository; 80 private RefreshTokenRepository refreshTokenRepository;
80 81
  82 + @Autowired
  83 + private SystemSecurityService systemSecurityService;
  84 +
81 85
82 @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") 86 @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
83 @RequestMapping(value = "/user/{userId}", method = RequestMethod.GET) 87 @RequestMapping(value = "/user/{userId}", method = RequestMethod.GET)
@@ -146,7 +150,7 @@ public class UserController extends BaseController { @@ -146,7 +150,7 @@ public class UserController extends BaseController {
146 if (sendEmail) { 150 if (sendEmail) {
147 SecurityUser authUser = getCurrentUser(); 151 SecurityUser authUser = getCurrentUser();
148 UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), savedUser.getId()); 152 UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), savedUser.getId());
149 - String baseUrl = MiscUtils.constructBaseUrl(request); 153 + String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
150 String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, 154 String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
151 userCredentials.getActivateToken()); 155 userCredentials.getActivateToken());
152 String email = savedUser.getEmail(); 156 String email = savedUser.getEmail();
@@ -186,7 +190,7 @@ public class UserController extends BaseController { @@ -186,7 +190,7 @@ public class UserController extends BaseController {
186 190
187 UserCredentials userCredentials = userService.findUserCredentialsByUserId(getCurrentUser().getTenantId(), user.getId()); 191 UserCredentials userCredentials = userService.findUserCredentialsByUserId(getCurrentUser().getTenantId(), user.getId());
188 if (!userCredentials.isEnabled()) { 192 if (!userCredentials.isEnabled()) {
189 - String baseUrl = MiscUtils.constructBaseUrl(request); 193 + String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
190 String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, 194 String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
191 userCredentials.getActivateToken()); 195 userCredentials.getActivateToken());
192 mailService.sendActivationEmail(activateUrl, email); 196 mailService.sendActivationEmail(activateUrl, email);
@@ -211,7 +215,7 @@ public class UserController extends BaseController { @@ -211,7 +215,7 @@ public class UserController extends BaseController {
211 SecurityUser authUser = getCurrentUser(); 215 SecurityUser authUser = getCurrentUser();
212 UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), user.getId()); 216 UserCredentials userCredentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), user.getId());
213 if (!userCredentials.isEnabled()) { 217 if (!userCredentials.isEnabled()) {
214 - String baseUrl = MiscUtils.constructBaseUrl(request); 218 + String baseUrl = systemSecurityService.getBaseUrl(getTenantId(), getCurrentUser().getCustomerId(), request);
215 String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl, 219 String activateUrl = String.format(ACTIVATE_URL_PATTERN, baseUrl,
216 userCredentials.getActivateToken()); 220 userCredentials.getActivateToken());
217 return activateUrl; 221 return activateUrl;
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.apiusage;
  17 +
  18 +import com.google.common.util.concurrent.FutureCallback;
  19 +import lombok.extern.slf4j.Slf4j;
  20 +import org.checkerframework.checker.nullness.qual.Nullable;
  21 +import org.springframework.beans.factory.annotation.Autowired;
  22 +import org.springframework.beans.factory.annotation.Value;
  23 +import org.springframework.boot.context.event.ApplicationReadyEvent;
  24 +import org.springframework.context.annotation.Lazy;
  25 +import org.springframework.context.event.EventListener;
  26 +import org.springframework.core.annotation.Order;
  27 +import org.springframework.data.util.Pair;
  28 +import org.springframework.stereotype.Service;
  29 +import org.thingsboard.server.common.data.ApiFeature;
  30 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
  31 +import org.thingsboard.server.common.data.ApiUsageState;
  32 +import org.thingsboard.server.common.data.ApiUsageStateValue;
  33 +import org.thingsboard.server.common.data.Tenant;
  34 +import org.thingsboard.server.common.data.TenantProfile;
  35 +import org.thingsboard.server.common.data.id.ApiUsageStateId;
  36 +import org.thingsboard.server.common.data.id.TenantId;
  37 +import org.thingsboard.server.common.data.id.TenantProfileId;
  38 +import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
  39 +import org.thingsboard.server.common.data.kv.LongDataEntry;
  40 +import org.thingsboard.server.common.data.kv.StringDataEntry;
  41 +import org.thingsboard.server.common.data.kv.TsKvEntry;
  42 +import org.thingsboard.server.common.data.page.PageDataIterable;
  43 +import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration;
  44 +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
  45 +import org.thingsboard.server.common.msg.queue.ServiceType;
  46 +import org.thingsboard.server.common.msg.queue.TbCallback;
  47 +import org.thingsboard.server.common.msg.tools.SchedulerUtils;
  48 +import org.thingsboard.server.dao.tenant.TenantService;
  49 +import org.thingsboard.server.dao.timeseries.TimeseriesService;
  50 +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
  51 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
  52 +import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto;
  53 +import org.thingsboard.server.queue.common.TbProtoQueueMsg;
  54 +import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  55 +import org.thingsboard.server.queue.discovery.PartitionService;
  56 +import org.thingsboard.server.queue.scheduler.SchedulerComponent;
  57 +import org.thingsboard.server.queue.util.TbCoreComponent;
  58 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
  59 +import org.thingsboard.server.service.queue.TbClusterService;
  60 +import org.thingsboard.server.service.telemetry.InternalTelemetryService;
  61 +
  62 +import javax.annotation.PostConstruct;
  63 +import java.util.ArrayList;
  64 +import java.util.HashMap;
  65 +import java.util.HashSet;
  66 +import java.util.List;
  67 +import java.util.Map;
  68 +import java.util.Set;
  69 +import java.util.UUID;
  70 +import java.util.concurrent.ConcurrentHashMap;
  71 +import java.util.concurrent.ExecutionException;
  72 +import java.util.concurrent.TimeUnit;
  73 +import java.util.concurrent.locks.Lock;
  74 +import java.util.concurrent.locks.ReentrantLock;
  75 +
  76 +@Slf4j
  77 +@Service
  78 +public class DefaultTbApiUsageStateService implements TbApiUsageStateService {
  79 +
  80 + public static final String HOURLY = "Hourly";
  81 + public static final FutureCallback<Integer> VOID_CALLBACK = new FutureCallback<Integer>() {
  82 + @Override
  83 + public void onSuccess(@Nullable Integer result) {
  84 + }
  85 +
  86 + @Override
  87 + public void onFailure(Throwable t) {
  88 + }
  89 + };
  90 + private final TbClusterService clusterService;
  91 + private final PartitionService partitionService;
  92 + private final TenantService tenantService;
  93 + private final TimeseriesService tsService;
  94 + private final ApiUsageStateService apiUsageStateService;
  95 + private final SchedulerComponent scheduler;
  96 + private final TbTenantProfileCache tenantProfileCache;
  97 +
  98 + @Lazy
  99 + @Autowired
  100 + private InternalTelemetryService tsWsService;
  101 +
  102 + // Tenants that should be processed on this server
  103 + private final Map<TenantId, TenantApiUsageState> myTenantStates = new ConcurrentHashMap<>();
  104 + // Tenants that should be processed on other servers
  105 + private final Map<TenantId, ApiUsageState> otherTenantStates = new ConcurrentHashMap<>();
  106 +
  107 + @Value("${usage.stats.report.enabled:true}")
  108 + private boolean enabled;
  109 +
  110 + @Value("${usage.stats.check.cycle:60000}")
  111 + private long nextCycleCheckInterval;
  112 +
  113 + private final Lock updateLock = new ReentrantLock();
  114 +
  115 + public DefaultTbApiUsageStateService(TbClusterService clusterService,
  116 + PartitionService partitionService,
  117 + TenantService tenantService,
  118 + TimeseriesService tsService,
  119 + ApiUsageStateService apiUsageStateService,
  120 + SchedulerComponent scheduler,
  121 + TbTenantProfileCache tenantProfileCache) {
  122 + this.clusterService = clusterService;
  123 + this.partitionService = partitionService;
  124 + this.tenantService = tenantService;
  125 + this.tsService = tsService;
  126 + this.apiUsageStateService = apiUsageStateService;
  127 + this.scheduler = scheduler;
  128 + this.tenantProfileCache = tenantProfileCache;
  129 + }
  130 +
  131 + @PostConstruct
  132 + public void init() {
  133 + if (enabled) {
  134 + log.info("Starting api usage service.");
  135 + scheduler.scheduleAtFixedRate(this::checkStartOfNextCycle, nextCycleCheckInterval, nextCycleCheckInterval, TimeUnit.MILLISECONDS);
  136 + log.info("Started api usage service.");
  137 + }
  138 + }
  139 +
  140 + @Override
  141 + public void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
  142 + ToUsageStatsServiceMsg statsMsg = msg.getValue();
  143 + TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB()));
  144 + TenantApiUsageState tenantState;
  145 + List<TsKvEntry> updatedEntries;
  146 + Map<ApiFeature, ApiUsageStateValue> result;
  147 + updateLock.lock();
  148 + try {
  149 + tenantState = getOrFetchState(tenantId);
  150 + long ts = tenantState.getCurrentCycleTs();
  151 + long hourTs = tenantState.getCurrentHourTs();
  152 + long newHourTs = SchedulerUtils.getStartOfCurrentHour();
  153 + if (newHourTs != hourTs) {
  154 + tenantState.setHour(newHourTs);
  155 + }
  156 + updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length);
  157 + Set<ApiFeature> apiFeatures = new HashSet<>();
  158 + for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) {
  159 + ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey());
  160 + long newValue = tenantState.add(recordKey, kvProto.getValue());
  161 + updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue)));
  162 + long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue());
  163 + updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue)));
  164 + apiFeatures.add(recordKey.getApiFeature());
  165 + }
  166 + result = tenantState.checkStateUpdatedDueToThreshold(apiFeatures);
  167 + } finally {
  168 + updateLock.unlock();
  169 + }
  170 + tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK);
  171 + if (!result.isEmpty()) {
  172 + persistAndNotify(tenantState, result);
  173 + }
  174 + callback.onSuccess();
  175 + }
  176 +
  177 + @Override
  178 + public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
  179 + if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) {
  180 + myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
  181 + otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
  182 + initStatesFromDataBase();
  183 + }
  184 + }
  185 +
  186 + @Override
  187 + public ApiUsageState getApiUsageState(TenantId tenantId) {
  188 + TenantApiUsageState tenantState = myTenantStates.get(tenantId);
  189 + if (tenantState != null) {
  190 + return tenantState.getApiUsageState();
  191 + } else {
  192 + ApiUsageState state = otherTenantStates.get(tenantId);
  193 + if (state != null) {
  194 + return state;
  195 + } else {
  196 + if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) {
  197 + return getOrFetchState(tenantId).getApiUsageState();
  198 + } else {
  199 + updateLock.lock();
  200 + try {
  201 + state = otherTenantStates.get(tenantId);
  202 + if (state == null) {
  203 + state = apiUsageStateService.findTenantApiUsageState(tenantId);
  204 + if (state != null) {
  205 + otherTenantStates.put(tenantId, state);
  206 + }
  207 + }
  208 + } finally {
  209 + updateLock.unlock();
  210 + }
  211 + return state;
  212 + }
  213 + }
  214 + }
  215 + }
  216 +
  217 + @Override
  218 + public void onApiUsageStateUpdate(TenantId tenantId) {
  219 + otherTenantStates.remove(tenantId);
  220 + }
  221 +
  222 + @Override
  223 + public void onTenantProfileUpdate(TenantProfileId tenantProfileId) {
  224 + TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId);
  225 + updateLock.lock();
  226 + try {
  227 + myTenantStates.values().forEach(state -> {
  228 + if (tenantProfile.getId().equals(state.getTenantProfileId())) {
  229 + updateTenantState(state, tenantProfile);
  230 + }
  231 + });
  232 + } finally {
  233 + updateLock.unlock();
  234 + }
  235 + }
  236 +
  237 + @Override
  238 + public void onTenantUpdate(TenantId tenantId) {
  239 + TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
  240 + updateLock.lock();
  241 + try {
  242 + TenantApiUsageState state = myTenantStates.get(tenantId);
  243 + if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) {
  244 + updateTenantState(state, tenantProfile);
  245 + }
  246 + } finally {
  247 + updateLock.unlock();
  248 + }
  249 + }
  250 +
  251 + private void updateTenantState(TenantApiUsageState state, TenantProfile tenantProfile) {
  252 + TenantProfileData oldProfileData = state.getTenantProfileData();
  253 + state.setTenantProfileId(tenantProfile.getId());
  254 + state.setTenantProfileData(tenantProfile.getProfileData());
  255 + Map<ApiFeature, ApiUsageStateValue> result = state.checkStateUpdatedDueToThresholds();
  256 + if (!result.isEmpty()) {
  257 + persistAndNotify(state, result);
  258 + }
  259 + updateProfileThresholds(state.getTenantId(), state.getApiUsageState().getId(),
  260 + oldProfileData.getConfiguration(), tenantProfile.getProfileData().getConfiguration());
  261 + }
  262 +
  263 + private void updateProfileThresholds(TenantId tenantId, ApiUsageStateId id,
  264 + TenantProfileConfiguration oldData, TenantProfileConfiguration newData) {
  265 + long ts = System.currentTimeMillis();
  266 + List<TsKvEntry> profileThresholds = new ArrayList<>();
  267 + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
  268 + long newProfileThreshold = newData.getProfileThreshold(key);
  269 + if (oldData == null || oldData.getProfileThreshold(key) != newProfileThreshold) {
  270 + log.info("[{}] Updating profile threshold [{}]:[{}]", tenantId, key, newProfileThreshold);
  271 + profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newProfileThreshold)));
  272 + }
  273 + }
  274 + if (!profileThresholds.isEmpty()) {
  275 + tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, VOID_CALLBACK);
  276 + }
  277 + }
  278 +
  279 + private void persistAndNotify(TenantApiUsageState state, Map<ApiFeature, ApiUsageStateValue> result) {
  280 + log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result);
  281 + apiUsageStateService.update(state.getApiUsageState());
  282 + clusterService.onApiStateChange(state.getApiUsageState(), null);
  283 + long ts = System.currentTimeMillis();
  284 + List<TsKvEntry> stateTelemetry = new ArrayList<>();
  285 + result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name())))));
  286 + tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK);
  287 + //TODO: notify tenant admin via email!
  288 + }
  289 +
  290 + private void checkStartOfNextCycle() {
  291 + updateLock.lock();
  292 + try {
  293 + long now = System.currentTimeMillis();
  294 + myTenantStates.values().forEach(state -> {
  295 + if ((state.getNextCycleTs() > now) && (state.getNextCycleTs() - now < TimeUnit.HOURS.toMillis(1))) {
  296 + state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth());
  297 + }
  298 + });
  299 + } finally {
  300 + updateLock.unlock();
  301 + }
  302 + }
  303 +
  304 + private TenantApiUsageState getOrFetchState(TenantId tenantId) {
  305 + TenantApiUsageState tenantState = myTenantStates.get(tenantId);
  306 + if (tenantState == null) {
  307 + ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
  308 + if (dbStateEntity == null) {
  309 + try {
  310 + dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId);
  311 + } catch (Exception e) {
  312 + dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
  313 + }
  314 + }
  315 + TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
  316 + tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity);
  317 + try {
  318 + List<TsKvEntry> dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get();
  319 + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
  320 + boolean cycleEntryFound = false;
  321 + boolean hourlyEntryFound = false;
  322 + for (TsKvEntry tsKvEntry : dbValues) {
  323 + if (tsKvEntry.getKey().equals(key.getApiCountKey())) {
  324 + cycleEntryFound = true;
  325 + tenantState.put(key, tsKvEntry.getTs() == tenantState.getCurrentCycleTs() ? tsKvEntry.getLongValue().get() : 0L);
  326 + } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) {
  327 + hourlyEntryFound = true;
  328 + tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L);
  329 + }
  330 + if (cycleEntryFound && hourlyEntryFound) {
  331 + break;
  332 + }
  333 + }
  334 + }
  335 + log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity);
  336 + myTenantStates.put(tenantId, tenantState);
  337 + } catch (InterruptedException | ExecutionException e) {
  338 + log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e);
  339 + }
  340 + }
  341 + return tenantState;
  342 + }
  343 +
  344 + private void initStatesFromDataBase() {
  345 + try {
  346 + log.info("Initializing tenant states.");
  347 + PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024);
  348 + for (Tenant tenant : tenantIterator) {
  349 + if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) {
  350 + log.debug("[{}] Initializing tenant state.", tenant.getId());
  351 + updateLock.lock();
  352 + try {
  353 + updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId()));
  354 + log.debug("[{}] Initialized tenant state.", tenant.getId());
  355 + } catch (Exception e) {
  356 + log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e);
  357 + } finally {
  358 + updateLock.unlock();
  359 + }
  360 + }
  361 + }
  362 + log.info("Initialized tenant states.");
  363 + } catch (Exception e) {
  364 + log.warn("Unknown failure", e);
  365 + }
  366 + }
  367 +
  368 +}
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.apiusage;
  17 +
  18 +import org.springframework.context.ApplicationListener;
  19 +import org.thingsboard.server.common.data.ApiUsageState;
  20 +import org.thingsboard.server.common.data.id.TenantId;
  21 +import org.thingsboard.server.common.data.id.TenantProfileId;
  22 +import org.thingsboard.server.common.msg.queue.TbCallback;
  23 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
  24 +import org.thingsboard.server.queue.common.TbProtoQueueMsg;
  25 +import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
  26 +
  27 +public interface TbApiUsageStateService extends ApplicationListener<PartitionChangeEvent> {
  28 +
  29 + void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback);
  30 +
  31 + ApiUsageState getApiUsageState(TenantId tenantId);
  32 +
  33 + void onTenantProfileUpdate(TenantProfileId tenantProfileId);
  34 +
  35 + void onTenantUpdate(TenantId tenantId);
  36 +
  37 + void onApiUsageStateUpdate(TenantId tenantId);
  38 +}
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.apiusage;
  17 +
  18 +import lombok.Getter;
  19 +import lombok.Setter;
  20 +import org.springframework.data.util.Pair;
  21 +import org.thingsboard.server.common.data.ApiFeature;
  22 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
  23 +import org.thingsboard.server.common.data.ApiUsageState;
  24 +import org.thingsboard.server.common.data.ApiUsageStateValue;
  25 +import org.thingsboard.server.common.data.TenantProfile;
  26 +import org.thingsboard.server.common.data.id.TenantId;
  27 +import org.thingsboard.server.common.data.id.TenantProfileId;
  28 +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
  29 +import org.thingsboard.server.common.msg.tools.SchedulerUtils;
  30 +
  31 +import java.util.Arrays;
  32 +import java.util.HashMap;
  33 +import java.util.HashSet;
  34 +import java.util.Map;
  35 +import java.util.Set;
  36 +import java.util.concurrent.ConcurrentHashMap;
  37 +
  38 +public class TenantApiUsageState {
  39 +
  40 + private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>();
  41 + private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>();
  42 +
  43 + @Getter
  44 + @Setter
  45 + private TenantProfileId tenantProfileId;
  46 + @Getter
  47 + @Setter
  48 + private TenantProfileData tenantProfileData;
  49 + @Getter
  50 + private final ApiUsageState apiUsageState;
  51 + @Getter
  52 + private volatile long currentCycleTs;
  53 + @Getter
  54 + private volatile long nextCycleTs;
  55 + @Getter
  56 + private volatile long currentHourTs;
  57 +
  58 + public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) {
  59 + this.tenantProfileId = tenantProfile.getId();
  60 + this.tenantProfileData = tenantProfile.getProfileData();
  61 + this.apiUsageState = apiUsageState;
  62 + this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
  63 + this.nextCycleTs = SchedulerUtils.getStartOfNextMonth();
  64 + this.currentHourTs = SchedulerUtils.getStartOfCurrentHour();
  65 + }
  66 +
  67 + public void put(ApiUsageRecordKey key, Long value) {
  68 + currentCycleValues.put(key, value);
  69 + }
  70 +
  71 + public void putHourly(ApiUsageRecordKey key, Long value) {
  72 + currentHourValues.put(key, value);
  73 + }
  74 +
  75 + public long add(ApiUsageRecordKey key, long value) {
  76 + long result = currentCycleValues.getOrDefault(key, 0L) + value;
  77 + currentCycleValues.put(key, result);
  78 + return result;
  79 + }
  80 +
  81 + public long get(ApiUsageRecordKey key) {
  82 + return currentCycleValues.getOrDefault(key, 0L);
  83 + }
  84 +
  85 + public long addToHourly(ApiUsageRecordKey key, long value) {
  86 + long result = currentHourValues.getOrDefault(key, 0L) + value;
  87 + currentHourValues.put(key, result);
  88 + return result;
  89 + }
  90 +
  91 + public void setHour(long currentHourTs) {
  92 + this.currentHourTs = currentHourTs;
  93 + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
  94 + currentHourValues.put(key, 0L);
  95 + }
  96 + }
  97 +
  98 + public void setCycles(long currentCycleTs, long nextCycleTs) {
  99 + this.currentCycleTs = currentCycleTs;
  100 + this.nextCycleTs = nextCycleTs;
  101 + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
  102 + currentCycleValues.put(key, 0L);
  103 + }
  104 + }
  105 +
  106 + public long getProfileThreshold(ApiUsageRecordKey key) {
  107 + return tenantProfileData.getConfiguration().getProfileThreshold(key);
  108 + }
  109 +
  110 + public long getProfileWarnThreshold(ApiUsageRecordKey key) {
  111 + return tenantProfileData.getConfiguration().getWarnThreshold(key);
  112 + }
  113 +
  114 + public TenantId getTenantId() {
  115 + return apiUsageState.getTenantId();
  116 + }
  117 +
  118 + public ApiUsageStateValue getFeatureValue(ApiFeature feature) {
  119 + switch (feature) {
  120 + case TRANSPORT:
  121 + return apiUsageState.getTransportState();
  122 + case RE:
  123 + return apiUsageState.getReExecState();
  124 + case DB:
  125 + return apiUsageState.getDbStorageState();
  126 + case JS:
  127 + return apiUsageState.getJsExecState();
  128 + default:
  129 + return ApiUsageStateValue.ENABLED;
  130 + }
  131 + }
  132 +
  133 + public boolean setFeatureValue(ApiFeature feature, ApiUsageStateValue value) {
  134 + ApiUsageStateValue currentValue = getFeatureValue(feature);
  135 + switch (feature) {
  136 + case TRANSPORT:
  137 + apiUsageState.setTransportState(value);
  138 + break;
  139 + case RE:
  140 + apiUsageState.setReExecState(value);
  141 + break;
  142 + case DB:
  143 + apiUsageState.setDbStorageState(value);
  144 + break;
  145 + case JS:
  146 + apiUsageState.setJsExecState(value);
  147 + break;
  148 + }
  149 + return !currentValue.equals(value);
  150 + }
  151 +
  152 + public Map<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThresholds() {
  153 + return checkStateUpdatedDueToThreshold(new HashSet<>(Arrays.asList(ApiFeature.values())));
  154 + }
  155 +
  156 + public Map<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(Set<ApiFeature> features) {
  157 + Map<ApiFeature, ApiUsageStateValue> result = new HashMap<>();
  158 + for (ApiFeature feature : features) {
  159 + Pair<ApiFeature, ApiUsageStateValue> tmp = checkStateUpdatedDueToThreshold(feature);
  160 + if (tmp != null) {
  161 + result.put(tmp.getFirst(), tmp.getSecond());
  162 + }
  163 + }
  164 + return result;
  165 + }
  166 +
  167 + public Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) {
  168 + ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED;
  169 + for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) {
  170 + long value = get(recordKey);
  171 + long threshold = getProfileThreshold(recordKey);
  172 + long warnThreshold = getProfileWarnThreshold(recordKey);
  173 + ApiUsageStateValue tmpValue;
  174 + if (threshold == 0 || value < warnThreshold) {
  175 + tmpValue = ApiUsageStateValue.ENABLED;
  176 + } else if (value < threshold) {
  177 + tmpValue = ApiUsageStateValue.WARNING;
  178 + } else {
  179 + tmpValue = ApiUsageStateValue.DISABLED;
  180 + }
  181 + featureValue = ApiUsageStateValue.toMoreRestricted(featureValue, tmpValue);
  182 + }
  183 + return setFeatureValue(feature, featureValue) ? Pair.of(feature, featureValue) : null;
  184 + }
  185 +
  186 +}
@@ -30,7 +30,8 @@ import org.thingsboard.server.common.data.Device; @@ -30,7 +30,8 @@ import org.thingsboard.server.common.data.Device;
30 import org.thingsboard.server.common.data.DeviceProfile; 30 import org.thingsboard.server.common.data.DeviceProfile;
31 import org.thingsboard.server.common.data.Tenant; 31 import org.thingsboard.server.common.data.Tenant;
32 import org.thingsboard.server.common.data.TenantProfile; 32 import org.thingsboard.server.common.data.TenantProfile;
33 -import org.thingsboard.server.common.data.TenantProfileData; 33 +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
  34 +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
34 import org.thingsboard.server.common.data.User; 35 import org.thingsboard.server.common.data.User;
35 import org.thingsboard.server.common.data.asset.Asset; 36 import org.thingsboard.server.common.data.asset.Asset;
36 import org.thingsboard.server.common.data.id.CustomerId; 37 import org.thingsboard.server.common.data.id.CustomerId;
@@ -127,6 +128,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @@ -127,6 +128,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
127 public void createDefaultTenantProfiles() throws Exception { 128 public void createDefaultTenantProfiles() throws Exception {
128 tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID); 129 tenantProfileService.findOrCreateDefaultTenantProfile(TenantId.SYS_TENANT_ID);
129 130
  131 + TenantProfileData tenantProfileData = new TenantProfileData();
  132 + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration());
  133 +
130 TenantProfile isolatedTbCoreProfile = new TenantProfile(); 134 TenantProfile isolatedTbCoreProfile = new TenantProfile();
131 isolatedTbCoreProfile.setDefault(false); 135 isolatedTbCoreProfile.setDefault(false);
132 isolatedTbCoreProfile.setName("Isolated TB Core"); 136 isolatedTbCoreProfile.setName("Isolated TB Core");
@@ -134,6 +138,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @@ -134,6 +138,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
134 isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile"); 138 isolatedTbCoreProfile.setDescription("Isolated TB Core tenant profile");
135 isolatedTbCoreProfile.setIsolatedTbCore(true); 139 isolatedTbCoreProfile.setIsolatedTbCore(true);
136 isolatedTbCoreProfile.setIsolatedTbRuleEngine(false); 140 isolatedTbCoreProfile.setIsolatedTbRuleEngine(false);
  141 + isolatedTbCoreProfile.setProfileData(tenantProfileData);
137 try { 142 try {
138 tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile); 143 tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreProfile);
139 } catch (DataValidationException e) { 144 } catch (DataValidationException e) {
@@ -147,6 +152,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @@ -147,6 +152,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
147 isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile"); 152 isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile");
148 isolatedTbRuleEngineProfile.setIsolatedTbCore(false); 153 isolatedTbRuleEngineProfile.setIsolatedTbCore(false);
149 isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true); 154 isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true);
  155 + isolatedTbRuleEngineProfile.setProfileData(tenantProfileData);
  156 +
150 try { 157 try {
151 tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile); 158 tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile);
152 } catch (DataValidationException e) { 159 } catch (DataValidationException e) {
@@ -160,6 +167,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @@ -160,6 +167,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
160 isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile"); 167 isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile");
161 isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true); 168 isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true);
162 isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true); 169 isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true);
  170 + isolatedTbCoreAndTbRuleEngineProfile.setProfileData(tenantProfileData);
  171 +
163 try { 172 try {
164 tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile); 173 tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile);
165 } catch (DataValidationException e) { 174 } catch (DataValidationException e) {
@@ -173,6 +182,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @@ -173,6 +182,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
173 generalSettings.setKey("general"); 182 generalSettings.setKey("general");
174 ObjectNode node = objectMapper.createObjectNode(); 183 ObjectNode node = objectMapper.createObjectNode();
175 node.put("baseUrl", "http://localhost:8080"); 184 node.put("baseUrl", "http://localhost:8080");
  185 + node.put("prohibitDifferentUrl", true);
176 generalSettings.setJsonValue(node); 186 generalSettings.setJsonValue(node);
177 adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); 187 adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings);
178 188
@@ -28,6 +28,7 @@ import org.thingsboard.server.dao.dashboard.DashboardService; @@ -28,6 +28,7 @@ import org.thingsboard.server.dao.dashboard.DashboardService;
28 import org.thingsboard.server.dao.device.DeviceProfileService; 28 import org.thingsboard.server.dao.device.DeviceProfileService;
29 import org.thingsboard.server.dao.device.DeviceService; 29 import org.thingsboard.server.dao.device.DeviceService;
30 import org.thingsboard.server.dao.tenant.TenantService; 30 import org.thingsboard.server.dao.tenant.TenantService;
  31 +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
31 import org.thingsboard.server.service.install.sql.SqlDbHelper; 32 import org.thingsboard.server.service.install.sql.SqlDbHelper;
32 33
33 import java.nio.charset.Charset; 34 import java.nio.charset.Charset;
@@ -96,6 +97,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @@ -96,6 +97,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
96 @Autowired 97 @Autowired
97 private DeviceProfileService deviceProfileService; 98 private DeviceProfileService deviceProfileService;
98 99
  100 + @Autowired
  101 + private ApiUsageStateService apiUsageStateService;
  102 +
99 103
100 @Override 104 @Override
101 public void upgradeDatabase(String fromVersion) throws Exception { 105 public void upgradeDatabase(String fromVersion) throws Exception {
@@ -352,6 +356,22 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @@ -352,6 +356,22 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
352 } catch (Exception e) { 356 } catch (Exception e) {
353 } 357 }
354 358
  359 + try {
  360 + conn.createStatement().execute("CREATE TABLE IF NOT EXISTS api_usage_state (" +
  361 + " id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY," +
  362 + " created_time bigint NOT NULL," +
  363 + " tenant_id uuid," +
  364 + " entity_type varchar(32)," +
  365 + " entity_id uuid," +
  366 + " transport varchar(32)," +
  367 + " db_storage varchar(32)," +
  368 + " re_exec varchar(32)," +
  369 + " js_exec varchar(32)," +
  370 + " CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id)\n" +
  371 + ");");
  372 + } catch (Exception e) {
  373 + }
  374 +
355 schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.1", "schema_update_before.sql"); 375 schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.1", "schema_update_before.sql");
356 loadSql(schemaUpdateFile, conn); 376 loadSql(schemaUpdateFile, conn);
357 377
@@ -367,6 +387,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @@ -367,6 +387,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
367 do { 387 do {
368 pageData = tenantService.findTenants(pageLink); 388 pageData = tenantService.findTenants(pageLink);
369 for (Tenant tenant : pageData.getData()) { 389 for (Tenant tenant : pageData.getData()) {
  390 + try {
  391 + apiUsageStateService.createDefaultApiUsageState(tenant.getId());
  392 + } catch (Exception e) {
  393 + }
370 List<EntitySubtype> deviceTypes = deviceService.findDeviceTypesByTenantId(tenant.getId()).get(); 394 List<EntitySubtype> deviceTypes = deviceService.findDeviceTypesByTenantId(tenant.getId()).get();
371 try { 395 try {
372 deviceProfileService.createDefaultDeviceProfile(tenant.getId()); 396 deviceProfileService.createDefaultDeviceProfile(tenant.getId());
@@ -26,11 +26,11 @@ import org.thingsboard.server.common.data.id.TenantId; @@ -26,11 +26,11 @@ import org.thingsboard.server.common.data.id.TenantId;
26 import org.thingsboard.server.dao.device.DeviceProfileService; 26 import org.thingsboard.server.dao.device.DeviceProfileService;
27 import org.thingsboard.server.dao.device.DeviceService; 27 import org.thingsboard.server.dao.device.DeviceService;
28 28
29 -import java.util.Map;  
30 import java.util.concurrent.ConcurrentHashMap; 29 import java.util.concurrent.ConcurrentHashMap;
31 import java.util.concurrent.ConcurrentMap; 30 import java.util.concurrent.ConcurrentMap;
32 import java.util.concurrent.locks.Lock; 31 import java.util.concurrent.locks.Lock;
33 import java.util.concurrent.locks.ReentrantLock; 32 import java.util.concurrent.locks.ReentrantLock;
  33 +import java.util.function.BiConsumer;
34 import java.util.function.Consumer; 34 import java.util.function.Consumer;
35 35
36 @Service 36 @Service
@@ -43,7 +43,8 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { @@ -43,7 +43,8 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
43 43
44 private final ConcurrentMap<DeviceProfileId, DeviceProfile> deviceProfilesMap = new ConcurrentHashMap<>(); 44 private final ConcurrentMap<DeviceProfileId, DeviceProfile> deviceProfilesMap = new ConcurrentHashMap<>();
45 private final ConcurrentMap<DeviceId, DeviceProfileId> devicesMap = new ConcurrentHashMap<>(); 45 private final ConcurrentMap<DeviceId, DeviceProfileId> devicesMap = new ConcurrentHashMap<>();
46 - private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, Consumer<DeviceProfile>>> listeners = new ConcurrentHashMap<>(); 46 + private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, Consumer<DeviceProfile>>> profileListeners = new ConcurrentHashMap<>();
  47 + private final ConcurrentMap<TenantId, ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>>> deviceProfileListeners = new ConcurrentHashMap<>();
47 48
48 public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) { 49 public DefaultTbDeviceProfileCache(DeviceProfileService deviceProfileService, DeviceService deviceService) {
49 this.deviceProfileService = deviceProfileService; 50 this.deviceProfileService = deviceProfileService;
@@ -88,32 +89,36 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { @@ -88,32 +89,36 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
88 } 89 }
89 90
90 @Override 91 @Override
91 - public void put(DeviceProfile profile) {  
92 - if (profile.getId() != null) {  
93 - deviceProfilesMap.put(profile.getId(), profile);  
94 - log.debug("[{}] pushed device profile to cache: {}", profile.getId(), profile);  
95 - notifyListeners(profile);  
96 - }  
97 - }  
98 -  
99 - @Override  
100 public void evict(TenantId tenantId, DeviceProfileId profileId) { 92 public void evict(TenantId tenantId, DeviceProfileId profileId) {
101 DeviceProfile oldProfile = deviceProfilesMap.remove(profileId); 93 DeviceProfile oldProfile = deviceProfilesMap.remove(profileId);
102 log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile); 94 log.debug("[{}] evict device profile from cache: {}", profileId, oldProfile);
103 DeviceProfile newProfile = get(tenantId, profileId); 95 DeviceProfile newProfile = get(tenantId, profileId);
104 if (newProfile != null) { 96 if (newProfile != null) {
105 - notifyListeners(newProfile); 97 + notifyProfileListeners(newProfile);
106 } 98 }
107 } 99 }
108 100
109 @Override 101 @Override
110 - public void evict(DeviceId deviceId) {  
111 - devicesMap.remove(deviceId); 102 + public void evict(TenantId tenantId, DeviceId deviceId) {
  103 + DeviceProfileId old = devicesMap.remove(deviceId);
  104 + if (old != null) {
  105 + DeviceProfile newProfile = get(tenantId, deviceId);
  106 + if (newProfile == null || !old.equals(newProfile.getId())) {
  107 + notifyDeviceListeners(tenantId, deviceId, newProfile);
  108 + }
  109 + }
112 } 110 }
113 111
114 @Override 112 @Override
115 - public void addListener(TenantId tenantId, EntityId listenerId, Consumer<DeviceProfile> listener) {  
116 - listeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, listener); 113 + public void addListener(TenantId tenantId, EntityId listenerId,
  114 + Consumer<DeviceProfile> profileListener,
  115 + BiConsumer<DeviceId, DeviceProfile> deviceListener) {
  116 + if (profileListener != null) {
  117 + profileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, profileListener);
  118 + }
  119 + if (deviceListener != null) {
  120 + deviceProfileListeners.computeIfAbsent(tenantId, id -> new ConcurrentHashMap<>()).put(listenerId, deviceListener);
  121 + }
117 } 122 }
118 123
119 @Override 124 @Override
@@ -128,17 +133,30 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache { @@ -128,17 +133,30 @@ public class DefaultTbDeviceProfileCache implements TbDeviceProfileCache {
128 133
129 @Override 134 @Override
130 public void removeListener(TenantId tenantId, EntityId listenerId) { 135 public void removeListener(TenantId tenantId, EntityId listenerId) {
131 - ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = listeners.get(tenantId); 136 + ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = profileListeners.get(tenantId);
132 if (tenantListeners != null) { 137 if (tenantListeners != null) {
133 tenantListeners.remove(listenerId); 138 tenantListeners.remove(listenerId);
134 } 139 }
  140 + ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>> deviceListeners = deviceProfileListeners.get(tenantId);
  141 + if (deviceListeners != null) {
  142 + deviceListeners.remove(listenerId);
  143 + }
135 } 144 }
136 145
137 - private void notifyListeners(DeviceProfile profile) {  
138 - ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = listeners.get(profile.getTenantId()); 146 + private void notifyProfileListeners(DeviceProfile profile) {
  147 + ConcurrentMap<EntityId, Consumer<DeviceProfile>> tenantListeners = profileListeners.get(profile.getTenantId());
139 if (tenantListeners != null) { 148 if (tenantListeners != null) {
140 tenantListeners.forEach((id, listener) -> listener.accept(profile)); 149 tenantListeners.forEach((id, listener) -> listener.accept(profile));
141 } 150 }
142 } 151 }
143 152
  153 + private void notifyDeviceListeners(TenantId tenantId, DeviceId deviceId, DeviceProfile profile) {
  154 + if (profile != null) {
  155 + ConcurrentMap<EntityId, BiConsumer<DeviceId, DeviceProfile>> tenantListeners = deviceProfileListeners.get(tenantId);
  156 + if (tenantListeners != null) {
  157 + tenantListeners.forEach((id, listener) -> listener.accept(deviceId, profile));
  158 + }
  159 + }
  160 + }
  161 +
144 } 162 }
@@ -23,11 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId; @@ -23,11 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId;
23 23
24 public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache { 24 public interface TbDeviceProfileCache extends RuleEngineDeviceProfileCache {
25 25
26 - void put(DeviceProfile profile);  
27 -  
28 void evict(TenantId tenantId, DeviceProfileId id); 26 void evict(TenantId tenantId, DeviceProfileId id);
29 27
30 - void evict(DeviceId id); 28 + void evict(TenantId tenantId, DeviceId id);
31 29
32 DeviceProfile find(DeviceProfileId deviceProfileId); 30 DeviceProfile find(DeviceProfileId deviceProfileId);
33 31
@@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value; @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value;
21 import org.springframework.scheduling.annotation.Scheduled; 21 import org.springframework.scheduling.annotation.Scheduled;
22 import org.springframework.stereotype.Service; 22 import org.springframework.stereotype.Service;
23 import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; 23 import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg;
  24 +import org.thingsboard.server.common.data.ApiUsageState;
24 import org.thingsboard.server.common.data.DeviceProfile; 25 import org.thingsboard.server.common.data.DeviceProfile;
25 import org.thingsboard.server.common.data.EntityType; 26 import org.thingsboard.server.common.data.EntityType;
26 import org.thingsboard.server.common.data.HasName; 27 import org.thingsboard.server.common.data.HasName;
@@ -47,6 +48,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica @@ -47,6 +48,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotifica
47 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 48 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
48 import org.thingsboard.server.queue.TbQueueCallback; 49 import org.thingsboard.server.queue.TbQueueCallback;
49 import org.thingsboard.server.queue.TbQueueProducer; 50 import org.thingsboard.server.queue.TbQueueProducer;
  51 +import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper;
50 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 52 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
51 import org.thingsboard.server.queue.discovery.PartitionService; 53 import org.thingsboard.server.queue.discovery.PartitionService;
52 import org.thingsboard.server.queue.provider.TbQueueProducerProvider; 54 import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
@@ -156,8 +158,16 @@ public class DefaultTbClusterService implements TbClusterService { @@ -156,8 +158,16 @@ public class DefaultTbClusterService implements TbClusterService {
156 private TbMsg transformMsg(TbMsg tbMsg, DeviceProfile deviceProfile) { 158 private TbMsg transformMsg(TbMsg tbMsg, DeviceProfile deviceProfile) {
157 if (deviceProfile != null) { 159 if (deviceProfile != null) {
158 RuleChainId targetRuleChainId = deviceProfile.getDefaultRuleChainId(); 160 RuleChainId targetRuleChainId = deviceProfile.getDefaultRuleChainId();
159 - if (targetRuleChainId != null && !targetRuleChainId.equals(tbMsg.getRuleChainId())) { 161 + String targetQueueName = deviceProfile.getDefaultQueueName();
  162 + boolean isRuleChainTransform = targetRuleChainId != null && !targetRuleChainId.equals(tbMsg.getRuleChainId());
  163 + boolean isQueueTransform = targetQueueName != null && !targetQueueName.equals(tbMsg.getQueueName());
  164 +
  165 + if (isRuleChainTransform && isQueueTransform) {
  166 + tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId, targetQueueName);
  167 + } else if (isRuleChainTransform) {
160 tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId); 168 tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId);
  169 + } else if (isQueueTransform) {
  170 + tbMsg = TbMsg.transformMsg(tbMsg, targetQueueName);
161 } 171 }
162 } 172 }
163 return tbMsg; 173 return tbMsg;
@@ -207,6 +217,12 @@ public class DefaultTbClusterService implements TbClusterService { @@ -207,6 +217,12 @@ public class DefaultTbClusterService implements TbClusterService {
207 } 217 }
208 218
209 @Override 219 @Override
  220 + public void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback) {
  221 + onEntityChange(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState, callback);
  222 + broadcast(new ComponentLifecycleMsg(apiUsageState.getTenantId(), apiUsageState.getId(), ComponentLifecycleEvent.UPDATED));
  223 + }
  224 +
  225 + @Override
210 public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) { 226 public void onDeviceProfileDelete(DeviceProfile entity, TbQueueCallback callback) {
211 onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback); 227 onEntityDelete(entity.getTenantId(), entity.getId(), entity.getName(), callback);
212 } 228 }
@@ -221,13 +237,14 @@ public class DefaultTbClusterService implements TbClusterService { @@ -221,13 +237,14 @@ public class DefaultTbClusterService implements TbClusterService {
221 onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback); 237 onEntityDelete(TenantId.SYS_TENANT_ID, entity.getId(), entity.getName(), callback);
222 } 238 }
223 239
224 - public <T extends HasName> void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) {  
225 - log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entity.getName()); 240 + public <T> void onEntityChange(TenantId tenantId, EntityId entityid, T entity, TbQueueCallback callback) {
  241 + String entityName = (entity instanceof HasName) ? ((HasName) entity).getName() : entity.getClass().getName();
  242 + log.trace("[{}][{}][{}] Processing [{}] change event", tenantId, entityid.getEntityType(), entityid.getId(), entityName);
226 TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder() 243 TransportProtos.EntityUpdateMsg entityUpdateMsg = TransportProtos.EntityUpdateMsg.newBuilder()
227 .setEntityType(entityid.getEntityType().name()) 244 .setEntityType(entityid.getEntityType().name())
228 .setData(ByteString.copyFrom(encodingService.encode(entity))).build(); 245 .setData(ByteString.copyFrom(encodingService.encode(entity))).build();
229 ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build(); 246 ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityUpdateMsg(entityUpdateMsg).build();
230 - broadcast(transportMsg); 247 + broadcast(transportMsg, callback);
231 } 248 }
232 249
233 private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) { 250 private void onEntityDelete(TenantId tenantId, EntityId entityId, String name, TbQueueCallback callback) {
@@ -238,15 +255,16 @@ public class DefaultTbClusterService implements TbClusterService { @@ -238,15 +255,16 @@ public class DefaultTbClusterService implements TbClusterService {
238 .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) 255 .setEntityIdLSB(entityId.getId().getLeastSignificantBits())
239 .build(); 256 .build();
240 ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build(); 257 ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setEntityDeleteMsg(entityDeleteMsg).build();
241 - broadcast(transportMsg); 258 + broadcast(transportMsg, callback);
242 } 259 }
243 260
244 - private void broadcast(ToTransportMsg transportMsg) { 261 + private void broadcast(ToTransportMsg transportMsg, TbQueueCallback callback) {
245 TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer(); 262 TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer();
246 Set<String> tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT); 263 Set<String> tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT);
  264 + TbQueueCallback proxyCallback = callback != null ? new MultipleTbQueueCallbackWrapper(tbTransportServices.size(), callback) : null;
247 for (String transportServiceId : tbTransportServices) { 265 for (String transportServiceId : tbTransportServices) {
248 TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId); 266 TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId);
249 - toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); 267 + toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), proxyCallback);
250 toTransportNfs.incrementAndGet(); 268 toTransportNfs.incrementAndGet();
251 } 269 }
252 } 270 }
@@ -256,7 +274,8 @@ public class DefaultTbClusterService implements TbClusterService { @@ -256,7 +274,8 @@ public class DefaultTbClusterService implements TbClusterService {
256 TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); 274 TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer();
257 Set<String> tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); 275 Set<String> tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE));
258 if (msg.getEntityId().getEntityType().equals(EntityType.TENANT) 276 if (msg.getEntityId().getEntityType().equals(EntityType.TENANT)
259 - || msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE)) { 277 + || msg.getEntityId().getEntityType().equals(EntityType.DEVICE_PROFILE)
  278 + || msg.getEntityId().getEntityType().equals(EntityType.API_USAGE_STATE)) {
260 TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); 279 TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer();
261 Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); 280 Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE);
262 for (String serviceId : tbCoreServices) { 281 for (String serviceId : tbCoreServices) {
@@ -17,41 +17,45 @@ package org.thingsboard.server.service.queue; @@ -17,41 +17,45 @@ package org.thingsboard.server.service.queue;
17 17
18 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
19 import org.springframework.beans.factory.annotation.Value; 19 import org.springframework.beans.factory.annotation.Value;
  20 +import org.springframework.boot.context.event.ApplicationReadyEvent;
  21 +import org.springframework.context.event.EventListener;
  22 +import org.springframework.core.annotation.Order;
20 import org.springframework.scheduling.annotation.Scheduled; 23 import org.springframework.scheduling.annotation.Scheduled;
21 import org.springframework.stereotype.Service; 24 import org.springframework.stereotype.Service;
  25 +import org.thingsboard.common.util.ThingsBoardThreadFactory;
22 import org.thingsboard.rule.engine.api.RpcError; 26 import org.thingsboard.rule.engine.api.RpcError;
23 import org.thingsboard.server.actors.ActorSystemContext; 27 import org.thingsboard.server.actors.ActorSystemContext;
24 -import org.thingsboard.server.common.data.EntityType;  
25 import org.thingsboard.server.common.data.alarm.Alarm; 28 import org.thingsboard.server.common.data.alarm.Alarm;
26 -import org.thingsboard.server.common.data.id.DeviceProfileId;  
27 import org.thingsboard.server.common.data.id.TenantId; 29 import org.thingsboard.server.common.data.id.TenantId;
28 import org.thingsboard.server.common.msg.MsgType; 30 import org.thingsboard.server.common.msg.MsgType;
29 import org.thingsboard.server.common.msg.TbActorMsg; 31 import org.thingsboard.server.common.msg.TbActorMsg;
30 -import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;  
31 import org.thingsboard.server.common.msg.queue.ServiceType; 32 import org.thingsboard.server.common.msg.queue.ServiceType;
32 import org.thingsboard.server.common.msg.queue.TbCallback; 33 import org.thingsboard.server.common.msg.queue.TbCallback;
  34 +import org.thingsboard.server.common.stats.StatsFactory;
  35 +import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
33 import org.thingsboard.server.dao.util.mapping.JacksonUtil; 36 import org.thingsboard.server.dao.util.mapping.JacksonUtil;
34 import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; 37 import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
35 import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; 38 import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
36 import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; 39 import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto;
37 import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto; 40 import org.thingsboard.server.gen.transport.TransportProtos.SubscriptionMgrMsgProto;
38 -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto;  
39 -import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto;  
40 -import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto;  
41 import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto; 41 import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto;
  42 +import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto;
  43 +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto;
  44 +import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto;
42 import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto; 45 import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseProto;
43 import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; 46 import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto;
44 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; 47 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
45 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; 48 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  49 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
46 import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; 50 import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
47 import org.thingsboard.server.queue.TbQueueConsumer; 51 import org.thingsboard.server.queue.TbQueueConsumer;
48 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 52 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
49 import org.thingsboard.server.queue.discovery.PartitionChangeEvent; 53 import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
50 import org.thingsboard.server.queue.provider.TbCoreQueueFactory; 54 import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
51 -import org.thingsboard.server.common.stats.StatsFactory;  
52 import org.thingsboard.server.queue.util.TbCoreComponent; 55 import org.thingsboard.server.queue.util.TbCoreComponent;
53 -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; 56 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
54 import org.thingsboard.server.service.profile.TbDeviceProfileCache; 57 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
  58 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
55 import org.thingsboard.server.service.queue.processing.AbstractConsumerService; 59 import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
56 import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; 60 import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
57 import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; 61 import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
@@ -70,6 +74,8 @@ import java.util.UUID; @@ -70,6 +74,8 @@ import java.util.UUID;
70 import java.util.concurrent.ConcurrentHashMap; 74 import java.util.concurrent.ConcurrentHashMap;
71 import java.util.concurrent.ConcurrentMap; 75 import java.util.concurrent.ConcurrentMap;
72 import java.util.concurrent.CountDownLatch; 76 import java.util.concurrent.CountDownLatch;
  77 +import java.util.concurrent.ExecutorService;
  78 +import java.util.concurrent.Executors;
73 import java.util.concurrent.TimeUnit; 79 import java.util.concurrent.TimeUnit;
74 import java.util.function.Function; 80 import java.util.function.Function;
75 import java.util.stream.Collectors; 81 import java.util.stream.Collectors;
@@ -88,32 +94,57 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore @@ -88,32 +94,57 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
88 94
89 private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer; 95 private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer;
90 private final DeviceStateService stateService; 96 private final DeviceStateService stateService;
  97 + private final TbApiUsageStateService statsService;
91 private final TbLocalSubscriptionService localSubscriptionService; 98 private final TbLocalSubscriptionService localSubscriptionService;
92 private final SubscriptionManagerService subscriptionManagerService; 99 private final SubscriptionManagerService subscriptionManagerService;
93 private final TbCoreDeviceRpcService tbCoreDeviceRpcService; 100 private final TbCoreDeviceRpcService tbCoreDeviceRpcService;
94 private final TbCoreConsumerStats stats; 101 private final TbCoreConsumerStats stats;
  102 + protected final TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
  103 +
  104 + protected volatile ExecutorService usageStatsExecutor;
95 105
96 - public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext,  
97 - DeviceStateService stateService, TbLocalSubscriptionService localSubscriptionService,  
98 - SubscriptionManagerService subscriptionManagerService, DataDecodingEncodingService encodingService,  
99 - TbCoreDeviceRpcService tbCoreDeviceRpcService, StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache) {  
100 - super(actorContext, encodingService, deviceProfileCache, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer()); 106 + public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory,
  107 + ActorSystemContext actorContext,
  108 + DeviceStateService stateService,
  109 + TbLocalSubscriptionService localSubscriptionService,
  110 + SubscriptionManagerService subscriptionManagerService,
  111 + DataDecodingEncodingService encodingService,
  112 + TbCoreDeviceRpcService tbCoreDeviceRpcService,
  113 + StatsFactory statsFactory,
  114 + TbDeviceProfileCache deviceProfileCache,
  115 + TbApiUsageStateService statsService,
  116 + TbTenantProfileCache tenantProfileCache,
  117 + TbApiUsageStateService apiUsageStateService) {
  118 + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
101 this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); 119 this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
  120 + this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
102 this.stateService = stateService; 121 this.stateService = stateService;
103 this.localSubscriptionService = localSubscriptionService; 122 this.localSubscriptionService = localSubscriptionService;
104 this.subscriptionManagerService = subscriptionManagerService; 123 this.subscriptionManagerService = subscriptionManagerService;
105 this.tbCoreDeviceRpcService = tbCoreDeviceRpcService; 124 this.tbCoreDeviceRpcService = tbCoreDeviceRpcService;
106 this.stats = new TbCoreConsumerStats(statsFactory); 125 this.stats = new TbCoreConsumerStats(statsFactory);
  126 + this.statsService = statsService;
107 } 127 }
108 128
109 @PostConstruct 129 @PostConstruct
110 public void init() { 130 public void init() {
111 super.init("tb-core-consumer", "tb-core-notifications-consumer"); 131 super.init("tb-core-consumer", "tb-core-notifications-consumer");
  132 + this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
112 } 133 }
113 134
114 @PreDestroy 135 @PreDestroy
115 public void destroy() { 136 public void destroy() {
116 super.destroy(); 137 super.destroy();
  138 + if (usageStatsExecutor != null) {
  139 + usageStatsExecutor.shutdownNow();
  140 + }
  141 + }
  142 +
  143 + @EventListener(ApplicationReadyEvent.class)
  144 + @Order(value = 2)
  145 + public void onApplicationEvent(ApplicationReadyEvent event) {
  146 + super.onApplicationEvent(event);
  147 + launchUsageStatsConsumer();
117 } 148 }
118 149
119 @Override 150 @Override
@@ -121,6 +152,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore @@ -121,6 +152,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
121 if (partitionChangeEvent.getServiceType().equals(getServiceType())) { 152 if (partitionChangeEvent.getServiceType().equals(getServiceType())) {
122 log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions()); 153 log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions());
123 this.mainConsumer.subscribe(partitionChangeEvent.getPartitions()); 154 this.mainConsumer.subscribe(partitionChangeEvent.getPartitions());
  155 + this.usageStatsConsumer.subscribe(
  156 + partitionChangeEvent
  157 + .getPartitions()
  158 + .stream()
  159 + .map(tpi -> tpi.newByTopic(usageStatsConsumer.getTopic()))
  160 + .collect(Collectors.toSet()));
124 } 161 }
125 } 162 }
126 163
@@ -223,6 +260,53 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore @@ -223,6 +260,53 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
223 } 260 }
224 } 261 }
225 262
  263 + private void launchUsageStatsConsumer() {
  264 + usageStatsExecutor.submit(() -> {
  265 + while (!stopped) {
  266 + try {
  267 + List<TbProtoQueueMsg<ToUsageStatsServiceMsg>> msgs = usageStatsConsumer.poll(getNotificationPollDuration());
  268 + if (msgs.isEmpty()) {
  269 + continue;
  270 + }
  271 + ConcurrentMap<UUID, TbProtoQueueMsg<ToUsageStatsServiceMsg>> pendingMap = msgs.stream().collect(
  272 + Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
  273 + CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
  274 + TbPackProcessingContext<TbProtoQueueMsg<ToUsageStatsServiceMsg>> ctx = new TbPackProcessingContext<>(
  275 + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
  276 + pendingMap.forEach((id, msg) -> {
  277 + log.trace("[{}] Creating usage stats callback for message: {}", id, msg.getValue());
  278 + TbCallback callback = new TbPackCallback<>(id, ctx);
  279 + try {
  280 + handleUsageStats(msg, callback);
  281 + } catch (Throwable e) {
  282 + log.warn("[{}] Failed to process usge stats: {}", id, msg, e);
  283 + callback.onFailure(e);
  284 + }
  285 + });
  286 + if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
  287 + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process usage stats: {}", id, msg.getValue()));
  288 + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process usage stats: {}", id, msg.getValue()));
  289 + }
  290 + usageStatsConsumer.commit();
  291 + } catch (Exception e) {
  292 + if (!stopped) {
  293 + log.warn("Failed to obtain usage stats from queue.", e);
  294 + try {
  295 + Thread.sleep(getNotificationPollDuration());
  296 + } catch (InterruptedException e2) {
  297 + log.trace("Failed to wait until the server has capacity to handle new usage stats", e2);
  298 + }
  299 + }
  300 + }
  301 + }
  302 + log.info("TB Usage Stats Consumer stopped.");
  303 + });
  304 + }
  305 +
  306 + private void handleUsageStats(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
  307 + statsService.process(msg, callback);
  308 + }
  309 +
226 private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) { 310 private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) {
227 RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null; 311 RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null;
228 FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB()) 312 FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB())
@@ -321,6 +405,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore @@ -321,6 +405,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
321 if (mainConsumer != null) { 405 if (mainConsumer != null) {
322 mainConsumer.unsubscribe(); 406 mainConsumer.unsubscribe();
323 } 407 }
  408 + if (usageStatsConsumer != null) {
  409 + usageStatsConsumer.unsubscribe();
  410 + }
324 } 411 }
325 412
326 } 413 }
@@ -15,7 +15,6 @@ @@ -15,7 +15,6 @@
15 */ 15 */
16 package org.thingsboard.server.service.queue; 16 package org.thingsboard.server.service.queue;
17 17
18 -import com.google.protobuf.ByteString;  
19 import com.google.protobuf.ProtocolStringList; 18 import com.google.protobuf.ProtocolStringList;
20 import lombok.extern.slf4j.Slf4j; 19 import lombok.extern.slf4j.Slf4j;
21 import org.springframework.beans.factory.annotation.Value; 20 import org.springframework.beans.factory.annotation.Value;
@@ -24,10 +23,16 @@ import org.springframework.stereotype.Service; @@ -24,10 +23,16 @@ import org.springframework.stereotype.Service;
24 import org.thingsboard.rule.engine.api.RpcError; 23 import org.thingsboard.rule.engine.api.RpcError;
25 import org.thingsboard.server.actors.ActorSystemContext; 24 import org.thingsboard.server.actors.ActorSystemContext;
26 import org.thingsboard.server.common.data.id.TenantId; 25 import org.thingsboard.server.common.data.id.TenantId;
27 -import org.thingsboard.server.common.msg.TbActorMsg;  
28 import org.thingsboard.server.common.msg.TbMsg; 26 import org.thingsboard.server.common.msg.TbMsg;
29 -import org.thingsboard.server.common.msg.queue.*; 27 +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
  28 +import org.thingsboard.server.common.msg.queue.RuleEngineException;
  29 +import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
  30 +import org.thingsboard.server.common.msg.queue.ServiceQueue;
  31 +import org.thingsboard.server.common.msg.queue.ServiceType;
  32 +import org.thingsboard.server.common.msg.queue.TbCallback;
  33 +import org.thingsboard.server.common.msg.queue.TbMsgCallback;
30 import org.thingsboard.server.common.stats.StatsFactory; 34 import org.thingsboard.server.common.stats.StatsFactory;
  35 +import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
31 import org.thingsboard.server.gen.transport.TransportProtos; 36 import org.thingsboard.server.gen.transport.TransportProtos;
32 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 37 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
33 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; 38 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@@ -38,17 +43,33 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; @@ -38,17 +43,33 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
38 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; 43 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
39 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; 44 import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
40 import org.thingsboard.server.queue.util.TbRuleEngineComponent; 45 import org.thingsboard.server.queue.util.TbRuleEngineComponent;
41 -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; 46 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
42 import org.thingsboard.server.service.profile.TbDeviceProfileCache; 47 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
43 -import org.thingsboard.server.service.queue.processing.*; 48 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
  49 +import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
  50 +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
  51 +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
  52 +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy;
  53 +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
  54 +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
  55 +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
44 import org.thingsboard.server.service.rpc.FromDeviceRpcResponse; 56 import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
45 import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; 57 import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
46 import org.thingsboard.server.service.stats.RuleEngineStatisticsService; 58 import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
47 59
48 import javax.annotation.PostConstruct; 60 import javax.annotation.PostConstruct;
49 import javax.annotation.PreDestroy; 61 import javax.annotation.PreDestroy;
50 -import java.util.*;  
51 -import java.util.concurrent.*; 62 +import java.util.Collections;
  63 +import java.util.HashSet;
  64 +import java.util.List;
  65 +import java.util.Map;
  66 +import java.util.Set;
  67 +import java.util.UUID;
  68 +import java.util.concurrent.ConcurrentHashMap;
  69 +import java.util.concurrent.ConcurrentMap;
  70 +import java.util.concurrent.ExecutorService;
  71 +import java.util.concurrent.Executors;
  72 +import java.util.concurrent.TimeUnit;
52 73
53 @Service 74 @Service
54 @TbRuleEngineComponent 75 @TbRuleEngineComponent
@@ -79,11 +100,16 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @@ -79,11 +100,16 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
79 public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, 100 public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory,
80 TbRuleEngineSubmitStrategyFactory submitStrategyFactory, 101 TbRuleEngineSubmitStrategyFactory submitStrategyFactory,
81 TbQueueRuleEngineSettings ruleEngineSettings, 102 TbQueueRuleEngineSettings ruleEngineSettings,
82 - TbRuleEngineQueueFactory tbRuleEngineQueueFactory, RuleEngineStatisticsService statisticsService,  
83 - ActorSystemContext actorContext, DataDecodingEncodingService encodingService, 103 + TbRuleEngineQueueFactory tbRuleEngineQueueFactory,
  104 + RuleEngineStatisticsService statisticsService,
  105 + ActorSystemContext actorContext,
  106 + DataDecodingEncodingService encodingService,
84 TbRuleEngineDeviceRpcService tbDeviceRpcService, 107 TbRuleEngineDeviceRpcService tbDeviceRpcService,
85 - StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache) {  
86 - super(actorContext, encodingService, deviceProfileCache, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); 108 + StatsFactory statsFactory,
  109 + TbDeviceProfileCache deviceProfileCache,
  110 + TbTenantProfileCache tenantProfileCache,
  111 + TbApiUsageStateService apiUsageStateService) {
  112 + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer());
87 this.statisticsService = statisticsService; 113 this.statisticsService = statisticsService;
88 this.ruleEngineSettings = ruleEngineSettings; 114 this.ruleEngineSettings = ruleEngineSettings;
89 this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; 115 this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory;
@@ -21,11 +21,10 @@ import org.springframework.stereotype.Service; @@ -21,11 +21,10 @@ import org.springframework.stereotype.Service;
21 import org.thingsboard.server.common.data.Tenant; 21 import org.thingsboard.server.common.data.Tenant;
22 import org.thingsboard.server.common.data.TenantProfile; 22 import org.thingsboard.server.common.data.TenantProfile;
23 import org.thingsboard.server.common.data.id.TenantId; 23 import org.thingsboard.server.common.data.id.TenantId;
24 -import org.thingsboard.server.dao.tenant.TenantProfileService;  
25 import org.thingsboard.server.dao.tenant.TenantService; 24 import org.thingsboard.server.dao.tenant.TenantService;
26 import org.thingsboard.server.queue.discovery.TenantRoutingInfo; 25 import org.thingsboard.server.queue.discovery.TenantRoutingInfo;
27 import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; 26 import org.thingsboard.server.queue.discovery.TenantRoutingInfoService;
28 -import org.thingsboard.server.service.profile.TbTenantProfileCache; 27 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
29 28
30 @Slf4j 29 @Slf4j
31 @Service 30 @Service
@@ -16,6 +16,7 @@ @@ -16,6 +16,7 @@
16 package org.thingsboard.server.service.queue; 16 package org.thingsboard.server.service.queue;
17 17
18 import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg; 18 import org.thingsboard.rule.engine.api.msg.ToDeviceActorNotificationMsg;
  19 +import org.thingsboard.server.common.data.ApiUsageState;
19 import org.thingsboard.server.common.data.DeviceProfile; 20 import org.thingsboard.server.common.data.DeviceProfile;
20 import org.thingsboard.server.common.data.Tenant; 21 import org.thingsboard.server.common.data.Tenant;
21 import org.thingsboard.server.common.data.TenantProfile; 22 import org.thingsboard.server.common.data.TenantProfile;
@@ -63,4 +64,6 @@ public interface TbClusterService { @@ -63,4 +64,6 @@ public interface TbClusterService {
63 void onTenantChange(Tenant tenant, TbQueueCallback callback); 64 void onTenantChange(Tenant tenant, TbQueueCallback callback);
64 65
65 void onTenantDelete(Tenant tenant, TbQueueCallback callback); 66 void onTenantDelete(Tenant tenant, TbQueueCallback callback);
  67 +
  68 + void onApiStateChange(ApiUsageState apiUsageState, TbQueueCallback callback);
66 } 69 }
@@ -20,11 +20,14 @@ import lombok.extern.slf4j.Slf4j; @@ -20,11 +20,14 @@ import lombok.extern.slf4j.Slf4j;
20 import org.springframework.boot.context.event.ApplicationReadyEvent; 20 import org.springframework.boot.context.event.ApplicationReadyEvent;
21 import org.springframework.context.ApplicationListener; 21 import org.springframework.context.ApplicationListener;
22 import org.springframework.context.event.EventListener; 22 import org.springframework.context.event.EventListener;
  23 +import org.springframework.core.annotation.Order;
23 import org.thingsboard.common.util.ThingsBoardThreadFactory; 24 import org.thingsboard.common.util.ThingsBoardThreadFactory;
24 import org.thingsboard.server.actors.ActorSystemContext; 25 import org.thingsboard.server.actors.ActorSystemContext;
25 import org.thingsboard.server.common.data.EntityType; 26 import org.thingsboard.server.common.data.EntityType;
26 import org.thingsboard.server.common.data.id.DeviceId; 27 import org.thingsboard.server.common.data.id.DeviceId;
27 import org.thingsboard.server.common.data.id.DeviceProfileId; 28 import org.thingsboard.server.common.data.id.DeviceProfileId;
  29 +import org.thingsboard.server.common.data.id.TenantProfileId;
  30 +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
28 import org.thingsboard.server.common.msg.TbActorMsg; 31 import org.thingsboard.server.common.msg.TbActorMsg;
29 import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; 32 import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
30 import org.thingsboard.server.common.msg.queue.ServiceType; 33 import org.thingsboard.server.common.msg.queue.ServiceType;
@@ -33,7 +36,9 @@ import org.thingsboard.server.queue.TbQueueConsumer; @@ -33,7 +36,9 @@ import org.thingsboard.server.queue.TbQueueConsumer;
33 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 36 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
34 import org.thingsboard.server.queue.discovery.PartitionChangeEvent; 37 import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
35 import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; 38 import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
  39 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
36 import org.thingsboard.server.service.profile.TbDeviceProfileCache; 40 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
  41 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
37 import org.thingsboard.server.service.queue.TbPackCallback; 42 import org.thingsboard.server.service.queue.TbPackCallback;
38 import org.thingsboard.server.service.queue.TbPackProcessingContext; 43 import org.thingsboard.server.service.queue.TbPackProcessingContext;
39 44
@@ -59,15 +64,19 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene @@ -59,15 +64,19 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
59 64
60 protected final ActorSystemContext actorContext; 65 protected final ActorSystemContext actorContext;
61 protected final DataDecodingEncodingService encodingService; 66 protected final DataDecodingEncodingService encodingService;
  67 + protected final TbTenantProfileCache tenantProfileCache;
62 protected final TbDeviceProfileCache deviceProfileCache; 68 protected final TbDeviceProfileCache deviceProfileCache;
  69 + protected final TbApiUsageStateService apiUsageStateService;
63 70
64 protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer; 71 protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer;
65 72
66 public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, 73 public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService,
67 - TbDeviceProfileCache deviceProfileCache, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) { 74 + TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbApiUsageStateService apiUsageStateService, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
68 this.actorContext = actorContext; 75 this.actorContext = actorContext;
69 this.encodingService = encodingService; 76 this.encodingService = encodingService;
  77 + this.tenantProfileCache = tenantProfileCache;
70 this.deviceProfileCache = deviceProfileCache; 78 this.deviceProfileCache = deviceProfileCache;
  79 + this.apiUsageStateService = apiUsageStateService;
71 this.nfConsumer = nfConsumer; 80 this.nfConsumer = nfConsumer;
72 } 81 }
73 82
@@ -77,6 +86,7 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene @@ -77,6 +86,7 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
77 } 86 }
78 87
79 @EventListener(ApplicationReadyEvent.class) 88 @EventListener(ApplicationReadyEvent.class)
  89 + @Order(value = 2)
80 public void onApplicationEvent(ApplicationReadyEvent event) { 90 public void onApplicationEvent(ApplicationReadyEvent event) {
81 log.info("Subscribing to notifications: {}", nfConsumer.getTopic()); 91 log.info("Subscribing to notifications: {}", nfConsumer.getTopic());
82 this.nfConsumer.subscribe(); 92 this.nfConsumer.subscribe();
@@ -143,10 +153,23 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene @@ -143,10 +153,23 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
143 TbActorMsg actorMsg = actorMsgOpt.get(); 153 TbActorMsg actorMsg = actorMsgOpt.get();
144 if (actorMsg instanceof ComponentLifecycleMsg) { 154 if (actorMsg instanceof ComponentLifecycleMsg) {
145 ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg; 155 ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg;
146 - if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { 156 + if (EntityType.TENANT_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
  157 + TenantProfileId tenantProfileId = new TenantProfileId(componentLifecycleMsg.getEntityId().getId());
  158 + tenantProfileCache.evict(tenantProfileId);
  159 + if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
  160 + apiUsageStateService.onTenantProfileUpdate(tenantProfileId);
  161 + }
  162 + } else if (EntityType.TENANT.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
  163 + tenantProfileCache.evict(componentLifecycleMsg.getTenantId());
  164 + if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
  165 + apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId());
  166 + }
  167 + } else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
147 deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId())); 168 deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId()));
148 } else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { 169 } else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
149 - deviceProfileCache.evict(new DeviceId(componentLifecycleMsg.getEntityId().getId())); 170 + deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId()));
  171 + } else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
  172 + apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId());
150 } 173 }
151 } 174 }
152 log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg); 175 log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);
@@ -19,6 +19,10 @@ import com.google.common.util.concurrent.Futures; @@ -19,6 +19,10 @@ import com.google.common.util.concurrent.Futures;
19 import com.google.common.util.concurrent.ListenableFuture; 19 import com.google.common.util.concurrent.ListenableFuture;
20 import lombok.extern.slf4j.Slf4j; 20 import lombok.extern.slf4j.Slf4j;
21 import org.thingsboard.common.util.ThingsBoardThreadFactory; 21 import org.thingsboard.common.util.ThingsBoardThreadFactory;
  22 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
  23 +import org.thingsboard.server.common.data.id.TenantId;
  24 +import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
  25 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
22 26
23 import java.util.Map; 27 import java.util.Map;
24 import java.util.UUID; 28 import java.util.UUID;
@@ -33,9 +37,16 @@ import java.util.concurrent.atomic.AtomicInteger; @@ -33,9 +37,16 @@ import java.util.concurrent.atomic.AtomicInteger;
33 @Slf4j 37 @Slf4j
34 public abstract class AbstractJsInvokeService implements JsInvokeService { 38 public abstract class AbstractJsInvokeService implements JsInvokeService {
35 39
  40 + private final TbApiUsageStateService apiUsageStateService;
  41 + private final TbApiUsageClient apiUsageClient;
36 protected ScheduledExecutorService timeoutExecutorService; 42 protected ScheduledExecutorService timeoutExecutorService;
37 protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>(); 43 protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>();
38 - protected Map<UUID, BlackListInfo> blackListedFunctions = new ConcurrentHashMap<>(); 44 + protected Map<UUID, DisableListInfo> disabledFunctions = new ConcurrentHashMap<>();
  45 +
  46 + protected AbstractJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) {
  47 + this.apiUsageStateService = apiUsageStateService;
  48 + this.apiUsageClient = apiUsageClient;
  49 + }
39 50
40 public void init(long maxRequestsTimeout) { 51 public void init(long maxRequestsTimeout) {
41 if (maxRequestsTimeout > 0) { 52 if (maxRequestsTimeout > 0) {
@@ -50,24 +61,33 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { @@ -50,24 +61,33 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
50 } 61 }
51 62
52 @Override 63 @Override
53 - public ListenableFuture<UUID> eval(JsScriptType scriptType, String scriptBody, String... argNames) {  
54 - UUID scriptId = UUID.randomUUID();  
55 - String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_');  
56 - String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames);  
57 - return doEval(scriptId, functionName, jsScript); 64 + public ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) {
  65 + if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
  66 + UUID scriptId = UUID.randomUUID();
  67 + String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_');
  68 + String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames);
  69 + return doEval(scriptId, functionName, jsScript);
  70 + } else {
  71 + return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!"));
  72 + }
58 } 73 }
59 74
60 @Override 75 @Override
61 - public ListenableFuture<Object> invokeFunction(UUID scriptId, Object... args) {  
62 - String functionName = scriptIdToNameMap.get(scriptId);  
63 - if (functionName == null) {  
64 - return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!"));  
65 - }  
66 - if (!isBlackListed(scriptId)) {  
67 - return doInvokeFunction(scriptId, functionName, args); 76 + public ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args) {
  77 + if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
  78 + String functionName = scriptIdToNameMap.get(scriptId);
  79 + if (functionName == null) {
  80 + return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!"));
  81 + }
  82 + if (!isDisabled(scriptId)) {
  83 + apiUsageClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1);
  84 + return doInvokeFunction(scriptId, functionName, args);
  85 + } else {
  86 + return Futures.immediateFailedFuture(
  87 + new RuntimeException("Script invocation is blocked due to maximum error count " + getMaxErrors() + "!"));
  88 + }
68 } else { 89 } else {
69 - return Futures.immediateFailedFuture(  
70 - new RuntimeException("Script is blacklisted due to maximum error count " + getMaxErrors() + "!")); 90 + return Futures.immediateFailedFuture(new RuntimeException("JS Execution is disabled due to API limits!"));
71 } 91 }
72 } 92 }
73 93
@@ -77,7 +97,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { @@ -77,7 +97,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
77 if (functionName != null) { 97 if (functionName != null) {
78 try { 98 try {
79 scriptIdToNameMap.remove(scriptId); 99 scriptIdToNameMap.remove(scriptId);
80 - blackListedFunctions.remove(scriptId); 100 + disabledFunctions.remove(scriptId);
81 doRelease(scriptId, functionName); 101 doRelease(scriptId, functionName);
82 } catch (Exception e) { 102 } catch (Exception e) {
83 return Futures.immediateFailedFuture(e); 103 return Futures.immediateFailedFuture(e);
@@ -97,7 +117,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { @@ -97,7 +117,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
97 protected abstract long getMaxBlacklistDuration(); 117 protected abstract long getMaxBlacklistDuration();
98 118
99 protected void onScriptExecutionError(UUID scriptId) { 119 protected void onScriptExecutionError(UUID scriptId) {
100 - blackListedFunctions.computeIfAbsent(scriptId, key -> new BlackListInfo()).incrementAndGet(); 120 + disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo()).incrementAndGet();
101 } 121 }
102 122
103 private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) { 123 private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) {
@@ -107,11 +127,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { @@ -107,11 +127,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
107 throw new RuntimeException("No script factory implemented for scriptType: " + scriptType); 127 throw new RuntimeException("No script factory implemented for scriptType: " + scriptType);
108 } 128 }
109 129
110 - private boolean isBlackListed(UUID scriptId) {  
111 - BlackListInfo errorCount = blackListedFunctions.get(scriptId); 130 + private boolean isDisabled(UUID scriptId) {
  131 + DisableListInfo errorCount = disabledFunctions.get(scriptId);
112 if (errorCount != null) { 132 if (errorCount != null) {
113 if (errorCount.getExpirationTime() <= System.currentTimeMillis()) { 133 if (errorCount.getExpirationTime() <= System.currentTimeMillis()) {
114 - blackListedFunctions.remove(scriptId); 134 + disabledFunctions.remove(scriptId);
115 return false; 135 return false;
116 } else { 136 } else {
117 return errorCount.get() >= getMaxErrors(); 137 return errorCount.get() >= getMaxErrors();
@@ -121,11 +141,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService { @@ -121,11 +141,11 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
121 } 141 }
122 } 142 }
123 143
124 - private class BlackListInfo { 144 + private class DisableListInfo {
125 private final AtomicInteger counter; 145 private final AtomicInteger counter;
126 private long expirationTime; 146 private long expirationTime;
127 147
128 - private BlackListInfo() { 148 + private DisableListInfo() {
129 this.counter = new AtomicInteger(0); 149 this.counter = new AtomicInteger(0);
130 } 150 }
131 151
@@ -24,10 +24,10 @@ import delight.nashornsandbox.NashornSandboxes; @@ -24,10 +24,10 @@ import delight.nashornsandbox.NashornSandboxes;
24 import jdk.nashorn.api.scripting.NashornScriptEngineFactory; 24 import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
25 import lombok.Getter; 25 import lombok.Getter;
26 import lombok.extern.slf4j.Slf4j; 26 import lombok.extern.slf4j.Slf4j;
27 -import org.springframework.beans.factory.annotation.Autowired;  
28 import org.springframework.beans.factory.annotation.Value; 27 import org.springframework.beans.factory.annotation.Value;
29 import org.springframework.scheduling.annotation.Scheduled; 28 import org.springframework.scheduling.annotation.Scheduled;
30 -import org.thingsboard.common.util.ThingsBoardThreadFactory; 29 +import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
  30 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
31 31
32 import javax.annotation.PostConstruct; 32 import javax.annotation.PostConstruct;
33 import javax.annotation.PreDestroy; 33 import javax.annotation.PreDestroy;
@@ -38,7 +38,6 @@ import java.util.UUID; @@ -38,7 +38,6 @@ import java.util.UUID;
38 import java.util.concurrent.ExecutionException; 38 import java.util.concurrent.ExecutionException;
39 import java.util.concurrent.ExecutorService; 39 import java.util.concurrent.ExecutorService;
40 import java.util.concurrent.Executors; 40 import java.util.concurrent.Executors;
41 -import java.util.concurrent.ScheduledExecutorService;  
42 import java.util.concurrent.TimeUnit; 41 import java.util.concurrent.TimeUnit;
43 import java.util.concurrent.atomic.AtomicInteger; 42 import java.util.concurrent.atomic.AtomicInteger;
44 43
@@ -57,9 +56,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer @@ -57,9 +56,8 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
57 private final FutureCallback<UUID> evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs); 56 private final FutureCallback<UUID> evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs);
58 private final FutureCallback<Object> invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs); 57 private final FutureCallback<Object> invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs);
59 58
60 - @Autowired  
61 @Getter 59 @Getter
62 - private JsExecutorService jsExecutor; 60 + private final JsExecutorService jsExecutor;
63 61
64 @Value("${js.local.max_requests_timeout:0}") 62 @Value("${js.local.max_requests_timeout:0}")
65 private long maxRequestsTimeout; 63 private long maxRequestsTimeout;
@@ -67,6 +65,11 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer @@ -67,6 +65,11 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
67 @Value("${js.local.stats.enabled:false}") 65 @Value("${js.local.stats.enabled:false}")
68 private boolean statsEnabled; 66 private boolean statsEnabled;
69 67
  68 + public AbstractNashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) {
  69 + super(apiUsageStateService, apiUsageClient);
  70 + this.jsExecutor = jsExecutor;
  71 + }
  72 +
70 @Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}") 73 @Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}")
71 public void printStats() { 74 public void printStats() {
72 if (statsEnabled) { 75 if (statsEnabled) {
@@ -27,7 +27,7 @@ public class JsExecutorService extends AbstractListeningExecutor { @@ -27,7 +27,7 @@ public class JsExecutorService extends AbstractListeningExecutor {
27 27
28 @Override 28 @Override
29 protected int getThreadPollSize() { 29 protected int getThreadPollSize() {
30 - return jsExecutorThreadPoolSize; 30 + return Math.max(jsExecutorThreadPoolSize, 1);
31 } 31 }
32 32
33 } 33 }
@@ -17,14 +17,15 @@ package org.thingsboard.server.service.script; @@ -17,14 +17,15 @@ package org.thingsboard.server.service.script;
17 17
18 import com.google.common.util.concurrent.ListenableFuture; 18 import com.google.common.util.concurrent.ListenableFuture;
19 import org.thingsboard.server.common.data.id.EntityId; 19 import org.thingsboard.server.common.data.id.EntityId;
  20 +import org.thingsboard.server.common.data.id.TenantId;
20 21
21 import java.util.UUID; 22 import java.util.UUID;
22 23
23 public interface JsInvokeService { 24 public interface JsInvokeService {
24 25
25 - ListenableFuture<UUID> eval(JsScriptType scriptType, String scriptBody, String... argNames); 26 + ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames);
26 27
27 - ListenableFuture<Object> invokeFunction(UUID scriptId, Object... args); 28 + ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args);
28 29
29 ListenableFuture<Void> release(UUID scriptId); 30 ListenableFuture<Void> release(UUID scriptId);
30 31
@@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j; @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j;
19 import org.springframework.beans.factory.annotation.Value; 19 import org.springframework.beans.factory.annotation.Value;
20 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 20 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
21 import org.springframework.stereotype.Service; 21 import org.springframework.stereotype.Service;
  22 +import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
  23 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
22 24
23 import java.util.concurrent.TimeUnit; 25 import java.util.concurrent.TimeUnit;
24 26
@@ -42,6 +44,10 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService { @@ -42,6 +44,10 @@ public class NashornJsInvokeService extends AbstractNashornJsInvokeService {
42 @Value("${js.local.max_black_list_duration_sec:60}") 44 @Value("${js.local.max_black_list_duration_sec:60}")
43 private int maxBlackListDurationSec; 45 private int maxBlackListDurationSec;
44 46
  47 + public NashornJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient, JsExecutorService jsExecutor) {
  48 + super(apiUsageStateService, apiUsageClient, jsExecutor);
  49 + }
  50 +
45 @Override 51 @Override
46 protected boolean useJsSandbox() { 52 protected boolean useJsSandbox() {
47 return useJsSandbox; 53 return useJsSandbox;
@@ -30,6 +30,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -30,6 +30,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
30 import org.thingsboard.server.queue.TbQueueRequestTemplate; 30 import org.thingsboard.server.queue.TbQueueRequestTemplate;
31 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; 31 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
32 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 32 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
  33 +import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
  34 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
33 35
34 import javax.annotation.Nullable; 36 import javax.annotation.Nullable;
35 import javax.annotation.PostConstruct; 37 import javax.annotation.PostConstruct;
@@ -68,6 +70,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { @@ -68,6 +70,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
68 private final AtomicInteger queueFailedMsgs = new AtomicInteger(0); 70 private final AtomicInteger queueFailedMsgs = new AtomicInteger(0);
69 private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0); 71 private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0);
70 72
  73 + public RemoteJsInvokeService(TbApiUsageStateService apiUsageStateService, TbApiUsageClient apiUsageClient) {
  74 + super(apiUsageStateService, apiUsageClient);
  75 + }
  76 +
71 @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}") 77 @Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}")
72 public void printStats() { 78 public void printStats() {
73 if (statsEnabled) { 79 if (statsEnabled) {
@@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors; @@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors;
25 import lombok.extern.slf4j.Slf4j; 25 import lombok.extern.slf4j.Slf4j;
26 import org.apache.commons.lang3.StringUtils; 26 import org.apache.commons.lang3.StringUtils;
27 import org.thingsboard.server.common.data.id.EntityId; 27 import org.thingsboard.server.common.data.id.EntityId;
  28 +import org.thingsboard.server.common.data.id.TenantId;
28 import org.thingsboard.server.common.msg.TbMsg; 29 import org.thingsboard.server.common.msg.TbMsg;
29 import org.thingsboard.server.common.msg.TbMsgMetaData; 30 import org.thingsboard.server.common.msg.TbMsgMetaData;
30 31
@@ -43,13 +44,15 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S @@ -43,13 +44,15 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
43 private final JsInvokeService sandboxService; 44 private final JsInvokeService sandboxService;
44 45
45 private final UUID scriptId; 46 private final UUID scriptId;
  47 + private final TenantId tenantId;
46 private final EntityId entityId; 48 private final EntityId entityId;
47 49
48 - public RuleNodeJsScriptEngine(JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) { 50 + public RuleNodeJsScriptEngine(TenantId tenantId, JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) {
  51 + this.tenantId = tenantId;
49 this.sandboxService = sandboxService; 52 this.sandboxService = sandboxService;
50 this.entityId = entityId; 53 this.entityId = entityId;
51 try { 54 try {
52 - this.scriptId = this.sandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, script, argNames).get(); 55 + this.scriptId = this.sandboxService.eval(tenantId, JsScriptType.RULE_NODE_SCRIPT, script, argNames).get();
53 } catch (Exception e) { 56 } catch (Exception e) {
54 Throwable t = e; 57 Throwable t = e;
55 if (e instanceof ExecutionException) { 58 if (e instanceof ExecutionException) {
@@ -203,7 +206,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S @@ -203,7 +206,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
203 private JsonNode executeScript(TbMsg msg) throws ScriptException { 206 private JsonNode executeScript(TbMsg msg) throws ScriptException {
204 try { 207 try {
205 String[] inArgs = prepareArgs(msg); 208 String[] inArgs = prepareArgs(msg);
206 - String eval = sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString(); 209 + String eval = sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString();
207 return mapper.readTree(eval); 210 return mapper.readTree(eval);
208 } catch (ExecutionException e) { 211 } catch (ExecutionException e) {
209 if (e.getCause() instanceof ScriptException) { 212 if (e.getCause() instanceof ScriptException) {
@@ -220,7 +223,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S @@ -220,7 +223,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
220 223
221 private ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) { 224 private ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) {
222 String[] inArgs = prepareArgs(msg); 225 String[] inArgs = prepareArgs(msg);
223 - return Futures.transformAsync(sandboxService.invokeFunction(this.scriptId, inArgs[0], inArgs[1], inArgs[2]), 226 + return Futures.transformAsync(sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]),
224 o -> { 227 o -> {
225 try { 228 try {
226 return Futures.immediateFuture(mapper.readTree(o.toString())); 229 return Futures.immediateFuture(mapper.readTree(o.toString()));
@@ -25,6 +25,7 @@ import org.springframework.http.ResponseEntity; @@ -25,6 +25,7 @@ import org.springframework.http.ResponseEntity;
25 import org.springframework.stereotype.Component; 25 import org.springframework.stereotype.Component;
26 import org.springframework.web.context.request.async.DeferredResult; 26 import org.springframework.web.context.request.async.DeferredResult;
27 import org.thingsboard.common.util.ThingsBoardThreadFactory; 27 import org.thingsboard.common.util.ThingsBoardThreadFactory;
  28 +import org.thingsboard.server.common.data.ApiUsageState;
28 import org.thingsboard.server.common.data.Customer; 29 import org.thingsboard.server.common.data.Customer;
29 import org.thingsboard.server.common.data.Device; 30 import org.thingsboard.server.common.data.Device;
30 import org.thingsboard.server.common.data.DeviceProfile; 31 import org.thingsboard.server.common.data.DeviceProfile;
@@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.Tenant; @@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.Tenant;
33 import org.thingsboard.server.common.data.User; 34 import org.thingsboard.server.common.data.User;
34 import org.thingsboard.server.common.data.asset.Asset; 35 import org.thingsboard.server.common.data.asset.Asset;
35 import org.thingsboard.server.common.data.exception.ThingsboardException; 36 import org.thingsboard.server.common.data.exception.ThingsboardException;
  37 +import org.thingsboard.server.common.data.id.ApiUsageStateId;
36 import org.thingsboard.server.common.data.id.AssetId; 38 import org.thingsboard.server.common.data.id.AssetId;
37 import org.thingsboard.server.common.data.id.CustomerId; 39 import org.thingsboard.server.common.data.id.CustomerId;
38 import org.thingsboard.server.common.data.id.DeviceId; 40 import org.thingsboard.server.common.data.id.DeviceId;
@@ -53,8 +55,10 @@ import org.thingsboard.server.dao.customer.CustomerService; @@ -53,8 +55,10 @@ import org.thingsboard.server.dao.customer.CustomerService;
53 import org.thingsboard.server.dao.device.DeviceProfileService; 55 import org.thingsboard.server.dao.device.DeviceProfileService;
54 import org.thingsboard.server.dao.device.DeviceService; 56 import org.thingsboard.server.dao.device.DeviceService;
55 import org.thingsboard.server.dao.entityview.EntityViewService; 57 import org.thingsboard.server.dao.entityview.EntityViewService;
  58 +import org.thingsboard.server.dao.exception.IncorrectParameterException;
56 import org.thingsboard.server.dao.rule.RuleChainService; 59 import org.thingsboard.server.dao.rule.RuleChainService;
57 import org.thingsboard.server.dao.tenant.TenantService; 60 import org.thingsboard.server.dao.tenant.TenantService;
  61 +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
58 import org.thingsboard.server.dao.user.UserService; 62 import org.thingsboard.server.dao.user.UserService;
59 import org.thingsboard.server.service.security.model.SecurityUser; 63 import org.thingsboard.server.service.security.model.SecurityUser;
60 import org.thingsboard.server.service.security.permission.AccessControlService; 64 import org.thingsboard.server.service.security.permission.AccessControlService;
@@ -111,6 +115,9 @@ public class AccessValidator { @@ -111,6 +115,9 @@ public class AccessValidator {
111 @Autowired 115 @Autowired
112 protected AccessControlService accessControlService; 116 protected AccessControlService accessControlService;
113 117
  118 + @Autowired
  119 + protected ApiUsageStateService apiUsageStateService;
  120 +
114 private ExecutorService executor; 121 private ExecutorService executor;
115 122
116 @PostConstruct 123 @PostConstruct
@@ -152,7 +159,11 @@ public class AccessValidator { @@ -152,7 +159,11 @@ public class AccessValidator {
152 new FutureCallback<DeferredResult<ResponseEntity>>() { 159 new FutureCallback<DeferredResult<ResponseEntity>>() {
153 @Override 160 @Override
154 public void onSuccess(@Nullable DeferredResult<ResponseEntity> result) { 161 public void onSuccess(@Nullable DeferredResult<ResponseEntity> result) {
155 - onSuccess.accept(response, currentUser.getTenantId(), entityId); 162 + try {
  163 + onSuccess.accept(response, currentUser.getTenantId(), entityId);
  164 + } catch (Exception e) {
  165 + onFailure(e);
  166 + }
156 } 167 }
157 168
158 @Override 169 @Override
@@ -193,6 +204,9 @@ public class AccessValidator { @@ -193,6 +204,9 @@ public class AccessValidator {
193 case ENTITY_VIEW: 204 case ENTITY_VIEW:
194 validateEntityView(currentUser, operation, entityId, callback); 205 validateEntityView(currentUser, operation, entityId, callback);
195 return; 206 return;
  207 + case API_USAGE_STATE:
  208 + validateApiUsageState(currentUser, operation, entityId, callback);
  209 + return;
196 default: 210 default:
197 //TODO: add support of other entities 211 //TODO: add support of other entities
198 throw new IllegalStateException("Not Implemented!"); 212 throw new IllegalStateException("Not Implemented!");
@@ -237,6 +251,27 @@ public class AccessValidator { @@ -237,6 +251,27 @@ public class AccessValidator {
237 } 251 }
238 } 252 }
239 253
  254 + private void validateApiUsageState(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
  255 + if (currentUser.isSystemAdmin()) {
  256 + callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));
  257 + } else {
  258 + if (!operation.equals(Operation.READ_TELEMETRY)) {
  259 + callback.onSuccess(ValidationResult.accessDenied("Allowed only READ_TELEMETRY operation!"));
  260 + }
  261 + ApiUsageState apiUsageState = apiUsageStateService.findApiUsageStateById(currentUser.getTenantId(), new ApiUsageStateId(entityId.getId()));
  262 + if (apiUsageState == null) {
  263 + callback.onSuccess(ValidationResult.entityNotFound("Api Usage State with requested id wasn't found!"));
  264 + } else {
  265 + try {
  266 + accessControlService.checkPermission(currentUser, Resource.API_USAGE_STATE, operation, entityId, apiUsageState);
  267 + } catch (ThingsboardException e) {
  268 + callback.onSuccess(ValidationResult.accessDenied(e.getMessage()));
  269 + }
  270 + callback.onSuccess(ValidationResult.ok(apiUsageState));
  271 + }
  272 + }
  273 + }
  274 +
240 private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) { 275 private void validateAsset(final SecurityUser currentUser, Operation operation, EntityId entityId, FutureCallback<ValidationResult> callback) {
241 if (currentUser.isSystemAdmin()) { 276 if (currentUser.isSystemAdmin()) {
242 callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION)); 277 callback.onSuccess(ValidationResult.accessDenied(SYSTEM_ADMINISTRATOR_IS_NOT_ALLOWED_TO_PERFORM_THIS_OPERATION));
@@ -404,9 +439,9 @@ public class AccessValidator { @@ -404,9 +439,9 @@ public class AccessValidator {
404 439
405 public static void handleError(Throwable e, final DeferredResult<ResponseEntity> response, HttpStatus defaultErrorStatus) { 440 public static void handleError(Throwable e, final DeferredResult<ResponseEntity> response, HttpStatus defaultErrorStatus) {
406 ResponseEntity responseEntity; 441 ResponseEntity responseEntity;
407 - if (e != null && e instanceof ToErrorResponseEntity) { 442 + if (e instanceof ToErrorResponseEntity) {
408 responseEntity = ((ToErrorResponseEntity) e).toErrorResponseEntity(); 443 responseEntity = ((ToErrorResponseEntity) e).toErrorResponseEntity();
409 - } else if (e != null && e instanceof IllegalArgumentException) { 444 + } else if (e instanceof IllegalArgumentException || e instanceof IncorrectParameterException) {
410 responseEntity = new ResponseEntity<>(e.getMessage(), HttpStatus.BAD_REQUEST); 445 responseEntity = new ResponseEntity<>(e.getMessage(), HttpStatus.BAD_REQUEST);
411 } else { 446 } else {
412 responseEntity = new ResponseEntity<>(defaultErrorStatus); 447 responseEntity = new ResponseEntity<>(defaultErrorStatus);
@@ -30,7 +30,7 @@ import java.nio.charset.StandardCharsets; @@ -30,7 +30,7 @@ import java.nio.charset.StandardCharsets;
30 30
31 @Component(value = "oauth2AuthenticationFailureHandler") 31 @Component(value = "oauth2AuthenticationFailureHandler")
32 @ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") 32 @ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true")
33 -public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler { 33 +public class Oauth2AuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler {
34 34
35 @Override 35 @Override
36 public void onAuthenticationFailure(HttpServletRequest request, 36 public void onAuthenticationFailure(HttpServletRequest request,
@@ -63,7 +63,6 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS @@ -63,7 +63,6 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS
63 public void onAuthenticationSuccess(HttpServletRequest request, 63 public void onAuthenticationSuccess(HttpServletRequest request,
64 HttpServletResponse response, 64 HttpServletResponse response,
65 Authentication authentication) throws IOException { 65 Authentication authentication) throws IOException {
66 -  
67 String baseUrl = MiscUtils.constructBaseUrl(request); 66 String baseUrl = MiscUtils.constructBaseUrl(request);
68 try { 67 try {
69 OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication; 68 OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication;
@@ -35,7 +35,8 @@ public enum Resource { @@ -35,7 +35,8 @@ public enum Resource {
35 OAUTH2_CONFIGURATION_INFO(), 35 OAUTH2_CONFIGURATION_INFO(),
36 OAUTH2_CONFIGURATION_TEMPLATE(), 36 OAUTH2_CONFIGURATION_TEMPLATE(),
37 TENANT_PROFILE(EntityType.TENANT_PROFILE), 37 TENANT_PROFILE(EntityType.TENANT_PROFILE),
38 - DEVICE_PROFILE(EntityType.DEVICE_PROFILE); 38 + DEVICE_PROFILE(EntityType.DEVICE_PROFILE),
  39 + API_USAGE_STATE(EntityType.API_USAGE_STATE);
39 40
40 private final EntityType entityType; 41 private final EntityType entityType;
41 42
@@ -40,6 +40,7 @@ public class TenantAdminPermissions extends AbstractPermissions { @@ -40,6 +40,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
40 put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); 40 put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker);
41 put(Resource.WIDGET_TYPE, widgetsPermissionChecker); 41 put(Resource.WIDGET_TYPE, widgetsPermissionChecker);
42 put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); 42 put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker);
  43 + put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker);
43 } 44 }
44 45
45 public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { 46 public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() {
@@ -40,17 +40,20 @@ import org.thingsboard.rule.engine.api.MailService; @@ -40,17 +40,20 @@ import org.thingsboard.rule.engine.api.MailService;
40 import org.thingsboard.server.common.data.AdminSettings; 40 import org.thingsboard.server.common.data.AdminSettings;
41 import org.thingsboard.server.common.data.User; 41 import org.thingsboard.server.common.data.User;
42 import org.thingsboard.server.common.data.exception.ThingsboardException; 42 import org.thingsboard.server.common.data.exception.ThingsboardException;
  43 +import org.thingsboard.server.common.data.id.CustomerId;
43 import org.thingsboard.server.common.data.id.TenantId; 44 import org.thingsboard.server.common.data.id.TenantId;
44 import org.thingsboard.server.common.data.security.UserCredentials; 45 import org.thingsboard.server.common.data.security.UserCredentials;
  46 +import org.thingsboard.server.common.data.security.model.SecuritySettings;
  47 +import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
45 import org.thingsboard.server.dao.exception.DataValidationException; 48 import org.thingsboard.server.dao.exception.DataValidationException;
46 import org.thingsboard.server.dao.settings.AdminSettingsService; 49 import org.thingsboard.server.dao.settings.AdminSettingsService;
47 import org.thingsboard.server.dao.user.UserService; 50 import org.thingsboard.server.dao.user.UserService;
48 import org.thingsboard.server.dao.user.UserServiceImpl; 51 import org.thingsboard.server.dao.user.UserServiceImpl;
49 import org.thingsboard.server.service.security.exception.UserPasswordExpiredException; 52 import org.thingsboard.server.service.security.exception.UserPasswordExpiredException;
50 -import org.thingsboard.server.common.data.security.model.SecuritySettings;  
51 -import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; 53 +import org.thingsboard.server.utils.MiscUtils;
52 54
53 import javax.annotation.Resource; 55 import javax.annotation.Resource;
  56 +import javax.servlet.http.HttpServletRequest;
54 import java.util.ArrayList; 57 import java.util.ArrayList;
55 import java.util.List; 58 import java.util.List;
56 import java.util.Map; 59 import java.util.Map;
@@ -146,7 +149,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService { @@ -146,7 +149,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
146 if (isPositiveInteger(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) { 149 if (isPositiveInteger(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) {
147 if ((userCredentials.getCreatedTime() 150 if ((userCredentials.getCreatedTime()
148 + TimeUnit.DAYS.toMillis(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays())) 151 + TimeUnit.DAYS.toMillis(securitySettings.getPasswordPolicy().getPasswordExpirationPeriodDays()))
149 - < System.currentTimeMillis()) { 152 + < System.currentTimeMillis()) {
150 userCredentials = userService.requestExpiredPasswordReset(tenantId, userCredentials.getId()); 153 userCredentials = userService.requestExpiredPasswordReset(tenantId, userCredentials.getId());
151 throw new UserPasswordExpiredException("User password expired!", userCredentials.getResetToken()); 154 throw new UserPasswordExpiredException("User password expired!", userCredentials.getResetToken());
152 } 155 }
@@ -197,6 +200,21 @@ public class DefaultSystemSecurityService implements SystemSecurityService { @@ -197,6 +200,21 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
197 } 200 }
198 } 201 }
199 202
  203 + @Override
  204 + public String getBaseUrl(TenantId tenantId, CustomerId customerId, HttpServletRequest httpServletRequest) {
  205 + String baseUrl;
  206 + AdminSettings generalSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "general");
  207 +
  208 + JsonNode prohibitDifferentUrl = generalSettings.getJsonValue().get("prohibitDifferentUrl");
  209 +
  210 + if (prohibitDifferentUrl != null && prohibitDifferentUrl.asBoolean()) {
  211 + baseUrl = generalSettings.getJsonValue().get("baseUrl").asText();
  212 + } else {
  213 + baseUrl = MiscUtils.constructBaseUrl(httpServletRequest);
  214 + }
  215 + return baseUrl;
  216 + }
  217 +
200 private static boolean isPositiveInteger(Integer val) { 218 private static boolean isPositiveInteger(Integer val) {
201 return val != null && val.intValue() > 0; 219 return val != null && val.intValue() > 0;
202 } 220 }
@@ -16,11 +16,14 @@ @@ -16,11 +16,14 @@
16 package org.thingsboard.server.service.security.system; 16 package org.thingsboard.server.service.security.system;
17 17
18 import org.springframework.security.core.AuthenticationException; 18 import org.springframework.security.core.AuthenticationException;
  19 +import org.thingsboard.server.common.data.id.CustomerId;
19 import org.thingsboard.server.common.data.id.TenantId; 20 import org.thingsboard.server.common.data.id.TenantId;
20 import org.thingsboard.server.common.data.security.UserCredentials; 21 import org.thingsboard.server.common.data.security.UserCredentials;
21 import org.thingsboard.server.dao.exception.DataValidationException; 22 import org.thingsboard.server.dao.exception.DataValidationException;
22 import org.thingsboard.server.common.data.security.model.SecuritySettings; 23 import org.thingsboard.server.common.data.security.model.SecuritySettings;
23 24
  25 +import javax.servlet.http.HttpServletRequest;
  26 +
24 public interface SystemSecurityService { 27 public interface SystemSecurityService {
25 28
26 SecuritySettings getSecuritySettings(TenantId tenantId); 29 SecuritySettings getSecuritySettings(TenantId tenantId);
@@ -31,4 +34,6 @@ public interface SystemSecurityService { @@ -31,4 +34,6 @@ public interface SystemSecurityService {
31 34
32 void validatePassword(TenantId tenantId, String password, UserCredentials userCredentials) throws DataValidationException; 35 void validatePassword(TenantId tenantId, String password, UserCredentials userCredentials) throws DataValidationException;
33 36
  37 + String getBaseUrl(TenantId tenantId, CustomerId customerId, HttpServletRequest httpServletRequest);
  38 +
34 } 39 }
@@ -48,9 +48,9 @@ import java.util.stream.Collectors; @@ -48,9 +48,9 @@ import java.util.stream.Collectors;
48 public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { 48 public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService {
49 49
50 public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; 50 public static final String TB_SERVICE_QUEUE = "TbServiceQueue";
51 - public static final FutureCallback<Void> CALLBACK = new FutureCallback<Void>() { 51 + public static final FutureCallback<Integer> CALLBACK = new FutureCallback<Integer>() {
52 @Override 52 @Override
53 - public void onSuccess(@Nullable Void result) { 53 + public void onSuccess(@Nullable Integer result) {
54 54
55 } 55 }
56 56
@@ -85,7 +85,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS @@ -85,7 +85,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
85 .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) 85 .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get())))
86 .collect(Collectors.toList()); 86 .collect(Collectors.toList());
87 if (!tsList.isEmpty()) { 87 if (!tsList.isEmpty()) {
88 - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); 88 + tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, CALLBACK);
89 } 89 }
90 } 90 }
91 } catch (DataValidationException e) { 91 } catch (DataValidationException e) {
@@ -97,7 +97,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS @@ -97,7 +97,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
97 ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { 97 ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> {
98 TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); 98 TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString()));
99 try { 99 try {
100 - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); 100 + tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK);
101 } catch (DataValidationException e2) { 101 } catch (DataValidationException e2) {
102 if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { 102 if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) {
103 throw e2; 103 throw e2;
@@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener @@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener
105 } 105 }
106 } 106 }
107 107
108 - protected void addWsCallback(ListenableFuture<List<Void>> saveFuture, Consumer<Void> callback) {  
109 - Futures.addCallback(saveFuture, new FutureCallback<List<Void>>() { 108 + protected <T> void addWsCallback(ListenableFuture<T> saveFuture, Consumer<T> callback) {
  109 + Futures.addCallback(saveFuture, new FutureCallback<T>() {
110 @Override 110 @Override
111 - public void onSuccess(@Nullable List<Void> result) { 111 + public void onSuccess(@Nullable T result) {
112 callback.accept(null); 112 callback.accept(null);
113 } 113 }
114 114
@@ -20,10 +20,9 @@ import com.google.common.util.concurrent.Futures; @@ -20,10 +20,9 @@ import com.google.common.util.concurrent.Futures;
20 import com.google.common.util.concurrent.ListenableFuture; 20 import com.google.common.util.concurrent.ListenableFuture;
21 import com.google.common.util.concurrent.MoreExecutors; 21 import com.google.common.util.concurrent.MoreExecutors;
22 import lombok.extern.slf4j.Slf4j; 22 import lombok.extern.slf4j.Slf4j;
23 -import org.springframework.beans.factory.annotation.Autowired;  
24 -import org.springframework.context.event.EventListener;  
25 import org.springframework.stereotype.Service; 23 import org.springframework.stereotype.Service;
26 import org.thingsboard.common.util.ThingsBoardThreadFactory; 24 import org.thingsboard.common.util.ThingsBoardThreadFactory;
  25 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
27 import org.thingsboard.server.common.data.EntityType; 26 import org.thingsboard.server.common.data.EntityType;
28 import org.thingsboard.server.common.data.EntityView; 27 import org.thingsboard.server.common.data.EntityView;
29 import org.thingsboard.server.common.data.id.EntityId; 28 import org.thingsboard.server.common.data.id.EntityId;
@@ -41,11 +40,12 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -41,11 +40,12 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
41 import org.thingsboard.server.dao.attributes.AttributesService; 40 import org.thingsboard.server.dao.attributes.AttributesService;
42 import org.thingsboard.server.dao.entityview.EntityViewService; 41 import org.thingsboard.server.dao.entityview.EntityViewService;
43 import org.thingsboard.server.dao.timeseries.TimeseriesService; 42 import org.thingsboard.server.dao.timeseries.TimeseriesService;
  43 +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
44 import org.thingsboard.server.gen.transport.TransportProtos; 44 import org.thingsboard.server.gen.transport.TransportProtos;
45 -import org.thingsboard.server.queue.discovery.PartitionChangeEvent;  
46 import org.thingsboard.server.queue.discovery.PartitionService; 45 import org.thingsboard.server.queue.discovery.PartitionService;
  46 +import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
  47 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
47 import org.thingsboard.server.service.queue.TbClusterService; 48 import org.thingsboard.server.service.queue.TbClusterService;
48 -import org.thingsboard.server.service.subscription.SubscriptionManagerService;  
49 import org.thingsboard.server.service.subscription.TbSubscriptionUtils; 49 import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
50 50
51 import javax.annotation.Nullable; 51 import javax.annotation.Nullable;
@@ -59,12 +59,8 @@ import java.util.HashMap; @@ -59,12 +59,8 @@ import java.util.HashMap;
59 import java.util.List; 59 import java.util.List;
60 import java.util.Map; 60 import java.util.Map;
61 import java.util.Optional; 61 import java.util.Optional;
62 -import java.util.Set;  
63 -import java.util.concurrent.ConcurrentHashMap;  
64 import java.util.concurrent.ExecutorService; 62 import java.util.concurrent.ExecutorService;
65 import java.util.concurrent.Executors; 63 import java.util.concurrent.Executors;
66 -import java.util.function.Consumer;  
67 -import java.util.stream.Collectors;  
68 64
69 /** 65 /**
70 * Created by ashvayka on 27.03.18. 66 * Created by ashvayka on 27.03.18.
@@ -76,6 +72,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @@ -76,6 +72,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
76 private final AttributesService attrService; 72 private final AttributesService attrService;
77 private final TimeseriesService tsService; 73 private final TimeseriesService tsService;
78 private final EntityViewService entityViewService; 74 private final EntityViewService entityViewService;
  75 + private final TbApiUsageClient apiUsageClient;
  76 + private final TbApiUsageStateService apiUsageStateService;
79 77
80 private ExecutorService tsCallBackExecutor; 78 private ExecutorService tsCallBackExecutor;
81 79
@@ -83,11 +81,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @@ -83,11 +81,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
83 TimeseriesService tsService, 81 TimeseriesService tsService,
84 EntityViewService entityViewService, 82 EntityViewService entityViewService,
85 TbClusterService clusterService, 83 TbClusterService clusterService,
86 - PartitionService partitionService) { 84 + PartitionService partitionService,
  85 + TbApiUsageClient apiUsageClient,
  86 + TbApiUsageStateService apiUsageStateService) {
87 super(clusterService, partitionService); 87 super(clusterService, partitionService);
88 this.attrService = attrService; 88 this.attrService = attrService;
89 this.tsService = tsService; 89 this.tsService = tsService;
90 this.entityViewService = entityViewService; 90 this.entityViewService = entityViewService;
  91 + this.apiUsageClient = apiUsageClient;
  92 + this.apiUsageStateService = apiUsageStateService;
91 } 93 }
92 94
93 @PostConstruct 95 @PostConstruct
@@ -116,7 +118,35 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @@ -116,7 +118,35 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
116 118
117 @Override 119 @Override
118 public void saveAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Void> callback) { 120 public void saveAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Void> callback) {
119 - ListenableFuture<List<Void>> saveFuture = tsService.save(tenantId, entityId, ts, ttl); 121 + checkInternalEntity(entityId);
  122 + if (apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) {
  123 + saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback<Integer>() {
  124 + @Override
  125 + public void onSuccess(Integer result) {
  126 + if (result != null && result > 0) {
  127 + apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result);
  128 + }
  129 + callback.onSuccess(null);
  130 + }
  131 +
  132 + @Override
  133 + public void onFailure(Throwable t) {
  134 + callback.onFailure(t);
  135 + }
  136 + });
  137 + } else{
  138 + callback.onFailure(new RuntimeException("DB storage writes are disabled due to API limits!"));
  139 + }
  140 + }
  141 +
  142 + @Override
  143 + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Integer> callback) {
  144 + saveAndNotifyInternal(tenantId, entityId, ts, 0L, callback);
  145 + }
  146 +
  147 + @Override
  148 + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Integer> callback) {
  149 + ListenableFuture<Integer> saveFuture = tsService.save(tenantId, entityId, ts, ttl);
120 addMainCallback(saveFuture, callback); 150 addMainCallback(saveFuture, callback);
121 addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); 151 addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
122 if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { 152 if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) {
@@ -141,9 +171,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @@ -141,9 +171,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
141 Optional<TsKvEntry> tsKvEntry = entries.stream() 171 Optional<TsKvEntry> tsKvEntry = entries.stream()
142 .filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs) 172 .filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs)
143 .max(Comparator.comparingLong(TsKvEntry::getTs)); 173 .max(Comparator.comparingLong(TsKvEntry::getTs));
144 - if (tsKvEntry.isPresent()) {  
145 - entityViewLatest.add(tsKvEntry.get());  
146 - } 174 + tsKvEntry.ifPresent(entityViewLatest::add);
147 } 175 }
148 } 176 }
149 if (!entityViewLatest.isEmpty()) { 177 if (!entityViewLatest.isEmpty()) {
@@ -176,29 +204,53 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @@ -176,29 +204,53 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
176 204
177 @Override 205 @Override
178 public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) { 206 public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) {
  207 + checkInternalEntity(entityId);
  208 + saveAndNotifyInternal(tenantId, entityId, scope, attributes, notifyDevice, callback);
  209 + }
  210 +
  211 + @Override
  212 + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback) {
179 ListenableFuture<List<Void>> saveFuture = attrService.save(tenantId, entityId, scope, attributes); 213 ListenableFuture<List<Void>> saveFuture = attrService.save(tenantId, entityId, scope, attributes);
180 - addMainCallback(saveFuture, callback); 214 + addVoidCallback(saveFuture, callback);
181 addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); 215 addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice));
182 } 216 }
183 217
184 @Override 218 @Override
185 public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) { 219 public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) {
  220 + checkInternalEntity(entityId);
  221 + saveLatestAndNotifyInternal(tenantId, entityId, ts, callback);
  222 + }
  223 +
  224 + @Override
  225 + public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) {
186 ListenableFuture<List<Void>> saveFuture = tsService.saveLatest(tenantId, entityId, ts); 226 ListenableFuture<List<Void>> saveFuture = tsService.saveLatest(tenantId, entityId, ts);
187 - addMainCallback(saveFuture, callback); 227 + addVoidCallback(saveFuture, callback);
188 addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); 228 addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
189 } 229 }
190 230
191 @Override 231 @Override
192 public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) { 232 public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) {
  233 + checkInternalEntity(entityId);
  234 + deleteAndNotifyInternal(tenantId, entityId, scope, keys, callback);
  235 + }
  236 +
  237 + @Override
  238 + public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) {
193 ListenableFuture<List<Void>> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); 239 ListenableFuture<List<Void>> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys);
194 - addMainCallback(deleteFuture, callback); 240 + addVoidCallback(deleteFuture, callback);
195 addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); 241 addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys));
196 } 242 }
197 243
198 @Override 244 @Override
199 public void deleteLatest(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) { 245 public void deleteLatest(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) {
  246 + checkInternalEntity(entityId);
  247 + deleteLatestInternal(tenantId, entityId, keys, callback);
  248 + }
  249 +
  250 + @Override
  251 + public void deleteLatestInternal(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) {
200 ListenableFuture<List<Void>> deleteFuture = tsService.removeLatest(tenantId, entityId, keys); 252 ListenableFuture<List<Void>> deleteFuture = tsService.removeLatest(tenantId, entityId, keys);
201 - addMainCallback(deleteFuture, callback); 253 + addVoidCallback(deleteFuture, callback);
202 } 254 }
203 255
204 @Override 256 @Override
@@ -283,7 +335,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @@ -283,7 +335,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
283 } 335 }
284 } 336 }
285 337
286 - private <S, R> void addMainCallback(ListenableFuture<S> saveFuture, final FutureCallback<R> callback) { 338 + private <S> void addVoidCallback(ListenableFuture<S> saveFuture, final FutureCallback<Void> callback) {
287 Futures.addCallback(saveFuture, new FutureCallback<S>() { 339 Futures.addCallback(saveFuture, new FutureCallback<S>() {
288 @Override 340 @Override
289 public void onSuccess(@Nullable S result) { 341 public void onSuccess(@Nullable S result) {
@@ -296,4 +348,25 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @@ -296,4 +348,25 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
296 } 348 }
297 }, tsCallBackExecutor); 349 }, tsCallBackExecutor);
298 } 350 }
  351 +
  352 + private <S> void addMainCallback(ListenableFuture<S> saveFuture, final FutureCallback<S> callback) {
  353 + Futures.addCallback(saveFuture, new FutureCallback<S>() {
  354 + @Override
  355 + public void onSuccess(@Nullable S result) {
  356 + callback.onSuccess(result);
  357 + }
  358 +
  359 + @Override
  360 + public void onFailure(Throwable t) {
  361 + callback.onFailure(t);
  362 + }
  363 + }, tsCallBackExecutor);
  364 + }
  365 +
  366 + private void checkInternalEntity(EntityId entityId) {
  367 + if (EntityType.API_USAGE_STATE.equals(entityId.getEntityType())) {
  368 + throw new RuntimeException("Can't update API Usage State!");
  369 + }
  370 + }
  371 +
299 } 372 }
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.telemetry;
  17 +
  18 +import com.google.common.util.concurrent.FutureCallback;
  19 +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
  20 +import org.thingsboard.server.common.data.id.EntityId;
  21 +import org.thingsboard.server.common.data.id.TenantId;
  22 +import org.thingsboard.server.common.data.kv.AttributeKvEntry;
  23 +import org.thingsboard.server.common.data.kv.TsKvEntry;
  24 +
  25 +import java.util.List;
  26 +
  27 +/**
  28 + * Created by ashvayka on 27.03.18.
  29 + */
  30 +public interface InternalTelemetryService extends RuleEngineTelemetryService {
  31 +
  32 + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Integer> callback);
  33 +
  34 + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Integer> callback);
  35 +
  36 + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes, boolean notifyDevice, FutureCallback<Void> callback);
  37 +
  38 + void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback);
  39 +
  40 + void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback);
  41 +
  42 + void deleteLatestInternal(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback);
  43 +
  44 +
  45 +
  46 +}
@@ -22,6 +22,6 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent; @@ -22,6 +22,6 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
22 /** 22 /**
23 * Created by ashvayka on 27.03.18. 23 * Created by ashvayka on 27.03.18.
24 */ 24 */
25 -public interface TelemetrySubscriptionService extends RuleEngineTelemetryService, ApplicationListener<PartitionChangeEvent> { 25 +public interface TelemetrySubscriptionService extends InternalTelemetryService, ApplicationListener<PartitionChangeEvent> {
26 26
27 } 27 }
@@ -25,6 +25,7 @@ import com.google.protobuf.ByteString; @@ -25,6 +25,7 @@ import com.google.protobuf.ByteString;
25 import lombok.extern.slf4j.Slf4j; 25 import lombok.extern.slf4j.Slf4j;
26 import org.springframework.stereotype.Service; 26 import org.springframework.stereotype.Service;
27 import org.springframework.util.StringUtils; 27 import org.springframework.util.StringUtils;
  28 +import org.thingsboard.server.common.data.ApiUsageState;
28 import org.thingsboard.server.common.data.DataConstants; 29 import org.thingsboard.server.common.data.DataConstants;
29 import org.thingsboard.server.common.data.Device; 30 import org.thingsboard.server.common.data.Device;
30 import org.thingsboard.server.common.data.DeviceProfile; 31 import org.thingsboard.server.common.data.DeviceProfile;
@@ -51,7 +52,6 @@ import org.thingsboard.server.dao.device.DeviceService; @@ -51,7 +52,6 @@ import org.thingsboard.server.dao.device.DeviceService;
51 import org.thingsboard.server.dao.device.provision.ProvisionRequest; 52 import org.thingsboard.server.dao.device.provision.ProvisionRequest;
52 import org.thingsboard.server.dao.device.provision.ProvisionResponse; 53 import org.thingsboard.server.dao.device.provision.ProvisionResponse;
53 import org.thingsboard.server.dao.relation.RelationService; 54 import org.thingsboard.server.dao.relation.RelationService;
54 -import org.thingsboard.server.dao.tenant.TenantService;  
55 import org.thingsboard.server.dao.util.mapping.JacksonUtil; 55 import org.thingsboard.server.dao.util.mapping.JacksonUtil;
56 import org.thingsboard.server.gen.transport.TransportProtos; 56 import org.thingsboard.server.gen.transport.TransportProtos;
57 import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; 57 import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
@@ -68,9 +68,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce @@ -68,9 +68,10 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce
68 import org.thingsboard.server.queue.common.TbProtoQueueMsg; 68 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
69 import org.thingsboard.server.queue.util.TbCoreComponent; 69 import org.thingsboard.server.queue.util.TbCoreComponent;
70 import org.thingsboard.server.dao.device.provision.ProvisionFailedException; 70 import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
  71 +import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
71 import org.thingsboard.server.service.executors.DbCallbackExecutorService; 72 import org.thingsboard.server.service.executors.DbCallbackExecutorService;
72 import org.thingsboard.server.service.profile.TbDeviceProfileCache; 73 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
73 -import org.thingsboard.server.service.profile.TbTenantProfileCache; 74 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
74 import org.thingsboard.server.service.queue.TbClusterService; 75 import org.thingsboard.server.service.queue.TbClusterService;
75 import org.thingsboard.server.service.state.DeviceStateService; 76 import org.thingsboard.server.service.state.DeviceStateService;
76 77
@@ -92,6 +93,7 @@ public class DefaultTransportApiService implements TransportApiService { @@ -92,6 +93,7 @@ public class DefaultTransportApiService implements TransportApiService {
92 93
93 private final TbDeviceProfileCache deviceProfileCache; 94 private final TbDeviceProfileCache deviceProfileCache;
94 private final TbTenantProfileCache tenantProfileCache; 95 private final TbTenantProfileCache tenantProfileCache;
  96 + private final TbApiUsageStateService apiUsageStateService;
95 private final DeviceService deviceService; 97 private final DeviceService deviceService;
96 private final RelationService relationService; 98 private final RelationService relationService;
97 private final DeviceCredentialsService deviceCredentialsService; 99 private final DeviceCredentialsService deviceCredentialsService;
@@ -104,13 +106,14 @@ public class DefaultTransportApiService implements TransportApiService { @@ -104,13 +106,14 @@ public class DefaultTransportApiService implements TransportApiService {
104 private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>(); 106 private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>();
105 107
106 public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache, 108 public DefaultTransportApiService(TbDeviceProfileCache deviceProfileCache,
107 - TbTenantProfileCache tenantProfileCache, DeviceService deviceService, 109 + TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, DeviceService deviceService,
108 RelationService relationService, DeviceCredentialsService deviceCredentialsService, 110 RelationService relationService, DeviceCredentialsService deviceCredentialsService,
109 DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService, 111 DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService,
110 TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService, 112 TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService,
111 DeviceProvisionService deviceProvisionService) { 113 DeviceProvisionService deviceProvisionService) {
112 this.deviceProfileCache = deviceProfileCache; 114 this.deviceProfileCache = deviceProfileCache;
113 this.tenantProfileCache = tenantProfileCache; 115 this.tenantProfileCache = tenantProfileCache;
  116 + this.apiUsageStateService = apiUsageStateService;
114 this.deviceService = deviceService; 117 this.deviceService = deviceService;
115 this.relationService = relationService; 118 this.relationService = relationService;
116 this.deviceCredentialsService = deviceCredentialsService; 119 this.deviceCredentialsService = deviceCredentialsService;
@@ -316,18 +319,21 @@ public class DefaultTransportApiService implements TransportApiService { @@ -316,18 +319,21 @@ public class DefaultTransportApiService implements TransportApiService {
316 private ListenableFuture<TransportApiResponseMsg> handle(GetEntityProfileRequestMsg requestMsg) { 319 private ListenableFuture<TransportApiResponseMsg> handle(GetEntityProfileRequestMsg requestMsg) {
317 EntityType entityType = EntityType.valueOf(requestMsg.getEntityType()); 320 EntityType entityType = EntityType.valueOf(requestMsg.getEntityType());
318 UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB()); 321 UUID entityUuid = new UUID(requestMsg.getEntityIdMSB(), requestMsg.getEntityIdLSB());
319 - ByteString data; 322 + GetEntityProfileResponseMsg.Builder builder = GetEntityProfileResponseMsg.newBuilder();
320 if (entityType.equals(EntityType.DEVICE_PROFILE)) { 323 if (entityType.equals(EntityType.DEVICE_PROFILE)) {
321 DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid); 324 DeviceProfileId deviceProfileId = new DeviceProfileId(entityUuid);
322 DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId); 325 DeviceProfile deviceProfile = deviceProfileCache.find(deviceProfileId);
323 - data = ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile)); 326 + builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceProfile)));
324 } else if (entityType.equals(EntityType.TENANT)) { 327 } else if (entityType.equals(EntityType.TENANT)) {
325 - TenantProfile tenantProfile = tenantProfileCache.get(new TenantId(entityUuid));  
326 - data = ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile)); 328 + TenantId tenantId = new TenantId(entityUuid);
  329 + TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
  330 + ApiUsageState state = apiUsageStateService.getApiUsageState(tenantId);
  331 + builder.setData(ByteString.copyFrom(dataDecodingEncodingService.encode(tenantProfile)));
  332 + builder.setApiState(ByteString.copyFrom(dataDecodingEncodingService.encode(state)));
327 } else { 333 } else {
328 throw new RuntimeException("Invalid entity profile request: " + entityType); 334 throw new RuntimeException("Invalid entity profile request: " + entityType);
329 } 335 }
330 - return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(GetEntityProfileResponseMsg.newBuilder().setData(data).build()).build()); 336 + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build());
331 } 337 }
332 338
333 private ListenableFuture<TransportApiResponseMsg> getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { 339 private ListenableFuture<TransportApiResponseMsg> getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) {
@@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j;
19 import org.springframework.beans.factory.annotation.Value; 19 import org.springframework.beans.factory.annotation.Value;
20 import org.springframework.boot.context.event.ApplicationReadyEvent; 20 import org.springframework.boot.context.event.ApplicationReadyEvent;
21 import org.springframework.context.event.EventListener; 21 import org.springframework.context.event.EventListener;
  22 +import org.springframework.core.annotation.Order;
22 import org.springframework.stereotype.Service; 23 import org.springframework.stereotype.Service;
23 import org.thingsboard.server.common.stats.MessagesStats; 24 import org.thingsboard.server.common.stats.MessagesStats;
24 import org.thingsboard.server.common.stats.StatsFactory; 25 import org.thingsboard.server.common.stats.StatsFactory;
@@ -90,6 +91,7 @@ public class TbCoreTransportApiService { @@ -90,6 +91,7 @@ public class TbCoreTransportApiService {
90 } 91 }
91 92
92 @EventListener(ApplicationReadyEvent.class) 93 @EventListener(ApplicationReadyEvent.class)
  94 + @Order(value = 2)
93 public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { 95 public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
94 log.info("Received application ready event. Starting polling for events."); 96 log.info("Received application ready event. Starting polling for events.");
95 transportApiTemplate.init(transportApiService); 97 transportApiTemplate.init(transportApiService);
@@ -673,6 +673,7 @@ queue: @@ -673,6 +673,7 @@ queue:
673 poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" 673 poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
674 partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" 674 partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
675 pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" 675 pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}"
  676 + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}"
676 stats: 677 stats:
677 enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" 678 enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
678 print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" 679 print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"
@@ -23,7 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired; @@ -23,7 +23,8 @@ import org.springframework.beans.factory.annotation.Autowired;
23 import org.thingsboard.server.common.data.EntityInfo; 23 import org.thingsboard.server.common.data.EntityInfo;
24 import org.thingsboard.server.common.data.Tenant; 24 import org.thingsboard.server.common.data.Tenant;
25 import org.thingsboard.server.common.data.TenantProfile; 25 import org.thingsboard.server.common.data.TenantProfile;
26 -import org.thingsboard.server.common.data.TenantProfileData; 26 +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
  27 +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
27 import org.thingsboard.server.common.data.id.TenantId; 28 import org.thingsboard.server.common.data.id.TenantId;
28 import org.thingsboard.server.common.data.page.PageData; 29 import org.thingsboard.server.common.data.page.PageData;
29 import org.thingsboard.server.common.data.page.PageLink; 30 import org.thingsboard.server.common.data.page.PageLink;
@@ -285,7 +286,9 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController @@ -285,7 +286,9 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController
285 TenantProfile tenantProfile = new TenantProfile(); 286 TenantProfile tenantProfile = new TenantProfile();
286 tenantProfile.setName(name); 287 tenantProfile.setName(name);
287 tenantProfile.setDescription(name + " Test"); 288 tenantProfile.setDescription(name + " Test");
288 - tenantProfile.setProfileData(new TenantProfileData()); 289 + TenantProfileData tenantProfileData = new TenantProfileData();
  290 + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration());
  291 + tenantProfile.setProfileData(tenantProfileData);
289 tenantProfile.setDefault(false); 292 tenantProfile.setDefault(false);
290 tenantProfile.setIsolatedTbCore(false); 293 tenantProfile.setIsolatedTbCore(false);
291 tenantProfile.setIsolatedTbRuleEngine(false); 294 tenantProfile.setIsolatedTbRuleEngine(false);
@@ -27,7 +27,7 @@ import java.util.Arrays; @@ -27,7 +27,7 @@ import java.util.Arrays;
27 @RunWith(ClasspathSuite.class) 27 @RunWith(ClasspathSuite.class)
28 @ClasspathSuite.ClassnameFilters({ 28 @ClasspathSuite.ClassnameFilters({
29 // "org.thingsboard.server.controller.sql.WebsocketApiSqlTest", 29 // "org.thingsboard.server.controller.sql.WebsocketApiSqlTest",
30 -// "org.thingsboard.server.controller.sql.EntityQueryControllerSqlTest", 30 +// "org.thingsboard.server.controller.sql.TenantProfileControllerSqlTest",
31 "org.thingsboard.server.controller.sql.*Test", 31 "org.thingsboard.server.controller.sql.*Test",
32 }) 32 })
33 public class ControllerSqlTestSuite { 33 public class ControllerSqlTestSuite {
1 -/**  
2 - * Copyright © 2016-2020 The Thingsboard Authors  
3 - *  
4 - * Licensed under the Apache License, Version 2.0 (the "License");  
5 - * you may not use this file except in compliance with the License.  
6 - * You may obtain a copy of the License at  
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - *  
10 - * Unless required by applicable law or agreed to in writing, software  
11 - * distributed under the License is distributed on an "AS IS" BASIS,  
12 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
13 - * See the License for the specific language governing permissions and  
14 - * limitations under the License.  
15 - */  
16 -package org.thingsboard.server.service.script;  
17 -  
18 -import com.datastax.oss.driver.api.core.uuid.Uuids;  
19 -import com.google.common.collect.Sets;  
20 -import org.junit.After;  
21 -import org.junit.Before;  
22 -import org.junit.Test;  
23 -import org.thingsboard.rule.engine.api.ScriptEngine;  
24 -import org.thingsboard.server.common.data.id.EntityId;  
25 -import org.thingsboard.server.common.data.id.RuleNodeId;  
26 -import org.thingsboard.server.common.msg.TbMsg;  
27 -import org.thingsboard.server.common.msg.TbMsgDataType;  
28 -import org.thingsboard.server.common.msg.TbMsgMetaData;  
29 -  
30 -import javax.script.ScriptException;  
31 -import java.util.Map;  
32 -import java.util.Set;  
33 -import java.util.UUID;  
34 -import java.util.concurrent.*;  
35 -import java.util.concurrent.atomic.AtomicInteger;  
36 -  
37 -import static org.junit.Assert.*;  
38 -  
39 -public class RuleNodeJsScriptEngineTest {  
40 -  
41 - private ScriptEngine scriptEngine;  
42 - private TestNashornJsInvokeService jsSandboxService;  
43 -  
44 - private EntityId ruleNodeId = new RuleNodeId(Uuids.timeBased());  
45 -  
46 - @Before  
47 - public void beforeTest() throws Exception {  
48 - jsSandboxService = new TestNashornJsInvokeService(false, 1, 100, 3);  
49 - }  
50 -  
51 - @After  
52 - public void afterTest() throws Exception {  
53 - jsSandboxService.stop();  
54 - }  
55 -  
56 - @Test  
57 - public void msgCanBeUpdated() throws ScriptException {  
58 - String function = "metadata.temp = metadata.temp * 10; return {metadata: metadata};";  
59 - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);  
60 -  
61 - TbMsgMetaData metaData = new TbMsgMetaData();  
62 - metaData.putValue("temp", "7");  
63 - metaData.putValue("humidity", "99");  
64 - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";  
65 -  
66 - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);  
67 -  
68 - TbMsg actual = scriptEngine.executeUpdate(msg);  
69 - assertEquals("70", actual.getMetaData().getValue("temp"));  
70 - scriptEngine.destroy();  
71 - }  
72 -  
73 - @Test  
74 - public void newAttributesCanBeAddedInMsg() throws ScriptException {  
75 - String function = "metadata.newAttr = metadata.humidity - msg.passed; return {metadata: metadata};";  
76 - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);  
77 - TbMsgMetaData metaData = new TbMsgMetaData();  
78 - metaData.putValue("temp", "7");  
79 - metaData.putValue("humidity", "99");  
80 - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";  
81 -  
82 - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);  
83 -  
84 - TbMsg actual = scriptEngine.executeUpdate(msg);  
85 - assertEquals("94", actual.getMetaData().getValue("newAttr"));  
86 - scriptEngine.destroy();  
87 - }  
88 -  
89 - @Test  
90 - public void payloadCanBeUpdated() throws ScriptException {  
91 - String function = "msg.passed = msg.passed * metadata.temp; msg.bigObj.newProp = 'Ukraine'; return {msg: msg};";  
92 - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);  
93 - TbMsgMetaData metaData = new TbMsgMetaData();  
94 - metaData.putValue("temp", "7");  
95 - metaData.putValue("humidity", "99");  
96 - String rawJson = "{\"name\":\"Vit\",\"passed\": 5,\"bigObj\":{\"prop\":42}}";  
97 -  
98 - TbMsg msg =TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson);  
99 -  
100 - TbMsg actual = scriptEngine.executeUpdate(msg);  
101 -  
102 - String expectedJson = "{\"name\":\"Vit\",\"passed\":35,\"bigObj\":{\"prop\":42,\"newProp\":\"Ukraine\"}}";  
103 - assertEquals(expectedJson, actual.getData());  
104 - scriptEngine.destroy();  
105 - }  
106 -  
107 - @Test  
108 - public void metadataAccessibleForFilter() throws ScriptException {  
109 - String function = "return metadata.humidity < 15;";  
110 - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);  
111 - TbMsgMetaData metaData = new TbMsgMetaData();  
112 - metaData.putValue("temp", "7");  
113 - metaData.putValue("humidity", "99");  
114 - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";  
115 -  
116 - TbMsg msg = TbMsg.newMsg("USER", null, metaData, TbMsgDataType.JSON, rawJson);  
117 - assertFalse(scriptEngine.executeFilter(msg));  
118 - scriptEngine.destroy();  
119 - }  
120 -  
121 - @Test  
122 - public void dataAccessibleForFilter() throws ScriptException {  
123 - String function = "return msg.passed < 15 && msg.name === 'Vit' && metadata.temp == 7 && msg.bigObj.prop == 42;";  
124 - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, function);  
125 - TbMsgMetaData metaData = new TbMsgMetaData();  
126 - metaData.putValue("temp", "7");  
127 - metaData.putValue("humidity", "99");  
128 - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";  
129 -  
130 - TbMsg msg = TbMsg.newMsg( "USER", null, metaData,TbMsgDataType.JSON, rawJson);  
131 - assertTrue(scriptEngine.executeFilter(msg));  
132 - scriptEngine.destroy();  
133 - }  
134 -  
135 - @Test  
136 - public void dataAccessibleForSwitch() throws ScriptException {  
137 - String jsCode = "function nextRelation(metadata, msg) {\n" +  
138 - " if(msg.passed == 5 && metadata.temp == 10)\n" +  
139 - " return 'one'\n" +  
140 - " else\n" +  
141 - " return 'two';\n" +  
142 - "};\n" +  
143 - "\n" +  
144 - "return nextRelation(metadata, msg);";  
145 - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode);  
146 - TbMsgMetaData metaData = new TbMsgMetaData();  
147 - metaData.putValue("temp", "10");  
148 - metaData.putValue("humidity", "99");  
149 - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";  
150 -  
151 - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);  
152 - Set<String> actual = scriptEngine.executeSwitch(msg);  
153 - assertEquals(Sets.newHashSet("one"), actual);  
154 - scriptEngine.destroy();  
155 - }  
156 -  
157 - @Test  
158 - public void multipleRelationsReturnedFromSwitch() throws ScriptException {  
159 - String jsCode = "function nextRelation(metadata, msg) {\n" +  
160 - " if(msg.passed == 5 && metadata.temp == 10)\n" +  
161 - " return ['three', 'one']\n" +  
162 - " else\n" +  
163 - " return 'two';\n" +  
164 - "};\n" +  
165 - "\n" +  
166 - "return nextRelation(metadata, msg);";  
167 - scriptEngine = new RuleNodeJsScriptEngine(jsSandboxService, ruleNodeId, jsCode);  
168 - TbMsgMetaData metaData = new TbMsgMetaData();  
169 - metaData.putValue("temp", "10");  
170 - metaData.putValue("humidity", "99");  
171 - String rawJson = "{\"name\": \"Vit\", \"passed\": 5, \"bigObj\": {\"prop\":42}}";  
172 -  
173 - TbMsg msg = TbMsg.newMsg( "USER", null, metaData, TbMsgDataType.JSON, rawJson);  
174 - Set<String> actual = scriptEngine.executeSwitch(msg);  
175 - assertEquals(Sets.newHashSet("one", "three"), actual);  
176 - scriptEngine.destroy();  
177 - }  
178 -  
179 - @Test  
180 - public void concurrentReleasedCorrectly() throws InterruptedException, ExecutionException {  
181 - String code = "metadata.temp = metadata.temp * 10; return {metadata: metadata};";  
182 -  
183 - int repeat = 1000;  
184 - ExecutorService service = Executors.newFixedThreadPool(repeat);  
185 - Map<UUID, Object> scriptIds = new ConcurrentHashMap<>();  
186 - CountDownLatch startLatch = new CountDownLatch(repeat);  
187 - CountDownLatch finishLatch = new CountDownLatch(repeat);  
188 - AtomicInteger failedCount = new AtomicInteger(0);  
189 -  
190 - for (int i = 0; i < repeat; i++) {  
191 - service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code));  
192 - }  
193 -  
194 - finishLatch.await();  
195 - assertTrue(scriptIds.size() == 1);  
196 - assertTrue(failedCount.get() == 0);  
197 -  
198 - CountDownLatch nextStart = new CountDownLatch(repeat);  
199 - CountDownLatch nextFinish = new CountDownLatch(repeat);  
200 - for (int i = 0; i < repeat; i++) {  
201 - service.submit(() -> runScript(nextStart, nextFinish, failedCount, scriptIds, code));  
202 - }  
203 -  
204 - nextFinish.await();  
205 - assertTrue(scriptIds.size() == 1);  
206 - assertTrue(failedCount.get() == 0);  
207 - service.shutdownNow();  
208 - }  
209 -  
210 - @Test  
211 - public void concurrentFailedEvaluationShouldThrowException() throws InterruptedException {  
212 - String code = "metadata.temp = metadata.temp * 10; urn {metadata: metadata};";  
213 -  
214 - int repeat = 10000;  
215 - ExecutorService service = Executors.newFixedThreadPool(repeat);  
216 - Map<UUID, Object> scriptIds = new ConcurrentHashMap<>();  
217 - CountDownLatch startLatch = new CountDownLatch(repeat);  
218 - CountDownLatch finishLatch = new CountDownLatch(repeat);  
219 - AtomicInteger failedCount = new AtomicInteger(0);  
220 - for (int i = 0; i < repeat; i++) {  
221 - service.submit(() -> {  
222 - service.submit(() -> runScript(startLatch, finishLatch, failedCount, scriptIds, code));  
223 - });  
224 - }  
225 -  
226 - finishLatch.await();  
227 - assertTrue(scriptIds.isEmpty());  
228 - assertEquals(repeat, failedCount.get());  
229 - service.shutdownNow();  
230 - }  
231 -  
232 - private void runScript(CountDownLatch startLatch, CountDownLatch finishLatch, AtomicInteger failedCount,  
233 - Map<UUID, Object> scriptIds, String code) {  
234 - try {  
235 - for (int k = 0; k < 10; k++) {  
236 - startLatch.countDown();  
237 - startLatch.await();  
238 - UUID scriptId = jsSandboxService.eval(JsScriptType.RULE_NODE_SCRIPT, code).get();  
239 - scriptIds.put(scriptId, new Object());  
240 - jsSandboxService.invokeFunction(scriptId, "{}", "{}", "TEXT").get();  
241 - jsSandboxService.release(scriptId).get();  
242 - }  
243 - } catch (Throwable th) {  
244 - failedCount.incrementAndGet();  
245 - } finally {  
246 - finishLatch.countDown();  
247 - }  
248 - }  
249 -  
250 -}  
1 -/**  
2 - * Copyright © 2016-2020 The Thingsboard Authors  
3 - *  
4 - * Licensed under the Apache License, Version 2.0 (the "License");  
5 - * you may not use this file except in compliance with the License.  
6 - * You may obtain a copy of the License at  
7 - *  
8 - * http://www.apache.org/licenses/LICENSE-2.0  
9 - *  
10 - * Unless required by applicable law or agreed to in writing, software  
11 - * distributed under the License is distributed on an "AS IS" BASIS,  
12 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
13 - * See the License for the specific language governing permissions and  
14 - * limitations under the License.  
15 - */  
16 -package org.thingsboard.server.service.script;  
17 -  
18 -public class TestNashornJsInvokeService extends AbstractNashornJsInvokeService {  
19 -  
20 - private boolean useJsSandbox;  
21 - private final int monitorThreadPoolSize;  
22 - private final long maxCpuTime;  
23 - private final int maxErrors;  
24 -  
25 - public TestNashornJsInvokeService(boolean useJsSandbox, int monitorThreadPoolSize, long maxCpuTime, int maxErrors) {  
26 - this.useJsSandbox = useJsSandbox;  
27 - this.monitorThreadPoolSize = monitorThreadPoolSize;  
28 - this.maxCpuTime = maxCpuTime;  
29 - this.maxErrors = maxErrors;  
30 - init();  
31 - }  
32 -  
33 - @Override  
34 - protected boolean useJsSandbox() {  
35 - return useJsSandbox;  
36 - }  
37 -  
38 - @Override  
39 - protected int getMonitorThreadPoolSize() {  
40 - return monitorThreadPoolSize;  
41 - }  
42 -  
43 - @Override  
44 - protected long getMaxCpuTime() {  
45 - return maxCpuTime;  
46 - }  
47 -  
48 - @Override  
49 - protected int getMaxErrors() {  
50 - return maxErrors;  
51 - }  
52 -  
53 - @Override  
54 - protected long getMaxBlacklistDuration() {  
55 - return 100000;  
56 - }  
57 -}  
@@ -31,4 +31,6 @@ public interface RuleNodeStateService { @@ -31,4 +31,6 @@ public interface RuleNodeStateService {
31 RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState); 31 RuleNodeState save(TenantId tenantId, RuleNodeState ruleNodeState);
32 32
33 void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId); 33 void removeByRuleNodeId(TenantId tenantId, RuleNodeId selfId);
  34 +
  35 + void removeByRuleNodeIdAndEntityId(TenantId tenantId, RuleNodeId selfId, EntityId entityId);
34 } 36 }
common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java renamed from application/src/main/java/org/thingsboard/server/service/profile/TbTenantProfileCache.java
@@ -13,7 +13,7 @@ @@ -13,7 +13,7 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.server.service.profile; 16 +package org.thingsboard.server.dao.tenant;
17 17
18 import org.thingsboard.server.common.data.TenantProfile; 18 import org.thingsboard.server.common.data.TenantProfile;
19 import org.thingsboard.server.common.data.id.TenantId; 19 import org.thingsboard.server.common.data.id.TenantId;
@@ -36,9 +36,9 @@ public interface TimeseriesService { @@ -36,9 +36,9 @@ public interface TimeseriesService {
36 36
37 ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId); 37 ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId);
38 38
39 - ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); 39 + ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry);
40 40
41 - ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry, long ttl); 41 + ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry, long ttl);
42 42
43 ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry); 43 ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry);
44 44
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.usagerecord;
  17 +
  18 +import org.thingsboard.server.common.data.ApiUsageState;
  19 +import org.thingsboard.server.common.data.id.ApiUsageStateId;
  20 +import org.thingsboard.server.common.data.id.TenantId;
  21 +
  22 +public interface ApiUsageStateService {
  23 +
  24 + ApiUsageState createDefaultApiUsageState(TenantId id);
  25 +
  26 + ApiUsageState update(ApiUsageState apiUsageState);
  27 +
  28 + ApiUsageState findTenantApiUsageState(TenantId tenantId);
  29 +
  30 + void deleteApiUsageStateByTenantId(TenantId tenantId);
  31 +
  32 + ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id);
  33 +}
common/data/src/main/java/org/thingsboard/server/common/data/ApiFeature.java renamed from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitType.java
@@ -13,35 +13,21 @@ @@ -13,35 +13,21 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.server.common.transport.limits; 16 +package org.thingsboard.server.common.data;
17 17
18 import lombok.Getter; 18 import lombok.Getter;
19 19
20 -public enum TransportRateLimitType { 20 +public enum ApiFeature {
  21 + TRANSPORT("transportApiState"),
  22 + DB("dbApiState"),
  23 + RE("ruleEngineApiState"),
  24 + JS("jsExecutionApiState");
21 25
22 - TENANT_MAX_MSGS("transport.tenant.msg", true, true),  
23 - TENANT_TELEMETRY_MSGS("transport.tenant.telemetry", true, true),  
24 - TENANT_MAX_DATA_POINTS("transport.tenant.dataPoints", true, false),  
25 - DEVICE_MAX_MSGS("transport.device.msg", false, true),  
26 - DEVICE_TELEMETRY_MSGS("transport.device.telemetry", false, true),  
27 - DEVICE_MAX_DATA_POINTS("transport.device.dataPoints", false, false);  
28 -  
29 - @Getter  
30 - private final String configurationKey;  
31 - @Getter  
32 - private final boolean tenantLevel;  
33 @Getter 26 @Getter
34 - private final boolean deviceLevel;  
35 - @Getter  
36 - private final boolean messageLevel;  
37 - @Getter  
38 - private final boolean dataPointLevel; 27 + private final String apiStateKey;
39 28
40 - TransportRateLimitType(String configurationKey, boolean tenantLevel, boolean messageLevel) {  
41 - this.configurationKey = configurationKey;  
42 - this.tenantLevel = tenantLevel;  
43 - this.deviceLevel = !tenantLevel;  
44 - this.messageLevel = messageLevel;  
45 - this.dataPointLevel = !messageLevel; 29 + ApiFeature(String apiStateKey) {
  30 + this.apiStateKey = apiStateKey;
46 } 31 }
  32 +
47 } 33 }
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data;
  17 +
  18 +import lombok.Getter;
  19 +
  20 +public enum ApiUsageRecordKey {
  21 +
  22 + TRANSPORT_MSG_COUNT(ApiFeature.TRANSPORT, "transportMsgCount", "transportMsgLimit"),
  23 + TRANSPORT_DP_COUNT(ApiFeature.TRANSPORT, "transportDataPointsCount", "transportDataPointsLimit"),
  24 + STORAGE_DP_COUNT(ApiFeature.DB, "storageDataPointsCount", "storageDataPointsLimit"),
  25 + RE_EXEC_COUNT(ApiFeature.RE, "ruleEngineExecutionCount", "ruleEngineExecutionLimit"),
  26 + JS_EXEC_COUNT(ApiFeature.JS, "jsExecutionCount", "jsExecutionLimit");
  27 + private static final ApiUsageRecordKey[] JS_RECORD_KEYS = {JS_EXEC_COUNT};
  28 + private static final ApiUsageRecordKey[] RE_RECORD_KEYS = {RE_EXEC_COUNT};
  29 + private static final ApiUsageRecordKey[] DB_RECORD_KEYS = {STORAGE_DP_COUNT};
  30 + private static final ApiUsageRecordKey[] TRANSPORT_RECORD_KEYS = {TRANSPORT_MSG_COUNT, TRANSPORT_DP_COUNT};
  31 +
  32 + @Getter
  33 + private final ApiFeature apiFeature;
  34 + @Getter
  35 + private final String apiCountKey;
  36 + @Getter
  37 + private final String apiLimitKey;
  38 +
  39 + ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey) {
  40 + this.apiFeature = apiFeature;
  41 + this.apiCountKey = apiCountKey;
  42 + this.apiLimitKey = apiLimitKey;
  43 + }
  44 +
  45 + public static ApiUsageRecordKey[] getKeys(ApiFeature feature) {
  46 + switch (feature) {
  47 + case TRANSPORT:
  48 + return TRANSPORT_RECORD_KEYS;
  49 + case DB:
  50 + return DB_RECORD_KEYS;
  51 + case RE:
  52 + return RE_RECORD_KEYS;
  53 + case JS:
  54 + return JS_RECORD_KEYS;
  55 + default:
  56 + return new ApiUsageRecordKey[]{};
  57 + }
  58 + }
  59 +
  60 +}
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data;
  17 +
  18 +import lombok.EqualsAndHashCode;
  19 +import lombok.Getter;
  20 +import lombok.Setter;
  21 +import lombok.ToString;
  22 +import org.thingsboard.server.common.data.id.EntityId;
  23 +import org.thingsboard.server.common.data.id.TenantId;
  24 +import org.thingsboard.server.common.data.id.ApiUsageStateId;
  25 +
  26 +@ToString
  27 +@EqualsAndHashCode(callSuper = true)
  28 +public class ApiUsageState extends BaseData<ApiUsageStateId> implements HasTenantId {
  29 +
  30 + private static final long serialVersionUID = 8250339805336035966L;
  31 +
  32 + @Getter
  33 + @Setter
  34 + private TenantId tenantId;
  35 + @Getter
  36 + @Setter
  37 + private EntityId entityId;
  38 + @Getter
  39 + @Setter
  40 + private ApiUsageStateValue transportState;
  41 + @Getter
  42 + @Setter
  43 + private ApiUsageStateValue dbStorageState;
  44 + @Getter
  45 + @Setter
  46 + private ApiUsageStateValue reExecState;
  47 + @Getter
  48 + @Setter
  49 + private ApiUsageStateValue jsExecState;
  50 +
  51 + public ApiUsageState() {
  52 + super();
  53 + }
  54 +
  55 + public ApiUsageState(ApiUsageStateId id) {
  56 + super(id);
  57 + }
  58 +
  59 + public ApiUsageState(ApiUsageState ur) {
  60 + super(ur);
  61 + this.tenantId = ur.getTenantId();
  62 + this.entityId = ur.getEntityId();
  63 + this.transportState = ur.getTransportState();
  64 + this.dbStorageState = ur.getDbStorageState();
  65 + this.reExecState = ur.getReExecState();
  66 + this.jsExecState = ur.getJsExecState();
  67 + }
  68 +
  69 + public boolean isTransportEnabled() {
  70 + return !ApiUsageStateValue.DISABLED.equals(transportState);
  71 + }
  72 +
  73 + public boolean isReExecEnabled() {
  74 + return !ApiUsageStateValue.DISABLED.equals(reExecState);
  75 + }
  76 +
  77 + public boolean isDbStorageEnabled() {
  78 + return !ApiUsageStateValue.DISABLED.equals(dbStorageState);
  79 + }
  80 +
  81 + public boolean isJsExecEnabled() {
  82 + return !ApiUsageStateValue.DISABLED.equals(jsExecState);
  83 + }
  84 +}
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data;
  17 +
  18 +public enum ApiUsageStateValue {
  19 +
  20 + ENABLED, WARNING, DISABLED;
  21 +
  22 +
  23 + public static ApiUsageStateValue toMoreRestricted(ApiUsageStateValue a, ApiUsageStateValue b) {
  24 + return a.ordinal() > b.ordinal() ? a : b;
  25 + }
  26 +}
@@ -43,6 +43,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H @@ -43,6 +43,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H
43 private DeviceTransportType transportType; 43 private DeviceTransportType transportType;
44 private DeviceProfileProvisionType provisionType; 44 private DeviceProfileProvisionType provisionType;
45 private RuleChainId defaultRuleChainId; 45 private RuleChainId defaultRuleChainId;
  46 + private String defaultQueueName;
46 private transient DeviceProfileData profileData; 47 private transient DeviceProfileData profileData;
47 @JsonIgnore 48 @JsonIgnore
48 private byte[] profileDataBytes; 49 private byte[] profileDataBytes;
@@ -63,6 +64,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H @@ -63,6 +64,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H
63 this.description = deviceProfile.getDescription(); 64 this.description = deviceProfile.getDescription();
64 this.isDefault = deviceProfile.isDefault(); 65 this.isDefault = deviceProfile.isDefault();
65 this.defaultRuleChainId = deviceProfile.getDefaultRuleChainId(); 66 this.defaultRuleChainId = deviceProfile.getDefaultRuleChainId();
  67 + this.defaultQueueName = deviceProfile.getDefaultQueueName();
66 this.setProfileData(deviceProfile.getProfileData()); 68 this.setProfileData(deviceProfile.getProfileData());
67 this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey(); 69 this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey();
68 } 70 }
@@ -19,5 +19,5 @@ package org.thingsboard.server.common.data; @@ -19,5 +19,5 @@ package org.thingsboard.server.common.data;
19 * @author Andrew Shvayka 19 * @author Andrew Shvayka
20 */ 20 */
21 public enum EntityType { 21 public enum EntityType {
22 - TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE 22 + TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE;
23 } 23 }
@@ -21,6 +21,8 @@ import lombok.Data; @@ -21,6 +21,8 @@ import lombok.Data;
21 import lombok.EqualsAndHashCode; 21 import lombok.EqualsAndHashCode;
22 import lombok.extern.slf4j.Slf4j; 22 import lombok.extern.slf4j.Slf4j;
23 import org.thingsboard.server.common.data.id.TenantProfileId; 23 import org.thingsboard.server.common.data.id.TenantProfileId;
  24 +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
  25 +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
24 26
25 import java.io.ByteArrayInputStream; 27 import java.io.ByteArrayInputStream;
26 import java.io.IOException; 28 import java.io.IOException;
@@ -78,15 +80,21 @@ public class TenantProfile extends SearchTextBased<TenantProfileId> implements H @@ -78,15 +80,21 @@ public class TenantProfile extends SearchTextBased<TenantProfileId> implements H
78 profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class); 80 profileData = mapper.readValue(new ByteArrayInputStream(profileDataBytes), TenantProfileData.class);
79 } catch (IOException e) { 81 } catch (IOException e) {
80 log.warn("Can't deserialize tenant profile data: ", e); 82 log.warn("Can't deserialize tenant profile data: ", e);
81 - return null; 83 + return createDefaultTenantProfileData();
82 } 84 }
83 return profileData; 85 return profileData;
84 } else { 86 } else {
85 - return null; 87 + return createDefaultTenantProfileData();
86 } 88 }
87 } 89 }
88 } 90 }
89 91
  92 + public TenantProfileData createDefaultTenantProfileData() {
  93 + TenantProfileData tpd = new TenantProfileData();
  94 + tpd.setConfiguration(new DefaultTenantProfileConfiguration());
  95 + return tpd;
  96 + }
  97 +
90 public void setProfileData(TenantProfileData data) { 98 public void setProfileData(TenantProfileData data) {
91 this.profileData = data; 99 this.profileData = data;
92 try { 100 try {
common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileType.java renamed from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportRateLimitFactory.java
@@ -13,12 +13,8 @@ @@ -13,12 +13,8 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.server.common.transport.limits;  
17 -  
18 -public interface TransportRateLimitFactory {  
19 -  
20 - TransportRateLimit create(TransportRateLimitType type, Object config);  
21 -  
22 - TransportRateLimit createDefault(TransportRateLimitType type); 16 +package org.thingsboard.server.common.data;
23 17
  18 +public enum TenantProfileType {
  19 + DEFAULT
24 } 20 }
@@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile; @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile;
18 import lombok.Data; 18 import lombok.Data;
19 import org.thingsboard.server.common.data.alarm.AlarmSeverity; 19 import org.thingsboard.server.common.data.alarm.AlarmSeverity;
20 20
  21 +import java.util.LinkedHashMap;
21 import java.util.List; 22 import java.util.List;
22 import java.util.Map; 23 import java.util.Map;
23 24
@@ -27,7 +28,7 @@ public class DeviceProfileAlarm { @@ -27,7 +28,7 @@ public class DeviceProfileAlarm {
27 private String id; 28 private String id;
28 private String alarmType; 29 private String alarmType;
29 30
30 - private Map<AlarmSeverity, AlarmRule> createRules; 31 + private LinkedHashMap<AlarmSeverity, AlarmRule> createRules;
31 private AlarmRule clearRule; 32 private AlarmRule clearRule;
32 33
33 // Hidden in advanced settings 34 // Hidden in advanced settings
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data.id;
  17 +
  18 +import com.fasterxml.jackson.annotation.JsonCreator;
  19 +import com.fasterxml.jackson.annotation.JsonIgnore;
  20 +import com.fasterxml.jackson.annotation.JsonProperty;
  21 +import org.thingsboard.server.common.data.EntityType;
  22 +
  23 +import java.util.UUID;
  24 +
  25 +public class ApiUsageStateId extends UUIDBased implements EntityId {
  26 +
  27 + @JsonCreator
  28 + public ApiUsageStateId(@JsonProperty("id") UUID id) {
  29 + super(id);
  30 + }
  31 +
  32 + public static ApiUsageStateId fromString(String userId) {
  33 + return new ApiUsageStateId(UUID.fromString(userId));
  34 + }
  35 +
  36 + @JsonIgnore
  37 + @Override
  38 + public EntityType getEntityType() {
  39 + return EntityType.API_USAGE_STATE;
  40 + }
  41 +
  42 +}
@@ -66,6 +66,8 @@ public class EntityIdFactory { @@ -66,6 +66,8 @@ public class EntityIdFactory {
66 return new DeviceProfileId(uuid); 66 return new DeviceProfileId(uuid);
67 case TENANT_PROFILE: 67 case TENANT_PROFILE:
68 return new TenantProfileId(uuid); 68 return new TenantProfileId(uuid);
  69 + case API_USAGE_STATE:
  70 + return new ApiUsageStateId(uuid);
69 } 71 }
70 throw new IllegalArgumentException("EntityType " + type + " is not supported!"); 72 throw new IllegalArgumentException("EntityType " + type + " is not supported!");
71 } 73 }
@@ -19,7 +19,7 @@ import java.util.Objects; @@ -19,7 +19,7 @@ import java.util.Objects;
19 import java.util.Optional; 19 import java.util.Optional;
20 20
21 public class BasicTsKvEntry implements TsKvEntry { 21 public class BasicTsKvEntry implements TsKvEntry {
22 - 22 + private static final int MAX_CHARS_PER_DATA_POINT = 512;
23 private final long ts; 23 private final long ts;
24 private final KvEntry kv; 24 private final KvEntry kv;
25 25
@@ -99,4 +99,21 @@ public class BasicTsKvEntry implements TsKvEntry { @@ -99,4 +99,21 @@ public class BasicTsKvEntry implements TsKvEntry {
99 public String getValueAsString() { 99 public String getValueAsString() {
100 return kv.getValueAsString(); 100 return kv.getValueAsString();
101 } 101 }
  102 +
  103 + @Override
  104 + public int getDataPoints() {
  105 + int length;
  106 + switch (getDataType()) {
  107 + case STRING:
  108 + length = getStrValue().get().length();
  109 + break;
  110 + case JSON:
  111 + length = getJsonValue().get().length();
  112 + break;
  113 + default:
  114 + return 1;
  115 + }
  116 + return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT);
  117 + }
  118 +
102 } 119 }
@@ -15,6 +15,8 @@ @@ -15,6 +15,8 @@
15 */ 15 */
16 package org.thingsboard.server.common.data.kv; 16 package org.thingsboard.server.common.data.kv;
17 17
  18 +import com.fasterxml.jackson.annotation.JsonIgnore;
  19 +
18 /** 20 /**
19 * Represents time series KV data entry 21 * Represents time series KV data entry
20 * 22 *
@@ -25,4 +27,7 @@ public interface TsKvEntry extends KvEntry { @@ -25,4 +27,7 @@ public interface TsKvEntry extends KvEntry {
25 27
26 long getTs(); 28 long getTs();
27 29
  30 + @JsonIgnore
  31 + int getDataPoints();
  32 +
28 } 33 }
@@ -21,5 +21,5 @@ import java.io.Serializable; @@ -21,5 +21,5 @@ import java.io.Serializable;
21 * @author Andrew Shvayka 21 * @author Andrew Shvayka
22 */ 22 */
23 public enum ComponentLifecycleEvent implements Serializable { 23 public enum ComponentLifecycleEvent implements Serializable {
24 - CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED 24 + CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED, ADDED_TO_ALLOW_LIST, ADDED_TO_DENY_LIST
25 } 25 }
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data.query;
  17 +
  18 +import lombok.Data;
  19 +import org.thingsboard.server.common.data.id.EntityId;
  20 +
  21 +@Data
  22 +public class ApiUsageStateFilter implements EntityFilter {
  23 + @Override
  24 + public EntityFilterType getType() {
  25 + return EntityFilterType.API_USAGE_STATE;
  26 + }
  27 +
  28 +}
@@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
32 @JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"), 32 @JsonSubTypes.Type(value = AssetTypeFilter.class, name = "assetType"),
33 @JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"), 33 @JsonSubTypes.Type(value = DeviceTypeFilter.class, name = "deviceType"),
34 @JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"), 34 @JsonSubTypes.Type(value = EntityViewTypeFilter.class, name = "entityViewType"),
  35 + @JsonSubTypes.Type(value = ApiUsageStateFilter.class, name = "apiUsageState"),
35 @JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"), 36 @JsonSubTypes.Type(value = RelationsQueryFilter.class, name = "relationsQuery"),
36 @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), 37 @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"),
37 @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), 38 @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"),
@@ -25,7 +25,8 @@ public enum EntityFilterType { @@ -25,7 +25,8 @@ public enum EntityFilterType {
25 RELATIONS_QUERY("relationsQuery"), 25 RELATIONS_QUERY("relationsQuery"),
26 ASSET_SEARCH_QUERY("assetSearchQuery"), 26 ASSET_SEARCH_QUERY("assetSearchQuery"),
27 DEVICE_SEARCH_QUERY("deviceSearchQuery"), 27 DEVICE_SEARCH_QUERY("deviceSearchQuery"),
28 - ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"); 28 + ENTITY_VIEW_SEARCH_QUERY("entityViewSearchQuery"),
  29 + API_USAGE_STATE("apiUsageState");
29 30
30 private final String label; 31 private final String label;
31 32
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data.tenant.profile;
  17 +
  18 +import lombok.Data;
  19 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
  20 +import org.thingsboard.server.common.data.TenantProfileType;
  21 +
  22 +@Data
  23 +public class DefaultTenantProfileConfiguration implements TenantProfileConfiguration {
  24 +
  25 + private long maxDevices;
  26 + private long maxAssets;
  27 +
  28 + private String transportTenantMsgRateLimit;
  29 + private String transportTenantTelemetryMsgRateLimit;
  30 + private String transportTenantTelemetryDataPointsRateLimit;
  31 + private String transportDeviceMsgRateLimit;
  32 + private String transportDeviceTelemetryMsgRateLimit;
  33 + private String transportDeviceTelemetryDataPointsRateLimit;
  34 +
  35 + private long maxTransportMessages;
  36 + private long maxTransportDataPoints;
  37 + private long maxREExecutions;
  38 + private long maxJSExecutions;
  39 + private long maxDPStorageDays;
  40 + private int maxRuleNodeExecutionsPerMessage;
  41 +
  42 + private double warnThreshold;
  43 +
  44 + @Override
  45 + public long getProfileThreshold(ApiUsageRecordKey key) {
  46 + switch (key) {
  47 + case TRANSPORT_MSG_COUNT:
  48 + return maxTransportMessages;
  49 + case TRANSPORT_DP_COUNT:
  50 + return maxTransportDataPoints;
  51 + case JS_EXEC_COUNT:
  52 + return maxJSExecutions;
  53 + case RE_EXEC_COUNT:
  54 + return maxREExecutions;
  55 + case STORAGE_DP_COUNT:
  56 + return maxDPStorageDays;
  57 + }
  58 + return 0L;
  59 + }
  60 +
  61 + @Override
  62 + public long getWarnThreshold(ApiUsageRecordKey key) {
  63 + return (long) (getProfileThreshold(key) * (warnThreshold > 0.0 ? warnThreshold : 0.8));
  64 + }
  65 +
  66 + @Override
  67 + public TenantProfileType getType() {
  68 + return TenantProfileType.DEFAULT;
  69 + }
  70 +
  71 + @Override
  72 + public int getMaxRuleNodeExecsPerMessage() {
  73 + return maxRuleNodeExecutionsPerMessage;
  74 + }
  75 +}
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data.tenant.profile;
  17 +
  18 +import com.fasterxml.jackson.annotation.JsonIgnore;
  19 +import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
  20 +import com.fasterxml.jackson.annotation.JsonSubTypes;
  21 +import com.fasterxml.jackson.annotation.JsonTypeInfo;
  22 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
  23 +import org.thingsboard.server.common.data.TenantProfileType;
  24 +
  25 +@JsonIgnoreProperties(ignoreUnknown = true)
  26 +@JsonTypeInfo(
  27 + use = JsonTypeInfo.Id.NAME,
  28 + include = JsonTypeInfo.As.PROPERTY,
  29 + property = "type")
  30 +@JsonSubTypes({
  31 + @JsonSubTypes.Type(value = DefaultTenantProfileConfiguration.class, name = "DEFAULT")})
  32 +public interface TenantProfileConfiguration {
  33 +
  34 + @JsonIgnore
  35 + TenantProfileType getType();
  36 +
  37 + @JsonIgnore
  38 + long getProfileThreshold(ApiUsageRecordKey key);
  39 +
  40 + @JsonIgnore
  41 + long getWarnThreshold(ApiUsageRecordKey key);
  42 +
  43 + @JsonIgnore
  44 + int getMaxRuleNodeExecsPerMessage();
  45 +
  46 +}
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java renamed from common/data/src/main/java/org/thingsboard/server/common/data/TenantProfileData.java
@@ -13,30 +13,13 @@ @@ -13,30 +13,13 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.server.common.data; 16 +package org.thingsboard.server.common.data.tenant.profile;
17 17
18 -import com.fasterxml.jackson.annotation.JsonAnyGetter;  
19 -import com.fasterxml.jackson.annotation.JsonAnySetter;  
20 -import com.fasterxml.jackson.annotation.JsonIgnore;  
21 import lombok.Data; 18 import lombok.Data;
22 19
23 -import java.util.HashMap;  
24 -import java.util.Map;  
25 -  
26 @Data 20 @Data
27 public class TenantProfileData { 21 public class TenantProfileData {
28 22
29 - @JsonIgnore  
30 - private Map<String, Object> properties = new HashMap<>();  
31 -  
32 - @JsonAnyGetter  
33 - public Map<String, Object> properties() {  
34 - return this.properties;  
35 - }  
36 -  
37 - @JsonAnySetter  
38 - public void put(String name, Object value) {  
39 - this.properties.put(name, value);  
40 - } 23 + private TenantProfileConfiguration configuration;
41 24
42 } 25 }
@@ -18,27 +18,27 @@ package org.thingsboard.server.common.msg; @@ -18,27 +18,27 @@ package org.thingsboard.server.common.msg;
18 import com.fasterxml.jackson.annotation.JsonIgnore; 18 import com.fasterxml.jackson.annotation.JsonIgnore;
19 import com.google.protobuf.ByteString; 19 import com.google.protobuf.ByteString;
20 import com.google.protobuf.InvalidProtocolBufferException; 20 import com.google.protobuf.InvalidProtocolBufferException;
  21 +import lombok.AccessLevel;
21 import lombok.Builder; 22 import lombok.Builder;
22 import lombok.Data; 23 import lombok.Data;
  24 +import lombok.Getter;
23 import lombok.extern.slf4j.Slf4j; 25 import lombok.extern.slf4j.Slf4j;
24 import org.thingsboard.server.common.data.id.EntityId; 26 import org.thingsboard.server.common.data.id.EntityId;
25 import org.thingsboard.server.common.data.id.EntityIdFactory; 27 import org.thingsboard.server.common.data.id.EntityIdFactory;
26 import org.thingsboard.server.common.data.id.RuleChainId; 28 import org.thingsboard.server.common.data.id.RuleChainId;
27 import org.thingsboard.server.common.data.id.RuleNodeId; 29 import org.thingsboard.server.common.data.id.RuleNodeId;
28 import org.thingsboard.server.common.msg.gen.MsgProtos; 30 import org.thingsboard.server.common.msg.gen.MsgProtos;
29 -import org.thingsboard.server.common.msg.queue.RuleNodeInfo;  
30 import org.thingsboard.server.common.msg.queue.ServiceQueue; 31 import org.thingsboard.server.common.msg.queue.ServiceQueue;
31 import org.thingsboard.server.common.msg.queue.TbMsgCallback; 32 import org.thingsboard.server.common.msg.queue.TbMsgCallback;
32 33
33 -import java.io.IOException;  
34 import java.io.Serializable; 34 import java.io.Serializable;
35 import java.util.UUID; 35 import java.util.UUID;
  36 +import java.util.concurrent.atomic.AtomicInteger;
36 37
37 /** 38 /**
38 * Created by ashvayka on 13.01.18. 39 * Created by ashvayka on 13.01.18.
39 */ 40 */
40 @Data 41 @Data
41 -@Builder  
42 @Slf4j 42 @Slf4j
43 public final class TbMsg implements Serializable { 43 public final class TbMsg implements Serializable {
44 44
@@ -52,51 +52,73 @@ public final class TbMsg implements Serializable { @@ -52,51 +52,73 @@ public final class TbMsg implements Serializable {
52 private final String data; 52 private final String data;
53 private final RuleChainId ruleChainId; 53 private final RuleChainId ruleChainId;
54 private final RuleNodeId ruleNodeId; 54 private final RuleNodeId ruleNodeId;
  55 + @Getter(value = AccessLevel.NONE)
  56 + private final AtomicInteger ruleNodeExecCounter;
  57 +
  58 + public int getAndIncrementRuleNodeCounter() {
  59 + return ruleNodeExecCounter.getAndIncrement();
  60 + }
  61 +
55 //This field is not serialized because we use queues and there is no need to do it 62 //This field is not serialized because we use queues and there is no need to do it
56 @JsonIgnore 63 @JsonIgnore
57 transient private final TbMsgCallback callback; 64 transient private final TbMsgCallback callback;
58 65
59 public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { 66 public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
60 - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); 67 + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator,
  68 + metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
61 } 69 }
62 70
63 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { 71 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) {
64 - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); 72 + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
65 } 73 }
66 74
  75 + // REALLY NEW MSG
  76 +
67 public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) { 77 public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) {
68 - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); 78 + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
69 } 79 }
70 80
71 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) { 81 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) {
72 - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, TbMsgCallback.EMPTY); 82 + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, 0, TbMsgCallback.EMPTY);
73 } 83 }
74 84
  85 + // For Tests only
  86 +
75 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { 87 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
76 - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); 88 + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
77 } 89 }
78 90
79 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) { 91 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) {
80 - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, callback); 92 + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, callback);
  93 + }
  94 +
  95 + public static TbMsg transformMsg(TbMsg tbMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) {
  96 + return new TbMsg(tbMsg.getQueueName(), tbMsg.getId(), tbMsg.getTs(), type, originator, metaData.copy(), tbMsg.getDataType(),
  97 + data, tbMsg.getRuleChainId(), tbMsg.getRuleNodeId(), tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
  98 + }
  99 +
  100 + public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId) {
  101 + return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
  102 + tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
81 } 103 }
82 104
83 - public static TbMsg transformMsg(TbMsg origMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) {  
84 - return new TbMsg(origMsg.getQueueName(), origMsg.getId(), origMsg.getTs(), type, originator, metaData.copy(), origMsg.getDataType(),  
85 - data, origMsg.getRuleChainId(), origMsg.getRuleNodeId(), origMsg.getCallback()); 105 + public static TbMsg transformMsg(TbMsg tbMsg, String queueName) {
  106 + return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
  107 + tbMsg.data, tbMsg.getRuleChainId(), null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
86 } 108 }
87 109
88 - public static TbMsg transformMsg(TbMsg origMsg, RuleChainId ruleChainId) {  
89 - return new TbMsg(origMsg.queueName, origMsg.id, origMsg.ts, origMsg.type, origMsg.originator, origMsg.metaData, origMsg.dataType,  
90 - origMsg.data, ruleChainId, null, origMsg.getCallback()); 110 + public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId, String queueName) {
  111 + return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
  112 + tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
91 } 113 }
92 114
93 public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { 115 public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
94 return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(), 116 return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(),
95 - tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, TbMsgCallback.EMPTY); 117 + tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, tbMsg.ruleNodeExecCounter.get(), TbMsgCallback.EMPTY);
96 } 118 }
97 119
98 private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, 120 private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
99 - RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgCallback callback) { 121 + RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) {
100 this.id = id; 122 this.id = id;
101 this.queueName = queueName; 123 this.queueName = queueName;
102 if (ts > 0) { 124 if (ts > 0) {
@@ -111,6 +133,7 @@ public final class TbMsg implements Serializable { @@ -111,6 +133,7 @@ public final class TbMsg implements Serializable {
111 this.data = data; 133 this.data = data;
112 this.ruleChainId = ruleChainId; 134 this.ruleChainId = ruleChainId;
113 this.ruleNodeId = ruleNodeId; 135 this.ruleNodeId = ruleNodeId;
  136 + this.ruleNodeExecCounter = new AtomicInteger(ruleNodeExecCounter);
114 if (callback != null) { 137 if (callback != null) {
115 this.callback = callback; 138 this.callback = callback;
116 } else { 139 } else {
@@ -147,6 +170,7 @@ public final class TbMsg implements Serializable { @@ -147,6 +170,7 @@ public final class TbMsg implements Serializable {
147 170
148 builder.setDataType(msg.getDataType().ordinal()); 171 builder.setDataType(msg.getDataType().ordinal());
149 builder.setData(msg.getData()); 172 builder.setData(msg.getData());
  173 + builder.setRuleNodeExecCounter(msg.ruleNodeExecCounter.get());
150 return builder.build().toByteArray(); 174 return builder.build().toByteArray();
151 } 175 }
152 176
@@ -164,18 +188,18 @@ public final class TbMsg implements Serializable { @@ -164,18 +188,18 @@ public final class TbMsg implements Serializable {
164 ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB())); 188 ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB()));
165 } 189 }
166 TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; 190 TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
167 - return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, callback); 191 + return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, proto.getRuleNodeExecCounter(), callback);
168 } catch (InvalidProtocolBufferException e) { 192 } catch (InvalidProtocolBufferException e) {
169 throw new IllegalStateException("Could not parse protobuf for TbMsg", e); 193 throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
170 } 194 }
171 } 195 }
172 196
173 public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) { 197 public TbMsg copyWithRuleChainId(RuleChainId ruleChainId) {
174 - return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, callback); 198 + return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, this.ruleNodeExecCounter.get(), callback);
175 } 199 }
176 200
177 public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { 201 public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
178 - return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); 202 + return new TbMsg(this.queueName, this.id, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ruleNodeExecCounter.get(), callback);
179 } 203 }
180 204
181 public TbMsgCallback getCallback() { 205 public TbMsgCallback getCallback() {
@@ -50,6 +50,10 @@ public class TopicPartitionInfo { @@ -50,6 +50,10 @@ public class TopicPartitionInfo {
50 this.fullTopicName = tmp; 50 this.fullTopicName = tmp;
51 } 51 }
52 52
  53 + public TopicPartitionInfo newByTopic(String topic) {
  54 + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition);
  55 + }
  56 +
53 public String getTopic() { 57 public String getTopic() {
54 return topic; 58 return topic;
55 } 59 }
@@ -15,16 +15,64 @@ @@ -15,16 +15,64 @@
15 */ 15 */
16 package org.thingsboard.server.common.msg.tools; 16 package org.thingsboard.server.common.msg.tools;
17 17
  18 +import java.time.LocalDate;
  19 +import java.time.LocalDateTime;
  20 +import java.time.LocalTime;
18 import java.time.ZoneId; 21 import java.time.ZoneId;
  22 +import java.time.ZoneOffset;
  23 +import java.time.temporal.ChronoUnit;
  24 +import java.time.temporal.TemporalAdjuster;
  25 +import java.time.temporal.TemporalAdjusters;
  26 +import java.time.temporal.TemporalUnit;
19 import java.util.concurrent.ConcurrentHashMap; 27 import java.util.concurrent.ConcurrentHashMap;
20 import java.util.concurrent.ConcurrentMap; 28 import java.util.concurrent.ConcurrentMap;
21 29
  30 +import static java.time.temporal.ChronoField.DAY_OF_MONTH;
  31 +import static java.time.temporal.ChronoUnit.MONTHS;
  32 +
22 public class SchedulerUtils { 33 public class SchedulerUtils {
23 34
  35 + private final static ZoneId UTC = ZoneId.of("UTC");
24 private static final ConcurrentMap<String, ZoneId> tzMap = new ConcurrentHashMap<>(); 36 private static final ConcurrentMap<String, ZoneId> tzMap = new ConcurrentHashMap<>();
25 37
26 public static ZoneId getZoneId(String tz) { 38 public static ZoneId getZoneId(String tz) {
27 return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of); 39 return tzMap.computeIfAbsent(tz == null || tz.isEmpty() ? "UTC" : tz, ZoneId::of);
28 } 40 }
29 41
  42 + public static long getStartOfCurrentHour() {
  43 + return getStartOfCurrentHour(UTC);
  44 + }
  45 +
  46 + public static long getStartOfCurrentHour(ZoneId zoneId) {
  47 + return LocalDateTime.now(ZoneOffset.UTC).atZone(zoneId).truncatedTo(ChronoUnit.HOURS).toInstant().toEpochMilli();
  48 + }
  49 +
  50 + public static long getStartOfCurrentMonth() {
  51 + return getStartOfCurrentMonth(UTC);
  52 + }
  53 +
  54 + public static long getStartOfCurrentMonth(ZoneId zoneId) {
  55 + return LocalDate.now().withDayOfMonth(1).atStartOfDay(zoneId).toInstant().toEpochMilli();
  56 + }
  57 +
  58 + public static long getStartOfNextMonth() {
  59 + return getStartOfNextMonth(UTC);
  60 + }
  61 +
  62 + public static long getStartOfNextMonth(ZoneId zoneId) {
  63 + return LocalDate.now().with(TemporalAdjusters.firstDayOfNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli();
  64 + }
  65 +
  66 + public static long getStartOfNextNextMonth() {
  67 + return getStartOfNextNextMonth(UTC);
  68 + }
  69 +
  70 + public static long getStartOfNextNextMonth(ZoneId zoneId) {
  71 + return LocalDate.now().with(firstDayOfNextNextMonth()).atStartOfDay(zoneId).toInstant().toEpochMilli();
  72 + }
  73 +
  74 + public static TemporalAdjuster firstDayOfNextNextMonth() {
  75 + return (temporal) -> temporal.with(DAY_OF_MONTH, 1).plus(2, MONTHS);
  76 + }
  77 +
30 } 78 }
@@ -45,4 +45,5 @@ message TbMsgProto { @@ -45,4 +45,5 @@ message TbMsgProto {
45 string data = 14; 45 string data = 14;
46 46
47 int64 ts = 15; 47 int64 ts = 15;
  48 + int32 ruleNodeExecCounter = 16;
48 } 49 }
common/queue/src/main/java/org/thingsboard/server/queue/common/MultipleTbQueueCallbackWrapper.java renamed from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitFactory.java
@@ -13,35 +13,33 @@ @@ -13,35 +13,33 @@
13 * See the License for the specific language governing permissions and 13 * See the License for the specific language governing permissions and
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 -package org.thingsboard.server.common.transport.limits; 16 +package org.thingsboard.server.queue.common;
17 17
18 -import lombok.extern.slf4j.Slf4j;  
19 -import org.springframework.stereotype.Component;  
20 -import org.springframework.util.StringUtils;  
21 -import org.thingsboard.server.common.msg.tools.TbRateLimits; 18 +import org.thingsboard.server.common.msg.queue.RuleEngineException;
  19 +import org.thingsboard.server.queue.TbQueueCallback;
  20 +import org.thingsboard.server.queue.TbQueueMsgMetadata;
22 21
23 -@Slf4j  
24 -@Component  
25 -public class DefaultTransportRateLimitFactory implements TransportRateLimitFactory { 22 +import java.util.concurrent.atomic.AtomicInteger;
26 23
27 - private static final DummyTransportRateLimit ALWAYS_TRUE = new DummyTransportRateLimit(); 24 +public class MultipleTbQueueCallbackWrapper implements TbQueueCallback {
  25 +
  26 + private final AtomicInteger tbQueueCallbackCount;
  27 + private final TbQueueCallback callback;
  28 +
  29 + public MultipleTbQueueCallbackWrapper(int tbQueueCallbackCount, TbQueueCallback callback) {
  30 + this.tbQueueCallbackCount = new AtomicInteger(tbQueueCallbackCount);
  31 + this.callback = callback;
  32 + }
28 33
29 @Override 34 @Override
30 - public TransportRateLimit create(TransportRateLimitType type, Object configuration) {  
31 - if (!StringUtils.isEmpty(configuration)) {  
32 - try {  
33 - return new SimpleTransportRateLimit(new TbRateLimits(configuration.toString()), configuration.toString());  
34 - } catch (Exception e) {  
35 - log.warn("[{}] Failed to init rate limit with configuration: {}", type, configuration, e);  
36 - return ALWAYS_TRUE;  
37 - }  
38 - } else {  
39 - return ALWAYS_TRUE; 35 + public void onSuccess(TbQueueMsgMetadata metadata) {
  36 + if (tbQueueCallbackCount.decrementAndGet() <= 0) {
  37 + callback.onSuccess(metadata);
40 } 38 }
41 } 39 }
42 40
43 @Override 41 @Override
44 - public TransportRateLimit createDefault(TransportRateLimitType type) {  
45 - return ALWAYS_TRUE; 42 + public void onFailure(Throwable t) {
  43 + callback.onFailure(new RuleEngineException(t.getMessage()));
46 } 44 }
47 } 45 }
@@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
20 import org.springframework.boot.context.event.ApplicationReadyEvent; 20 import org.springframework.boot.context.event.ApplicationReadyEvent;
21 import org.springframework.context.annotation.DependsOn; 21 import org.springframework.context.annotation.DependsOn;
22 import org.springframework.context.event.EventListener; 22 import org.springframework.context.event.EventListener;
  23 +import org.springframework.core.annotation.Order;
23 import org.springframework.stereotype.Service; 24 import org.springframework.stereotype.Service;
24 25
25 import java.util.Collections; 26 import java.util.Collections;
@@ -40,6 +41,7 @@ public class DummyDiscoveryService implements DiscoveryService { @@ -40,6 +41,7 @@ public class DummyDiscoveryService implements DiscoveryService {
40 } 41 }
41 42
42 @EventListener(ApplicationReadyEvent.class) 43 @EventListener(ApplicationReadyEvent.class)
  44 + @Order(value = 1)
43 public void onApplicationEvent(ApplicationReadyEvent event) { 45 public void onApplicationEvent(ApplicationReadyEvent event) {
44 partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); 46 partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList());
45 } 47 }
@@ -34,6 +34,7 @@ import org.springframework.beans.factory.annotation.Value; @@ -34,6 +34,7 @@ import org.springframework.beans.factory.annotation.Value;
34 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 34 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
35 import org.springframework.boot.context.event.ApplicationReadyEvent; 35 import org.springframework.boot.context.event.ApplicationReadyEvent;
36 import org.springframework.context.event.EventListener; 36 import org.springframework.context.event.EventListener;
  37 +import org.springframework.core.annotation.Order;
37 import org.springframework.stereotype.Service; 38 import org.springframework.stereotype.Service;
38 import org.springframework.util.Assert; 39 import org.springframework.util.Assert;
39 import org.thingsboard.common.util.ThingsBoardThreadFactory; 40 import org.thingsboard.common.util.ThingsBoardThreadFactory;
@@ -112,6 +113,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi @@ -112,6 +113,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
112 } 113 }
113 114
114 @EventListener(ApplicationReadyEvent.class) 115 @EventListener(ApplicationReadyEvent.class)
  116 + @Order(value = 1)
115 public void onApplicationEvent(ApplicationReadyEvent event) { 117 public void onApplicationEvent(ApplicationReadyEvent event) {
116 if (stopped) { 118 if (stopped) {
117 log.debug("Ignoring application ready event. Service is stopped."); 119 log.debug("Ignoring application ready event. Service is stopped.");
@@ -22,7 +22,7 @@ import org.springframework.stereotype.Component; @@ -22,7 +22,7 @@ import org.springframework.stereotype.Component;
22 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
23 import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; 23 import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
24 import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; 24 import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
25 -import org.thingsboard.server.gen.transport.TransportProtos; 25 +import org.thingsboard.server.gen.transport.TransportProtos.*;
26 import org.thingsboard.server.queue.TbQueueAdmin; 26 import org.thingsboard.server.queue.TbQueueAdmin;
27 import org.thingsboard.server.queue.TbQueueConsumer; 27 import org.thingsboard.server.queue.TbQueueConsumer;
28 import org.thingsboard.server.queue.TbQueueProducer; 28 import org.thingsboard.server.queue.TbQueueProducer;
@@ -91,64 +91,64 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -91,64 +91,64 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
91 } 91 }
92 92
93 @Override 93 @Override
94 - public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToTransportMsg>> createTransportNotificationsMsgProducer() { 94 + public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsMsgProducer() {
95 return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic()); 95 return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, transportNotificationSettings.getNotificationsTopic());
96 } 96 }
97 97
98 @Override 98 @Override
99 - public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createRuleEngineMsgProducer() { 99 + public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> createRuleEngineMsgProducer() {
100 return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); 100 return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic());
101 } 101 }
102 102
103 @Override 103 @Override
104 - public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() { 104 + public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createRuleEngineNotificationsMsgProducer() {
105 return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic()); 105 return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic());
106 } 106 }
107 107
108 @Override 108 @Override
109 - public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createTbCoreMsgProducer() { 109 + public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> createTbCoreMsgProducer() {
110 return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); 110 return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic());
111 } 111 }
112 112
113 @Override 113 @Override
114 - public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { 114 + public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() {
115 return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic()); 115 return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic());
116 } 116 }
117 117
118 @Override 118 @Override
119 - public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) { 119 + public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> createToRuleEngineMsgConsumer(TbRuleEngineQueueConfiguration configuration) {
120 return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(), 120 return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, ruleEngineSettings.getTopic(),
121 - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); 121 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
122 } 122 }
123 123
124 @Override 124 @Override
125 - public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() { 125 + public TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> createToRuleEngineNotificationsMsgConsumer() {
126 return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, 126 return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings,
127 partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), 127 partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(),
128 - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); 128 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
129 } 129 }
130 130
131 @Override 131 @Override
132 - public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> createToCoreMsgConsumer() { 132 + public TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> createToCoreMsgConsumer() {
133 return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(), 133 return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getTopic(),
134 - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); 134 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
135 } 135 }
136 136
137 @Override 137 @Override
138 - public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() { 138 + public TbQueueConsumer<TbProtoQueueMsg<ToCoreNotificationMsg>> createToCoreNotificationsMsgConsumer() {
139 return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, 139 return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings,
140 partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), 140 partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(),
141 - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); 141 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
142 } 142 }
143 143
144 @Override 144 @Override
145 - public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg>> createTransportApiRequestConsumer() { 145 + public TbQueueConsumer<TbProtoQueueMsg<TransportApiRequestMsg>> createTransportApiRequestConsumer() {
146 return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(), 146 return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getRequestsTopic(),
147 - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); 147 + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
148 } 148 }
149 149
150 @Override 150 @Override
151 - public TbQueueProducer<TbProtoQueueMsg<TransportProtos.TransportApiResponseMsg>> createTransportApiResponseProducer() { 151 + public TbQueueProducer<TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiResponseProducer() {
152 return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic()); 152 return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, transportApiSettings.getResponsesTopic());
153 } 153 }
154 154
@@ -175,6 +175,17 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -175,6 +175,17 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
175 return builder.build(); 175 return builder.build();
176 } 176 }
177 177
  178 + @Override
  179 + public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  180 + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
  181 + }
  182 +
  183 + @Override
  184 + public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
  185 + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(),
  186 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  187 + }
  188 +
178 @PreDestroy 189 @PreDestroy
179 private void destroy() { 190 private void destroy() {
180 if (coreAdmin != null) { 191 if (coreAdmin != null) {
@@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean;
21 import org.springframework.stereotype.Component; 21 import org.springframework.stereotype.Component;
22 import org.thingsboard.server.common.msg.queue.ServiceType; 22 import org.thingsboard.server.common.msg.queue.ServiceType;
23 import org.thingsboard.server.gen.js.JsInvokeProtos; 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
  24 +import org.thingsboard.server.gen.transport.TransportProtos;
24 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; 26 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
@@ -163,6 +164,17 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @@ -163,6 +164,17 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
163 return builder.build(); 164 return builder.build();
164 } 165 }
165 166
  167 + @Override
  168 + public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  169 + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
  170 + }
  171 +
  172 + @Override
  173 + public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
  174 + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(),
  175 + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  176 + }
  177 +
166 @PreDestroy 178 @PreDestroy
167 private void destroy() { 179 private void destroy() {
168 if (coreAdmin != null) { 180 if (coreAdmin != null) {
@@ -143,6 +143,11 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @@ -143,6 +143,11 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
143 return builder.build(); 143 return builder.build();
144 } 144 }
145 145
  146 + @Override
  147 + public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  148 + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
  149 + }
  150 +
146 @PreDestroy 151 @PreDestroy
147 private void destroy() { 152 private void destroy() {
148 if (coreAdmin != null) { 153 if (coreAdmin != null) {
@@ -158,4 +163,5 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @@ -158,4 +163,5 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
158 notificationAdmin.destroy(); 163 notificationAdmin.destroy();
159 } 164 }
160 } 165 }
  166 +
161 } 167 }
@@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider;
18 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
20 import org.springframework.stereotype.Component; 20 import org.springframework.stereotype.Component;
  21 +import org.thingsboard.server.gen.transport.TransportProtos;
21 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; 22 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
22 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 23 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
23 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 24 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
@@ -109,6 +110,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { @@ -109,6 +110,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
109 msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); 110 msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
110 } 111 }
111 112
  113 + @Override
  114 + public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  115 + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
  116 + }
  117 +
112 @PreDestroy 118 @PreDestroy
113 private void destroy() { 119 private void destroy() {
114 if (coreAdmin != null) { 120 if (coreAdmin != null) {
@@ -125,8 +125,20 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE @@ -125,8 +125,20 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
125 return null; 125 return null;
126 } 126 }
127 127
  128 + @Override
  129 + public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
  130 + return new InMemoryTbQueueConsumer<>(coreSettings.getUsageStatsTopic());
  131 + }
  132 +
  133 + @Override
  134 + public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  135 + return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic());
  136 + }
  137 +
128 @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") 138 @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}")
129 private void printInMemoryStats() { 139 private void printInMemoryStats() {
130 storage.printStats(); 140 storage.printStats();
131 } 141 }
  142 +
  143 +
132 } 144 }
@@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider;
18 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; 19 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
20 import org.springframework.stereotype.Component; 20 import org.springframework.stereotype.Component;
  21 +import org.thingsboard.server.gen.transport.TransportProtos;
21 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; 22 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
22 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 23 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
23 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 24 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
@@ -96,4 +97,10 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory @@ -96,4 +97,10 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory
96 public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() { 97 public TbQueueConsumer<TbProtoQueueMsg<ToTransportMsg>> createTransportNotificationsConsumer() {
97 return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()); 98 return new InMemoryTbQueueConsumer<>(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId());
98 } 99 }
  100 +
  101 + @Override
  102 + public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  103 + return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic());
  104 + }
  105 +
99 } 106 }
@@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 28 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
  29 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
29 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; 30 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
30 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; 31 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
31 import org.thingsboard.server.queue.TbQueueAdmin; 32 import org.thingsboard.server.queue.TbQueueAdmin;
@@ -249,6 +250,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @@ -249,6 +250,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
249 return builder.build(); 250 return builder.build();
250 } 251 }
251 252
  253 + @Override
  254 + public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
  255 + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder();
  256 + consumerBuilder.settings(kafkaSettings);
  257 + consumerBuilder.topic(coreSettings.getUsageStatsTopic());
  258 + consumerBuilder.clientId("monolith-us-consumer-" + serviceInfoProvider.getServiceId());
  259 + consumerBuilder.groupId("monolith-us-consumer-" + serviceInfoProvider.getServiceId());
  260 + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  261 + consumerBuilder.admin(coreAdmin);
  262 + return consumerBuilder.build();
  263 + }
  264 +
  265 + @Override
  266 + public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  267 + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
  268 + requestBuilder.settings(kafkaSettings);
  269 + requestBuilder.clientId("monolith-us-producer-" + serviceInfoProvider.getServiceId());
  270 + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
  271 + requestBuilder.admin(coreAdmin);
  272 + return requestBuilder.build();
  273 + }
  274 +
252 @PreDestroy 275 @PreDestroy
253 private void destroy() { 276 private void destroy() {
254 if (coreAdmin != null) { 277 if (coreAdmin != null) {
@@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 28 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
  29 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
29 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; 30 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
30 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; 31 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
31 import org.thingsboard.server.queue.TbQueueAdmin; 32 import org.thingsboard.server.queue.TbQueueAdmin;
@@ -219,6 +220,28 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { @@ -219,6 +220,28 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
219 return builder.build(); 220 return builder.build();
220 } 221 }
221 222
  223 + @Override
  224 + public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
  225 + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder();
  226 + consumerBuilder.settings(kafkaSettings);
  227 + consumerBuilder.topic(coreSettings.getUsageStatsTopic());
  228 + consumerBuilder.clientId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId());
  229 + consumerBuilder.groupId("tb-core-us-consumer-" + serviceInfoProvider.getServiceId());
  230 + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  231 + consumerBuilder.admin(coreAdmin);
  232 + return consumerBuilder.build();
  233 + }
  234 +
  235 + @Override
  236 + public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  237 + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
  238 + requestBuilder.settings(kafkaSettings);
  239 + requestBuilder.clientId("tb-core-us-producer-" + serviceInfoProvider.getServiceId());
  240 + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
  241 + requestBuilder.admin(coreAdmin);
  242 + return requestBuilder.build();
  243 + }
  244 +
222 @PreDestroy 245 @PreDestroy
223 private void destroy() { 246 private void destroy() {
224 if (coreAdmin != null) { 247 if (coreAdmin != null) {
@@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 28 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
  29 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
29 import org.thingsboard.server.queue.TbQueueAdmin; 30 import org.thingsboard.server.queue.TbQueueAdmin;
30 import org.thingsboard.server.queue.TbQueueConsumer; 31 import org.thingsboard.server.queue.TbQueueConsumer;
31 import org.thingsboard.server.queue.TbQueueProducer; 32 import org.thingsboard.server.queue.TbQueueProducer;
@@ -192,6 +193,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @@ -192,6 +193,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
192 return builder.build(); 193 return builder.build();
193 } 194 }
194 195
  196 + @Override
  197 + public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  198 + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
  199 + requestBuilder.settings(kafkaSettings);
  200 + requestBuilder.clientId("tb-rule-engine-us-producer-" + serviceInfoProvider.getServiceId());
  201 + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
  202 + requestBuilder.admin(coreAdmin);
  203 + return requestBuilder.build();
  204 + }
  205 +
195 @PreDestroy 206 @PreDestroy
196 private void destroy() { 207 private void destroy() {
197 if (coreAdmin != null) { 208 if (coreAdmin != null) {
@@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component;
21 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; 21 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
22 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 22 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
23 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 23 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
  24 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
24 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; 25 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
25 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; 26 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
26 import org.thingsboard.server.queue.TbQueueAdmin; 27 import org.thingsboard.server.queue.TbQueueAdmin;
@@ -138,6 +139,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { @@ -138,6 +139,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
138 return responseBuilder.build(); 139 return responseBuilder.build();
139 } 140 }
140 141
  142 + @Override
  143 + public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  144 + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
  145 + requestBuilder.settings(kafkaSettings);
  146 + requestBuilder.clientId("transport-node-us-producer-" + serviceInfoProvider.getServiceId());
  147 + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic());
  148 + requestBuilder.admin(coreAdmin);
  149 + return requestBuilder.build();
  150 + }
  151 +
141 @PreDestroy 152 @PreDestroy
142 private void destroy() { 153 private void destroy() {
143 if (coreAdmin != null) { 154 if (coreAdmin != null) {
@@ -27,6 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs @@ -27,6 +27,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
  30 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
30 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; 31 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
31 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; 32 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
32 import org.thingsboard.server.queue.TbQueueAdmin; 33 import org.thingsboard.server.queue.TbQueueAdmin;
@@ -183,6 +184,17 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @@ -183,6 +184,17 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
183 return builder.build(); 184 return builder.build();
184 } 185 }
185 186
  187 + @Override
  188 + public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
  189 + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(),
  190 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  191 + }
  192 +
  193 + @Override
  194 + public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  195 + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
  196 + }
  197 +
186 @PreDestroy 198 @PreDestroy
187 private void destroy() { 199 private void destroy() {
188 if (coreAdmin != null) { 200 if (coreAdmin != null) {
@@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; 26 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; 28 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
  29 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
29 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; 30 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
30 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; 31 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
31 import org.thingsboard.server.queue.TbQueueAdmin; 32 import org.thingsboard.server.queue.TbQueueAdmin;
@@ -157,6 +158,17 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @@ -157,6 +158,17 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
157 return builder.build(); 158 return builder.build();
158 } 159 }
159 160
  161 + @Override
  162 + public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
  163 + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(),
  164 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  165 + }
  166 +
  167 + @Override
  168 + public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  169 + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
  170 + }
  171 +
160 @PreDestroy 172 @PreDestroy
161 private void destroy() { 173 private void destroy() {
162 if (coreAdmin != null) { 174 if (coreAdmin != null) {