简柏林 / thingskit · Commits

Commit 47994e1a66152fc784a04d4cb58cdba1072073d7

Authored by Viacheslav Klimov
2 parents 4ce81807 1d24a08c

Merge branch 'master' into master-to-snmp 

# Conflicts:
#	application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
#	common/queue/src/main/proto/queue.proto
#	common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java
#	common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java
#	dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
Inline Side-by-side
Showing 95 changed files with 2467 additions and 455 deletions

Too many changes to show.
Reload with full diff Plain diff Email patch

To preserve performance only 95 of 219 files are displayed.

... ... @@ -31,7 +31,7 @@ CREATE TABLE IF NOT EXISTS edge (
31 31 tenant_id uuid,
32 32 CONSTRAINT edge_name_unq_key UNIQUE (tenant_id, name),
33 33 CONSTRAINT edge_routing_key_unq_key UNIQUE (routing_key)
34   -);
  34 + );
35 35
36 36 CREATE TABLE IF NOT EXISTS edge_event (
37 37 id uuid NOT NULL CONSTRAINT edge_event_pkey PRIMARY KEY,
... ... @@ -44,4 +44,57 @@ CREATE TABLE IF NOT EXISTS edge_event (
44 44 body varchar(10000000),
45 45 tenant_id uuid,
46 46 ts bigint NOT NULL
  47 + );
  48 +
  49 +CREATE TABLE IF NOT EXISTS resource (
  50 + id uuid NOT NULL CONSTRAINT resource_pkey PRIMARY KEY,
  51 + created_time bigint NOT NULL,
  52 + tenant_id uuid NOT NULL,
  53 + title varchar(255) NOT NULL,
  54 + resource_type varchar(32) NOT NULL,
  55 + resource_key varchar(255) NOT NULL,
  56 + search_text varchar(255),
  57 + file_name varchar(255) NOT NULL,
  58 + data varchar,
  59 + CONSTRAINT resource_unq_key UNIQUE (tenant_id, resource_type, resource_key)
  60 +);
  61 +
  62 +CREATE TABLE IF NOT EXISTS firmware (
  63 + id uuid NOT NULL CONSTRAINT firmware_pkey PRIMARY KEY,
  64 + created_time bigint NOT NULL,
  65 + tenant_id uuid NOT NULL,
  66 + title varchar(255) NOT NULL,
  67 + version varchar(255) NOT NULL,
  68 + file_name varchar(255),
  69 + content_type varchar(255),
  70 + checksum_algorithm varchar(32),
  71 + checksum varchar(1020),
  72 + data bytea,
  73 + data_size bigint,
  74 + additional_info varchar,
  75 + search_text varchar(255),
  76 + CONSTRAINT firmware_tenant_title_version_unq_key UNIQUE (tenant_id, title, version)
47 77 );
  78 +
  79 +ALTER TABLE device_profile
  80 + ADD COLUMN IF NOT EXISTS firmware_id uuid;
  81 +
  82 +ALTER TABLE device
  83 + ADD COLUMN IF NOT EXISTS firmware_id uuid;
  84 +
  85 +DO $$
  86 + BEGIN
  87 + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_firmware_device_profile') THEN
  88 + ALTER TABLE device_profile
  89 + ADD CONSTRAINT fk_firmware_device_profile
  90 + FOREIGN KEY (firmware_id) REFERENCES firmware(id);
  91 + END IF;
  92 +
  93 + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_firmware_device') THEN
  94 + ALTER TABLE device
  95 + ADD CONSTRAINT fk_firmware_device
  96 + FOREIGN KEY (firmware_id) REFERENCES firmware(id);
  97 + END IF;
  98 + END;
  99 +$$;
  100 +
... ...
... ... @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.EntityType;
41 41 import org.thingsboard.server.common.data.TenantProfile;
42 42 import org.thingsboard.server.common.data.alarm.Alarm;
43 43 import org.thingsboard.server.common.data.asset.Asset;
  44 +import org.thingsboard.server.common.data.id.CustomerId;
44 45 import org.thingsboard.server.common.data.id.DeviceId;
45 46 import org.thingsboard.server.common.data.id.EdgeId;
46 47 import org.thingsboard.server.common.data.id.EntityId;
... ... @@ -269,7 +270,12 @@ class DefaultTbContext implements TbContext {
269 270
270 271 @Override
271 272 public TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) {
272   - return TbMsg.newMsg(queueName, type, originator, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId());
  273 + return newMsg(queueName, type, originator, null, metaData, data);
  274 + }
  275 +
  276 + @Override
  277 + public TbMsg newMsg(String queueName, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
  278 + return TbMsg.newMsg(queueName, type, originator, customerId, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId());
273 279 }
274 280
275 281 @Override
... ...
... ... @@ -106,7 +106,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
106 106 int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
107 107 int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
108 108 if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
109   - apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
  109 + apiUsageClient.report(tenantId, tbMsg.getCustomerId(), ApiUsageRecordKey.RE_EXEC_COUNT);
110 110 if (ruleNode.isDebugMode()) {
111 111 systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), "Self");
112 112 }
... ... @@ -127,7 +127,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
127 127 int ruleNodeCount = tbMsg.getAndIncrementRuleNodeCounter();
128 128 int maxRuleNodeExecutionsPerMessage = getTenantProfileConfiguration().getMaxRuleNodeExecsPerMessage();
129 129 if (maxRuleNodeExecutionsPerMessage == 0 || ruleNodeCount < maxRuleNodeExecutionsPerMessage) {
130   - apiUsageClient.report(tenantId, ApiUsageRecordKey.RE_EXEC_COUNT);
  130 + apiUsageClient.report(tenantId, tbMsg.getCustomerId(), ApiUsageRecordKey.RE_EXEC_COUNT);
131 131 if (ruleNode.isDebugMode()) {
132 132 systemContext.persistDebugInput(tenantId, entityId, msg.getMsg(), msg.getFromRelationType());
133 133 }
... ...
... ... @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j;
24 24 import org.apache.commons.lang3.StringUtils;
25 25 import org.springframework.beans.factory.annotation.Autowired;
26 26 import org.springframework.beans.factory.annotation.Value;
  27 +import org.springframework.http.MediaType;
27 28 import org.springframework.security.core.Authentication;
28 29 import org.springframework.security.core.context.SecurityContextHolder;
29 30 import org.springframework.web.bind.annotation.ExceptionHandler;
... ... @@ -38,6 +39,8 @@ import org.thingsboard.server.common.data.EdgeUtils;
38 39 import org.thingsboard.server.common.data.EntityType;
39 40 import org.thingsboard.server.common.data.EntityView;
40 41 import org.thingsboard.server.common.data.EntityViewInfo;
  42 +import org.thingsboard.server.common.data.Firmware;
  43 +import org.thingsboard.server.common.data.FirmwareInfo;
41 44 import org.thingsboard.server.common.data.HasName;
42 45 import org.thingsboard.server.common.data.HasTenantId;
43 46 import org.thingsboard.server.common.data.TbResourceInfo;
... ... @@ -67,6 +70,7 @@ import org.thingsboard.server.common.data.id.EdgeId;
67 70 import org.thingsboard.server.common.data.id.EntityId;
68 71 import org.thingsboard.server.common.data.id.EntityIdFactory;
69 72 import org.thingsboard.server.common.data.id.EntityViewId;
  73 +import org.thingsboard.server.common.data.id.FirmwareId;
70 74 import org.thingsboard.server.common.data.id.TbResourceId;
71 75 import org.thingsboard.server.common.data.id.RuleChainId;
72 76 import org.thingsboard.server.common.data.id.RuleNodeId;
... ... @@ -106,6 +110,7 @@ import org.thingsboard.server.dao.edge.EdgeService;
106 110 import org.thingsboard.server.dao.entityview.EntityViewService;
107 111 import org.thingsboard.server.dao.exception.DataValidationException;
108 112 import org.thingsboard.server.dao.exception.IncorrectParameterException;
  113 +import org.thingsboard.server.dao.firmware.FirmwareService;
109 114 import org.thingsboard.server.dao.model.ModelConstants;
110 115 import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
111 116 import org.thingsboard.server.dao.oauth2.OAuth2Service;
... ... @@ -123,6 +128,7 @@ import org.thingsboard.server.queue.discovery.PartitionService;
123 128 import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
124 129 import org.thingsboard.server.queue.util.TbCoreComponent;
125 130 import org.thingsboard.server.service.component.ComponentDiscoveryService;
  131 +import org.thingsboard.server.service.firmware.FirmwareStateService;
126 132 import org.thingsboard.server.service.edge.EdgeNotificationService;
127 133 import org.thingsboard.server.service.edge.rpc.EdgeGrpcService;
128 134 import org.thingsboard.server.service.edge.rpc.init.SyncEdgeService;
... ... @@ -246,6 +252,12 @@ public abstract class BaseController {
246 252 protected TbResourceService resourceService;
247 253
248 254 @Autowired
  255 + protected FirmwareService firmwareService;
  256 +
  257 + @Autowired
  258 + protected FirmwareStateService firmwareStateService;
  259 +
  260 + @Autowired
249 261 protected TbQueueProducerProvider producerProvider;
250 262
251 263 @Autowired
... ... @@ -501,6 +513,9 @@ public abstract class BaseController {
501 513 case TB_RESOURCE:
502 514 checkResourceId(new TbResourceId(entityId.getId()), operation);
503 515 return;
  516 + case FIRMWARE:
  517 + checkFirmwareId(new FirmwareId(entityId.getId()), operation);
  518 + return;
504 519 default:
505 520 throw new IllegalArgumentException("Unsupported entity type: " + entityId.getEntityType());
506 521 }
... ... @@ -756,6 +771,30 @@ public abstract class BaseController {
756 771 }
757 772 }
758 773
  774 + Firmware checkFirmwareId(FirmwareId firmwareId, Operation operation) throws ThingsboardException {
  775 + try {
  776 + validateId(firmwareId, "Incorrect firmwareId " + firmwareId);
  777 + Firmware firmware = firmwareService.findFirmwareById(getCurrentUser().getTenantId(), firmwareId);
  778 + checkNotNull(firmware);
  779 + accessControlService.checkPermission(getCurrentUser(), Resource.FIRMWARE, operation, firmwareId, firmware);
  780 + return firmware;
  781 + } catch (Exception e) {
  782 + throw handleException(e, false);
  783 + }
  784 + }
  785 +
  786 + FirmwareInfo checkFirmwareInfoId(FirmwareId firmwareId, Operation operation) throws ThingsboardException {
  787 + try {
  788 + validateId(firmwareId, "Incorrect firmwareId " + firmwareId);
  789 + FirmwareInfo firmwareInfo = firmwareService.findFirmwareInfoById(getCurrentUser().getTenantId(), firmwareId);
  790 + checkNotNull(firmwareInfo);
  791 + accessControlService.checkPermission(getCurrentUser(), Resource.FIRMWARE, operation, firmwareId, firmwareInfo);
  792 + return firmwareInfo;
  793 + } catch (Exception e) {
  794 + throw handleException(e, false);
  795 + }
  796 + }
  797 +
759 798 @SuppressWarnings("unchecked")
760 799 protected <I extends EntityId> I emptyId(EntityType entityType) {
761 800 return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID);
... ... @@ -919,7 +958,7 @@ public abstract class BaseController {
919 958 entityNode.put("endTs", extractParameter(Long.class, 2, additionalInfo));
920 959 }
921 960 }
922   - TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode));
  961 + TbMsg tbMsg = TbMsg.newMsg(msgType, entityId, customerId, metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode));
923 962 TenantId tenantId = user.getTenantId();
924 963 if (tenantId.isNullUid()) {
925 964 if (entity instanceof HasTenantId) {
... ... @@ -1082,4 +1121,11 @@ public abstract class BaseController {
1082 1121 }
1083 1122 }
1084 1123
  1124 + protected MediaType parseMediaType(String contentType) {
  1125 + try {
  1126 + return MediaType.parseMediaType(contentType);
  1127 + } catch (Exception e) {
  1128 + return MediaType.APPLICATION_OCTET_STREAM;
  1129 + }
  1130 + }
1085 1131 }
... ...
... ... @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.id.EdgeId;
38 38 import org.thingsboard.server.common.data.id.TenantId;
39 39 import org.thingsboard.server.common.data.page.PageData;
40 40 import org.thingsboard.server.common.data.page.PageLink;
  41 +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
41 42 import org.thingsboard.server.queue.util.TbCoreComponent;
42 43 import org.thingsboard.server.service.security.permission.Operation;
43 44 import org.thingsboard.server.service.security.permission.Resource;
... ... @@ -148,6 +149,7 @@ public class CustomerController extends BaseController {
148 149 ActionType.DELETED, null, strCustomerId);
149 150
150 151 sendDeleteNotificationMsg(getTenantId(), customerId, relatedEdgeIds);
  152 + tbClusterService.onEntityStateChange(getTenantId(), customerId, ComponentLifecycleEvent.DELETED);
151 153 } catch (Exception e) {
152 154
153 155 logEntityAction(emptyId(EntityType.CUSTOMER),
... ...
... ... @@ -75,6 +75,7 @@ import javax.annotation.Nullable;
75 75 import java.io.IOException;
76 76 import java.util.ArrayList;
77 77 import java.util.List;
  78 +import java.util.Objects;
78 79 import java.util.stream.Collectors;
79 80
80 81 import static org.thingsboard.server.controller.EdgeController.EDGE_ID;
... ... @@ -124,15 +125,22 @@ public class DeviceController extends BaseController {
124 125
125 126 checkEntity(device.getId(), device, Resource.DEVICE);
126 127
  128 + boolean created = device.getId() == null;
  129 + Device oldDevice;
  130 + if (!created) {
  131 + oldDevice = deviceService.findDeviceById(getTenantId(), device.getId());
  132 + } else {
  133 + oldDevice = null;
  134 + }
  135 +
127 136 Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken));
128 137
129 138 tbClusterService.onDeviceChange(savedDevice, null);
130 139 tbClusterService.pushMsgToCore(new DeviceNameOrTypeUpdateMsg(savedDevice.getTenantId(),
131 140 savedDevice.getId(), savedDevice.getName(), savedDevice.getType()), null);
132   - tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(),
133   - device.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
  141 + tbClusterService.onEntityStateChange(savedDevice.getTenantId(), savedDevice.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
134 142
135   - if (device.getId() != null) {
  143 + if (!created) {
136 144 sendEntityNotificationMsg(savedDevice.getTenantId(), savedDevice.getId(), EdgeEventActionType.UPDATED);
137 145 }
138 146
... ... @@ -145,12 +153,17 @@ public class DeviceController extends BaseController {
145 153 } else {
146 154 deviceStateService.onDeviceUpdated(savedDevice);
147 155 }
  156 +
  157 + firmwareStateService.update(savedDevice, oldDevice);
  158 +
148 159 return savedDevice;
149   - } catch (Exception e) {
  160 + } catch (
  161 + Exception e) {
150 162 logEntityAction(emptyId(EntityType.DEVICE), device,
151 163 null, device.getId() == null ? ActionType.ADDED : ActionType.UPDATED, e);
152 164 throw handleException(e);
153 165 }
  166 +
154 167 }
155 168
156 169 @PreAuthorize("hasAuthority('TENANT_ADMIN')")
... ... @@ -645,7 +658,7 @@ public class DeviceController extends BaseController {
645 658 private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) {
646 659 String data = entityToStr(assignedDevice);
647 660 if (data != null) {
648   - TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_ASSIGNED_FROM_TENANT, assignedDevice.getId(), getMetaDataForAssignedFrom(currentTenant), TbMsgDataType.JSON, data);
  661 + TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_ASSIGNED_FROM_TENANT, assignedDevice.getId(), assignedDevice.getCustomerId(), getMetaDataForAssignedFrom(currentTenant), TbMsgDataType.JSON, data);
649 662 tbClusterService.pushMsgToRuleEngine(newTenantId, assignedDevice.getId(), tbMsg, null);
650 663 }
651 664 }
... ...
... ... @@ -44,6 +44,7 @@ import org.thingsboard.server.service.security.permission.Operation;
44 44 import org.thingsboard.server.service.security.permission.Resource;
45 45
46 46 import java.util.List;
  47 +import java.util.Objects;
47 48 import java.util.UUID;
48 49
49 50 @RestController
... ... @@ -144,6 +145,15 @@ public class DeviceProfileController extends BaseController {
144 145
145 146 checkEntity(deviceProfile.getId(), deviceProfile, Resource.DEVICE_PROFILE);
146 147
  148 + boolean isFirmwareChanged = false;
  149 +
  150 + if (!created) {
  151 + DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId());
  152 + if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) {
  153 + isFirmwareChanged = true;
  154 + }
  155 + }
  156 +
147 157 DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile));
148 158
149 159 tbClusterService.onDeviceProfileChange(savedDeviceProfile, null);
... ... @@ -154,9 +164,11 @@ public class DeviceProfileController extends BaseController {
154 164 null,
155 165 created ? ActionType.ADDED : ActionType.UPDATED, null);
156 166
  167 + if (isFirmwareChanged) {
  168 + firmwareStateService.update(savedDeviceProfile);
  169 + }
157 170 sendEntityNotificationMsg(getTenantId(), savedDeviceProfile.getId(),
158 171 deviceProfile.getId() == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED);
159   -
160 172 return savedDeviceProfile;
161 173 } catch (Exception e) {
162 174 logEntityAction(emptyId(EntityType.DEVICE_PROFILE), deviceProfile,
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.controller;
  17 +
  18 +import com.google.common.hash.Hashing;
  19 +import lombok.extern.slf4j.Slf4j;
  20 +import org.apache.commons.lang3.StringUtils;
  21 +import org.springframework.core.io.ByteArrayResource;
  22 +import org.springframework.http.HttpHeaders;
  23 +import org.springframework.http.ResponseEntity;
  24 +import org.springframework.security.access.prepost.PreAuthorize;
  25 +import org.springframework.web.bind.annotation.PathVariable;
  26 +import org.springframework.web.bind.annotation.RequestBody;
  27 +import org.springframework.web.bind.annotation.RequestMapping;
  28 +import org.springframework.web.bind.annotation.RequestMethod;
  29 +import org.springframework.web.bind.annotation.RequestParam;
  30 +import org.springframework.web.bind.annotation.ResponseBody;
  31 +import org.springframework.web.bind.annotation.RestController;
  32 +import org.springframework.web.multipart.MultipartFile;
  33 +import org.thingsboard.server.common.data.Firmware;
  34 +import org.thingsboard.server.common.data.FirmwareInfo;
  35 +import org.thingsboard.server.common.data.exception.ThingsboardException;
  36 +import org.thingsboard.server.common.data.id.FirmwareId;
  37 +import org.thingsboard.server.common.data.page.PageData;
  38 +import org.thingsboard.server.common.data.page.PageLink;
  39 +import org.thingsboard.server.queue.util.TbCoreComponent;
  40 +import org.thingsboard.server.service.security.permission.Operation;
  41 +import org.thingsboard.server.service.security.permission.Resource;
  42 +
  43 +import java.nio.ByteBuffer;
  44 +
  45 +@Slf4j
  46 +@RestController
  47 +@TbCoreComponent
  48 +@RequestMapping("/api")
  49 +public class FirmwareController extends BaseController {
  50 +
  51 + public static final String FIRMWARE_ID = "firmwareId";
  52 +
  53 + @PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')")
  54 + @RequestMapping(value = "/firmware/{firmwareId}/download", method = RequestMethod.GET)
  55 + @ResponseBody
  56 + public ResponseEntity<org.springframework.core.io.Resource> downloadFirmware(@PathVariable(FIRMWARE_ID) String strFirmwareId) throws ThingsboardException {
  57 + checkParameter(FIRMWARE_ID, strFirmwareId);
  58 + try {
  59 + FirmwareId firmwareId = new FirmwareId(toUUID(strFirmwareId));
  60 + Firmware firmware = checkFirmwareId(firmwareId, Operation.READ);
  61 +
  62 + ByteArrayResource resource = new ByteArrayResource(firmware.getData().array());
  63 + return ResponseEntity.ok()
  64 + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + firmware.getFileName())
  65 + .header("x-filename", firmware.getFileName())
  66 + .contentLength(resource.contentLength())
  67 + .contentType(parseMediaType(firmware.getContentType()))
  68 + .body(resource);
  69 + } catch (Exception e) {
  70 + throw handleException(e);
  71 + }
  72 + }
  73 +
  74 + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
  75 + @RequestMapping(value = "/firmware/info/{firmwareId}", method = RequestMethod.GET)
  76 + @ResponseBody
  77 + public FirmwareInfo getFirmwareInfoById(@PathVariable(FIRMWARE_ID) String strFirmwareId) throws ThingsboardException {
  78 + checkParameter(FIRMWARE_ID, strFirmwareId);
  79 + try {
  80 + FirmwareId firmwareId = new FirmwareId(toUUID(strFirmwareId));
  81 + return checkFirmwareInfoId(firmwareId, Operation.READ);
  82 + } catch (Exception e) {
  83 + throw handleException(e);
  84 + }
  85 + }
  86 +
  87 + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
  88 + @RequestMapping(value = "/firmware/{firmwareId}", method = RequestMethod.GET)
  89 + @ResponseBody
  90 + public Firmware getFirmwareById(@PathVariable(FIRMWARE_ID) String strFirmwareId) throws ThingsboardException {
  91 + checkParameter(FIRMWARE_ID, strFirmwareId);
  92 + try {
  93 + FirmwareId firmwareId = new FirmwareId(toUUID(strFirmwareId));
  94 + return checkFirmwareId(firmwareId, Operation.READ);
  95 + } catch (Exception e) {
  96 + throw handleException(e);
  97 + }
  98 + }
  99 +
  100 + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
  101 + @RequestMapping(value = "/firmware", method = RequestMethod.POST)
  102 + @ResponseBody
  103 + public FirmwareInfo saveFirmwareInfo(@RequestBody FirmwareInfo firmwareInfo) throws ThingsboardException {
  104 + firmwareInfo.setTenantId(getTenantId());
  105 + checkEntity(firmwareInfo.getId(), firmwareInfo, Resource.FIRMWARE);
  106 + try {
  107 + return firmwareService.saveFirmwareInfo(firmwareInfo);
  108 + } catch (Exception e) {
  109 + throw handleException(e);
  110 + }
  111 + }
  112 +
  113 + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
  114 + @RequestMapping(value = "/firmware/{firmwareId}", method = RequestMethod.POST)
  115 + @ResponseBody
  116 + public Firmware saveFirmwareData(@PathVariable(FIRMWARE_ID) String strFirmwareId,
  117 + @RequestParam(required = false) String checksum,
  118 + @RequestParam(required = false) String checksumAlgorithm,
  119 + @RequestBody MultipartFile file) throws ThingsboardException {
  120 + checkParameter(FIRMWARE_ID, strFirmwareId);
  121 + try {
  122 + FirmwareId firmwareId = new FirmwareId(toUUID(strFirmwareId));
  123 + FirmwareInfo info = checkFirmwareInfoId(firmwareId, Operation.READ);
  124 +
  125 + Firmware firmware = new Firmware(firmwareId);
  126 + firmware.setCreatedTime(info.getCreatedTime());
  127 + firmware.setTenantId(getTenantId());
  128 + firmware.setTitle(info.getTitle());
  129 + firmware.setVersion(info.getVersion());
  130 + firmware.setAdditionalInfo(info.getAdditionalInfo());
  131 +
  132 + byte[] data = file.getBytes();
  133 + if (StringUtils.isEmpty(checksumAlgorithm)) {
  134 + checksumAlgorithm = "sha256";
  135 + checksum = Hashing.sha256().hashBytes(data).toString();
  136 + }
  137 +
  138 + firmware.setChecksumAlgorithm(checksumAlgorithm);
  139 + firmware.setChecksum(checksum);
  140 + firmware.setFileName(file.getOriginalFilename());
  141 + firmware.setContentType(file.getContentType());
  142 + firmware.setData(ByteBuffer.wrap(data));
  143 + firmware.setDataSize((long) data.length);
  144 + return firmwareService.saveFirmware(firmware);
  145 + } catch (Exception e) {
  146 + throw handleException(e);
  147 + }
  148 + }
  149 +
  150 + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
  151 + @RequestMapping(value = "/firmwares", method = RequestMethod.GET)
  152 + @ResponseBody
  153 + public PageData<FirmwareInfo> getFirmwares(@RequestParam int pageSize,
  154 + @RequestParam int page,
  155 + @RequestParam(required = false) String textSearch,
  156 + @RequestParam(required = false) String sortProperty,
  157 + @RequestParam(required = false) String sortOrder) throws ThingsboardException {
  158 + try {
  159 + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
  160 + return checkNotNull(firmwareService.findTenantFirmwaresByTenantId(getTenantId(), pageLink));
  161 + } catch (Exception e) {
  162 + throw handleException(e);
  163 + }
  164 + }
  165 +
  166 + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
  167 + @RequestMapping(value = "/firmwares/{hasData}", method = RequestMethod.GET)
  168 + @ResponseBody
  169 + public PageData<FirmwareInfo> getFirmwares(@PathVariable("hasData") boolean hasData,
  170 + @RequestParam int pageSize,
  171 + @RequestParam int page,
  172 + @RequestParam(required = false) String textSearch,
  173 + @RequestParam(required = false) String sortProperty,
  174 + @RequestParam(required = false) String sortOrder) throws ThingsboardException {
  175 + try {
  176 + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
  177 + return checkNotNull(firmwareService.findTenantFirmwaresByTenantIdAndHasData(getTenantId(), hasData, pageLink));
  178 + } catch (Exception e) {
  179 + throw handleException(e);
  180 + }
  181 + }
  182 +
  183 + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
  184 + @RequestMapping(value = "/firmware/{firmwareId}", method = RequestMethod.DELETE)
  185 + @ResponseBody
  186 + public void deleteResource(@PathVariable("firmwareId") String strFirmwareId) throws ThingsboardException {
  187 + checkParameter(FIRMWARE_ID, strFirmwareId);
  188 + try {
  189 + FirmwareId firmwareId = new FirmwareId(toUUID(strFirmwareId));
  190 + checkFirmwareInfoId(firmwareId, Operation.DELETE);
  191 + firmwareService.deleteFirmware(getTenantId(), firmwareId);
  192 + } catch (Exception e) {
  193 + throw handleException(e);
  194 + }
  195 + }
  196 +
  197 +}
... ...
... ... @@ -15,6 +15,7 @@
15 15 */
16 16 package org.thingsboard.server.controller;
17 17
  18 +import org.springframework.beans.factory.annotation.Autowired;
18 19 import org.springframework.security.access.prepost.PreAuthorize;
19 20 import org.springframework.web.bind.annotation.RequestMapping;
20 21 import org.springframework.web.bind.annotation.RequestMethod;
... ... @@ -23,17 +24,23 @@ import org.springframework.web.bind.annotation.ResponseBody;
23 24 import org.springframework.web.bind.annotation.RestController;
24 25 import org.thingsboard.server.common.data.exception.ThingsboardException;
25 26 import org.thingsboard.server.common.msg.queue.ServiceType;
  27 +import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
  28 +import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
26 29 import org.thingsboard.server.queue.util.TbCoreComponent;
27 30
28 31 import java.util.Arrays;
29 32 import java.util.Collections;
30 33 import java.util.List;
  34 +import java.util.stream.Collectors;
31 35
32 36 @RestController
33 37 @TbCoreComponent
34 38 @RequestMapping("/api")
35 39 public class QueueController extends BaseController {
36 40
  41 + @Autowired(required = false)
  42 + private TbQueueRuleEngineSettings ruleEngineSettings;
  43 +
37 44 @PreAuthorize("hasAuthority('TENANT_ADMIN')")
38 45 @RequestMapping(value = "/tenant/queues", params = {"serviceType"}, method = RequestMethod.GET)
39 46 @ResponseBody
... ... @@ -43,7 +50,12 @@ public class QueueController extends BaseController {
43 50 ServiceType type = ServiceType.valueOf(serviceType);
44 51 switch (type) {
45 52 case TB_RULE_ENGINE:
46   - return Arrays.asList("Main", "HighPriority", "SequentialByOriginator");
  53 + if (ruleEngineSettings == null) {
  54 + return Arrays.asList("Main", "HighPriority", "SequentialByOriginator");
  55 + }
  56 + return ruleEngineSettings.getQueues().stream()
  57 + .map(TbRuleEngineQueueConfiguration::getName)
  58 + .collect(Collectors.toList());
47 59 default:
48 60 return Collections.emptyList();
49 61 }
... ...
... ... @@ -119,7 +119,7 @@ public class RpcController extends BaseController {
119 119 expTime,
120 120 body
121 121 );
122   - deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse));
  122 + deviceRpcService.processRestApiRpcRequest(rpcRequest, fromDeviceRpcResponse -> reply(new LocalRequestMetaData(rpcRequest, currentUser, result), fromDeviceRpcResponse), currentUser);
123 123 }
124 124
125 125 @Override
... ...
... ... @@ -166,7 +166,7 @@ public class TbResourceController extends BaseController {
166 166 @RequestMapping(value = "/resource/{resourceId}", method = RequestMethod.DELETE)
167 167 @ResponseBody
168 168 public void deleteResource(@PathVariable("resourceId") String strResourceId) throws ThingsboardException {
169   - checkParameter("resourceId", strResourceId);
  169 + checkParameter(RESOURCE_ID, strResourceId);
170 170 try {
171 171 TbResourceId resourceId = new TbResourceId(toUUID(strResourceId));
172 172 TbResource tbResource = checkResourceId(resourceId, Operation.DELETE);
... ...
... ... @@ -48,6 +48,7 @@ import org.thingsboard.server.common.data.EntityType;
48 48 import org.thingsboard.server.common.data.TenantProfile;
49 49 import org.thingsboard.server.common.data.audit.ActionType;
50 50 import org.thingsboard.server.common.data.exception.ThingsboardException;
  51 +import org.thingsboard.server.common.data.id.CustomerId;
51 52 import org.thingsboard.server.common.data.id.DeviceId;
52 53 import org.thingsboard.server.common.data.id.EntityId;
53 54 import org.thingsboard.server.common.data.id.EntityIdFactory;
... ... @@ -449,7 +450,7 @@ public class TelemetryController extends BaseController {
449 450 TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
450 451 tenantTtl = TimeUnit.DAYS.toSeconds(((DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration()).getDefaultStorageTtlDays());
451 452 }
452   - tsSubService.saveAndNotify(tenantId, entityId, entries, tenantTtl, new FutureCallback<Void>() {
  453 + tsSubService.saveAndNotify(tenantId, user.getCustomerId(), entityId, entries, tenantTtl, new FutureCallback<Void>() {
453 454 @Override
454 455 public void onSuccess(@Nullable Void tmp) {
455 456 logTelemetryUpdated(user, entityId, entries, null);
... ...
... ... @@ -232,7 +232,6 @@ public class ThingsboardInstallService {
232 232 systemDataLoaderService.createAdminSettings();
233 233 systemDataLoaderService.loadSystemWidgets();
234 234 systemDataLoaderService.createOAuth2Templates();
235   - systemDataLoaderService.loadSystemLwm2mResources();
236 235 // systemDataLoaderService.loadSystemPlugins();
237 236 // systemDataLoaderService.loadSystemRules();
238 237
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.apiusage;
  17 +
  18 +import lombok.Getter;
  19 +import org.springframework.data.util.Pair;
  20 +import org.thingsboard.server.common.data.ApiFeature;
  21 +import org.thingsboard.server.common.data.ApiUsageRecordKey;
  22 +import org.thingsboard.server.common.data.ApiUsageState;
  23 +import org.thingsboard.server.common.data.ApiUsageStateValue;
  24 +import org.thingsboard.server.common.data.EntityType;
  25 +import org.thingsboard.server.common.data.id.EntityId;
  26 +import org.thingsboard.server.common.data.id.TenantId;
  27 +import org.thingsboard.server.common.msg.tools.SchedulerUtils;
  28 +
  29 +import java.util.Arrays;
  30 +import java.util.HashMap;
  31 +import java.util.HashSet;
  32 +import java.util.Map;
  33 +import java.util.Set;
  34 +import java.util.concurrent.ConcurrentHashMap;
  35 +
  36 +public abstract class BaseApiUsageState {
  37 + private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>();
  38 + private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>();
  39 +
  40 + @Getter
  41 + private final ApiUsageState apiUsageState;
  42 + @Getter
  43 + private volatile long currentCycleTs;
  44 + @Getter
  45 + private volatile long nextCycleTs;
  46 + @Getter
  47 + private volatile long currentHourTs;
  48 +
  49 + public BaseApiUsageState(ApiUsageState apiUsageState) {
  50 + this.apiUsageState = apiUsageState;
  51 + this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
  52 + this.nextCycleTs = SchedulerUtils.getStartOfNextMonth();
  53 + this.currentHourTs = SchedulerUtils.getStartOfCurrentHour();
  54 + }
  55 +
  56 + public void put(ApiUsageRecordKey key, Long value) {
  57 + currentCycleValues.put(key, value);
  58 + }
  59 +
  60 + public void putHourly(ApiUsageRecordKey key, Long value) {
  61 + currentHourValues.put(key, value);
  62 + }
  63 +
  64 + public long add(ApiUsageRecordKey key, long value) {
  65 + long result = currentCycleValues.getOrDefault(key, 0L) + value;
  66 + currentCycleValues.put(key, result);
  67 + return result;
  68 + }
  69 +
  70 + public long get(ApiUsageRecordKey key) {
  71 + return currentCycleValues.getOrDefault(key, 0L);
  72 + }
  73 +
  74 + public long addToHourly(ApiUsageRecordKey key, long value) {
  75 + long result = currentHourValues.getOrDefault(key, 0L) + value;
  76 + currentHourValues.put(key, result);
  77 + return result;
  78 + }
  79 +
  80 + public void setHour(long currentHourTs) {
  81 + this.currentHourTs = currentHourTs;
  82 + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
  83 + currentHourValues.put(key, 0L);
  84 + }
  85 + }
  86 +
  87 + public void setCycles(long currentCycleTs, long nextCycleTs) {
  88 + this.currentCycleTs = currentCycleTs;
  89 + this.nextCycleTs = nextCycleTs;
  90 + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
  91 + currentCycleValues.put(key, 0L);
  92 + }
  93 + }
  94 +
  95 + public ApiUsageStateValue getFeatureValue(ApiFeature feature) {
  96 + switch (feature) {
  97 + case TRANSPORT:
  98 + return apiUsageState.getTransportState();
  99 + case RE:
  100 + return apiUsageState.getReExecState();
  101 + case DB:
  102 + return apiUsageState.getDbStorageState();
  103 + case JS:
  104 + return apiUsageState.getJsExecState();
  105 + case EMAIL:
  106 + return apiUsageState.getEmailExecState();
  107 + case SMS:
  108 + return apiUsageState.getSmsExecState();
  109 + default:
  110 + return ApiUsageStateValue.ENABLED;
  111 + }
  112 + }
  113 +
  114 + public boolean setFeatureValue(ApiFeature feature, ApiUsageStateValue value) {
  115 + ApiUsageStateValue currentValue = getFeatureValue(feature);
  116 + switch (feature) {
  117 + case TRANSPORT:
  118 + apiUsageState.setTransportState(value);
  119 + break;
  120 + case RE:
  121 + apiUsageState.setReExecState(value);
  122 + break;
  123 + case DB:
  124 + apiUsageState.setDbStorageState(value);
  125 + break;
  126 + case JS:
  127 + apiUsageState.setJsExecState(value);
  128 + break;
  129 + case EMAIL:
  130 + apiUsageState.setEmailExecState(value);
  131 + break;
  132 + case SMS:
  133 + apiUsageState.setSmsExecState(value);
  134 + break;
  135 + }
  136 + return !currentValue.equals(value);
  137 + }
  138 +
  139 + public abstract EntityType getEntityType();
  140 +
  141 + public TenantId getTenantId() {
  142 + return getApiUsageState().getTenantId();
  143 + }
  144 +
  145 + public EntityId getEntityId() {
  146 + return getApiUsageState().getEntityId();
  147 + }
  148 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.apiusage;
  17 +
  18 +import org.thingsboard.server.common.data.ApiUsageState;
  19 +import org.thingsboard.server.common.data.EntityType;
  20 +
  21 +public class CustomerApiUsageState extends BaseApiUsageState {
  22 + public CustomerApiUsageState(ApiUsageState apiUsageState) {
  23 + super(apiUsageState);
  24 + }
  25 +
  26 + @Override
  27 + public EntityType getEntityType() {
  28 + return EntityType.CUSTOMER;
  29 + }
  30 +}
... ...
... ... @@ -29,10 +29,13 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey;
29 29 import org.thingsboard.server.common.data.ApiUsageState;
30 30 import org.thingsboard.server.common.data.ApiUsageStateMailMessage;
31 31 import org.thingsboard.server.common.data.ApiUsageStateValue;
  32 +import org.thingsboard.server.common.data.EntityType;
32 33 import org.thingsboard.server.common.data.Tenant;
33 34 import org.thingsboard.server.common.data.TenantProfile;
34 35 import org.thingsboard.server.common.data.exception.ThingsboardException;
35 36 import org.thingsboard.server.common.data.id.ApiUsageStateId;
  37 +import org.thingsboard.server.common.data.id.CustomerId;
  38 +import org.thingsboard.server.common.data.id.EntityId;
36 39 import org.thingsboard.server.common.data.id.TenantId;
37 40 import org.thingsboard.server.common.data.id.TenantProfileId;
38 41 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
... ... @@ -45,6 +48,7 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
45 48 import org.thingsboard.server.common.msg.queue.ServiceType;
46 49 import org.thingsboard.server.common.msg.queue.TbCallback;
47 50 import org.thingsboard.server.common.msg.tools.SchedulerUtils;
  51 +import org.thingsboard.server.dao.customer.CustomerService;
48 52 import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
49 53 import org.thingsboard.server.dao.tenant.TenantService;
50 54 import org.thingsboard.server.dao.timeseries.TimeseriesService;
... ... @@ -63,6 +67,7 @@ import javax.annotation.PostConstruct;
63 67 import javax.annotation.PreDestroy;
64 68 import java.util.ArrayList;
65 69 import java.util.Arrays;
  70 +import java.util.Collections;
66 71 import java.util.HashSet;
67 72 import java.util.List;
68 73 import java.util.Map;
... ... @@ -95,6 +100,7 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
95 100 private final TbClusterService clusterService;
96 101 private final PartitionService partitionService;
97 102 private final TenantService tenantService;
  103 + private final CustomerService customerService;
98 104 private final TimeseriesService tsService;
99 105 private final ApiUsageStateService apiUsageStateService;
100 106 private final SchedulerComponent scheduler;
... ... @@ -105,10 +111,12 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
105 111 @Autowired
106 112 private InternalTelemetryService tsWsService;
107 113
108   - // Tenants that should be processed on this server
109   - private final Map<TenantId, TenantApiUsageState> myTenantStates = new ConcurrentHashMap<>();
110   - // Tenants that should be processed on other servers
111   - private final Map<TenantId, ApiUsageState> otherTenantStates = new ConcurrentHashMap<>();
  114 + // Entities that should be processed on this server
  115 + private final Map<EntityId, BaseApiUsageState> myUsageStates = new ConcurrentHashMap<>();
  116 + // Entities that should be processed on other servers
  117 + private final Map<EntityId, ApiUsageState> otherUsageStates = new ConcurrentHashMap<>();
  118 +
  119 + private final Set<EntityId> deletedEntities = Collections.newSetFromMap(new ConcurrentHashMap<>());
112 120
113 121 @Value("${usage.stats.report.enabled:true}")
114 122 private boolean enabled;
... ... @@ -123,13 +131,16 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
123 131 public DefaultTbApiUsageStateService(TbClusterService clusterService,
124 132 PartitionService partitionService,
125 133 TenantService tenantService,
  134 + CustomerService customerService,
126 135 TimeseriesService tsService,
127 136 ApiUsageStateService apiUsageStateService,
128 137 SchedulerComponent scheduler,
129   - TbTenantProfileCache tenantProfileCache, MailService mailService) {
  138 + TbTenantProfileCache tenantProfileCache,
  139 + MailService mailService) {
130 140 this.clusterService = clusterService;
131 141 this.partitionService = partitionService;
132 142 this.tenantService = tenantService;
  143 + this.customerService = customerService;
133 144 this.tsService = tsService;
134 145 this.apiUsageStateService = apiUsageStateService;
135 146 this.scheduler = scheduler;
... ... @@ -150,74 +161,92 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
150 161 @Override
151 162 public void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
152 163 ToUsageStatsServiceMsg statsMsg = msg.getValue();
153   - TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB()));
154 164
155   - if (tenantProfileCache.get(tenantId) == null) {
156   - return;
  165 + TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB()));
  166 + EntityId entityId;
  167 + if (statsMsg.getCustomerIdMSB() != 0 && statsMsg.getCustomerIdLSB() != 0) {
  168 + entityId = new CustomerId(new UUID(statsMsg.getCustomerIdMSB(), statsMsg.getCustomerIdLSB()));
  169 + } else {
  170 + entityId = tenantId;
157 171 }
158 172
159   - TenantApiUsageState tenantState;
  173 + processEntityUsageStats(tenantId, entityId, statsMsg.getValuesList());
  174 + callback.onSuccess();
  175 + }
  176 +
  177 + private void processEntityUsageStats(TenantId tenantId, EntityId entityId, List<UsageStatsKVProto> values) {
  178 + if (deletedEntities.contains(entityId)) return;
  179 +
  180 + BaseApiUsageState usageState;
160 181 List<TsKvEntry> updatedEntries;
161 182 Map<ApiFeature, ApiUsageStateValue> result;
  183 +
162 184 updateLock.lock();
163 185 try {
164   - tenantState = getOrFetchState(tenantId);
165   - long ts = tenantState.getCurrentCycleTs();
166   - long hourTs = tenantState.getCurrentHourTs();
  186 + usageState = getOrFetchState(tenantId, entityId);
  187 + long ts = usageState.getCurrentCycleTs();
  188 + long hourTs = usageState.getCurrentHourTs();
167 189 long newHourTs = SchedulerUtils.getStartOfCurrentHour();
168 190 if (newHourTs != hourTs) {
169   - tenantState.setHour(newHourTs);
  191 + usageState.setHour(newHourTs);
170 192 }
171 193 updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length);
172 194 Set<ApiFeature> apiFeatures = new HashSet<>();
173   - for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) {
  195 + for (UsageStatsKVProto kvProto : values) {
174 196 ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey());
175   - long newValue = tenantState.add(recordKey, kvProto.getValue());
  197 + long newValue = usageState.add(recordKey, kvProto.getValue());
176 198 updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue)));
177   - long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue());
  199 + long newHourlyValue = usageState.addToHourly(recordKey, kvProto.getValue());
178 200 updatedEntries.add(new BasicTsKvEntry(newHourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue)));
179 201 apiFeatures.add(recordKey.getApiFeature());
180 202 }
181   - result = tenantState.checkStateUpdatedDueToThreshold(apiFeatures);
  203 + if (usageState.getEntityType() == EntityType.TENANT && !usageState.getEntityId().equals(TenantId.SYS_TENANT_ID)) {
  204 + result = ((TenantApiUsageState) usageState).checkStateUpdatedDueToThreshold(apiFeatures);
  205 + } else {
  206 + result = Collections.emptyMap();
  207 + }
182 208 } finally {
183 209 updateLock.unlock();
184 210 }
185   - tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK);
  211 + tsWsService.saveAndNotifyInternal(tenantId, usageState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK);
186 212 if (!result.isEmpty()) {
187   - persistAndNotify(tenantState, result);
  213 + persistAndNotify(usageState, result);
188 214 }
189   - callback.onSuccess();
190 215 }
191 216
192 217 @Override
193 218 protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
194 219 if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) {
195   - myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
196   - otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
  220 + myUsageStates.entrySet().removeIf(entry -> {
  221 + return !partitionService.resolve(ServiceType.TB_CORE, entry.getValue().getTenantId(), entry.getKey()).isMyPartition();
  222 + });
  223 + otherUsageStates.entrySet().removeIf(entry -> {
  224 + return partitionService.resolve(ServiceType.TB_CORE, entry.getValue().getTenantId(), entry.getKey()).isMyPartition();
  225 + });
197 226 initStatesFromDataBase();
198 227 }
199 228 }
200 229
201 230 @Override
202 231 public ApiUsageState getApiUsageState(TenantId tenantId) {
203   - TenantApiUsageState tenantState = myTenantStates.get(tenantId);
  232 + TenantApiUsageState tenantState = (TenantApiUsageState) myUsageStates.get(tenantId);
204 233 if (tenantState != null) {
205 234 return tenantState.getApiUsageState();
206 235 } else {
207   - ApiUsageState state = otherTenantStates.get(tenantId);
  236 + ApiUsageState state = otherUsageStates.get(tenantId);
208 237 if (state != null) {
209 238 return state;
210 239 } else {
211 240 if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) {
212   - return getOrFetchState(tenantId).getApiUsageState();
  241 + return getOrFetchState(tenantId, tenantId).getApiUsageState();
213 242 } else {
214 243 updateLock.lock();
215 244 try {
216   - state = otherTenantStates.get(tenantId);
  245 + state = otherUsageStates.get(tenantId);
217 246 if (state == null) {
218 247 state = apiUsageStateService.findTenantApiUsageState(tenantId);
219 248 if (state != null) {
220   - otherTenantStates.put(tenantId, state);
  249 + otherUsageStates.put(tenantId, state);
221 250 }
222 251 }
223 252 } finally {
... ... @@ -231,7 +260,7 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
231 260
232 261 @Override
233 262 public void onApiUsageStateUpdate(TenantId tenantId) {
234   - otherTenantStates.remove(tenantId);
  263 + otherUsageStates.remove(tenantId);
235 264 }
236 265
237 266 @Override
... ... @@ -240,11 +269,14 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
240 269 TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId);
241 270 updateLock.lock();
242 271 try {
243   - myTenantStates.values().forEach(state -> {
244   - if (tenantProfile.getId().equals(state.getTenantProfileId())) {
245   - updateTenantState(state, tenantProfile);
246   - }
247   - });
  272 + myUsageStates.values().stream()
  273 + .filter(state -> state.getEntityType() == EntityType.TENANT)
  274 + .map(state -> (TenantApiUsageState) state)
  275 + .forEach(state -> {
  276 + if (tenantProfile.getId().equals(state.getTenantProfileId())) {
  277 + updateTenantState(state, tenantProfile);
  278 + }
  279 + });
248 280 } finally {
249 281 updateLock.unlock();
250 282 }
... ... @@ -256,7 +288,7 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
256 288 TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
257 289 updateLock.lock();
258 290 try {
259   - TenantApiUsageState state = myTenantStates.get(tenantId);
  291 + TenantApiUsageState state = (TenantApiUsageState) myUsageStates.get(tenantId);
260 292 if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) {
261 293 updateTenantState(state, tenantProfile);
262 294 }
... ... @@ -293,29 +325,42 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
293 325 }
294 326 }
295 327
296   - private void persistAndNotify(TenantApiUsageState state, Map<ApiFeature, ApiUsageStateValue> result) {
297   - log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result);
  328 + public void onTenantDelete(TenantId tenantId) {
  329 + deletedEntities.add(tenantId);
  330 + myUsageStates.remove(tenantId);
  331 + otherUsageStates.remove(tenantId);
  332 + }
  333 +
  334 + @Override
  335 + public void onCustomerDelete(CustomerId customerId) {
  336 + deletedEntities.add(customerId);
  337 + myUsageStates.remove(customerId);
  338 + }
  339 +
  340 + private void persistAndNotify(BaseApiUsageState state, Map<ApiFeature, ApiUsageStateValue> result) {
  341 + log.info("[{}] Detected update of the API state for {}: {}", state.getEntityId(), state.getEntityType(), result);
298 342 apiUsageStateService.update(state.getApiUsageState());
299 343 clusterService.onApiStateChange(state.getApiUsageState(), null);
300 344 long ts = System.currentTimeMillis();
301 345 List<TsKvEntry> stateTelemetry = new ArrayList<>();
302   - result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name())))));
  346 + result.forEach((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name()))));
303 347 tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK);
304 348
305   - String email = tenantService.findTenantById(state.getTenantId()).getEmail();
306   -
307   - if (StringUtils.isNotEmpty(email)) {
308   - result.forEach((apiFeature, stateValue) -> {
309   - mailExecutor.submit(() -> {
310   - try {
311   - mailService.sendApiFeatureStateEmail(apiFeature, stateValue, email, createStateMailMessage(state, apiFeature, stateValue));
312   - } catch (ThingsboardException e) {
313   - log.warn("[{}] Can't send update of the API state to tenant with provided email [{}]", state.getTenantId(), email, e);
314   - }
  349 + if (state.getEntityType() == EntityType.TENANT && !state.getEntityId().equals(TenantId.SYS_TENANT_ID)) {
  350 + String email = tenantService.findTenantById(state.getTenantId()).getEmail();
  351 + if (StringUtils.isNotEmpty(email)) {
  352 + result.forEach((apiFeature, stateValue) -> {
  353 + mailExecutor.submit(() -> {
  354 + try {
  355 + mailService.sendApiFeatureStateEmail(apiFeature, stateValue, email, createStateMailMessage((TenantApiUsageState) state, apiFeature, stateValue));
  356 + } catch (ThingsboardException e) {
  357 + log.warn("[{}] Can't send update of the API state to tenant with provided email [{}]", state.getTenantId(), email, e);
  358 + }
  359 + });
315 360 });
316   - });
317   - } else {
318   - log.warn("[{}] Can't send update of the API state to tenant with empty email!", state.getTenantId());
  361 + } else {
  362 + log.warn("[{}] Can't send update of the API state to tenant with empty email!", state.getTenantId());
  363 + }
319 364 }
320 365 }
321 366
... ... @@ -350,12 +395,14 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
350 395 updateLock.lock();
351 396 try {
352 397 long now = System.currentTimeMillis();
353   - myTenantStates.values().forEach(state -> {
  398 + myUsageStates.values().forEach(state -> {
354 399 if ((state.getNextCycleTs() < now) && (now - state.getNextCycleTs() < TimeUnit.HOURS.toMillis(1))) {
355   - TenantId tenantId = state.getTenantId();
356 400 state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth());
357 401 saveNewCounts(state, Arrays.asList(ApiUsageRecordKey.values()));
358   - updateTenantState(state, tenantProfileCache.get(tenantId));
  402 + if (state.getEntityType() == EntityType.TENANT && !state.getEntityId().equals(TenantId.SYS_TENANT_ID)) {
  403 + TenantId tenantId = state.getTenantId();
  404 + updateTenantState((TenantApiUsageState) state, tenantProfileCache.get(tenantId));
  405 + }
359 406 }
360 407 });
361 408 } finally {
... ... @@ -363,7 +410,7 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
363 410 }
364 411 }
365 412
366   - private void saveNewCounts(TenantApiUsageState state, List<ApiUsageRecordKey> keys) {
  413 + private void saveNewCounts(BaseApiUsageState state, List<ApiUsageRecordKey> keys) {
367 414 List<TsKvEntry> counts = keys.stream()
368 415 .map(key -> new BasicTsKvEntry(state.getCurrentCycleTs(), new LongDataEntry(key.getApiCountKey(), 0L)))
369 416 .collect(Collectors.toList());
... ... @@ -371,52 +418,66 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
371 418 tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), counts, VOID_CALLBACK);
372 419 }
373 420
374   - private TenantApiUsageState getOrFetchState(TenantId tenantId) {
375   - TenantApiUsageState tenantState = myTenantStates.get(tenantId);
376   - if (tenantState == null) {
377   - ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
378   - if (dbStateEntity == null) {
379   - try {
380   - dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId);
381   - } catch (Exception e) {
382   - dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId);
383   - }
384   - }
385   - TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
386   - tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity);
387   - List<ApiUsageRecordKey> newCounts = new ArrayList<>();
  421 + private BaseApiUsageState getOrFetchState(TenantId tenantId, EntityId entityId) {
  422 + if (entityId == null || entityId.isNullUid()) {
  423 + entityId = tenantId;
  424 + }
  425 + BaseApiUsageState state = myUsageStates.get(entityId);
  426 + if (state != null) {
  427 + return state;
  428 + }
  429 +
  430 + ApiUsageState storedState = apiUsageStateService.findApiUsageStateByEntityId(entityId);
  431 + if (storedState == null) {
388 432 try {
389   - List<TsKvEntry> dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get();
390   - for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
391   - boolean cycleEntryFound = false;
392   - boolean hourlyEntryFound = false;
393   - for (TsKvEntry tsKvEntry : dbValues) {
394   - if (tsKvEntry.getKey().equals(key.getApiCountKey())) {
395   - cycleEntryFound = true;
396   -
397   - boolean oldCount = tsKvEntry.getTs() == tenantState.getCurrentCycleTs();
398   - tenantState.put(key, oldCount ? tsKvEntry.getLongValue().get() : 0L);
399   -
400   - if (!oldCount) {
401   - newCounts.add(key);
402   - }
403   - } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) {
404   - hourlyEntryFound = true;
405   - tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L);
406   - }
407   - if (cycleEntryFound && hourlyEntryFound) {
408   - break;
  433 + storedState = apiUsageStateService.createDefaultApiUsageState(tenantId, entityId);
  434 + } catch (Exception e) {
  435 + storedState = apiUsageStateService.findApiUsageStateByEntityId(entityId);
  436 + }
  437 + }
  438 + if (entityId.getEntityType() == EntityType.TENANT) {
  439 + if (!entityId.equals(TenantId.SYS_TENANT_ID)) {
  440 + state = new TenantApiUsageState(tenantProfileCache.get((TenantId) entityId), storedState);
  441 + } else {
  442 + state = new TenantApiUsageState(storedState);
  443 + }
  444 + } else {
  445 + state = new CustomerApiUsageState(storedState);
  446 + }
  447 +
  448 + List<ApiUsageRecordKey> newCounts = new ArrayList<>();
  449 + try {
  450 + List<TsKvEntry> dbValues = tsService.findAllLatest(tenantId, storedState.getId()).get();
  451 + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
  452 + boolean cycleEntryFound = false;
  453 + boolean hourlyEntryFound = false;
  454 + for (TsKvEntry tsKvEntry : dbValues) {
  455 + if (tsKvEntry.getKey().equals(key.getApiCountKey())) {
  456 + cycleEntryFound = true;
  457 +
  458 + boolean oldCount = tsKvEntry.getTs() == state.getCurrentCycleTs();
  459 + state.put(key, oldCount ? tsKvEntry.getLongValue().get() : 0L);
  460 +
  461 + if (!oldCount) {
  462 + newCounts.add(key);
409 463 }
  464 + } else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) {
  465 + hourlyEntryFound = true;
  466 + state.putHourly(key, tsKvEntry.getTs() == state.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L);
  467 + }
  468 + if (cycleEntryFound && hourlyEntryFound) {
  469 + break;
410 470 }
411 471 }
412   - log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity);
413   - myTenantStates.put(tenantId, tenantState);
414   - saveNewCounts(tenantState, newCounts);
415   - } catch (InterruptedException | ExecutionException e) {
416   - log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e);
417 472 }
  473 + log.debug("[{}] Initialized state: {}", entityId, storedState);
  474 + myUsageStates.put(entityId, state);
  475 + saveNewCounts(state, newCounts);
  476 + } catch (InterruptedException | ExecutionException e) {
  477 + log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e);
418 478 }
419   - return tenantState;
  479 +
  480 + return state;
420 481 }
421 482
422 483 private void initStatesFromDataBase() {
... ... @@ -429,11 +490,11 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener<Pa
429 490 PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024);
430 491 List<Future<?>> futures = new ArrayList<>();
431 492 for (Tenant tenant : tenantIterator) {
432   - if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) {
  493 + if (!myUsageStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) {
433 494 log.debug("[{}] Initializing tenant state.", tenant.getId());
434 495 futures.add(tmpInitExecutor.submit(() -> {
435 496 try {
436   - updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId()));
  497 + updateTenantState((TenantApiUsageState) getOrFetchState(tenant.getId(), tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId()));
437 498 log.debug("[{}] Initialized tenant state.", tenant.getId());
438 499 } catch (Exception e) {
439 500 log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e);
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.service.apiusage;
17 17
18 18 import org.springframework.context.ApplicationListener;
19 19 import org.thingsboard.server.common.data.ApiUsageState;
  20 +import org.thingsboard.server.common.data.id.CustomerId;
20 21 import org.thingsboard.server.common.data.id.TenantId;
21 22 import org.thingsboard.server.common.data.id.TenantProfileId;
22 23 import org.thingsboard.server.common.msg.queue.TbCallback;
... ... @@ -34,5 +35,9 @@ public interface TbApiUsageStateService extends ApplicationListener<PartitionCha
34 35
35 36 void onTenantUpdate(TenantId tenantId);
36 37
  38 + void onTenantDelete(TenantId tenantId);
  39 +
  40 + void onCustomerDelete(CustomerId customerId);
  41 +
37 42 void onApiUsageStateUpdate(TenantId tenantId);
38 43 }
... ...
... ... @@ -22,85 +22,33 @@ import org.thingsboard.server.common.data.ApiFeature;
22 22 import org.thingsboard.server.common.data.ApiUsageRecordKey;
23 23 import org.thingsboard.server.common.data.ApiUsageState;
24 24 import org.thingsboard.server.common.data.ApiUsageStateValue;
  25 +import org.thingsboard.server.common.data.EntityType;
25 26 import org.thingsboard.server.common.data.TenantProfile;
26   -import org.thingsboard.server.common.data.id.TenantId;
27 27 import org.thingsboard.server.common.data.id.TenantProfileId;
28 28 import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
29   -import org.thingsboard.server.common.msg.tools.SchedulerUtils;
30 29
31 30 import java.util.Arrays;
32 31 import java.util.HashMap;
33 32 import java.util.HashSet;
34 33 import java.util.Map;
35 34 import java.util.Set;
36   -import java.util.concurrent.ConcurrentHashMap;
37   -
38   -public class TenantApiUsageState {
39   -
40   - private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>();
41   - private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>();
42 35
  36 +public class TenantApiUsageState extends BaseApiUsageState {
43 37 @Getter
44 38 @Setter
45 39 private TenantProfileId tenantProfileId;
46 40 @Getter
47 41 @Setter
48 42 private TenantProfileData tenantProfileData;
49   - @Getter
50   - private final ApiUsageState apiUsageState;
51   - @Getter
52   - private volatile long currentCycleTs;
53   - @Getter
54   - private volatile long nextCycleTs;
55   - @Getter
56   - private volatile long currentHourTs;
57 43
58 44 public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) {
  45 + super(apiUsageState);
59 46 this.tenantProfileId = tenantProfile.getId();
60 47 this.tenantProfileData = tenantProfile.getProfileData();
61   - this.apiUsageState = apiUsageState;
62   - this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth();
63   - this.nextCycleTs = SchedulerUtils.getStartOfNextMonth();
64   - this.currentHourTs = SchedulerUtils.getStartOfCurrentHour();
65   - }
66   -
67   - public void put(ApiUsageRecordKey key, Long value) {
68   - currentCycleValues.put(key, value);
69 48 }
70 49
71   - public void putHourly(ApiUsageRecordKey key, Long value) {
72   - currentHourValues.put(key, value);
73   - }
74   -
75   - public long add(ApiUsageRecordKey key, long value) {
76   - long result = currentCycleValues.getOrDefault(key, 0L) + value;
77   - currentCycleValues.put(key, result);
78   - return result;
79   - }
80   -
81   - public long get(ApiUsageRecordKey key) {
82   - return currentCycleValues.getOrDefault(key, 0L);
83   - }
84   -
85   - public long addToHourly(ApiUsageRecordKey key, long value) {
86   - long result = currentHourValues.getOrDefault(key, 0L) + value;
87   - currentHourValues.put(key, result);
88   - return result;
89   - }
90   -
91   - public void setHour(long currentHourTs) {
92   - this.currentHourTs = currentHourTs;
93   - for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
94   - currentHourValues.put(key, 0L);
95   - }
96   - }
97   -
98   - public void setCycles(long currentCycleTs, long nextCycleTs) {
99   - this.currentCycleTs = currentCycleTs;
100   - this.nextCycleTs = nextCycleTs;
101   - for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
102   - currentCycleValues.put(key, 0L);
103   - }
  50 + public TenantApiUsageState(ApiUsageState apiUsageState) {
  51 + super(apiUsageState);
104 52 }
105 53
106 54 public long getProfileThreshold(ApiUsageRecordKey key) {
... ... @@ -111,53 +59,25 @@ public class TenantApiUsageState {
111 59 return tenantProfileData.getConfiguration().getWarnThreshold(key);
112 60 }
113 61
114   - public TenantId getTenantId() {
115   - return apiUsageState.getTenantId();
116   - }
117   -
118   - public ApiUsageStateValue getFeatureValue(ApiFeature feature) {
119   - switch (feature) {
120   - case TRANSPORT:
121   - return apiUsageState.getTransportState();
122   - case RE:
123   - return apiUsageState.getReExecState();
124   - case DB:
125   - return apiUsageState.getDbStorageState();
126   - case JS:
127   - return apiUsageState.getJsExecState();
128   - case EMAIL:
129   - return apiUsageState.getEmailExecState();
130   - case SMS:
131   - return apiUsageState.getSmsExecState();
132   - default:
133   - return ApiUsageStateValue.ENABLED;
  62 + private Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) {
  63 + ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED;
  64 + for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) {
  65 + long value = get(recordKey);
  66 + long threshold = getProfileThreshold(recordKey);
  67 + long warnThreshold = getProfileWarnThreshold(recordKey);
  68 + ApiUsageStateValue tmpValue;
  69 + if (threshold == 0 || value == 0 || value < warnThreshold) {
  70 + tmpValue = ApiUsageStateValue.ENABLED;
  71 + } else if (value < threshold) {
  72 + tmpValue = ApiUsageStateValue.WARNING;
  73 + } else {
  74 + tmpValue = ApiUsageStateValue.DISABLED;
  75 + }
  76 + featureValue = ApiUsageStateValue.toMoreRestricted(featureValue, tmpValue);
134 77 }
  78 + return setFeatureValue(feature, featureValue) ? Pair.of(feature, featureValue) : null;
135 79 }
136 80
137   - public boolean setFeatureValue(ApiFeature feature, ApiUsageStateValue value) {
138   - ApiUsageStateValue currentValue = getFeatureValue(feature);
139   - switch (feature) {
140   - case TRANSPORT:
141   - apiUsageState.setTransportState(value);
142   - break;
143   - case RE:
144   - apiUsageState.setReExecState(value);
145   - break;
146   - case DB:
147   - apiUsageState.setDbStorageState(value);
148   - break;
149   - case JS:
150   - apiUsageState.setJsExecState(value);
151   - break;
152   - case EMAIL:
153   - apiUsageState.setEmailExecState(value);
154   - break;
155   - case SMS:
156   - apiUsageState.setSmsExecState(value);
157   - break;
158   - }
159   - return !currentValue.equals(value);
160   - }
161 81
162 82 public Map<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThresholds() {
163 83 return checkStateUpdatedDueToThreshold(new HashSet<>(Arrays.asList(ApiFeature.values())));
... ... @@ -174,23 +94,9 @@ public class TenantApiUsageState {
174 94 return result;
175 95 }
176 96
177   - public Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) {
178   - ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED;
179   - for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) {
180   - long value = get(recordKey);
181   - long threshold = getProfileThreshold(recordKey);
182   - long warnThreshold = getProfileWarnThreshold(recordKey);
183   - ApiUsageStateValue tmpValue;
184   - if (threshold == 0 || value == 0 || value < warnThreshold) {
185   - tmpValue = ApiUsageStateValue.ENABLED;
186   - } else if (value < threshold) {
187   - tmpValue = ApiUsageStateValue.WARNING;
188   - } else {
189   - tmpValue = ApiUsageStateValue.DISABLED;
190   - }
191   - featureValue = ApiUsageStateValue.toMoreRestricted(featureValue, tmpValue);
192   - }
193   - return setFeatureValue(feature, featureValue) ? Pair.of(feature, featureValue) : null;
  97 + @Override
  98 + public EntityType getEntityType() {
  99 + return EntityType.TENANT;
194 100 }
195 101
196 102 }
... ...
... ... @@ -230,7 +230,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
230 230 private void pushProvisionEventToRuleEngine(ProvisionRequest request, Device device, String type) {
231 231 try {
232 232 JsonNode entityNode = JacksonUtil.valueToTree(request);
233   - TbMsg msg = TbMsg.newMsg(type, device.getId(), createTbMsgMetaData(device), JacksonUtil.toString(entityNode));
  233 + TbMsg msg = TbMsg.newMsg(type, device.getId(), device.getCustomerId(), createTbMsgMetaData(device), JacksonUtil.toString(entityNode));
234 234 sendToRuleEngine(device.getTenantId(), msg, null);
235 235 } catch (IllegalArgumentException e) {
236 236 log.warn("[{}] Failed to push device action to rule engine: {}", device.getId(), type, e);
... ... @@ -240,7 +240,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService {
240 240 private void pushDeviceCreatedEventToRuleEngine(Device device) {
241 241 try {
242 242 ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(device);
243   - TbMsg msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, device.getId(), createTbMsgMetaData(device), JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode));
  243 + TbMsg msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, device.getId(), device.getCustomerId(), createTbMsgMetaData(device), JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode));
244 244 sendToRuleEngine(device.getTenantId(), msg, null);
245 245 } catch (JsonProcessingException | IllegalArgumentException e) {
246 246 log.warn("[{}] Failed to push device action to rule engine: {}", device.getId(), DataConstants.ENTITY_CREATED, e);
... ...
... ... @@ -920,7 +920,7 @@ public final class EdgeGrpcSession implements Closeable {
920 920 try {
921 921 if (uplinkMsg.getEntityDataCount() > 0) {
922 922 for (EntityDataProto entityData : uplinkMsg.getEntityDataList()) {
923   - result.addAll(ctx.getTelemetryProcessor().onTelemetryUpdate(edge.getTenantId(), entityData));
  923 + result.addAll(ctx.getTelemetryProcessor().onTelemetryUpdate(edge.getTenantId(), edge.getCustomerId(), entityData));
924 924 }
925 925 }
926 926 if (uplinkMsg.getDeviceUpdateMsgCount() > 0) {
... ...
... ... @@ -231,7 +231,7 @@ public class DeviceProcessor extends BaseProcessor {
231 231 try {
232 232 DeviceId deviceId = device.getId();
233 233 ObjectNode entityNode = mapper.valueToTree(device);
234   - TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId,
  234 + TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, device.getCustomerId(),
235 235 getActionTbMsgMetaData(edge, device.getCustomerId()), TbMsgDataType.JSON, mapper.writeValueAsString(entityNode));
236 236 tbClusterService.pushMsgToRuleEngine(tenantId, deviceId, tbMsg, new TbQueueCallback() {
237 237 @Override
... ...
... ... @@ -70,7 +70,7 @@ public class TelemetryProcessor extends BaseProcessor {
70 70
71 71 private final Gson gson = new Gson();
72 72
73   - public List<ListenableFuture<Void>> onTelemetryUpdate(TenantId tenantId, EntityDataProto entityData) {
  73 + public List<ListenableFuture<Void>> onTelemetryUpdate(TenantId tenantId, CustomerId customerId, EntityDataProto entityData) {
74 74 log.trace("[{}] onTelemetryUpdate [{}]", tenantId, entityData);
75 75 List<ListenableFuture<Void>> result = new ArrayList<>();
76 76 EntityId entityId = constructEntityId(entityData);
... ... @@ -80,14 +80,14 @@ public class TelemetryProcessor extends BaseProcessor {
80 80 TbMsgMetaData metaData = new TbMsgMetaData();
81 81 metaData.putValue(DataConstants.MSG_SOURCE_KEY, DataConstants.EDGE_MSG_SOURCE);
82 82 if (entityData.hasPostAttributesMsg()) {
83   - result.add(processPostAttributes(tenantId, entityId, entityData.getPostAttributesMsg(), metaData));
  83 + result.add(processPostAttributes(tenantId, customerId, entityId, entityData.getPostAttributesMsg(), metaData));
84 84 }
85 85 if (entityData.hasAttributesUpdatedMsg()) {
86 86 metaData.putValue("scope", entityData.getPostAttributeScope());
87   - result.add(processAttributesUpdate(tenantId, entityId, entityData.getAttributesUpdatedMsg(), metaData));
  87 + result.add(processAttributesUpdate(tenantId, customerId, entityId, entityData.getAttributesUpdatedMsg(), metaData));
88 88 }
89 89 if (entityData.hasPostTelemetryMsg()) {
90   - result.add(processPostTelemetry(tenantId, entityId, entityData.getPostTelemetryMsg(), metaData));
  90 + result.add(processPostTelemetry(tenantId, customerId, entityId, entityData.getPostTelemetryMsg(), metaData));
91 91 }
92 92 }
93 93 if (entityData.hasAttributeDeleteMsg()) {
... ... @@ -148,7 +148,7 @@ public class TelemetryProcessor extends BaseProcessor {
148 148 }
149 149 }
150 150
151   - private ListenableFuture<Void> processPostTelemetry(TenantId tenantId, EntityId entityId, TransportProtos.PostTelemetryMsg msg, TbMsgMetaData metaData) {
  151 + private ListenableFuture<Void> processPostTelemetry(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostTelemetryMsg msg, TbMsgMetaData metaData) {
152 152 SettableFuture<Void> futureToSet = SettableFuture.create();
153 153 for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) {
154 154 JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList());
... ... @@ -156,7 +156,7 @@ public class TelemetryProcessor extends BaseProcessor {
156 156 Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
157 157 String queueName = defaultQueueAndRuleChain.getKey();
158 158 RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
159   - TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_TELEMETRY_REQUEST.name(), entityId, metaData, gson.toJson(json), ruleChainId, null);
  159 + TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_TELEMETRY_REQUEST.name(), entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
160 160 tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
161 161 @Override
162 162 public void onSuccess(TbQueueMsgMetadata metadata) {
... ... @@ -173,13 +173,13 @@ public class TelemetryProcessor extends BaseProcessor {
173 173 return futureToSet;
174 174 }
175 175
176   - private ListenableFuture<Void> processPostAttributes(TenantId tenantId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
  176 + private ListenableFuture<Void> processPostAttributes(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
177 177 SettableFuture<Void> futureToSet = SettableFuture.create();
178 178 JsonObject json = JsonUtils.getJsonObject(msg.getKvList());
179 179 Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
180 180 String queueName = defaultQueueAndRuleChain.getKey();
181 181 RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
182   - TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), entityId, metaData, gson.toJson(json), ruleChainId, null);
  182 + TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
183 183 tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
184 184 @Override
185 185 public void onSuccess(TbQueueMsgMetadata metadata) {
... ... @@ -195,7 +195,7 @@ public class TelemetryProcessor extends BaseProcessor {
195 195 return futureToSet;
196 196 }
197 197
198   - private ListenableFuture<Void> processAttributesUpdate(TenantId tenantId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
  198 + private ListenableFuture<Void> processAttributesUpdate(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
199 199 SettableFuture<Void> futureToSet = SettableFuture.create();
200 200 JsonObject json = JsonUtils.getJsonObject(msg.getKvList());
201 201 Set<AttributeKvEntry> attributes = JsonConverter.convertToAttributes(json);
... ... @@ -206,7 +206,7 @@ public class TelemetryProcessor extends BaseProcessor {
206 206 Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
207 207 String queueName = defaultQueueAndRuleChain.getKey();
208 208 RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
209   - TbMsg tbMsg = TbMsg.newMsg(queueName, DataConstants.ATTRIBUTES_UPDATED, entityId, metaData, gson.toJson(json), ruleChainId, null);
  209 + TbMsg tbMsg = TbMsg.newMsg(queueName, DataConstants.ATTRIBUTES_UPDATED, entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
210 210 tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
211 211 @Override
212 212 public void onSuccess(TbQueueMsgMetadata metadata) {
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.firmware;
  17 +
  18 +import com.google.common.util.concurrent.FutureCallback;
  19 +import lombok.extern.slf4j.Slf4j;
  20 +import org.springframework.stereotype.Service;
  21 +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
  22 +import org.thingsboard.server.common.data.DataConstants;
  23 +import org.thingsboard.server.common.data.Device;
  24 +import org.thingsboard.server.common.data.DeviceProfile;
  25 +import org.thingsboard.server.common.data.Firmware;
  26 +import org.thingsboard.server.common.data.FirmwareInfo;
  27 +import org.thingsboard.server.common.data.id.DeviceId;
  28 +import org.thingsboard.server.common.data.id.FirmwareId;
  29 +import org.thingsboard.server.common.data.id.TenantId;
  30 +import org.thingsboard.server.common.data.kv.AttributeKvEntry;
  31 +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
  32 +import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
  33 +import org.thingsboard.server.common.data.kv.LongDataEntry;
  34 +import org.thingsboard.server.common.data.kv.StringDataEntry;
  35 +import org.thingsboard.server.common.data.kv.TsKvEntry;
  36 +import org.thingsboard.server.common.data.page.PageData;
  37 +import org.thingsboard.server.common.data.page.PageLink;
  38 +import org.thingsboard.server.common.msg.queue.TbCallback;
  39 +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
  40 +import org.thingsboard.server.dao.device.DeviceProfileService;
  41 +import org.thingsboard.server.dao.device.DeviceService;
  42 +import org.thingsboard.server.dao.firmware.FirmwareService;
  43 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
  44 +import org.thingsboard.server.queue.TbQueueProducer;
  45 +import org.thingsboard.server.queue.common.TbProtoQueueMsg;
  46 +import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
  47 +import org.thingsboard.server.queue.util.TbCoreComponent;
  48 +
  49 +import javax.annotation.Nullable;
  50 +import java.util.ArrayList;
  51 +import java.util.Arrays;
  52 +import java.util.Collections;
  53 +import java.util.List;
  54 +import java.util.UUID;
  55 +import java.util.function.Consumer;
  56 +
  57 +import static org.thingsboard.server.common.data.DataConstants.FIRMWARE_CHECKSUM;
  58 +import static org.thingsboard.server.common.data.DataConstants.FIRMWARE_CHECKSUM_ALGORITHM;
  59 +import static org.thingsboard.server.common.data.DataConstants.FIRMWARE_SIZE;
  60 +import static org.thingsboard.server.common.data.DataConstants.FIRMWARE_TITLE;
  61 +import static org.thingsboard.server.common.data.DataConstants.FIRMWARE_VERSION;
  62 +
  63 +@Slf4j
  64 +@Service
  65 +@TbCoreComponent
  66 +public class DefaultFirmwareStateService implements FirmwareStateService {
  67 +
  68 + private final FirmwareService firmwareService;
  69 + private final DeviceService deviceService;
  70 + private final DeviceProfileService deviceProfileService;
  71 + private final RuleEngineTelemetryService telemetryService;
  72 + private final TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> fwStateMsgProducer;
  73 +
  74 + public DefaultFirmwareStateService(FirmwareService firmwareService,
  75 + DeviceService deviceService,
  76 + DeviceProfileService deviceProfileService,
  77 + RuleEngineTelemetryService telemetryService,
  78 + TbCoreQueueFactory coreQueueFactory) {
  79 + this.firmwareService = firmwareService;
  80 + this.deviceService = deviceService;
  81 + this.deviceProfileService = deviceProfileService;
  82 + this.telemetryService = telemetryService;
  83 + this.fwStateMsgProducer = coreQueueFactory.createToFirmwareStateServiceMsgProducer();
  84 + }
  85 +
  86 + @Override
  87 + public void update(Device device, Device oldDevice) {
  88 + FirmwareId newFirmwareId = device.getFirmwareId();
  89 + if (newFirmwareId == null) {
  90 + DeviceProfile newDeviceProfile = deviceProfileService.findDeviceProfileById(device.getTenantId(), device.getDeviceProfileId());
  91 + newFirmwareId = newDeviceProfile.getFirmwareId();
  92 + }
  93 + if (oldDevice != null) {
  94 + if (newFirmwareId != null) {
  95 + FirmwareId oldFirmwareId = oldDevice.getFirmwareId();
  96 + if (oldFirmwareId == null) {
  97 + DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(oldDevice.getTenantId(), oldDevice.getDeviceProfileId());
  98 + oldFirmwareId = oldDeviceProfile.getFirmwareId();
  99 + }
  100 + if (!newFirmwareId.equals(oldFirmwareId)) {
  101 + // Device was updated and new firmware is different from previous firmware.
  102 + send(device.getTenantId(), device.getId(), newFirmwareId, System.currentTimeMillis());
  103 + }
  104 + } else {
  105 + // Device was updated and new firmware is not set.
  106 + remove(device);
  107 + }
  108 + } else if (newFirmwareId != null) {
  109 + // Device was created and firmware is defined.
  110 + send(device.getTenantId(), device.getId(), newFirmwareId, System.currentTimeMillis());
  111 + }
  112 + }
  113 +
  114 + @Override
  115 + public void update(DeviceProfile deviceProfile) {
  116 + TenantId tenantId = deviceProfile.getTenantId();
  117 +
  118 + Consumer<Device> updateConsumer;
  119 + if (deviceProfile.getFirmwareId() != null) {
  120 + long ts = System.currentTimeMillis();
  121 + updateConsumer = d -> send(d.getTenantId(), d.getId(), deviceProfile.getFirmwareId(), ts);
  122 + } else {
  123 + updateConsumer = this::remove;
  124 + }
  125 +
  126 + PageLink pageLink = new PageLink(100);
  127 + PageData<Device> pageData;
  128 + do {
  129 + pageData = deviceService.findDevicesByTenantIdAndTypeAndEmptyFirmware(tenantId, deviceProfile.getName(), pageLink);
  130 +
  131 + pageData.getData().forEach(updateConsumer);
  132 +
  133 + if (pageData.hasNext()) {
  134 + pageLink = pageLink.nextPageLink();
  135 + }
  136 + } while (pageData.hasNext());
  137 + }
  138 +
  139 + @Override
  140 + public boolean process(ToFirmwareStateServiceMsg msg) {
  141 + boolean isSuccess = false;
  142 + FirmwareId targetFirmwareId = new FirmwareId(new UUID(msg.getFirmwareIdMSB(), msg.getFirmwareIdLSB()));
  143 + DeviceId deviceId = new DeviceId(new UUID(msg.getDeviceIdMSB(), msg.getDeviceIdLSB()));
  144 + TenantId tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB()));
  145 + long ts = msg.getTs();
  146 +
  147 + Device device = deviceService.findDeviceById(tenantId, deviceId);
  148 + if (device == null) {
  149 + log.warn("[{}] [{}] Device was removed during firmware update msg was queued!", tenantId, deviceId);
  150 + } else {
  151 + FirmwareId currentFirmwareId = device.getFirmwareId();
  152 +
  153 + if (currentFirmwareId == null) {
  154 + currentFirmwareId = deviceProfileService.findDeviceProfileById(tenantId, device.getDeviceProfileId()).getFirmwareId();
  155 + }
  156 +
  157 + if (targetFirmwareId.equals(currentFirmwareId)) {
  158 + update(device, firmwareService.findFirmwareById(device.getTenantId(), targetFirmwareId), ts);
  159 + isSuccess = true;
  160 + } else {
  161 + log.warn("[{}] [{}] Can`t update firmware for the device, target firmwareId: [{}], current firmwareId: [{}]!", tenantId, deviceId, targetFirmwareId, currentFirmwareId);
  162 + }
  163 + }
  164 + return isSuccess;
  165 + }
  166 +
  167 + private void send(TenantId tenantId, DeviceId deviceId, FirmwareId firmwareId, long ts) {
  168 + ToFirmwareStateServiceMsg msg = ToFirmwareStateServiceMsg.newBuilder()
  169 + .setTenantIdMSB(tenantId.getId().getMostSignificantBits())
  170 + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
  171 + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
  172 + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
  173 + .setFirmwareIdMSB(firmwareId.getId().getMostSignificantBits())
  174 + .setFirmwareIdLSB(firmwareId.getId().getLeastSignificantBits())
  175 + .setTs(ts)
  176 + .build();
  177 +
  178 + FirmwareInfo firmware = firmwareService.findFirmwareInfoById(tenantId, firmwareId);
  179 + if (firmware == null) {
  180 + log.warn("[{}] Failed to send firmware update because firmware was already deleted", firmwareId);
  181 + return;
  182 + }
  183 +
  184 + TopicPartitionInfo tpi = new TopicPartitionInfo(fwStateMsgProducer.getDefaultTopic(), null, null, false);
  185 + fwStateMsgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null);
  186 +
  187 + List<TsKvEntry> telemetry = new ArrayList<>();
  188 + telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.TARGET_FIRMWARE_TITLE, firmware.getTitle())));
  189 + telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.TARGET_FIRMWARE_VERSION, firmware.getVersion())));
  190 + telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_STATE, FirmwareUpdateStatus.QUEUED.name())));
  191 +
  192 + telemetryService.saveAndNotify(tenantId, deviceId, telemetry, new FutureCallback<>() {
  193 + @Override
  194 + public void onSuccess(@Nullable Void tmp) {
  195 + log.trace("[{}] Success save firmware status!", deviceId);
  196 + }
  197 +
  198 + @Override
  199 + public void onFailure(Throwable t) {
  200 + log.error("[{}] Failed to save firmware status!", deviceId, t);
  201 + }
  202 + });
  203 + }
  204 +
  205 +
  206 + private void update(Device device, FirmwareInfo firmware, long ts) {
  207 + TenantId tenantId = device.getTenantId();
  208 + DeviceId deviceId = device.getId();
  209 +
  210 + BasicTsKvEntry status = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(DataConstants.FIRMWARE_STATE, FirmwareUpdateStatus.INITIATED.name()));
  211 +
  212 + telemetryService.saveAndNotify(tenantId, deviceId, Collections.singletonList(status), new FutureCallback<>() {
  213 + @Override
  214 + public void onSuccess(@Nullable Void tmp) {
  215 + log.trace("[{}] Success save telemetry with target firmware for device!", deviceId);
  216 + }
  217 +
  218 + @Override
  219 + public void onFailure(Throwable t) {
  220 + log.error("[{}] Failed to save telemetry with target firmware for device!", deviceId, t);
  221 + }
  222 + });
  223 +
  224 + List<AttributeKvEntry> attributes = new ArrayList<>();
  225 +
  226 + attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_TITLE, firmware.getTitle())));
  227 + attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_VERSION, firmware.getVersion())));
  228 +
  229 + attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(FIRMWARE_SIZE, firmware.getDataSize())));
  230 + attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_CHECKSUM_ALGORITHM, firmware.getChecksumAlgorithm())));
  231 + attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_CHECKSUM, firmware.getChecksum())));
  232 + telemetryService.saveAndNotify(tenantId, deviceId, DataConstants.SHARED_SCOPE, attributes, new FutureCallback<>() {
  233 + @Override
  234 + public void onSuccess(@Nullable Void tmp) {
  235 + log.trace("[{}] Success save attributes with target firmware!", deviceId);
  236 + }
  237 +
  238 + @Override
  239 + public void onFailure(Throwable t) {
  240 + log.error("[{}] Failed to save attributes with target firmware!", deviceId, t);
  241 + }
  242 + });
  243 + }
  244 +
  245 + private void remove(Device device) {
  246 + telemetryService.deleteAndNotify(device.getTenantId(), device.getId(), DataConstants.SHARED_SCOPE,
  247 + Arrays.asList(FIRMWARE_TITLE, FIRMWARE_VERSION, FIRMWARE_SIZE, FIRMWARE_CHECKSUM_ALGORITHM, FIRMWARE_CHECKSUM),
  248 + new FutureCallback<>() {
  249 + @Override
  250 + public void onSuccess(@Nullable Void tmp) {
  251 + log.trace("[{}] Success remove target firmware attributes!", device.getId());
  252 + }
  253 +
  254 + @Override
  255 + public void onFailure(Throwable t) {
  256 + log.error("[{}] Failed to remove target firmware attributes!", device.getId(), t);
  257 + }
  258 + });
  259 + }
  260 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.firmware;
  17 +
  18 +import org.thingsboard.server.common.data.Device;
  19 +import org.thingsboard.server.common.data.DeviceProfile;
  20 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
  21 +
  22 +public interface FirmwareStateService {
  23 +
  24 + void update(Device device, Device oldDevice);
  25 +
  26 + void update(DeviceProfile deviceProfile);
  27 +
  28 + boolean process(ToFirmwareStateServiceMsg msg);
  29 +
  30 +}
... ...
application/src/main/java/org/thingsboard/server/service/firmware/FirmwareUpdateStatus.java renamed from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/TypeServer.java
... ... @@ -13,17 +13,8 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.transport.lwm2m.utils;
  16 +package org.thingsboard.server.service.firmware;
17 17
18   -public enum TypeServer {
19   - BOOTSTRAP(0, "bootstrap"),
20   - CLIENT(1, "client");
21   -
22   - public int code;
23   - public String type;
24   -
25   - TypeServer(int code, String type) {
26   - this.code = code;
27   - this.type = type;
28   - }
  18 +public enum FirmwareUpdateStatus {
  19 + QUEUED, INITIATED, DOWNLOADING, DOWNLOADED, VERIFIED, UPDATING, UPDATED, FAILED
29 20 }
... ...
... ... @@ -444,11 +444,6 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
444 444 installScripts.loadSystemWidgets();
445 445 }
446 446
447   - @Override
448   - public void loadSystemLwm2mResources() throws Exception {
449   - installScripts.loadSystemLwm2mResources();
450   - }
451   -
452 447 private User createUser(Authority authority,
453 448 TenantId tenantId,
454 449 CustomerId customerId,
... ...
... ... @@ -22,8 +22,6 @@ import org.springframework.beans.factory.annotation.Value;
22 22 import org.springframework.stereotype.Component;
23 23 import org.springframework.util.StringUtils;
24 24 import org.thingsboard.server.common.data.Dashboard;
25   -import org.thingsboard.server.common.data.ResourceType;
26   -import org.thingsboard.server.common.data.TbResource;
27 25 import org.thingsboard.server.common.data.id.CustomerId;
28 26 import org.thingsboard.server.common.data.id.EntityId;
29 27 import org.thingsboard.server.common.data.id.TenantId;
... ... @@ -33,7 +31,6 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
33 31 import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
34 32 import org.thingsboard.server.common.data.widget.WidgetsBundle;
35 33 import org.thingsboard.server.dao.dashboard.DashboardService;
36   -import org.thingsboard.server.dao.exception.DataValidationException;
37 34 import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
38 35 import org.thingsboard.server.dao.resource.ResourceService;
39 36 import org.thingsboard.server.dao.rule.RuleChainService;
... ... @@ -45,7 +42,6 @@ import java.nio.file.DirectoryStream;
45 42 import java.nio.file.Files;
46 43 import java.nio.file.Path;
47 44 import java.nio.file.Paths;
48   -import java.util.Base64;
49 45 import java.util.Optional;
50 46
51 47 import static org.thingsboard.server.service.install.DatabaseHelper.objectMapper;
... ... @@ -210,29 +206,6 @@ public class InstallScripts {
210 206 }
211 207 }
212 208
213   - public void loadSystemLwm2mResources() throws Exception {
214   - Path modelsDir = Paths.get(getDataDir(), MODELS_DIR);
215   - if (Files.isDirectory(modelsDir)) {
216   - try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(modelsDir, path -> path.toString().endsWith(XML_EXT))) {
217   - dirStream.forEach(
218   - path -> {
219   - try {
220   - byte[] fileBytes = Files.readAllBytes(path);
221   - TbResource resource = new TbResource();
222   - resource.setFileName(path.getFileName().toString());
223   - resource.setTenantId(TenantId.SYS_TENANT_ID);
224   - resource.setResourceType(ResourceType.LWM2M_MODEL);
225   - resource.setData(Base64.getEncoder().encodeToString(fileBytes));
226   - resourceService.saveResource(resource);
227   - } catch (Exception e) {
228   - throw new DataValidationException(String.format("Could not parse the XML of objectModel with name %s", path.toString()));
229   - }
230   - }
231   - );
232   - }
233   - }
234   - }
235   -
236 209 public void loadDashboards(TenantId tenantId, CustomerId customerId) throws Exception {
237 210 Path dashboardsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR);
238 211 try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(dashboardsDir, path -> path.toString().endsWith(JSON_EXT))) {
... ...
... ... @@ -390,7 +390,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
390 390 pageData = tenantService.findTenants(pageLink);
391 391 for (Tenant tenant : pageData.getData()) {
392 392 try {
393   - apiUsageStateService.createDefaultApiUsageState(tenant.getId());
  393 + apiUsageStateService.createDefaultApiUsageState(tenant.getId(), null);
394 394 } catch (Exception e) {
395 395 }
396 396 List<EntitySubtype> deviceTypes = deviceService.findDeviceTypesByTenantId(tenant.getId()).get();
... ... @@ -450,37 +450,19 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
450 450 try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
451 451 log.info("Updating schema ...");
452 452 try {
453   - conn.createStatement().execute("CREATE TABLE IF NOT EXISTS resource ( " +
454   - "id uuid NOT NULL CONSTRAINT resource_pkey PRIMARY KEY, " +
455   - "created_time bigint NOT NULL, " +
456   - "tenant_id uuid NOT NULL, " +
457   - "title varchar(255) NOT NULL, " +
458   - "resource_type varchar(32) NOT NULL, " +
459   - "resource_key varchar(255) NOT NULL, " +
460   - "search_text varchar(255), " +
461   - "file_name varchar(255) NOT NULL, " +
462   - "data varchar, " +
463   - "CONSTRAINT resource_unq_key UNIQUE (tenant_id, resource_type, resource_key)" +
464   - ");");
465   -
466   - conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003000;");
467   - installScripts.loadSystemLwm2mResources();
468   -
469   - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", SCHEMA_UPDATE_SQL);
470   - loadSql(schemaUpdateFile, conn);
471   - try {
472   - conn.createStatement().execute("ALTER TABLE rule_chain ADD COLUMN type varchar(255) DEFAULT 'CORE'"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
473   - } catch (Exception ignored) {}
474   -
475   - log.info("Load Edge TTL functions ...");
476   - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_ttl.sql");
477   - loadSql(schemaUpdateFile, conn);
478   - log.info("Edge TTL functions successfully loaded!");
479   -
480   - } catch (Exception e) {
481   - log.error("Failed updating schema!!!", e);
  453 + conn.createStatement().execute("ALTER TABLE rule_chain ADD COLUMN type varchar(255) DEFAULT 'CORE'"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
  454 + } catch (Exception ignored) {
482 455 }
  456 + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", SCHEMA_UPDATE_SQL);
  457 + loadSql(schemaUpdateFile, conn);
  458 + log.info("Load Edge TTL functions ...");
  459 + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_ttl.sql");
  460 + loadSql(schemaUpdateFile, conn);
  461 + log.info("Edge TTL functions successfully loaded!");
  462 + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003000;");
483 463 log.info("Schema updated.");
  464 + } catch (Exception e) {
  465 + log.error("Failed updating schema!!!", e);
484 466 }
485 467 break;
486 468 default:
... ...
... ... @@ -33,6 +33,4 @@ public interface SystemDataLoaderService {
33 33
34 34 void deleteSystemWidgetBundle(String bundleAlias) throws Exception;
35 35
36   - void loadSystemLwm2mResources() throws Exception;
37   -
38 36 }
... ...
... ... @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.ApiUsageStateMailMessage;
36 36 import org.thingsboard.server.common.data.ApiUsageStateValue;
37 37 import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
38 38 import org.thingsboard.server.common.data.exception.ThingsboardException;
  39 +import org.thingsboard.server.common.data.id.CustomerId;
39 40 import org.thingsboard.server.common.data.id.EntityId;
40 41 import org.thingsboard.server.common.data.id.TenantId;
41 42 import org.thingsboard.server.dao.exception.IncorrectParameterException;
... ... @@ -233,7 +234,7 @@ public class DefaultMailService implements MailService {
233 234 }
234 235
235 236 @Override
236   - public void send(TenantId tenantId, String from, String to, String cc, String bcc, String subject, String body) throws MessagingException {
  237 + public void send(TenantId tenantId, CustomerId customerId, String from, String to, String cc, String bcc, String subject, String body) throws MessagingException {
237 238 if (apiUsageStateService.getApiUsageState(tenantId).isEmailSendEnabled()) {
238 239 MimeMessage mailMsg = mailSender.createMimeMessage();
239 240 MimeMessageHelper helper = new MimeMessageHelper(mailMsg, "UTF-8");
... ... @@ -248,7 +249,7 @@ public class DefaultMailService implements MailService {
248 249 helper.setSubject(subject);
249 250 helper.setText(body);
250 251 mailSender.send(helper.getMimeMessage());
251   - apiUsageClient.report(tenantId, ApiUsageRecordKey.EMAIL_EXEC_COUNT, 1);
  252 + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.EMAIL_EXEC_COUNT, 1);
252 253 } else {
253 254 throw new RuntimeException("Email sending is disabled due to API limits!");
254 255 }
... ...
... ... @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.HasName;
29 29 import org.thingsboard.server.common.data.TbResource;
30 30 import org.thingsboard.server.common.data.Tenant;
31 31 import org.thingsboard.server.common.data.TenantProfile;
  32 +import org.thingsboard.server.common.data.id.CustomerId;
32 33 import org.thingsboard.server.common.data.id.DeviceId;
33 34 import org.thingsboard.server.common.data.id.DeviceProfileId;
34 35 import org.thingsboard.server.common.data.id.EdgeId;
... ...
... ... @@ -24,6 +24,7 @@ import org.springframework.context.event.EventListener;
24 24 import org.springframework.core.annotation.Order;
25 25 import org.springframework.scheduling.annotation.Scheduled;
26 26 import org.springframework.stereotype.Service;
  27 +import org.thingsboard.common.util.JacksonUtil;
27 28 import org.thingsboard.common.util.ThingsBoardThreadFactory;
28 29 import org.thingsboard.rule.engine.api.RpcError;
29 30 import org.thingsboard.server.actors.ActorSystemContext;
... ... @@ -35,7 +36,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
35 36 import org.thingsboard.server.common.msg.queue.TbCallback;
36 37 import org.thingsboard.server.common.stats.StatsFactory;
37 38 import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
38   -import org.thingsboard.common.util.JacksonUtil;
  39 +import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
39 40 import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
40 41 import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto;
41 42 import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
... ... @@ -49,6 +50,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionCloseP
49 50 import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto;
50 51 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
51 52 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  53 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
52 54 import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
53 55 import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
54 56 import org.thingsboard.server.queue.TbQueueConsumer;
... ... @@ -58,8 +60,8 @@ import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
58 60 import org.thingsboard.server.queue.util.TbCoreComponent;
59 61 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
60 62 import org.thingsboard.server.service.edge.EdgeNotificationService;
  63 +import org.thingsboard.server.service.firmware.FirmwareStateService;
61 64 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
62   -import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
63 65 import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
64 66 import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
65 67 import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
... ... @@ -97,6 +99,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
97 99 @Value("${queue.core.stats.enabled:false}")
98 100 private boolean statsEnabled;
99 101
  102 + @Value("${queue.core.firmware.pack-interval-ms:60000}")
  103 + private long firmwarePackInterval;
  104 + @Value("${queue.core.firmware.pack-size:100}")
  105 + private int firmwarePackSize;
  106 +
100 107 private final TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> mainConsumer;
101 108 private final DeviceStateService stateService;
102 109 private final TbApiUsageStateService statsService;
... ... @@ -104,11 +111,15 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
104 111 private final SubscriptionManagerService subscriptionManagerService;
105 112 private final TbCoreDeviceRpcService tbCoreDeviceRpcService;
106 113 private final EdgeNotificationService edgeNotificationService;
  114 + private final FirmwareStateService firmwareStateService;
107 115 private final TbCoreConsumerStats stats;
108 116 protected final TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
  117 + private final TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> firmwareStatesConsumer;
109 118
110 119 protected volatile ExecutorService usageStatsExecutor;
111 120
  121 + private volatile ExecutorService firmwareStatesExecutor;
  122 +
112 123 public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory,
113 124 ActorSystemContext actorContext,
114 125 DeviceStateService stateService,
... ... @@ -121,10 +132,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
121 132 TbApiUsageStateService statsService,
122 133 TbTenantProfileCache tenantProfileCache,
123 134 TbApiUsageStateService apiUsageStateService,
124   - EdgeNotificationService edgeNotificationService) {
  135 + EdgeNotificationService edgeNotificationService,
  136 + FirmwareStateService firmwareStateService) {
125 137 super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
126 138 this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
127 139 this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
  140 + this.firmwareStatesConsumer = tbCoreQueueFactory.createToFirmwareStateServiceMsgConsumer();
128 141 this.stateService = stateService;
129 142 this.localSubscriptionService = localSubscriptionService;
130 143 this.subscriptionManagerService = subscriptionManagerService;
... ... @@ -132,12 +145,14 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
132 145 this.edgeNotificationService = edgeNotificationService;
133 146 this.stats = new TbCoreConsumerStats(statsFactory);
134 147 this.statsService = statsService;
  148 + this.firmwareStateService = firmwareStateService;
135 149 }
136 150
137 151 @PostConstruct
138 152 public void init() {
139 153 super.init("tb-core-consumer", "tb-core-notifications-consumer");
140   - this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
  154 + this.usageStatsExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
  155 + this.firmwareStatesExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-firmware-notifications-consumer"));
141 156 }
142 157
143 158 @PreDestroy
... ... @@ -146,6 +161,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
146 161 if (usageStatsExecutor != null) {
147 162 usageStatsExecutor.shutdownNow();
148 163 }
  164 + if (firmwareStatesExecutor != null) {
  165 + firmwareStatesExecutor.shutdownNow();
  166 + }
149 167 }
150 168
151 169 @EventListener(ApplicationReadyEvent.class)
... ... @@ -153,6 +171,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
153 171 public void onApplicationEvent(ApplicationReadyEvent event) {
154 172 super.onApplicationEvent(event);
155 173 launchUsageStatsConsumer();
  174 + launchFirmwareUpdateNotificationConsumer();
156 175 }
157 176
158 177 @Override
... ... @@ -167,6 +186,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
167 186 .map(tpi -> tpi.newByTopic(usageStatsConsumer.getTopic()))
168 187 .collect(Collectors.toSet()));
169 188 }
  189 + this.firmwareStatesConsumer.subscribe();
170 190 }
171 191
172 192 @Override
... ... @@ -336,10 +356,59 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
336 356 });
337 357 }
338 358
  359 + private void launchFirmwareUpdateNotificationConsumer() {
  360 + long maxProcessingTimeoutPerRecord = firmwarePackInterval / firmwarePackSize;
  361 + firmwareStatesExecutor.submit(() -> {
  362 + while (!stopped) {
  363 + try {
  364 + List<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> msgs = firmwareStatesConsumer.poll(getNotificationPollDuration());
  365 + if (msgs.isEmpty()) {
  366 + continue;
  367 + }
  368 + long timeToSleep = maxProcessingTimeoutPerRecord;
  369 + for (TbProtoQueueMsg<ToFirmwareStateServiceMsg> msg : msgs) {
  370 + try {
  371 + long startTime = System.currentTimeMillis();
  372 + boolean isSuccessUpdate = handleFirmwareUpdates(msg);
  373 + long endTime = System.currentTimeMillis();
  374 + long spentTime = endTime - startTime;
  375 + timeToSleep = timeToSleep - spentTime;
  376 + if (isSuccessUpdate) {
  377 + if (timeToSleep > 0) {
  378 + log.debug("Spent time per record is: [{}]!", spentTime);
  379 + Thread.sleep(timeToSleep);
  380 + timeToSleep = 0;
  381 + }
  382 + timeToSleep += maxProcessingTimeoutPerRecord;
  383 + }
  384 + } catch (Throwable e) {
  385 + log.warn("Failed to process firmware update msg: {}", msg, e);
  386 + }
  387 + }
  388 + firmwareStatesConsumer.commit();
  389 + } catch (Exception e) {
  390 + if (!stopped) {
  391 + log.warn("Failed to obtain usage stats from queue.", e);
  392 + try {
  393 + Thread.sleep(getNotificationPollDuration());
  394 + } catch (InterruptedException e2) {
  395 + log.trace("Failed to wait until the server has capacity to handle new firmware updates", e2);
  396 + }
  397 + }
  398 + }
  399 + }
  400 + log.info("TB Firmware States Consumer stopped.");
  401 + });
  402 + }
  403 +
339 404 private void handleUsageStats(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) {
340 405 statsService.process(msg, callback);
341 406 }
342 407
  408 + private boolean handleFirmwareUpdates(TbProtoQueueMsg<ToFirmwareStateServiceMsg> msg) {
  409 + return firmwareStateService.process(msg.getValue());
  410 + }
  411 +
343 412 private void forwardToCoreRpcService(FromDeviceRPCResponseProto proto, TbCallback callback) {
344 413 RpcError error = proto.getError() > 0 ? RpcError.values()[proto.getError()] : null;
345 414 FromDeviceRpcResponse response = new FromDeviceRpcResponse(new UUID(proto.getRequestIdMSB(), proto.getRequestIdLSB())
... ... @@ -448,6 +517,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
448 517 if (usageStatsConsumer != null) {
449 518 usageStatsConsumer.unsubscribe();
450 519 }
  520 + if (firmwareStatesConsumer != null) {
  521 + firmwareStatesConsumer.unsubscribe();
  522 + }
451 523 }
452 524
453 525 }
... ...
... ... @@ -18,14 +18,15 @@ package org.thingsboard.server.service.queue.processing;
18 18 import com.google.protobuf.ByteString;
19 19 import lombok.extern.slf4j.Slf4j;
20 20 import org.springframework.boot.context.event.ApplicationReadyEvent;
21   -import org.springframework.context.ApplicationListener;
22 21 import org.springframework.context.event.EventListener;
23 22 import org.springframework.core.annotation.Order;
24 23 import org.thingsboard.common.util.ThingsBoardThreadFactory;
25 24 import org.thingsboard.server.actors.ActorSystemContext;
26 25 import org.thingsboard.server.common.data.EntityType;
  26 +import org.thingsboard.server.common.data.id.CustomerId;
27 27 import org.thingsboard.server.common.data.id.DeviceId;
28 28 import org.thingsboard.server.common.data.id.DeviceProfileId;
  29 +import org.thingsboard.server.common.data.id.TenantId;
29 30 import org.thingsboard.server.common.data.id.TenantProfileId;
30 31 import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
31 32 import org.thingsboard.server.common.msg.TbActorMsg;
... ... @@ -72,7 +73,8 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
72 73 protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer;
73 74
74 75 public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService,
75   - TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbApiUsageStateService apiUsageStateService, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
  76 + TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache,
  77 + TbApiUsageStateService apiUsageStateService, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
76 78 this.actorContext = actorContext;
77 79 this.encodingService = encodingService;
78 80 this.tenantProfileCache = tenantProfileCache;
... ... @@ -166,6 +168,8 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
166 168 tenantProfileCache.evict(componentLifecycleMsg.getTenantId());
167 169 if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
168 170 apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId());
  171 + } else if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.DELETED)) {
  172 + apiUsageStateService.onTenantDelete((TenantId) componentLifecycleMsg.getEntityId());
169 173 }
170 174 } else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
171 175 deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId()));
... ... @@ -173,6 +177,10 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
173 177 deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId()));
174 178 } else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
175 179 apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId());
  180 + } else if (EntityType.CUSTOMER.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
  181 + if (componentLifecycleMsg.getEvent() == ComponentLifecycleEvent.DELETED) {
  182 + apiUsageStateService.onCustomerDelete((CustomerId) componentLifecycleMsg.getEntityId());
  183 + }
176 184 }
177 185 }
178 186 log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg);
... ...
... ... @@ -31,7 +31,7 @@ public interface TbResourceService {
31 31
32 32 TbResource saveResource(TbResource resource) throws ThingsboardException;
33 33
34   - TbResource getResource(TenantId tenantId, ResourceType resourceType, String resourceId);
  34 + TbResource getResource(TenantId tenantId, ResourceType resourceType, String resourceKey);
35 35
36 36 TbResource findResourceById(TenantId tenantId, TbResourceId resourceId);
37 37
... ...
... ... @@ -34,6 +34,7 @@ import org.thingsboard.server.dao.device.DeviceService;
34 34 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
35 35 import org.thingsboard.server.queue.util.TbCoreComponent;
36 36 import org.thingsboard.server.service.queue.TbClusterService;
  37 +import org.thingsboard.server.service.security.model.SecurityUser;
37 38
38 39 import javax.annotation.PostConstruct;
39 40 import javax.annotation.PreDestroy;
... ... @@ -95,11 +96,11 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
95 96 }
96 97
97 98 @Override
98   - public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer) {
  99 + public void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer, SecurityUser currentUser) {
99 100 log.trace("[{}][{}] Processing REST API call to rule engine [{}]", request.getTenantId(), request.getId(), request.getDeviceId());
100 101 UUID requestId = request.getId();
101 102 localToRuleEngineRpcRequests.put(requestId, responseConsumer);
102   - sendRpcRequestToRuleEngine(request);
  103 + sendRpcRequestToRuleEngine(request, currentUser);
103 104 scheduleToRuleEngineTimeout(request, requestId);
104 105 }
105 106
... ... @@ -149,7 +150,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
149 150 }
150 151 }
151 152
152   - private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg) {
  153 + private void sendRpcRequestToRuleEngine(ToDeviceRpcRequest msg, SecurityUser currentUser) {
153 154 ObjectNode entityNode = json.createObjectNode();
154 155 TbMsgMetaData metaData = new TbMsgMetaData();
155 156 metaData.putValue("requestUUID", msg.getId().toString());
... ... @@ -167,7 +168,7 @@ public class DefaultTbCoreDeviceRpcService implements TbCoreDeviceRpcService {
167 168 entityNode.put("params", msg.getBody().getParams());
168 169
169 170 try {
170   - TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode));
  171 + TbMsg tbMsg = TbMsg.newMsg(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE, msg.getDeviceId(), currentUser.getCustomerId(), metaData, TbMsgDataType.JSON, json.writeValueAsString(entityNode));
171 172 clusterService.pushMsgToRuleEngine(msg.getTenantId(), msg.getDeviceId(), tbMsg, null);
172 173 } catch (JsonProcessingException e) {
173 174 throw new RuntimeException(e);
... ...
... ... @@ -16,6 +16,7 @@
16 16 package org.thingsboard.server.service.rpc;
17 17
18 18 import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
  19 +import org.thingsboard.server.service.security.model.SecurityUser;
19 20
20 21 import java.util.function.Consumer;
21 22
... ... @@ -27,11 +28,11 @@ public interface TbCoreDeviceRpcService {
27 28 /**
28 29 * Handles REST API calls that contain RPC requests to Device and pushes them to Rule Engine.
29 30 * Schedules the timeout for the RPC call based on the {@link ToDeviceRpcRequest}
30   - *
31   - * @param request the RPC request
  31 + * @param request the RPC request
32 32 * @param responseConsumer the consumer of the RPC response
  33 + * @param currentUser
33 34 */
34   - void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer);
  35 + void processRestApiRpcRequest(ToDeviceRpcRequest request, Consumer<FromDeviceRpcResponse> responseConsumer, SecurityUser currentUser);
35 36
36 37 /**
37 38 * Handles the RPC response from the Rule Engine.
... ...
... ... @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture;
20 20 import lombok.extern.slf4j.Slf4j;
21 21 import org.thingsboard.common.util.ThingsBoardThreadFactory;
22 22 import org.thingsboard.server.common.data.ApiUsageRecordKey;
  23 +import org.thingsboard.server.common.data.id.CustomerId;
23 24 import org.thingsboard.server.common.data.id.TenantId;
24 25 import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
25 26 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
... ... @@ -73,14 +74,14 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
73 74 }
74 75
75 76 @Override
76   - public ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args) {
  77 + public ListenableFuture<Object> invokeFunction(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args) {
77 78 if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
78 79 String functionName = scriptIdToNameMap.get(scriptId);
79 80 if (functionName == null) {
80 81 return Futures.immediateFailedFuture(new RuntimeException("No compiled script found for scriptId: [" + scriptId + "]!"));
81 82 }
82 83 if (!isDisabled(scriptId)) {
83   - apiUsageClient.report(tenantId, ApiUsageRecordKey.JS_EXEC_COUNT, 1);
  84 + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.JS_EXEC_COUNT, 1);
84 85 return doInvokeFunction(scriptId, functionName, args);
85 86 } else {
86 87 return Futures.immediateFailedFuture(
... ...
... ... @@ -16,7 +16,7 @@
16 16 package org.thingsboard.server.service.script;
17 17
18 18 import com.google.common.util.concurrent.ListenableFuture;
19   -import org.thingsboard.server.common.data.id.EntityId;
  19 +import org.thingsboard.server.common.data.id.CustomerId;
20 20 import org.thingsboard.server.common.data.id.TenantId;
21 21
22 22 import java.util.UUID;
... ... @@ -25,7 +25,7 @@ public interface JsInvokeService {
25 25
26 26 ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames);
27 27
28   - ListenableFuture<Object> invokeFunction(TenantId tenantId, UUID scriptId, Object... args);
  28 + ListenableFuture<Object> invokeFunction(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args);
29 29
30 30 ListenableFuture<Void> release(UUID scriptId);
31 31
... ...
... ... @@ -218,7 +218,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
218 218 private JsonNode executeScript(TbMsg msg) throws ScriptException {
219 219 try {
220 220 String[] inArgs = prepareArgs(msg);
221   - String eval = sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString();
  221 + String eval = sandboxService.invokeFunction(tenantId, msg.getCustomerId(), this.scriptId, inArgs[0], inArgs[1], inArgs[2]).get().toString();
222 222 return mapper.readTree(eval);
223 223 } catch (ExecutionException e) {
224 224 if (e.getCause() instanceof ScriptException) {
... ... @@ -235,7 +235,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
235 235
236 236 private ListenableFuture<JsonNode> executeScriptAsync(TbMsg msg) {
237 237 String[] inArgs = prepareArgs(msg);
238   - return Futures.transformAsync(sandboxService.invokeFunction(tenantId, this.scriptId, inArgs[0], inArgs[1], inArgs[2]),
  238 + return Futures.transformAsync(sandboxService.invokeFunction(tenantId, msg.getCustomerId(), this.scriptId, inArgs[0], inArgs[1], inArgs[2]),
239 239 o -> {
240 240 try {
241 241 return Futures.immediateFuture(mapper.readTree(o.toString()));
... ...
... ... @@ -38,6 +38,7 @@ public enum Resource {
38 38 DEVICE_PROFILE(EntityType.DEVICE_PROFILE),
39 39 API_USAGE_STATE(EntityType.API_USAGE_STATE),
40 40 TB_RESOURCE(EntityType.TB_RESOURCE),
  41 + FIRMWARE(EntityType.FIRMWARE),
41 42 EDGE(EntityType.EDGE);
42 43
43 44 private final EntityType entityType;
... ...
... ... @@ -42,6 +42,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
42 42 put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker);
43 43 put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker);
44 44 put(Resource.TB_RESOURCE, tbResourcePermissionChecker);
  45 + put(Resource.FIRMWARE, tenantEntityPermissionChecker);
45 46 put(Resource.EDGE, tenantEntityPermissionChecker);
46 47 }
47 48
... ...
... ... @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service;
22 22 import org.thingsboard.rule.engine.api.SmsService;
23 23 import org.thingsboard.rule.engine.api.sms.SmsSender;
24 24 import org.thingsboard.rule.engine.api.sms.SmsSenderFactory;
  25 +import org.thingsboard.server.common.data.id.CustomerId;
25 26 import org.thingsboard.server.common.data.sms.config.SmsProviderConfiguration;
26 27 import org.thingsboard.server.common.data.sms.config.TestSmsRequest;
27 28 import org.thingsboard.server.common.data.AdminSettings;
... ... @@ -94,7 +95,7 @@ public class DefaultSmsService implements SmsService {
94 95 }
95 96
96 97 @Override
97   - public void sendSms(TenantId tenantId, String[] numbersTo, String message) throws ThingsboardException {
  98 + public void sendSms(TenantId tenantId, CustomerId customerId, String[] numbersTo, String message) throws ThingsboardException {
98 99 if (apiUsageStateService.getApiUsageState(tenantId).isSmsSendEnabled()) {
99 100 int smsCount = 0;
100 101 try {
... ... @@ -103,7 +104,7 @@ public class DefaultSmsService implements SmsService {
103 104 }
104 105 } finally {
105 106 if (smsCount > 0) {
106   - apiUsageClient.report(tenantId, ApiUsageRecordKey.SMS_EXEC_COUNT, smsCount);
  107 + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.SMS_EXEC_COUNT, smsCount);
107 108 }
108 109 }
109 110 } else {
... ...
... ... @@ -468,6 +468,7 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
468 468 md.putValue("deviceName", device.getName());
469 469 md.putValue("deviceType", device.getType());
470 470 return DeviceStateData.builder()
  471 + .customerId(device.getCustomerId())
471 472 .tenantId(device.getTenantId())
472 473 .deviceId(device.getId())
473 474 .deviceCreationTime(device.getCreatedTime())
... ... @@ -507,7 +508,7 @@ public class DefaultDeviceStateService extends TbApplicationEventListener<Partit
507 508 if(!persistToTelemetry){
508 509 md.putValue(DataConstants.SCOPE, SERVER_SCOPE);
509 510 }
510   - TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), md, TbMsgDataType.JSON, data);
  511 + TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getCustomerId(), md, TbMsgDataType.JSON, data);
511 512 clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null);
512 513 } catch (Exception e) {
513 514 log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e);
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.service.state;
17 17
18 18 import lombok.Builder;
19 19 import lombok.Data;
  20 +import org.thingsboard.server.common.data.id.CustomerId;
20 21 import org.thingsboard.server.common.data.id.DeviceId;
21 22 import org.thingsboard.server.common.data.id.TenantId;
22 23 import org.thingsboard.server.common.msg.TbMsgMetaData;
... ... @@ -29,6 +30,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
29 30 class DeviceStateData {
30 31
31 32 private final TenantId tenantId;
  33 + private final CustomerId customerId;
32 34 private final DeviceId deviceId;
33 35 private final long deviceCreationTime;
34 36 private TbMsgMetaData metaData;
... ...
... ... @@ -25,7 +25,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
25 25 import org.thingsboard.server.common.data.ApiUsageRecordKey;
26 26 import org.thingsboard.server.common.data.EntityType;
27 27 import org.thingsboard.server.common.data.EntityView;
28   -import org.thingsboard.server.common.data.TenantProfile;
  28 +import org.thingsboard.server.common.data.id.CustomerId;
29 29 import org.thingsboard.server.common.data.id.EntityId;
30 30 import org.thingsboard.server.common.data.id.TenantId;
31 31 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
... ... @@ -35,13 +35,11 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry;
35 35 import org.thingsboard.server.common.data.kv.LongDataEntry;
36 36 import org.thingsboard.server.common.data.kv.StringDataEntry;
37 37 import org.thingsboard.server.common.data.kv.TsKvEntry;
38   -import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
39 38 import org.thingsboard.server.common.msg.queue.ServiceType;
40 39 import org.thingsboard.server.common.msg.queue.TbCallback;
41 40 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
42 41 import org.thingsboard.server.dao.attributes.AttributesService;
43 42 import org.thingsboard.server.dao.entityview.EntityViewService;
44   -import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
45 43 import org.thingsboard.server.dao.timeseries.TimeseriesService;
46 44 import org.thingsboard.server.gen.transport.TransportProtos;
47 45 import org.thingsboard.server.queue.discovery.PartitionService;
... ... @@ -115,11 +113,11 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
115 113
116 114 @Override
117 115 public void saveAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) {
118   - saveAndNotify(tenantId, entityId, ts, 0L, callback);
  116 + saveAndNotify(tenantId, null, entityId, ts, 0L, callback);
119 117 }
120 118
121 119 @Override
122   - public void saveAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Void> callback) {
  120 + public void saveAndNotify(TenantId tenantId, CustomerId customerId, EntityId entityId, List<TsKvEntry> ts, long ttl, FutureCallback<Void> callback) {
123 121 checkInternalEntity(entityId);
124 122 boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null;
125 123 if (sysTenant || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) {
... ... @@ -127,7 +125,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
127 125 @Override
128 126 public void onSuccess(Integer result) {
129 127 if (!sysTenant && result != null && result > 0) {
130   - apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result);
  128 + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result);
131 129 }
132 130 callback.onSuccess(null);
133 131 }
... ...
... ... @@ -23,17 +23,20 @@ import com.google.common.util.concurrent.ListenableFuture;
23 23 import com.google.common.util.concurrent.MoreExecutors;
24 24 import com.google.protobuf.ByteString;
25 25 import lombok.extern.slf4j.Slf4j;
  26 +import org.springframework.cache.CacheManager;
26 27 import org.springframework.stereotype.Service;
27 28 import org.springframework.util.StringUtils;
28 29 import org.thingsboard.common.util.JacksonUtil;
  30 +import org.thingsboard.server.cache.firmware.FirmwareCacheWriter;
29 31 import org.thingsboard.server.common.data.ApiUsageState;
30 32 import org.thingsboard.server.common.data.DataConstants;
31 33 import org.thingsboard.server.common.data.Device;
32 34 import org.thingsboard.server.common.data.DeviceProfile;
33 35 import org.thingsboard.server.common.data.DeviceTransportType;
34 36 import org.thingsboard.server.common.data.EntityType;
35   -import org.thingsboard.server.common.data.TbResource;
  37 +import org.thingsboard.server.common.data.Firmware;
36 38 import org.thingsboard.server.common.data.ResourceType;
  39 +import org.thingsboard.server.common.data.TbResource;
37 40 import org.thingsboard.server.common.data.TenantProfile;
38 41 import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials;
39 42 import org.thingsboard.server.common.data.device.credentials.ProvisionDeviceCredentialsData;
... ... @@ -41,6 +44,7 @@ import org.thingsboard.server.common.data.device.profile.ProvisionDeviceProfileC
41 44 import org.thingsboard.server.common.data.id.CustomerId;
42 45 import org.thingsboard.server.common.data.id.DeviceId;
43 46 import org.thingsboard.server.common.data.id.DeviceProfileId;
  47 +import org.thingsboard.server.common.data.id.FirmwareId;
44 48 import org.thingsboard.server.common.data.id.TenantId;
45 49 import org.thingsboard.server.common.data.relation.EntityRelation;
46 50 import org.thingsboard.server.common.data.security.DeviceCredentials;
... ... @@ -56,6 +60,7 @@ import org.thingsboard.server.dao.device.DeviceService;
56 60 import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
57 61 import org.thingsboard.server.dao.device.provision.ProvisionRequest;
58 62 import org.thingsboard.server.dao.device.provision.ProvisionResponse;
  63 +import org.thingsboard.server.dao.firmware.FirmwareService;
59 64 import org.thingsboard.server.dao.relation.RelationService;
60 65 import org.thingsboard.server.dao.resource.ResourceService;
61 66 import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
... ... @@ -71,7 +76,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesReques
71 76 import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesResponseMsg;
72 77 import org.thingsboard.server.gen.transport.TransportProtos.GetResourceRequestMsg;
73 78 import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg;
74   -import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus;
75 79 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
76 80 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
77 81 import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg;
... ... @@ -84,9 +88,9 @@ import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
84 88 import org.thingsboard.server.service.executors.DbCallbackExecutorService;
85 89 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
86 90 import org.thingsboard.server.service.queue.TbClusterService;
  91 +import org.thingsboard.server.service.resource.TbResourceService;
87 92 import org.thingsboard.server.service.state.DeviceStateService;
88 93
89   -import java.util.Collections;
90 94 import java.util.List;
91 95 import java.util.Optional;
92 96 import java.util.UUID;
... ... @@ -117,7 +121,9 @@ public class DefaultTransportApiService implements TransportApiService {
117 121 private final TbClusterService tbClusterService;
118 122 private final DataDecodingEncodingService dataDecodingEncodingService;
119 123 private final DeviceProvisionService deviceProvisionService;
120   - private final ResourceService resourceService;
  124 + private final TbResourceService resourceService;
  125 + private final FirmwareService firmwareService;
  126 + private final FirmwareCacheWriter firmwareCacheWriter;
121 127
122 128 private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>();
123 129
... ... @@ -126,7 +132,7 @@ public class DefaultTransportApiService implements TransportApiService {
126 132 RelationService relationService, DeviceCredentialsService deviceCredentialsService,
127 133 DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService,
128 134 TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService,
129   - DeviceProvisionService deviceProvisionService, ResourceService resourceService) {
  135 + DeviceProvisionService deviceProvisionService, TbResourceService resourceService, FirmwareService firmwareService, FirmwareCacheWriter firmwareCacheWriter) {
130 136 this.deviceProfileCache = deviceProfileCache;
131 137 this.tenantProfileCache = tenantProfileCache;
132 138 this.apiUsageStateService = apiUsageStateService;
... ... @@ -139,6 +145,8 @@ public class DefaultTransportApiService implements TransportApiService {
139 145 this.dataDecodingEncodingService = dataDecodingEncodingService;
140 146 this.deviceProvisionService = deviceProvisionService;
141 147 this.resourceService = resourceService;
  148 + this.firmwareService = firmwareService;
  149 + this.firmwareCacheWriter = firmwareCacheWriter;
142 150 }
143 151
144 152 @Override
... ... @@ -174,6 +182,8 @@ public class DefaultTransportApiService implements TransportApiService {
174 182 result = handle(transportApiRequestMsg.getDeviceRequestMsg());
175 183 } else if (transportApiRequestMsg.hasDeviceCredentialsRequestMsg()) {
176 184 result = handle(transportApiRequestMsg.getDeviceCredentialsRequestMsg());
  185 + } else if (transportApiRequestMsg.hasFirmwareRequestMsg()) {
  186 + result = handle(transportApiRequestMsg.getFirmwareRequestMsg());
177 187 }
178 188
179 189 return Futures.transform(Optional.ofNullable(result).orElseGet(this::getEmptyTransportApiResponseFuture),
... ... @@ -282,7 +292,7 @@ public class DefaultTransportApiService implements TransportApiService {
282 292
283 293 DeviceId deviceId = device.getId();
284 294 ObjectNode entityNode = mapper.valueToTree(device);
285   - TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, metaData, TbMsgDataType.JSON, mapper.writeValueAsString(entityNode));
  295 + TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, customerId, metaData, TbMsgDataType.JSON, mapper.writeValueAsString(entityNode));
286 296 tbClusterService.pushMsgToRuleEngine(tenantId, deviceId, tbMsg, null);
287 297 }
288 298 GetOrCreateDeviceFromGatewayResponseMsg.Builder builder = GetOrCreateDeviceFromGatewayResponseMsg.newBuilder()
... ... @@ -323,14 +333,14 @@ public class DefaultTransportApiService implements TransportApiService {
323 333 } catch (ProvisionFailedException e) {
324 334 return Futures.immediateFuture(getTransportApiResponseMsg(
325 335 new DeviceCredentials(),
326   - TransportProtos.ProvisionResponseStatus.valueOf(e.getMessage())));
  336 + TransportProtos.ResponseStatus.valueOf(e.getMessage())));
327 337 }
328   - return Futures.transform(provisionResponseFuture, provisionResponse -> getTransportApiResponseMsg(provisionResponse.getDeviceCredentials(), TransportProtos.ProvisionResponseStatus.SUCCESS),
  338 + return Futures.transform(provisionResponseFuture, provisionResponse -> getTransportApiResponseMsg(provisionResponse.getDeviceCredentials(), TransportProtos.ResponseStatus.SUCCESS),
329 339 dbCallbackExecutorService);
330 340 }
331 341
332   - private TransportApiResponseMsg getTransportApiResponseMsg(DeviceCredentials deviceCredentials, TransportProtos.ProvisionResponseStatus status) {
333   - if (!status.equals(ProvisionResponseStatus.SUCCESS)) {
  342 + private TransportApiResponseMsg getTransportApiResponseMsg(DeviceCredentials deviceCredentials, TransportProtos.ResponseStatus status) {
  343 + if (!status.equals(TransportProtos.ResponseStatus.SUCCESS)) {
334 344 return TransportApiResponseMsg.newBuilder().setProvisionDeviceResponseMsg(TransportProtos.ProvisionDeviceResponseMsg.newBuilder().setStatus(status).build()).build();
335 345 }
336 346 TransportProtos.ProvisionDeviceResponseMsg.Builder provisionResponse = TransportProtos.ProvisionDeviceResponseMsg.newBuilder()
... ... @@ -468,6 +478,8 @@ public class DefaultTransportApiService implements TransportApiService {
468 478 return DeviceInfoProto.newBuilder()
469 479 .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits())
470 480 .setTenantIdLSB(device.getTenantId().getId().getLeastSignificantBits())
  481 + .setCustomerIdMSB(Optional.ofNullable(device.getCustomerId()).map(customerId -> customerId.getId().getMostSignificantBits()).orElse(0L))
  482 + .setCustomerIdLSB(Optional.ofNullable(device.getCustomerId()).map(customerId -> customerId.getId().getLeastSignificantBits()).orElse(0L))
471 483 .setDeviceIdMSB(device.getId().getId().getMostSignificantBits())
472 484 .setDeviceIdLSB(device.getId().getId().getLeastSignificantBits())
473 485 .setDeviceName(device.getName())
... ... @@ -495,6 +507,48 @@ public class DefaultTransportApiService implements TransportApiService {
495 507 }
496 508 }
497 509
  510 + private ListenableFuture<TransportApiResponseMsg> handle(TransportProtos.GetFirmwareRequestMsg requestMsg) {
  511 + TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB()));
  512 + DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB()));
  513 + Device device = deviceService.findDeviceById(tenantId, deviceId);
  514 +
  515 + if (device == null) {
  516 + return getEmptyTransportApiResponseFuture();
  517 + }
  518 +
  519 + FirmwareId firmwareId = device.getFirmwareId();
  520 +
  521 + if (firmwareId == null) {
  522 + firmwareId = deviceProfileCache.find(device.getDeviceProfileId()).getFirmwareId();
  523 + }
  524 +
  525 + TransportProtos.GetFirmwareResponseMsg.Builder builder = TransportProtos.GetFirmwareResponseMsg.newBuilder();
  526 +
  527 + if (firmwareId == null) {
  528 + builder.setResponseStatus(TransportProtos.ResponseStatus.NOT_FOUND);
  529 + } else {
  530 + Firmware firmware = firmwareService.findFirmwareById(tenantId, firmwareId);
  531 +
  532 + if (firmware == null) {
  533 + builder.setResponseStatus(TransportProtos.ResponseStatus.NOT_FOUND);
  534 + } else {
  535 + builder.setResponseStatus(TransportProtos.ResponseStatus.SUCCESS);
  536 + builder.setFirmwareIdMSB(firmwareId.getId().getMostSignificantBits());
  537 + builder.setFirmwareIdLSB(firmwareId.getId().getLeastSignificantBits());
  538 + builder.setTitle(firmware.getTitle());
  539 + builder.setVersion(firmware.getVersion());
  540 + builder.setFileName(firmware.getFileName());
  541 + builder.setContentType(firmware.getContentType());
  542 + firmwareCacheWriter.put(firmwareId.toString(), firmware.getData().array());
  543 + }
  544 + }
  545 +
  546 + return Futures.immediateFuture(
  547 + TransportApiResponseMsg.newBuilder()
  548 + .setFirmwareResponseMsg(builder.build())
  549 + .build());
  550 + }
  551 +
498 552 private ListenableFuture<TransportApiResponseMsg> handleRegistration(TransportProtos.LwM2MRegistrationRequestMsg msg) {
499 553 TenantId tenantId = new TenantId(UUID.fromString(msg.getTenantId()));
500 554 String deviceName = msg.getEndpoint();
... ...
... ... @@ -124,6 +124,7 @@ usage:
124 124 stats:
125 125 report:
126 126 enabled: "${USAGE_STATS_REPORT_ENABLED:true}"
  127 + enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
127 128 interval: "${USAGE_STATS_REPORT_INTERVAL:10}"
128 129 check:
129 130 cycle: "${USAGE_STATS_CHECK_CYCLE:60000}"
... ... @@ -370,6 +371,9 @@ caffeine:
370 371 tokensOutdatageTime:
371 372 timeToLiveInMinutes: 20000
372 373 maxSize: 10000
  374 + firmwares:
  375 + timeToLiveInMinutes: 1440
  376 + maxSize: 100
373 377 edges:
374 378 timeToLiveInMinutes: 1440
375 379 maxSize: 0
... ... @@ -445,6 +449,9 @@ spring.resources.chain:
445 449 content:
446 450 enabled: "true"
447 451
  452 +spring.servlet.multipart.max-file-size: "50MB"
  453 +spring.servlet.multipart.max-request-size: "50MB"
  454 +
448 455 spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true"
449 456 spring.jpa.properties.hibernate.order_by.default_null_ordering: "last"
450 457
... ... @@ -730,6 +737,7 @@ queue:
730 737 bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}"
731 738 acks: "${TB_KAFKA_ACKS:all}"
732 739 retries: "${TB_KAFKA_RETRIES:1}"
  740 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip
733 741 batch.size: "${TB_KAFKA_BATCH_SIZE:16384}"
734 742 linger.ms: "${TB_KAFKA_LINGER_MS:1}"
735 743 buffer.memory: "${TB_BUFFER_MEMORY:33554432}"
... ... @@ -744,6 +752,10 @@ queue:
744 752 sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}"
745 753 sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}"
746 754 security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}"
  755 + consumer-properties-per-topic:
  756 + tb_firmware:
  757 + - key: max.poll.records
  758 + value: 10
747 759 other:
748 760 topic-properties:
749 761 rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
... ... @@ -751,6 +763,7 @@ queue:
751 763 transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
752 764 notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
753 765 js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}"
  766 + fw-updates: "${TB_QUEUE_KAFKA_FW_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}"
754 767 consumer-stats:
755 768 enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
756 769 print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}"
... ... @@ -821,6 +834,10 @@ queue:
821 834 poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
822 835 partitions: "${TB_QUEUE_CORE_PARTITIONS:10}"
823 836 pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}"
  837 + firmware:
  838 + topic: "${TB_QUEUE_CORE_FW_TOPIC:tb_firmware}"
  839 + pack-interval-ms: "${TB_QUEUE_CORE_FW_PACK_INTERVAL_MS:60000}"
  840 + pack-size: "${TB_QUEUE_CORE_FW_PACK_SIZE:100}"
824 841 usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}"
825 842 stats:
826 843 enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.controller;
  17 +
  18 +import com.fasterxml.jackson.core.type.TypeReference;
  19 +import org.junit.After;
  20 +import org.junit.Assert;
  21 +import org.junit.Before;
  22 +import org.junit.Test;
  23 +import org.springframework.mock.web.MockMultipartFile;
  24 +import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder;
  25 +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
  26 +import org.thingsboard.common.util.JacksonUtil;
  27 +import org.thingsboard.server.common.data.Firmware;
  28 +import org.thingsboard.server.common.data.FirmwareInfo;
  29 +import org.thingsboard.server.common.data.Tenant;
  30 +import org.thingsboard.server.common.data.User;
  31 +import org.thingsboard.server.common.data.page.PageData;
  32 +import org.thingsboard.server.common.data.page.PageLink;
  33 +import org.thingsboard.server.common.data.security.Authority;
  34 +
  35 +import java.nio.ByteBuffer;
  36 +import java.util.ArrayList;
  37 +import java.util.Collections;
  38 +import java.util.List;
  39 +
  40 +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
  41 +
  42 +public abstract class BaseFirmwareControllerTest extends AbstractControllerTest {
  43 +
  44 + private IdComparator<FirmwareInfo> idComparator = new IdComparator<>();
  45 +
  46 + public static final String TITLE = "My firmware";
  47 + private static final String FILE_NAME = "filename.txt";
  48 + private static final String VERSION = "v1.0";
  49 + private static final String CONTENT_TYPE = "text/plain";
  50 + private static final String CHECKSUM_ALGORITHM = "sha256";
  51 + private static final String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a";
  52 + private static final ByteBuffer DATA = ByteBuffer.wrap(new byte[]{1});
  53 +
  54 + private Tenant savedTenant;
  55 + private User tenantAdmin;
  56 +
  57 + @Before
  58 + public void beforeTest() throws Exception {
  59 + loginSysAdmin();
  60 +
  61 + Tenant tenant = new Tenant();
  62 + tenant.setTitle("My tenant");
  63 + savedTenant = doPost("/api/tenant", tenant, Tenant.class);
  64 + Assert.assertNotNull(savedTenant);
  65 +
  66 + tenantAdmin = new User();
  67 + tenantAdmin.setAuthority(Authority.TENANT_ADMIN);
  68 + tenantAdmin.setTenantId(savedTenant.getId());
  69 + tenantAdmin.setEmail("tenant2@thingsboard.org");
  70 + tenantAdmin.setFirstName("Joe");
  71 + tenantAdmin.setLastName("Downs");
  72 +
  73 + tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1");
  74 + }
  75 +
  76 + @After
  77 + public void afterTest() throws Exception {
  78 + loginSysAdmin();
  79 +
  80 + doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
  81 + .andExpect(status().isOk());
  82 + }
  83 +
  84 + @Test
  85 + public void testSaveFirmware() throws Exception {
  86 + FirmwareInfo firmwareInfo = new FirmwareInfo();
  87 + firmwareInfo.setTitle(TITLE);
  88 + firmwareInfo.setVersion(VERSION);
  89 +
  90 + FirmwareInfo savedFirmwareInfo = save(firmwareInfo);
  91 +
  92 + Assert.assertNotNull(savedFirmwareInfo);
  93 + Assert.assertNotNull(savedFirmwareInfo.getId());
  94 + Assert.assertTrue(savedFirmwareInfo.getCreatedTime() > 0);
  95 + Assert.assertEquals(savedTenant.getId(), savedFirmwareInfo.getTenantId());
  96 + Assert.assertEquals(firmwareInfo.getTitle(), savedFirmwareInfo.getTitle());
  97 + Assert.assertEquals(firmwareInfo.getVersion(), savedFirmwareInfo.getVersion());
  98 +
  99 + savedFirmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode());
  100 +
  101 + save(savedFirmwareInfo);
  102 +
  103 + FirmwareInfo foundFirmwareInfo = doGet("/api/firmware/info/" + savedFirmwareInfo.getId().getId().toString(), FirmwareInfo.class);
  104 + Assert.assertEquals(foundFirmwareInfo.getTitle(), savedFirmwareInfo.getTitle());
  105 + }
  106 +
  107 + @Test
  108 + public void testSaveFirmwareData() throws Exception {
  109 + FirmwareInfo firmwareInfo = new FirmwareInfo();
  110 + firmwareInfo.setTitle(TITLE);
  111 + firmwareInfo.setVersion(VERSION);
  112 +
  113 + FirmwareInfo savedFirmwareInfo = save(firmwareInfo);
  114 +
  115 + Assert.assertNotNull(savedFirmwareInfo);
  116 + Assert.assertNotNull(savedFirmwareInfo.getId());
  117 + Assert.assertTrue(savedFirmwareInfo.getCreatedTime() > 0);
  118 + Assert.assertEquals(savedTenant.getId(), savedFirmwareInfo.getTenantId());
  119 + Assert.assertEquals(firmwareInfo.getTitle(), savedFirmwareInfo.getTitle());
  120 + Assert.assertEquals(firmwareInfo.getVersion(), savedFirmwareInfo.getVersion());
  121 +
  122 + savedFirmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode());
  123 +
  124 + save(savedFirmwareInfo);
  125 +
  126 + FirmwareInfo foundFirmwareInfo = doGet("/api/firmware/info/" + savedFirmwareInfo.getId().getId().toString(), FirmwareInfo.class);
  127 + Assert.assertEquals(foundFirmwareInfo.getTitle(), savedFirmwareInfo.getTitle());
  128 +
  129 + MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array());
  130 +
  131 + Firmware savedFirmware = savaData("/api/firmware/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM);
  132 +
  133 + Assert.assertEquals(FILE_NAME, savedFirmware.getFileName());
  134 + Assert.assertEquals(CONTENT_TYPE, savedFirmware.getContentType());
  135 + }
  136 +
  137 + @Test
  138 + public void testUpdateFirmwareFromDifferentTenant() throws Exception {
  139 + FirmwareInfo firmwareInfo = new FirmwareInfo();
  140 + firmwareInfo.setTitle(TITLE);
  141 + firmwareInfo.setVersion(VERSION);
  142 +
  143 + FirmwareInfo savedFirmwareInfo = save(firmwareInfo);
  144 +
  145 + loginDifferentTenant();
  146 + doPost("/api/firmware", savedFirmwareInfo, FirmwareInfo.class, status().isForbidden());
  147 + deleteDifferentTenant();
  148 + }
  149 +
  150 + @Test
  151 + public void testFindFirmwareInfoById() throws Exception {
  152 + FirmwareInfo firmwareInfo = new FirmwareInfo();
  153 + firmwareInfo.setTitle(TITLE);
  154 + firmwareInfo.setVersion(VERSION);
  155 +
  156 + FirmwareInfo savedFirmwareInfo = save(firmwareInfo);
  157 +
  158 + FirmwareInfo foundFirmware = doGet("/api/firmware/info/" + savedFirmwareInfo.getId().getId().toString(), FirmwareInfo.class);
  159 + Assert.assertNotNull(foundFirmware);
  160 + Assert.assertEquals(savedFirmwareInfo, foundFirmware);
  161 + }
  162 +
  163 + @Test
  164 + public void testFindFirmwareById() throws Exception {
  165 + FirmwareInfo firmwareInfo = new FirmwareInfo();
  166 + firmwareInfo.setTitle(TITLE);
  167 + firmwareInfo.setVersion(VERSION);
  168 +
  169 + FirmwareInfo savedFirmwareInfo = save(firmwareInfo);
  170 +
  171 + MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array());
  172 +
  173 + Firmware savedFirmware = savaData("/api/firmware/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM);
  174 +
  175 + Firmware foundFirmware = doGet("/api/firmware/" + savedFirmwareInfo.getId().getId().toString(), Firmware.class);
  176 + Assert.assertNotNull(foundFirmware);
  177 + Assert.assertEquals(savedFirmware, foundFirmware);
  178 + }
  179 +
  180 + @Test
  181 + public void testDeleteFirmware() throws Exception {
  182 + FirmwareInfo firmwareInfo = new FirmwareInfo();
  183 + firmwareInfo.setTitle(TITLE);
  184 + firmwareInfo.setVersion(VERSION);
  185 +
  186 + FirmwareInfo savedFirmwareInfo = save(firmwareInfo);
  187 +
  188 + doDelete("/api/firmware/" + savedFirmwareInfo.getId().getId().toString())
  189 + .andExpect(status().isOk());
  190 +
  191 + doGet("/api/firmware/info/" + savedFirmwareInfo.getId().getId().toString())
  192 + .andExpect(status().isNotFound());
  193 + }
  194 +
  195 + @Test
  196 + public void testFindTenantFirmwares() throws Exception {
  197 + List<FirmwareInfo> firmwares = new ArrayList<>();
  198 + for (int i = 0; i < 165; i++) {
  199 + FirmwareInfo firmwareInfo = new FirmwareInfo();
  200 + firmwareInfo.setTitle(TITLE);
  201 + firmwareInfo.setVersion(VERSION + i);
  202 +
  203 + FirmwareInfo savedFirmwareInfo = save(firmwareInfo);
  204 +
  205 + if (i > 100) {
  206 + MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array());
  207 +
  208 + Firmware savedFirmware = savaData("/api/firmware/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM);
  209 + firmwares.add(new FirmwareInfo(savedFirmware));
  210 + } else {
  211 + firmwares.add(savedFirmwareInfo);
  212 + }
  213 + }
  214 +
  215 + List<FirmwareInfo> loadedFirmwares = new ArrayList<>();
  216 + PageLink pageLink = new PageLink(24);
  217 + PageData<FirmwareInfo> pageData;
  218 + do {
  219 + pageData = doGetTypedWithPageLink("/api/firmwares?",
  220 + new TypeReference<>() {
  221 + }, pageLink);
  222 + loadedFirmwares.addAll(pageData.getData());
  223 + if (pageData.hasNext()) {
  224 + pageLink = pageLink.nextPageLink();
  225 + }
  226 + } while (pageData.hasNext());
  227 +
  228 + Collections.sort(firmwares, idComparator);
  229 + Collections.sort(loadedFirmwares, idComparator);
  230 +
  231 + Assert.assertEquals(firmwares, loadedFirmwares);
  232 + }
  233 +
  234 + @Test
  235 + public void testFindTenantFirmwaresByHasData() throws Exception {
  236 + List<FirmwareInfo> firmwaresWithData = new ArrayList<>();
  237 + List<FirmwareInfo> firmwaresWithoutData = new ArrayList<>();
  238 +
  239 + for (int i = 0; i < 165; i++) {
  240 + FirmwareInfo firmwareInfo = new FirmwareInfo();
  241 + firmwareInfo.setTitle(TITLE);
  242 + firmwareInfo.setVersion(VERSION + i);
  243 +
  244 + FirmwareInfo savedFirmwareInfo = save(firmwareInfo);
  245 +
  246 + if (i > 100) {
  247 + MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array());
  248 +
  249 + Firmware savedFirmware = savaData("/api/firmware/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM);
  250 + firmwaresWithData.add(new FirmwareInfo(savedFirmware));
  251 + } else {
  252 + firmwaresWithoutData.add(savedFirmwareInfo);
  253 + }
  254 + }
  255 +
  256 + List<FirmwareInfo> loadedFirmwaresWithData = new ArrayList<>();
  257 + PageLink pageLink = new PageLink(24);
  258 + PageData<FirmwareInfo> pageData;
  259 + do {
  260 + pageData = doGetTypedWithPageLink("/api/firmwares/true?",
  261 + new TypeReference<>() {
  262 + }, pageLink);
  263 + loadedFirmwaresWithData.addAll(pageData.getData());
  264 + if (pageData.hasNext()) {
  265 + pageLink = pageLink.nextPageLink();
  266 + }
  267 + } while (pageData.hasNext());
  268 +
  269 + List<FirmwareInfo> loadedFirmwaresWithoutData = new ArrayList<>();
  270 + pageLink = new PageLink(24);
  271 + do {
  272 + pageData = doGetTypedWithPageLink("/api/firmwares/false?",
  273 + new TypeReference<>() {
  274 + }, pageLink);
  275 + loadedFirmwaresWithoutData.addAll(pageData.getData());
  276 + if (pageData.hasNext()) {
  277 + pageLink = pageLink.nextPageLink();
  278 + }
  279 + } while (pageData.hasNext());
  280 +
  281 + Collections.sort(firmwaresWithData, idComparator);
  282 + Collections.sort(firmwaresWithoutData, idComparator);
  283 + Collections.sort(loadedFirmwaresWithData, idComparator);
  284 + Collections.sort(loadedFirmwaresWithoutData, idComparator);
  285 +
  286 + Assert.assertEquals(firmwaresWithData, loadedFirmwaresWithData);
  287 + Assert.assertEquals(firmwaresWithoutData, loadedFirmwaresWithoutData);
  288 + }
  289 +
  290 +
  291 + private FirmwareInfo save(FirmwareInfo firmwareInfo) throws Exception {
  292 + return doPost("/api/firmware", firmwareInfo, FirmwareInfo.class);
  293 + }
  294 +
  295 + protected Firmware savaData(String urlTemplate, MockMultipartFile content, String... params) throws Exception {
  296 + MockMultipartHttpServletRequestBuilder postRequest = MockMvcRequestBuilders.multipart(urlTemplate, params);
  297 + postRequest.file(content);
  298 + setJwtToken(postRequest);
  299 + return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), Firmware.class);
  300 + }
  301 +
  302 +}
... ...
... ... @@ -21,12 +21,12 @@ import org.checkerframework.checker.nullness.qual.Nullable;
21 21 import org.junit.After;
22 22 import org.junit.Assert;
23 23 import org.junit.Before;
24   -import org.junit.Ignore;
25 24 import org.junit.Test;
26 25 import org.springframework.beans.factory.annotation.Autowired;
27 26 import org.thingsboard.server.common.data.Device;
28 27 import org.thingsboard.server.common.data.Tenant;
29 28 import org.thingsboard.server.common.data.User;
  29 +import org.thingsboard.server.common.data.id.CustomerId;
30 30 import org.thingsboard.server.common.data.kv.Aggregation;
31 31 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
32 32 import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
... ... @@ -750,7 +750,7 @@ public class BaseWebsocketApiTest extends AbstractWebsocketTest {
750 750
751 751 private void sendTelemetry(Device device, List<TsKvEntry> tsData) throws InterruptedException {
752 752 CountDownLatch latch = new CountDownLatch(1);
753   - tsService.saveAndNotify(device.getTenantId(), device.getId(), tsData, 0, new FutureCallback<Void>() {
  753 + tsService.saveAndNotify(device.getTenantId(), null, device.getId(), tsData, 0, new FutureCallback<Void>() {
754 754 @Override
755 755 public void onSuccess(@Nullable Void result) {
756 756 latch.countDown();
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.controller.sql;
  17 +
  18 +import org.thingsboard.server.controller.BaseFirmwareControllerTest;
  19 +import org.thingsboard.server.dao.service.DaoSqlTest;
  20 +
  21 +@DaoSqlTest
  22 +public class FirmwareControllerSqlTest extends BaseFirmwareControllerTest {
  23 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.cache.firmware;
  17 +
  18 +import org.springframework.data.redis.connection.RedisConnectionFactory;
  19 +
  20 +import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE;
  21 +
  22 +public abstract class AbstractRedisFirmwareCache {
  23 +
  24 + protected final RedisConnectionFactory redisConnectionFactory;
  25 +
  26 + protected AbstractRedisFirmwareCache(RedisConnectionFactory redisConnectionFactory) {
  27 + this.redisConnectionFactory = redisConnectionFactory;
  28 + }
  29 +
  30 + protected byte[] toFirmwareCacheKey(String key) {
  31 + return String.format("%s::%s", FIRMWARE_CACHE, key).getBytes();
  32 + }
  33 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.cache.firmware;
  17 +
  18 +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  19 +import org.springframework.cache.CacheManager;
  20 +import org.springframework.stereotype.Service;
  21 +
  22 +import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE;
  23 +
  24 +@Service
  25 +@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport') && ('${cache.type:null}'=='caffeine' || '${cache.type:null}'=='null')")
  26 +public class CaffeineFirmwareCacheReader implements FirmwareCacheReader {
  27 +
  28 + private final CacheManager cacheManager;
  29 +
  30 + public CaffeineFirmwareCacheReader(CacheManager cacheManager) {
  31 + this.cacheManager = cacheManager;
  32 + }
  33 +
  34 + @Override
  35 + public byte[] get(String key) {
  36 + return get(key, 0, 0);
  37 + }
  38 +
  39 + @Override
  40 + public byte[] get(String key, int chunkSize, int chunk) {
  41 + byte[] data = cacheManager.getCache(FIRMWARE_CACHE).get(key, byte[].class);
  42 +
  43 + if (chunkSize < 1) {
  44 + return data;
  45 + }
  46 +
  47 + if (data != null && data.length > 0) {
  48 + int startIndex = chunkSize * chunk;
  49 +
  50 + int size = Math.min(data.length - startIndex, chunkSize);
  51 +
  52 + if (startIndex < data.length && size > 0) {
  53 + byte[] result = new byte[size];
  54 + System.arraycopy(data, startIndex, result, 0, size);
  55 + return result;
  56 + }
  57 + }
  58 + return new byte[0];
  59 + }
  60 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.cache.firmware;
  17 +
  18 +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  19 +import org.springframework.cache.CacheManager;
  20 +import org.springframework.stereotype.Service;
  21 +
  22 +import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE;
  23 +
  24 +@Service
  25 +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && ('${cache.type:null}'=='caffeine' || '${cache.type:null}'=='null')")
  26 +public class CaffeineFirmwareCacheWriter implements FirmwareCacheWriter {
  27 +
  28 + private final CacheManager cacheManager;
  29 +
  30 + public CaffeineFirmwareCacheWriter(CacheManager cacheManager) {
  31 + this.cacheManager = cacheManager;
  32 + }
  33 +
  34 + @Override
  35 + public void put(String key, byte[] value) {
  36 + cacheManager.getCache(FIRMWARE_CACHE).putIfAbsent(key, value);
  37 + }
  38 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.cache.firmware;
  17 +
  18 +public interface FirmwareCacheReader {
  19 + byte[] get(String key);
  20 +
  21 + byte[] get(String key, int chunkSize, int chunk);
  22 +}
... ...
common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheWriter.java renamed from common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/ResultsResourceValue.java
... ... @@ -13,20 +13,8 @@
13 13 * See the License for the specific language governing permissions and
14 14 * limitations under the License.
15 15 */
16   -package org.thingsboard.server.transport.lwm2m.server.client;
  16 +package org.thingsboard.server.cache.firmware;
17 17
18   -import lombok.Data;
19   -import org.thingsboard.server.common.data.kv.DataType;
20   -
21   -@Data
22   -public class ResultsResourceValue {
23   - DataType dataType;
24   - Object value;
25   - String resourceName;
26   -
27   - public ResultsResourceValue (DataType dataType, Object value, String resourceName) {
28   - this.dataType = dataType;
29   - this.value = value;
30   - this.resourceName = resourceName;
31   - }
  18 +public interface FirmwareCacheWriter {
  19 + void put(String key, byte[] value);
32 20 }
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.cache.firmware;
  17 +
  18 +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  19 +import org.springframework.data.redis.connection.RedisConnection;
  20 +import org.springframework.data.redis.connection.RedisConnectionFactory;
  21 +import org.springframework.stereotype.Service;
  22 +
  23 +@Service
  24 +@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport') && '${cache.type:null}'=='redis'")
  25 +public class RedisFirmwareCacheReader extends AbstractRedisFirmwareCache implements FirmwareCacheReader {
  26 +
  27 + public RedisFirmwareCacheReader(RedisConnectionFactory redisConnectionFactory) {
  28 + super(redisConnectionFactory);
  29 + }
  30 +
  31 + @Override
  32 + public byte[] get(String key) {
  33 + return get(key, 0, 0);
  34 + }
  35 +
  36 + @Override
  37 + public byte[] get(String key, int chunkSize, int chunk) {
  38 + try (RedisConnection connection = redisConnectionFactory.getConnection()) {
  39 + if (chunkSize == 0) {
  40 + return connection.get(toFirmwareCacheKey(key));
  41 + }
  42 +
  43 + int startIndex = chunkSize * chunk;
  44 + int endIndex = startIndex + chunkSize - 1;
  45 + return connection.getRange(toFirmwareCacheKey(key), startIndex, endIndex);
  46 + }
  47 + }
  48 +
  49 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.cache.firmware;
  17 +
  18 +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  19 +import org.springframework.data.redis.connection.RedisConnection;
  20 +import org.springframework.data.redis.connection.RedisConnectionFactory;
  21 +import org.springframework.stereotype.Service;
  22 +
  23 +@Service
  24 +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && '${cache.type:null}'=='redis'")
  25 +public class RedisFirmwareCacheWriter extends AbstractRedisFirmwareCache implements FirmwareCacheWriter {
  26 +
  27 + public RedisFirmwareCacheWriter(RedisConnectionFactory redisConnectionFactory) {
  28 + super(redisConnectionFactory);
  29 + }
  30 +
  31 + @Override
  32 + public void put(String key, byte[] value) {
  33 + try (RedisConnection connection = redisConnectionFactory.getConnection()) {
  34 + connection.set(toFirmwareCacheKey(key), value);
  35 + }
  36 + }
  37 +
  38 +}
... ...
... ... @@ -63,6 +63,8 @@ public interface DeviceService {
63 63
64 64 PageData<Device> findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink);
65 65
  66 + PageData<Device> findDevicesByTenantIdAndTypeAndEmptyFirmware(TenantId tenantId, String type, PageLink pageLink);
  67 +
66 68 PageData<DeviceInfo> findDeviceInfosByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink);
67 69
68 70 PageData<DeviceInfo> findDeviceInfosByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink);
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.firmware;
  17 +
  18 +import org.thingsboard.server.common.data.Firmware;
  19 +import org.thingsboard.server.common.data.FirmwareInfo;
  20 +import org.thingsboard.server.common.data.id.FirmwareId;
  21 +import org.thingsboard.server.common.data.id.TenantId;
  22 +import org.thingsboard.server.common.data.page.PageData;
  23 +import org.thingsboard.server.common.data.page.PageLink;
  24 +
  25 +public interface FirmwareService {
  26 +
  27 + FirmwareInfo saveFirmwareInfo(FirmwareInfo firmwareInfo);
  28 +
  29 + Firmware saveFirmware(Firmware firmware);
  30 +
  31 + Firmware findFirmwareById(TenantId tenantId, FirmwareId firmwareId);
  32 +
  33 + FirmwareInfo findFirmwareInfoById(TenantId tenantId, FirmwareId firmwareId);
  34 +
  35 + PageData<FirmwareInfo> findTenantFirmwaresByTenantId(TenantId tenantId, PageLink pageLink);
  36 +
  37 + PageData<FirmwareInfo> findTenantFirmwaresByTenantIdAndHasData(TenantId tenantId, boolean hasData, PageLink pageLink);
  38 +
  39 + void deleteFirmware(TenantId tenantId, FirmwareId firmwareId);
  40 +
  41 + void deleteFirmwaresByTenantId(TenantId tenantId);
  42 +}
... ...
... ... @@ -17,17 +17,22 @@ package org.thingsboard.server.dao.usagerecord;
17 17
18 18 import org.thingsboard.server.common.data.ApiUsageState;
19 19 import org.thingsboard.server.common.data.id.ApiUsageStateId;
  20 +import org.thingsboard.server.common.data.id.EntityId;
20 21 import org.thingsboard.server.common.data.id.TenantId;
21 22
22 23 public interface ApiUsageStateService {
23 24
24   - ApiUsageState createDefaultApiUsageState(TenantId id);
  25 + ApiUsageState createDefaultApiUsageState(TenantId id, EntityId entityId);
25 26
26 27 ApiUsageState update(ApiUsageState apiUsageState);
27 28
28 29 ApiUsageState findTenantApiUsageState(TenantId tenantId);
29 30
  31 + ApiUsageState findApiUsageStateByEntityId(EntityId entityId);
  32 +
30 33 void deleteApiUsageStateByTenantId(TenantId tenantId);
31 34
  35 + void deleteApiUsageStateByEntityId(EntityId entityId);
  36 +
32 37 ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id);
33 38 }
... ...
... ... @@ -29,4 +29,5 @@ public class CacheConstants {
29 29 public static final String DEVICE_PROFILE_CACHE = "deviceProfiles";
30 30 public static final String ATTRIBUTES_CACHE = "attributes";
31 31 public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime";
  32 + public static final String FIRMWARE_CACHE = "firmwares";
32 33 }
... ...
... ... @@ -93,6 +93,21 @@ public class DataConstants {
93 93 public static final String USERNAME = "username";
94 94 public static final String PASSWORD = "password";
95 95
  96 + //firmware
  97 + //telemetry
  98 + public static final String CURRENT_FIRMWARE_TITLE = "cur_fw_title";
  99 + public static final String CURRENT_FIRMWARE_VERSION = "cur_fw_version";
  100 + public static final String TARGET_FIRMWARE_TITLE = "target_fw_title";
  101 + public static final String TARGET_FIRMWARE_VERSION = "target_fw_version";
  102 + public static final String FIRMWARE_STATE = "fw_state";
  103 +
  104 + //attributes
  105 + //telemetry
  106 + public static final String FIRMWARE_TITLE = "fw_title";
  107 + public static final String FIRMWARE_VERSION = "fw_version";
  108 + public static final String FIRMWARE_SIZE = "fw_size";
  109 + public static final String FIRMWARE_CHECKSUM = "fw_checksum";
  110 + public static final String FIRMWARE_CHECKSUM_ALGORITHM = "fw_checksum_algorithm";
96 111 public static final String EDGE_MSG_SOURCE = "edge";
97 112 public static final String MSG_SOURCE_KEY = "source";
98 113 }
... ...
... ... @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.device.data.DeviceData;
23 23 import org.thingsboard.server.common.data.id.CustomerId;
24 24 import org.thingsboard.server.common.data.id.DeviceId;
25 25 import org.thingsboard.server.common.data.id.DeviceProfileId;
  26 +import org.thingsboard.server.common.data.id.FirmwareId;
26 27 import org.thingsboard.server.common.data.id.TenantId;
27 28 import org.thingsboard.server.common.data.validation.NoXss;
28 29
... ... @@ -48,6 +49,8 @@ public class Device extends SearchTextBasedWithAdditionalInfo<DeviceId> implemen
48 49 @JsonIgnore
49 50 private byte[] deviceDataBytes;
50 51
  52 + private FirmwareId firmwareId;
  53 +
51 54 public Device() {
52 55 super();
53 56 }
... ... @@ -65,6 +68,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo<DeviceId> implemen
65 68 this.label = device.getLabel();
66 69 this.deviceProfileId = device.getDeviceProfileId();
67 70 this.setDeviceData(device.getDeviceData());
  71 + this.firmwareId = device.getFirmwareId();
68 72 }
69 73
70 74 public Device updateDevice(Device device) {
... ... @@ -159,6 +163,14 @@ public class Device extends SearchTextBasedWithAdditionalInfo<DeviceId> implemen
159 163 return getName();
160 164 }
161 165
  166 + public FirmwareId getFirmwareId() {
  167 + return firmwareId;
  168 + }
  169 +
  170 + public void setFirmwareId(FirmwareId firmwareId) {
  171 + this.firmwareId = firmwareId;
  172 + }
  173 +
162 174 @Override
163 175 public String toString() {
164 176 StringBuilder builder = new StringBuilder();
... ... @@ -175,6 +187,8 @@ public class Device extends SearchTextBasedWithAdditionalInfo<DeviceId> implemen
175 187 builder.append(", deviceProfileId=");
176 188 builder.append(deviceProfileId);
177 189 builder.append(", deviceData=");
  190 + builder.append(firmwareId);
  191 + builder.append(", firmwareId=");
178 192 builder.append(deviceData);
179 193 builder.append(", additionalInfo=");
180 194 builder.append(getAdditionalInfo());
... ...
... ... @@ -22,6 +22,7 @@ import lombok.EqualsAndHashCode;
22 22 import lombok.extern.slf4j.Slf4j;
23 23 import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
24 24 import org.thingsboard.server.common.data.id.DeviceProfileId;
  25 +import org.thingsboard.server.common.data.id.FirmwareId;
25 26 import org.thingsboard.server.common.data.id.RuleChainId;
26 27 import org.thingsboard.server.common.data.id.TenantId;
27 28 import org.thingsboard.server.common.data.validation.NoXss;
... ... @@ -56,6 +57,8 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H
56 57 @NoXss
57 58 private String provisionDeviceKey;
58 59
  60 + private FirmwareId firmwareId;
  61 +
59 62 public DeviceProfile() {
60 63 super();
61 64 }
... ...
... ... @@ -19,5 +19,5 @@ package org.thingsboard.server.common.data;
19 19 * @author Andrew Shvayka
20 20 */
21 21 public enum EntityType {
22   - TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE, TB_RESOURCE, EDGE;
  22 + TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE, TB_RESOURCE, FIRMWARE, EDGE;
23 23 }
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data;
  17 +
  18 +import lombok.Data;
  19 +import lombok.EqualsAndHashCode;
  20 +import org.thingsboard.server.common.data.id.FirmwareId;
  21 +
  22 +import java.nio.ByteBuffer;
  23 +
  24 +@Data
  25 +@EqualsAndHashCode(callSuper = true)
  26 +public class Firmware extends FirmwareInfo {
  27 +
  28 + private static final long serialVersionUID = 3091601761339422546L;
  29 +
  30 + private transient ByteBuffer data;
  31 +
  32 + public Firmware() {
  33 + super();
  34 + }
  35 +
  36 + public Firmware(FirmwareId id) {
  37 + super(id);
  38 + }
  39 +
  40 + public Firmware(Firmware firmware) {
  41 + super(firmware);
  42 + this.data = firmware.getData();
  43 + }
  44 +}
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data;
  17 +
  18 +import lombok.Data;
  19 +import lombok.EqualsAndHashCode;
  20 +import lombok.extern.slf4j.Slf4j;
  21 +import org.thingsboard.server.common.data.id.FirmwareId;
  22 +import org.thingsboard.server.common.data.id.TenantId;
  23 +
  24 +@Slf4j
  25 +@Data
  26 +@EqualsAndHashCode(callSuper = true)
  27 +public class FirmwareInfo extends SearchTextBasedWithAdditionalInfo<FirmwareId> implements HasTenantId {
  28 +
  29 + private static final long serialVersionUID = 3168391583570815419L;
  30 +
  31 + private TenantId tenantId;
  32 + private String title;
  33 + private String version;
  34 + private boolean hasData;
  35 + private String fileName;
  36 + private String contentType;
  37 + private String checksumAlgorithm;
  38 + private String checksum;
  39 + private Long dataSize;
  40 +
  41 +
  42 + public FirmwareInfo() {
  43 + super();
  44 + }
  45 +
  46 + public FirmwareInfo(FirmwareId id) {
  47 + super(id);
  48 + }
  49 +
  50 + public FirmwareInfo(FirmwareInfo firmwareInfo) {
  51 + super(firmwareInfo);
  52 + this.tenantId = firmwareInfo.getTenantId();
  53 + this.title = firmwareInfo.getTitle();
  54 + this.version = firmwareInfo.getVersion();
  55 + this.hasData = firmwareInfo.isHasData();
  56 + this.fileName = firmwareInfo.getFileName();
  57 + this.contentType = firmwareInfo.getContentType();
  58 + this.checksumAlgorithm = firmwareInfo.getChecksumAlgorithm();
  59 + this.checksum = firmwareInfo.getChecksum();
  60 + this.dataSize = firmwareInfo.getDataSize();
  61 + }
  62 +
  63 + @Override
  64 + public String getSearchText() {
  65 + return title;
  66 + }
  67 +}
... ...
... ... @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.validation.NoXss;
27 27 @EqualsAndHashCode(callSuper = true)
28 28 public class TbResourceInfo extends SearchTextBased<TbResourceId> implements HasTenantId {
29 29
  30 + private static final long serialVersionUID = 7282664529021651736L;
  31 +
30 32 private TenantId tenantId;
31 33 @NoXss
32 34 private String title;
... ...
... ... @@ -30,6 +30,9 @@ public class MqttTopics {
30 30 private static final String CLAIM = "/claim";
31 31 private static final String SUB_TOPIC = "+";
32 32 private static final String PROVISION = "/provision";
  33 + private static final String FIRMWARE = "/fw";
  34 + private static final String CHUNK = "/chunk/";
  35 + private static final String ERROR = "/error";
33 36
34 37 private static final String ATTRIBUTES_RESPONSE = ATTRIBUTES + RESPONSE;
35 38 private static final String ATTRIBUTES_REQUEST = ATTRIBUTES + REQUEST;
... ... @@ -69,6 +72,13 @@ public class MqttTopics {
69 72 public static final String GATEWAY_ATTRIBUTES_REQUEST_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_REQUEST;
70 73 public static final String GATEWAY_ATTRIBUTES_RESPONSE_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_RESPONSE;
71 74
  75 + // v2 topics
  76 + public static final String BASE_DEVICE_API_TOPIC_V2 = "v2";
  77 +
  78 + public static final String DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + RESPONSE + "/";
  79 + public static final String DEVICE_FIRMWARE_RESPONSES_TOPIC = DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX + SUB_TOPIC + CHUNK + SUB_TOPIC;
  80 + public static final String DEVICE_FIRMWARE_ERROR_TOPIC = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + ERROR;
  81 +
72 82 private MqttTopics() {
73 83 }
74 84 }
... ...
... ... @@ -71,6 +71,8 @@ public class EntityIdFactory {
71 71 return new ApiUsageStateId(uuid);
72 72 case TB_RESOURCE:
73 73 return new TbResourceId(uuid);
  74 + case FIRMWARE:
  75 + return new FirmwareId(uuid);
74 76 case EDGE:
75 77 return new EdgeId(uuid);
76 78 }
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.common.data.id;
  17 +
  18 +import com.fasterxml.jackson.annotation.JsonCreator;
  19 +import com.fasterxml.jackson.annotation.JsonIgnore;
  20 +import com.fasterxml.jackson.annotation.JsonProperty;
  21 +import org.thingsboard.server.common.data.EntityType;
  22 +
  23 +import java.util.UUID;
  24 +
  25 +public class FirmwareId extends UUIDBased implements EntityId {
  26 +
  27 + private static final long serialVersionUID = 1L;
  28 +
  29 + @JsonCreator
  30 + public FirmwareId(@JsonProperty("id") UUID id) {
  31 + super(id);
  32 + }
  33 +
  34 + public static FirmwareId fromString(String firmwareId) {
  35 + return new FirmwareId(UUID.fromString(firmwareId));
  36 + }
  37 +
  38 + @JsonIgnore
  39 + @Override
  40 + public EntityType getEntityType() {
  41 + return EntityType.FIRMWARE;
  42 + }
  43 +
  44 +}
... ...
... ... @@ -16,10 +16,13 @@
16 16 package org.thingsboard.server.common.data.query;
17 17
18 18 import lombok.Data;
19   -import org.thingsboard.server.common.data.id.EntityId;
  19 +import org.thingsboard.server.common.data.id.CustomerId;
20 20
21 21 @Data
22 22 public class ApiUsageStateFilter implements EntityFilter {
  23 +
  24 + private CustomerId customerId;
  25 +
23 26 @Override
24 27 public EntityFilterType getType() {
25 28 return EntityFilterType.API_USAGE_STATE;
... ...
... ... @@ -19,10 +19,11 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
19 19 import com.google.protobuf.ByteString;
20 20 import com.google.protobuf.InvalidProtocolBufferException;
21 21 import lombok.AccessLevel;
22   -import lombok.Builder;
23 22 import lombok.Data;
24 23 import lombok.Getter;
25 24 import lombok.extern.slf4j.Slf4j;
  25 +import org.thingsboard.server.common.data.EntityType;
  26 +import org.thingsboard.server.common.data.id.CustomerId;
26 27 import org.thingsboard.server.common.data.id.EntityId;
27 28 import org.thingsboard.server.common.data.id.EntityIdFactory;
28 29 import org.thingsboard.server.common.data.id.RuleChainId;
... ... @@ -47,6 +48,7 @@ public final class TbMsg implements Serializable {
47 48 private final long ts;
48 49 private final String type;
49 50 private final EntityId originator;
  51 + private final CustomerId customerId;
50 52 private final TbMsgMetaData metaData;
51 53 private final TbMsgDataType dataType;
52 54 private final String data;
... ... @@ -55,6 +57,7 @@ public final class TbMsg implements Serializable {
55 57 @Getter(value = AccessLevel.NONE)
56 58 private final AtomicInteger ruleNodeExecCounter;
57 59
  60 +
58 61 public int getAndIncrementRuleNodeCounter() {
59 62 return ruleNodeExecCounter.getAndIncrement();
60 63 }
... ... @@ -64,60 +67,81 @@ public final class TbMsg implements Serializable {
64 67 transient private final TbMsgCallback callback;
65 68
66 69 public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
67   - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator,
  70 + return newMsg(queueName, type, originator, null, metaData, data, ruleChainId, ruleNodeId);
  71 + }
  72 +
  73 + public static TbMsg newMsg(String queueName, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
  74 + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
68 75 metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
69 76 }
70 77
71 78 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) {
72   - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
  79 + return newMsg(type, originator, null, metaData, data);
  80 + }
  81 +
  82 + public static TbMsg newMsg(String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
  83 + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
73 84 }
74 85
75 86 // REALLY NEW MSG
76 87
77 88 public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) {
78   - return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
  89 + return newMsg(queueName, type, originator, null, metaData, data);
  90 + }
  91 +
  92 + public static TbMsg newMsg(String queueName, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
  93 + return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, TbMsgCallback.EMPTY);
  94 + }
  95 +
  96 + public static TbMsg newMsg(String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data) {
  97 + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId, metaData.copy(), dataType, data, null, null, 0, TbMsgCallback.EMPTY);
79 98 }
80 99
81 100 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data) {
82   - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, null, null, 0, TbMsgCallback.EMPTY);
  101 + return newMsg(type, originator, null, metaData, dataType, data);
83 102 }
84 103
85 104 // For Tests only
86 105
87 106 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
88   - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
  107 + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, null, metaData.copy(), dataType, data, ruleChainId, ruleNodeId, 0, TbMsgCallback.EMPTY);
89 108 }
90 109
91 110 public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) {
92   - return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, callback);
  111 + return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, null, metaData.copy(), TbMsgDataType.JSON, data, null, null, 0, callback);
93 112 }
94 113
95 114 public static TbMsg transformMsg(TbMsg tbMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) {
96   - return new TbMsg(tbMsg.getQueueName(), tbMsg.getId(), tbMsg.getTs(), type, originator, metaData.copy(), tbMsg.getDataType(),
97   - data, tbMsg.getRuleChainId(), tbMsg.getRuleNodeId(), tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
  115 + return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, type, originator, tbMsg.customerId, metaData.copy(), tbMsg.dataType,
  116 + data, tbMsg.ruleChainId, tbMsg.ruleNodeId, tbMsg.ruleNodeExecCounter.get(), tbMsg.callback);
  117 + }
  118 +
  119 + public static TbMsg transformMsg(TbMsg tbMsg, CustomerId customerId) {
  120 + return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, customerId, tbMsg.metaData, tbMsg.dataType,
  121 + tbMsg.data, tbMsg.ruleChainId, tbMsg.ruleNodeId, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
98 122 }
99 123
100 124 public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId) {
101   - return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
  125 + return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
102 126 tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
103 127 }
104 128
105 129 public static TbMsg transformMsg(TbMsg tbMsg, String queueName) {
106   - return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
  130 + return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
107 131 tbMsg.data, tbMsg.getRuleChainId(), null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
108 132 }
109 133
110 134 public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId, String queueName) {
111   - return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.metaData, tbMsg.dataType,
  135 + return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
112 136 tbMsg.data, ruleChainId, null, tbMsg.ruleNodeExecCounter.get(), tbMsg.getCallback());
113 137 }
114 138
115 139 public static TbMsg newMsg(TbMsg tbMsg, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
116   - return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.getMetaData().copy(),
  140 + return new TbMsg(tbMsg.getQueueName(), UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.customerId, tbMsg.getMetaData().copy(),
117 141 tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, tbMsg.ruleNodeExecCounter.get(), TbMsgCallback.EMPTY);
118 142 }
119 143
120   - private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
  144 + private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
121 145 RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) {
122 146 this.id = id;
123 147 this.queueName = queueName != null ? queueName : ServiceQueue.MAIN;
... ... @@ -128,6 +152,15 @@ public final class TbMsg implements Serializable {
128 152 }
129 153 this.type = type;
130 154 this.originator = originator;
  155 + if (customerId == null || customerId.isNullUid()) {
  156 + if (originator != null && originator.getEntityType() == EntityType.CUSTOMER) {
  157 + this.customerId = (CustomerId) originator;
  158 + } else {
  159 + this.customerId = null;
  160 + }
  161 + } else {
  162 + this.customerId = customerId;
  163 + }
131 164 this.metaData = metaData;
132 165 this.dataType = dataType;
133 166 this.data = data;
... ... @@ -154,6 +187,11 @@ public final class TbMsg implements Serializable {
154 187 builder.setEntityIdMSB(msg.getOriginator().getId().getMostSignificantBits());
155 188 builder.setEntityIdLSB(msg.getOriginator().getId().getLeastSignificantBits());
156 189
  190 + if (msg.getCustomerId() != null) {
  191 + builder.setCustomerIdMSB(msg.getCustomerId().getId().getMostSignificantBits());
  192 + builder.setCustomerIdLSB(msg.getCustomerId().getId().getLeastSignificantBits());
  193 + }
  194 +
157 195 if (msg.getRuleChainId() != null) {
158 196 builder.setRuleChainIdMSB(msg.getRuleChainId().getId().getMostSignificantBits());
159 197 builder.setRuleChainIdLSB(msg.getRuleChainId().getId().getLeastSignificantBits());
... ... @@ -179,16 +217,21 @@ public final class TbMsg implements Serializable {
179 217 MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data);
180 218 TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap());
181 219 EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
  220 + CustomerId customerId = null;
182 221 RuleChainId ruleChainId = null;
183 222 RuleNodeId ruleNodeId = null;
  223 + if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) {
  224 + customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB()));
  225 + }
184 226 if (proto.getRuleChainIdMSB() != 0L && proto.getRuleChainIdLSB() != 0L) {
185 227 ruleChainId = new RuleChainId(new UUID(proto.getRuleChainIdMSB(), proto.getRuleChainIdLSB()));
186 228 }
187 229 if (proto.getRuleNodeIdMSB() != 0L && proto.getRuleNodeIdLSB() != 0L) {
188 230 ruleNodeId = new RuleNodeId(new UUID(proto.getRuleNodeIdMSB(), proto.getRuleNodeIdLSB()));
189 231 }
  232 +
190 233 TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
191   - return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, proto.getRuleNodeExecCounter(), callback);
  234 + return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, customerId, metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, proto.getRuleNodeExecCounter(), callback);
192 235 } catch (InvalidProtocolBufferException e) {
193 236 throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
194 237 }
... ... @@ -199,11 +242,11 @@ public final class TbMsg implements Serializable {
199 242 }
200 243
201 244 public TbMsg copyWithRuleChainId(RuleChainId ruleChainId, UUID msgId) {
202   - return new TbMsg(this.queueName, msgId, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, null, this.ruleNodeExecCounter.get(), callback);
  245 + return new TbMsg(this.queueName, msgId, this.ts, this.type, this.originator, this.customerId, this.metaData, this.dataType, this.data, ruleChainId, null, this.ruleNodeExecCounter.get(), callback);
203 246 }
204 247
205 248 public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID msgId) {
206   - return new TbMsg(this.queueName, msgId, this.ts, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ruleNodeExecCounter.get(), callback);
  249 + return new TbMsg(this.queueName, msgId, this.ts, this.type, this.originator, this.customerId, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ruleNodeExecCounter.get(), callback);
207 250 }
208 251
209 252 public TbMsgCallback getCallback() {
... ...
... ... @@ -46,4 +46,7 @@ message TbMsgProto {
46 46
47 47 int64 ts = 15;
48 48 int32 ruleNodeExecCounter = 16;
  49 +
  50 + int64 customerIdMSB = 17;
  51 + int64 customerIdLSB = 18;
49 52 }
\ No newline at end of file
... ...
... ... @@ -69,7 +69,7 @@ public class TbKafkaConsumerStatsService {
69 69 this.adminClient = AdminClient.create(kafkaSettings.toAdminProps());
70 70 this.statsPrintScheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("kafka-consumer-stats"));
71 71
72   - Properties consumerProps = kafkaSettings.toConsumerProps();
  72 + Properties consumerProps = kafkaSettings.toConsumerProps(null);
73 73 consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer-stats-loader-client");
74 74 consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "consumer-stats-loader-client-group");
75 75 this.consumer = new KafkaConsumer<>(consumerProps);
... ...
... ... @@ -50,7 +50,7 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
50 50 String clientId, String groupId, String topic,
51 51 TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) {
52 52 super(topic);
53   - Properties props = settings.toConsumerProps();
  53 + Properties props = settings.toConsumerProps(topic);
54 54 props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId);
55 55 if (groupId != null) {
56 56 props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
... ...
... ... @@ -31,7 +31,9 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
31 31 import org.springframework.boot.context.properties.ConfigurationProperties;
32 32 import org.springframework.stereotype.Component;
33 33
  34 +import java.util.Collections;
34 35 import java.util.List;
  36 +import java.util.Map;
35 37 import java.util.Properties;
36 38
37 39 /**
... ... @@ -52,6 +54,9 @@ public class TbKafkaSettings {
52 54 @Value("${queue.kafka.retries}")
53 55 private int retries;
54 56
  57 + @Value("${queue.kafka.compression.type:none}")
  58 + private String compressionType;
  59 +
55 60 @Value("${queue.kafka.batch.size}")
56 61 private int batchSize;
57 62
... ... @@ -95,6 +100,9 @@ public class TbKafkaSettings {
95 100 @Setter
96 101 private List<TbKafkaProperty> other;
97 102
  103 + @Setter
  104 + private Map<String, List<TbKafkaProperty>> consumerPropertiesPerTopic = Collections.emptyMap();
  105 +
98 106 public Properties toAdminProps() {
99 107 Properties props = toProps();
100 108 props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
... ... @@ -103,7 +111,7 @@ public class TbKafkaSettings {
103 111 return props;
104 112 }
105 113
106   - public Properties toConsumerProps() {
  114 + public Properties toConsumerProps(String topic) {
107 115 Properties props = toProps();
108 116 props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
109 117 props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
... ... @@ -113,6 +121,10 @@ public class TbKafkaSettings {
113 121
114 122 props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
115 123 props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class);
  124 +
  125 + consumerPropertiesPerTopic
  126 + .getOrDefault(topic, Collections.emptyList())
  127 + .forEach(kv -> props.put(kv.getKey(), kv.getValue()));
116 128 return props;
117 129 }
118 130
... ... @@ -126,6 +138,7 @@ public class TbKafkaSettings {
126 138 props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
127 139 props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
128 140 props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
  141 + props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType);
129 142 return props;
130 143 }
131 144
... ...
... ... @@ -19,6 +19,7 @@ import lombok.Getter;
19 19 import org.springframework.beans.factory.annotation.Value;
20 20 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
21 21 import org.springframework.stereotype.Component;
  22 +import org.thingsboard.server.common.data.StringUtils;
22 23
23 24 import javax.annotation.PostConstruct;
24 25 import java.util.HashMap;
... ... @@ -37,6 +38,8 @@ public class TbKafkaTopicConfigs {
37 38 private String notificationsProperties;
38 39 @Value("${queue.kafka.topic-properties.js-executor}")
39 40 private String jsExecutorProperties;
  41 + @Value("${queue.kafka.topic-properties.fw-updates:}")
  42 + private String fwUpdatesProperties;
40 43
41 44 @Getter
42 45 private Map<String, String> coreConfigs;
... ... @@ -48,6 +51,8 @@ public class TbKafkaTopicConfigs {
48 51 private Map<String, String> notificationsConfigs;
49 52 @Getter
50 53 private Map<String, String> jsExecutorConfigs;
  54 + @Getter
  55 + private Map<String, String> fwUpdatesConfigs;
51 56
52 57 @PostConstruct
53 58 private void init() {
... ... @@ -56,15 +61,18 @@ public class TbKafkaTopicConfigs {
56 61 transportApiConfigs = getConfigs(transportApiProperties);
57 62 notificationsConfigs = getConfigs(notificationsProperties);
58 63 jsExecutorConfigs = getConfigs(jsExecutorProperties);
  64 + fwUpdatesConfigs = getConfigs(fwUpdatesProperties);
59 65 }
60 66
61 67 private Map<String, String> getConfigs(String properties) {
62 68 Map<String, String> configs = new HashMap<>();
63   - for (String property : properties.split(";")) {
64   - int delimiterPosition = property.indexOf(":");
65   - String key = property.substring(0, delimiterPosition);
66   - String value = property.substring(delimiterPosition + 1);
67   - configs.put(key, value);
  69 + if (StringUtils.isNotEmpty(properties)) {
  70 + for (String property : properties.split(";")) {
  71 + int delimiterPosition = property.indexOf(":");
  72 + String key = property.substring(0, delimiterPosition);
  73 + String value = property.substring(delimiterPosition + 1);
  74 + configs.put(key, value);
  75 + }
68 76 }
69 77 return configs;
70 78 }
... ...
... ... @@ -186,6 +186,17 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
186 186 msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
187 187 }
188 188
  189 + @Override
  190 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  191 + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getFirmwareTopic(),
  192 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  193 + }
  194 +
  195 + @Override
  196 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  197 + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getFirmwareTopic());
  198 + }
  199 +
189 200 @PreDestroy
190 201 private void destroy() {
191 202 if (coreAdmin != null) {
... ...
... ... @@ -21,12 +21,13 @@ import org.springframework.context.annotation.Bean;
21 21 import org.springframework.stereotype.Component;
22 22 import org.thingsboard.server.common.msg.queue.ServiceType;
23 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
24   -import org.thingsboard.server.gen.transport.TransportProtos;
25 24 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
26 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  26 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
27 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
28 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
29 29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
  30 +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
30 31 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
31 32 import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
32 33 import org.thingsboard.server.queue.TbQueueAdmin;
... ... @@ -165,14 +166,25 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
165 166 }
166 167
167 168 @Override
168   - public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
  169 + public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
169 170 return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic());
170 171 }
171 172
172 173 @Override
173   - public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
  174 + public TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer() {
174 175 return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getUsageStatsTopic(),
175   - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  176 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  177 + }
  178 +
  179 + @Override
  180 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  181 + return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, coreSettings.getFirmwareTopic(),
  182 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  183 + }
  184 +
  185 + @Override
  186 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  187 + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getFirmwareTopic());
176 188 }
177 189
178 190 @PreDestroy
... ...
... ... @@ -131,6 +131,16 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
131 131 }
132 132
133 133 @Override
  134 + public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  135 + return new InMemoryTbQueueConsumer<>(coreSettings.getFirmwareTopic());
  136 + }
  137 +
  138 + @Override
  139 + public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  140 + return new InMemoryTbQueueProducer<>(coreSettings.getFirmwareTopic());
  141 + }
  142 +
  143 + @Override
134 144 public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
135 145 return new InMemoryTbQueueProducer<>(coreSettings.getUsageStatsTopic());
136 146 }
... ...
... ... @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
23 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
24 24 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
25 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  26 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
26 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
... ... @@ -73,6 +74,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
73 74 private final TbQueueAdmin jsExecutorAdmin;
74 75 private final TbQueueAdmin transportApiAdmin;
75 76 private final TbQueueAdmin notificationAdmin;
  77 + private final TbQueueAdmin fwUpdatesAdmin;
76 78
77 79 public KafkaMonolithQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings,
78 80 TbServiceInfoProvider serviceInfoProvider,
... ... @@ -98,6 +100,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
98 100 this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs());
99 101 this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs());
100 102 this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs());
  103 + this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs());
101 104 }
102 105
103 106 @Override
... ... @@ -274,6 +277,29 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
274 277 }
275 278
276 279 @Override
  280 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  281 + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder();
  282 + consumerBuilder.settings(kafkaSettings);
  283 + consumerBuilder.topic(coreSettings.getFirmwareTopic());
  284 + consumerBuilder.clientId("monolith-fw-consumer-" + serviceInfoProvider.getServiceId());
  285 + consumerBuilder.groupId("monolith-fw-consumer");
  286 + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  287 + consumerBuilder.admin(fwUpdatesAdmin);
  288 + consumerBuilder.statsService(consumerStatsService);
  289 + return consumerBuilder.build();
  290 + }
  291 +
  292 + @Override
  293 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  294 + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
  295 + requestBuilder.settings(kafkaSettings);
  296 + requestBuilder.clientId("monolith-fw-producer-" + serviceInfoProvider.getServiceId());
  297 + requestBuilder.defaultTopic(coreSettings.getFirmwareTopic());
  298 + requestBuilder.admin(fwUpdatesAdmin);
  299 + return requestBuilder.build();
  300 + }
  301 +
  302 + @Override
277 303 public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
278 304 TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
279 305 requestBuilder.settings(kafkaSettings);
... ...
... ... @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
23 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
24 24 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
25 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  26 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
26 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
... ... @@ -70,6 +71,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
70 71 private final TbQueueAdmin jsExecutorAdmin;
71 72 private final TbQueueAdmin transportApiAdmin;
72 73 private final TbQueueAdmin notificationAdmin;
  74 + private final TbQueueAdmin fwUpdatesAdmin;
73 75
74 76 public KafkaTbCoreQueueFactory(PartitionService partitionService, TbKafkaSettings kafkaSettings,
75 77 TbServiceInfoProvider serviceInfoProvider,
... ... @@ -93,6 +95,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
93 95 this.jsExecutorAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorConfigs());
94 96 this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs());
95 97 this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs());
  98 + this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs());
96 99 }
97 100
98 101 @Override
... ... @@ -242,6 +245,29 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
242 245 }
243 246
244 247 @Override
  248 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  249 + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder();
  250 + consumerBuilder.settings(kafkaSettings);
  251 + consumerBuilder.topic(coreSettings.getFirmwareTopic());
  252 + consumerBuilder.clientId("tb-core-fw-consumer-" + serviceInfoProvider.getServiceId());
  253 + consumerBuilder.groupId("tb-core-fw-consumer");
  254 + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  255 + consumerBuilder.admin(fwUpdatesAdmin);
  256 + consumerBuilder.statsService(consumerStatsService);
  257 + return consumerBuilder.build();
  258 + }
  259 +
  260 + @Override
  261 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  262 + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
  263 + requestBuilder.settings(kafkaSettings);
  264 + requestBuilder.clientId("tb-core-fw-producer-" + serviceInfoProvider.getServiceId());
  265 + requestBuilder.defaultTopic(coreSettings.getFirmwareTopic());
  266 + requestBuilder.admin(fwUpdatesAdmin);
  267 + return requestBuilder.build();
  268 + }
  269 +
  270 + @Override
245 271 public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
246 272 TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder();
247 273 requestBuilder.settings(kafkaSettings);
... ...
... ... @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component;
22 22 import org.thingsboard.server.common.msg.queue.ServiceType;
23 23 import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
24 24 import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
  25 +import org.thingsboard.server.gen.transport.TransportProtos.*;
25 26 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
26 27 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
27 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
... ... @@ -191,6 +192,17 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
191 192 }
192 193
193 194 @Override
  195 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  196 + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getFirmwareTopic(),
  197 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  198 + }
  199 +
  200 + @Override
  201 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  202 + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getFirmwareTopic());
  203 + }
  204 +
  205 + @Override
194 206 public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
195 207 return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
196 208 }
... ...
... ... @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
23 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
24 24 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
25 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  26 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
26 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
... ... @@ -165,6 +166,17 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
165 166 }
166 167
167 168 @Override
  169 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  170 + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getFirmwareTopic(),
  171 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  172 + }
  173 +
  174 + @Override
  175 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  176 + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getFirmwareTopic());
  177 + }
  178 +
  179 + @Override
168 180 public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
169 181 return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic());
170 182 }
... ...
... ... @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
24 24 import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
25 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
26 26 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  27 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
27 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
28 29 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
29 30 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
... ... @@ -189,6 +190,17 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
189 190 }
190 191
191 192 @Override
  193 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  194 + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getFirmwareTopic(),
  195 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  196 + }
  197 +
  198 + @Override
  199 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  200 + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getFirmwareTopic());
  201 + }
  202 +
  203 + @Override
192 204 public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
193 205 return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic());
194 206 }
... ...
... ... @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
23 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
24 24 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
25 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  26 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
26 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
... ... @@ -171,6 +172,17 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
171 172 }
172 173
173 174 @Override
  175 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  176 + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getFirmwareTopic(),
  177 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  178 + }
  179 +
  180 + @Override
  181 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  182 + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getFirmwareTopic());
  183 + }
  184 +
  185 + @Override
174 186 public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
175 187 return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic());
176 188 }
... ...
... ... @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
23 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
24 24 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
25 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  26 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
26 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
... ... @@ -188,6 +189,17 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
188 189 }
189 190
190 191 @Override
  192 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  193 + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getFirmwareTopic(),
  194 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  195 + }
  196 +
  197 + @Override
  198 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  199 + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getFirmwareTopic());
  200 + }
  201 +
  202 + @Override
191 203 public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
192 204 return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic());
193 205 }
... ...
... ... @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
23 23 import org.thingsboard.server.gen.js.JsInvokeProtos;
24 24 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
25 25 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
  26 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
26 27 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
27 28 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
28 29 import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
... ... @@ -171,6 +172,17 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
171 172 }
172 173
173 174 @Override
  175 + public TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer() {
  176 + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getFirmwareTopic(),
  177 + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToFirmwareStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
  178 + }
  179 +
  180 + @Override
  181 + public TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer() {
  182 + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getFirmwareTopic());
  183 + }
  184 +
  185 + @Override
174 186 public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() {
175 187 return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic());
176 188 }
... ...
... ... @@ -16,6 +16,7 @@
16 16 package org.thingsboard.server.queue.provider;
17 17
18 18 import org.thingsboard.server.gen.js.JsInvokeProtos;
  19 +import org.thingsboard.server.gen.transport.TransportProtos.ToFirmwareStateServiceMsg;
19 20 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
20 21 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
21 22 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
... ... @@ -86,6 +87,20 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory {
86 87 TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgConsumer();
87 88
88 89 /**
  90 + * Used to consume messages about firmware update notifications by TB Core Service
  91 + *
  92 + * @return
  93 + */
  94 + TbQueueConsumer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgConsumer();
  95 +
  96 + /**
  97 + * Used to consume messages about firmware update notifications by TB Core Service
  98 + *
  99 + * @return
  100 + */
  101 + TbQueueProducer<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> createToFirmwareStateServiceMsgProducer();
  102 +
  103 + /**
89 104 * Used to consume high priority messages by TB Core Service
90 105 *
91 106 * @return
... ...
... ... @@ -26,6 +26,9 @@ public class TbQueueCoreSettings {
26 26 @Value("${queue.core.topic}")
27 27 private String topic;
28 28
  29 + @Value("${queue.core.firmware.topic:tb_firmware}")
  30 + private String firmwareTopic;
  31 +
29 32 @Value("${queue.core.usage-stats-topic:tb_usage_stats}")
30 33 private String usageStatsTopic;
31 34
... ...
... ... @@ -15,10 +15,14 @@
15 15 */
16 16 package org.thingsboard.server.queue.usagestats;
17 17
  18 +import lombok.Data;
18 19 import lombok.extern.slf4j.Slf4j;
19 20 import org.springframework.beans.factory.annotation.Value;
20 21 import org.springframework.stereotype.Component;
21 22 import org.thingsboard.server.common.data.ApiUsageRecordKey;
  23 +import org.thingsboard.server.common.data.EntityType;
  24 +import org.thingsboard.server.common.data.id.CustomerId;
  25 +import org.thingsboard.server.common.data.id.EntityId;
22 26 import org.thingsboard.server.common.data.id.TenantId;
23 27 import org.thingsboard.server.common.msg.queue.ServiceType;
24 28 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
... ... @@ -31,6 +35,8 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
31 35 import org.thingsboard.server.queue.scheduler.SchedulerComponent;
32 36
33 37 import javax.annotation.PostConstruct;
  38 +import java.util.EnumMap;
  39 +import java.util.Optional;
34 40 import java.util.Random;
35 41 import java.util.UUID;
36 42 import java.util.concurrent.ConcurrentHashMap;
... ... @@ -44,11 +50,13 @@ public class DefaultTbApiUsageClient implements TbApiUsageClient {
44 50
45 51 @Value("${usage.stats.report.enabled:true}")
46 52 private boolean enabled;
  53 + @Value("${usage.stats.report.enabled_per_customer:false}")
  54 + private boolean enabledPerCustomer;
47 55 @Value("${usage.stats.report.interval:10}")
48 56 private int interval;
49 57
50   - @SuppressWarnings("unchecked")
51   - private final ConcurrentMap<TenantId, AtomicLong>[] values = new ConcurrentMap[ApiUsageRecordKey.values().length];
  58 + private final EnumMap<ApiUsageRecordKey, ConcurrentMap<OwnerId, AtomicLong>> stats = new EnumMap<>(ApiUsageRecordKey.class);
  59 +
52 60 private final PartitionService partitionService;
53 61 private final SchedulerComponent scheduler;
54 62 private final TbQueueProducerProvider producerProvider;
... ... @@ -65,55 +73,93 @@ public class DefaultTbApiUsageClient implements TbApiUsageClient {
65 73 if (enabled) {
66 74 msgProducer = this.producerProvider.getTbUsageStatsMsgProducer();
67 75 for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
68   - values[key.ordinal()] = new ConcurrentHashMap<>();
  76 + stats.put(key, new ConcurrentHashMap<>());
69 77 }
70   - scheduler.scheduleWithFixedDelay(this::reportStats, new Random().nextInt(interval), interval, TimeUnit.SECONDS);
  78 + scheduler.scheduleWithFixedDelay(() -> {
  79 + try {
  80 + reportStats();
  81 + } catch (Exception e) {
  82 + log.warn("Failed to report statistics: ", e);
  83 + }
  84 + }, new Random().nextInt(interval), interval, TimeUnit.SECONDS);
71 85 }
72 86 }
73 87
74 88 private void reportStats() {
75   - try {
76   - ConcurrentMap<TenantId, ToUsageStatsServiceMsg.Builder> report = new ConcurrentHashMap<>();
  89 + ConcurrentMap<OwnerId, ToUsageStatsServiceMsg.Builder> report = new ConcurrentHashMap<>();
77 90
78   - for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
79   - values[key.ordinal()].forEach(((tenantId, atomicLong) -> {
80   - long value = atomicLong.getAndSet(0);
81   - if (value > 0) {
82   - ToUsageStatsServiceMsg.Builder msgBuilder = report.computeIfAbsent(tenantId, id -> {
83   - ToUsageStatsServiceMsg.Builder msg = ToUsageStatsServiceMsg.newBuilder();
84   - msg.setTenantIdMSB(tenantId.getId().getMostSignificantBits());
85   - msg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits());
86   - return msg;
87   - });
88   - msgBuilder.addValues(UsageStatsKVProto.newBuilder().setKey(key.name()).setValue(value).build());
  91 + for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
  92 + ConcurrentMap<OwnerId, AtomicLong> statsForKey = stats.get(key);
  93 + statsForKey.forEach((ownerId, statsValue) -> {
  94 + long value = statsValue.get();
  95 + if (value == 0) return;
  96 +
  97 + ToUsageStatsServiceMsg.Builder statsMsgBuilder = report.computeIfAbsent(ownerId, id -> {
  98 + ToUsageStatsServiceMsg.Builder newStatsMsgBuilder = ToUsageStatsServiceMsg.newBuilder();
  99 +
  100 + TenantId tenantId = ownerId.getTenantId();
  101 + newStatsMsgBuilder.setTenantIdMSB(tenantId.getId().getMostSignificantBits());
  102 + newStatsMsgBuilder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits());
  103 +
  104 + EntityId entityId = ownerId.getEntityId();
  105 + if (entityId != null && entityId.getEntityType() == EntityType.CUSTOMER) {
  106 + newStatsMsgBuilder.setCustomerIdMSB(entityId.getId().getMostSignificantBits());
  107 + newStatsMsgBuilder.setCustomerIdLSB(entityId.getId().getLeastSignificantBits());
89 108 }
90   - }));
91   - }
92 109
93   - report.forEach(((tenantId, builder) -> {
94   - //TODO: figure out how to minimize messages into the queue. Maybe group by 100s of messages?
95   - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).newByTopic(msgProducer.getDefaultTopic());
96   - msgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), builder.build()), null);
97   - }));
98   - if (!report.isEmpty()) {
99   - log.info("Report statistics for: {} tenants", report.size());
100   - }
101   - } catch (Exception e) {
102   - log.warn("Failed to report statistics: ", e);
  110 + return newStatsMsgBuilder;
  111 + });
  112 +
  113 + statsMsgBuilder.addValues(UsageStatsKVProto.newBuilder().setKey(key.name()).setValue(value).build());
  114 + });
  115 + statsForKey.clear();
  116 + }
  117 +
  118 + report.forEach(((ownerId, statsMsg) -> {
  119 + //TODO: figure out how to minimize messages into the queue. Maybe group by 100s of messages?
  120 +
  121 + TenantId tenantId = ownerId.getTenantId();
  122 + EntityId entityId = Optional.ofNullable(ownerId.getEntityId()).orElse(tenantId);
  123 + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId).newByTopic(msgProducer.getDefaultTopic());
  124 + msgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), statsMsg.build()), null);
  125 + }));
  126 +
  127 + if (!report.isEmpty()) {
  128 + log.info("Reporting API usage statistics for {} tenants and customers", report.size());
103 129 }
104 130 }
105 131
106 132 @Override
107   - public void report(TenantId tenantId, ApiUsageRecordKey key, long value) {
  133 + public void report(TenantId tenantId, CustomerId customerId, ApiUsageRecordKey key, long value) {
108 134 if (enabled) {
109   - ConcurrentMap<TenantId, AtomicLong> map = values[key.ordinal()];
110   - AtomicLong atomicValue = map.computeIfAbsent(tenantId, id -> new AtomicLong());
111   - atomicValue.addAndGet(value);
  135 + ConcurrentMap<OwnerId, AtomicLong> statsForKey = stats.get(key);
  136 +
  137 + statsForKey.computeIfAbsent(new OwnerId(tenantId), id -> new AtomicLong()).addAndGet(value);
  138 + statsForKey.computeIfAbsent(new OwnerId(TenantId.SYS_TENANT_ID), id -> new AtomicLong()).addAndGet(value);
  139 +
  140 + if (enabledPerCustomer && customerId != null && !customerId.isNullUid()) {
  141 + statsForKey.computeIfAbsent(new OwnerId(tenantId, customerId), id -> new AtomicLong()).addAndGet(value);
  142 + }
112 143 }
113 144 }
114 145
115 146 @Override
116   - public void report(TenantId tenantId, ApiUsageRecordKey key) {
117   - report(tenantId, key, 1L);
  147 + public void report(TenantId tenantId, CustomerId customerId, ApiUsageRecordKey key) {
  148 + report(tenantId, customerId, key, 1);
  149 + }
  150 +
  151 + @Data
  152 + private static class OwnerId {
  153 + private TenantId tenantId;
  154 + private EntityId entityId;
  155 +
  156 + public OwnerId(TenantId tenantId) {
  157 + this.tenantId = tenantId;
  158 + }
  159 +
  160 + public OwnerId(TenantId tenantId, EntityId entityId) {
  161 + this.tenantId = tenantId;
  162 + this.entityId = entityId;
  163 + }
118 164 }
119 165 }
... ...
... ... @@ -16,12 +16,13 @@
16 16 package org.thingsboard.server.queue.usagestats;
17 17
18 18 import org.thingsboard.server.common.data.ApiUsageRecordKey;
  19 +import org.thingsboard.server.common.data.id.CustomerId;
19 20 import org.thingsboard.server.common.data.id.TenantId;
20 21
21 22 public interface TbApiUsageClient {
22 23
23   - void report(TenantId tenantId, ApiUsageRecordKey key, long value);
  24 + void report(TenantId tenantId, CustomerId customerId, ApiUsageRecordKey key, long value);
24 25
25   - void report(TenantId tenantId, ApiUsageRecordKey key);
  26 + void report(TenantId tenantId, CustomerId customerId, ApiUsageRecordKey key);
26 27
27 28 }
... ...
... ... @@ -54,6 +54,8 @@ message SessionInfoProto {
54 54 int64 gwSessionIdLSB = 11;
55 55 int64 deviceProfileIdMSB = 12;
56 56 int64 deviceProfileIdLSB = 13;
  57 + int64 customerIdMSB = 14;
  58 + int64 customerIdLSB = 15;
57 59 }
58 60
59 61 enum SessionEvent {
... ... @@ -111,6 +113,8 @@ message DeviceInfoProto {
111 113 string additionalInfo = 7;
112 114 int64 deviceProfileIdMSB = 8;
113 115 int64 deviceProfileIdLSB = 9;
  116 + int64 customerIdMSB = 10;
  117 + int64 customerIdLSB = 11;
114 118 }
115 119
116 120 /**
... ... @@ -365,17 +369,35 @@ message ProvisionDeviceCredentialsMsg {
365 369 }
366 370
367 371 message ProvisionDeviceResponseMsg {
368   - ProvisionResponseStatus status = 1;
  372 + ResponseStatus status = 1;
369 373 CredentialsType credentialsType = 2;
370 374 string credentialsValue = 3;
371 375 }
372 376
373   -enum ProvisionResponseStatus {
  377 +enum ResponseStatus {
374 378 UNKNOWN = 0;
375 379 SUCCESS = 1;
376 380 NOT_FOUND = 2;
377 381 FAILURE = 3;
378 382 }
  383 +
  384 +message GetFirmwareRequestMsg {
  385 + int64 deviceIdMSB = 1;
  386 + int64 deviceIdLSB = 2;
  387 + int64 tenantIdMSB = 3;
  388 + int64 tenantIdLSB = 4;
  389 +}
  390 +
  391 +message GetFirmwareResponseMsg {
  392 + ResponseStatus responseStatus = 1;
  393 + int64 firmwareIdMSB = 2;
  394 + int64 firmwareIdLSB = 3;
  395 + string title = 4;
  396 + string version = 5;
  397 + string contentType = 6;
  398 + string fileName = 7;
  399 +}
  400 +
379 401 //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level.
380 402 message SubscriptionInfoProto {
381 403 int64 lastActivityTime = 1;
... ... @@ -596,9 +618,10 @@ message TransportApiRequestMsg {
596 618 ProvisionDeviceRequestMsg provisionDeviceRequestMsg = 7;
597 619 ValidateDeviceLwM2MCredentialsRequestMsg validateDeviceLwM2MCredentialsRequestMsg = 8;
598 620 GetResourceRequestMsg resourceRequestMsg = 9;
599   - GetSnmpDevicesRequestMsg snmpDevicesRequestMsg = 10;
600   - GetDeviceRequestMsg deviceRequestMsg = 11;
601   - GetDeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 12;
  621 + GetFirmwareRequestMsg firmwareRequestMsg = 10;
  622 + GetSnmpDevicesRequestMsg snmpDevicesRequestMsg = 11;
  623 + GetDeviceRequestMsg deviceRequestMsg = 12;
  624 + GetDeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 13;
602 625 }
603 626
604 627 /* Response from ThingsBoard Core Service to Transport Service */
... ... @@ -610,8 +633,9 @@ message TransportApiResponseMsg {
610 633 GetSnmpDevicesResponseMsg snmpDevicesResponseMsg = 5;
611 634 LwM2MResponseMsg lwM2MResponseMsg = 6;
612 635 GetResourceResponseMsg resourceResponseMsg = 7;
613   - GetDeviceResponseMsg deviceResponseMsg = 8;
614   - GetDeviceCredentialsResponseMsg deviceCredentialsResponseMsg = 9;
  636 + GetFirmwareResponseMsg firmwareResponseMsg = 8;
  637 + GetDeviceResponseMsg deviceResponseMsg = 9;
  638 + GetDeviceCredentialsResponseMsg deviceCredentialsResponseMsg = 10;
615 639 }
616 640
617 641 /* Messages that are handled by ThingsBoard Core Service */
... ... @@ -673,4 +697,16 @@ message ToUsageStatsServiceMsg {
673 697 int64 entityIdMSB = 3;
674 698 int64 entityIdLSB = 4;
675 699 repeated UsageStatsKVProto values = 5;
  700 + int64 customerIdMSB = 6;
  701 + int64 customerIdLSB = 7;
  702 +}
  703 +
  704 +message ToFirmwareStateServiceMsg {
  705 + int64 ts = 1;
  706 + int64 tenantIdMSB = 2;
  707 + int64 tenantIdLSB = 3;
  708 + int64 deviceIdMSB = 4;
  709 + int64 deviceIdLSB = 5;
  710 + int64 firmwareIdMSB = 6;
  711 + int64 firmwareIdLSB = 7;
676 712 }
... ...
... ... @@ -407,7 +407,7 @@ public class CoapTransportResource extends AbstractCoapTransportResource {
407 407 @Override
408 408 public void onSuccess(TransportProtos.ProvisionDeviceResponseMsg msg) {
409 409 CoAP.ResponseCode responseCode = CoAP.ResponseCode.CREATED;
410   - if (!msg.getStatus().equals(TransportProtos.ProvisionResponseStatus.SUCCESS)) {
  410 + if (!msg.getStatus().equals(TransportProtos.ResponseStatus.SUCCESS)) {
411 411 responseCode = CoAP.ResponseCode.BAD_REQUEST;
412 412 }
413 413 if (payloadType.equals(TransportPayloadType.JSON)) {
... ...