简柏林 / thingskit · Commits

Commit 38f6e314b1000c8c7d8a1b58fd3a2f2aa8ba1107

Authored by Viacheslav Klimov
1 parent 5dc3d075

Refactor

Inline Side-by-side
Showing 8 changed files with 199 additions and 108 deletions
... ... @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
26 26 import org.thingsboard.server.queue.util.TbCoreComponent;
27 27 import org.thingsboard.server.service.action.EntityActionService;
28 28 import org.thingsboard.server.service.importing.AbstractBulkImportService;
  29 +import org.thingsboard.server.service.importing.BulkImportColumnType;
29 30 import org.thingsboard.server.service.importing.BulkImportRequest;
30 31 import org.thingsboard.server.service.importing.ImportedEntityInfo;
31 32 import org.thingsboard.server.service.security.AccessValidator;
... ... @@ -49,12 +50,12 @@ public class AssetBulkImportService extends AbstractBulkImportService<Asset> {
49 50 }
50 51
51 52 @Override
52   - protected ImportedEntityInfo<Asset> saveEntity(BulkImportRequest importRequest, Map<BulkImportRequest.ColumnMapping, String> entityData, SecurityUser user) {
  53 + protected ImportedEntityInfo<Asset> saveEntity(BulkImportRequest importRequest, Map<BulkImportColumnType, String> fields, SecurityUser user) {
53 54 ImportedEntityInfo<Asset> importedEntityInfo = new ImportedEntityInfo<>();
54 55
55 56 Asset asset = new Asset();
56 57 asset.setTenantId(user.getTenantId());
57   - setAssetFields(asset, entityData);
  58 + setAssetFields(asset, fields);
58 59
59 60 Asset existingAsset = assetService.findAssetByTenantIdAndName(user.getTenantId(), asset.getName());
60 61 if (existingAsset != null && importRequest.getMapping().getUpdate()) {
... ... @@ -69,10 +70,10 @@ public class AssetBulkImportService extends AbstractBulkImportService<Asset> {
69 70 return importedEntityInfo;
70 71 }
71 72
72   - private void setAssetFields(Asset asset, Map<BulkImportRequest.ColumnMapping, String> data) {
  73 + private void setAssetFields(Asset asset, Map<BulkImportColumnType, String> fields) {
73 74 ObjectNode additionalInfo = (ObjectNode) Optional.ofNullable(asset.getAdditionalInfo()).orElseGet(JacksonUtil::newObjectNode);
74   - data.forEach((columnMapping, value) -> {
75   - switch (columnMapping.getType()) {
  75 + fields.forEach((columnType, value) -> {
  76 + switch (columnType) {
76 77 case NAME:
77 78 asset.setName(value);
78 79 break;
... ...
... ... @@ -58,7 +58,6 @@ import java.util.EnumSet;
58 58 import java.util.Map;
59 59 import java.util.Optional;
60 60 import java.util.Set;
61   -import java.util.stream.Collectors;
62 61 import java.util.stream.Stream;
63 62
64 63 @Service
... ... @@ -80,12 +79,12 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
80 79 }
81 80
82 81 @Override
83   - protected ImportedEntityInfo<Device> saveEntity(BulkImportRequest importRequest, Map<BulkImportRequest.ColumnMapping, String> entityData, SecurityUser user) {
  82 + protected ImportedEntityInfo<Device> saveEntity(BulkImportRequest importRequest, Map<BulkImportColumnType, String> fields, SecurityUser user) {
84 83 ImportedEntityInfo<Device> importedEntityInfo = new ImportedEntityInfo<>();
85 84
86 85 Device device = new Device();
87 86 device.setTenantId(user.getTenantId());
88   - setDeviceFields(device, entityData);
  87 + setDeviceFields(device, fields);
89 88
90 89 Device existingDevice = deviceService.findDeviceByTenantIdAndName(user.getTenantId(), device.getName());
91 90 if (existingDevice != null && importRequest.getMapping().getUpdate()) {
... ... @@ -95,7 +94,7 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
95 94 device = existingDevice;
96 95 }
97 96
98   - DeviceCredentials deviceCredentials = createDeviceCredentials(entityData);
  97 + DeviceCredentials deviceCredentials = createDeviceCredentials(fields);
99 98 if (deviceCredentials.getCredentialsType() != null) {
100 99 if (deviceCredentials.getCredentialsType() == DeviceCredentialsType.LWM2M_CREDENTIALS) {
101 100 setUpLwM2mDeviceProfile(user.getTenantId(), device);
... ... @@ -121,10 +120,10 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
121 120 return importedEntityInfo;
122 121 }
123 122
124   - private void setDeviceFields(Device device, Map<BulkImportRequest.ColumnMapping, String> data) {
  123 + private void setDeviceFields(Device device, Map<BulkImportColumnType, String> fields) {
125 124 ObjectNode additionalInfo = (ObjectNode) Optional.ofNullable(device.getAdditionalInfo()).orElseGet(JacksonUtil::newObjectNode);
126   - data.forEach((columnMapping, value) -> {
127   - switch (columnMapping.getType()) {
  125 + fields.forEach((columnType, value) -> {
  126 + switch (columnType) {
128 127 case NAME:
129 128 device.setName(value);
130 129 break;
... ... @@ -146,25 +145,25 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
146 145 }
147 146
148 147 @SneakyThrows
149   - private DeviceCredentials createDeviceCredentials(Map<BulkImportRequest.ColumnMapping, String> data) {
150   - Set<BulkImportColumnType> columns = data.keySet().stream().map(BulkImportRequest.ColumnMapping::getType).collect(Collectors.toSet());
  148 + private DeviceCredentials createDeviceCredentials(Map<BulkImportColumnType, String> fields) {
  149 + Set<BulkImportColumnType> columns = fields.keySet();
151 150
152 151 DeviceCredentials credentials = new DeviceCredentials();
153 152
154 153 if (columns.contains(BulkImportColumnType.ACCESS_TOKEN)) {
155 154 credentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN);
156   - credentials.setCredentialsId(getByColumnType(BulkImportColumnType.ACCESS_TOKEN, data));
  155 + credentials.setCredentialsId(fields.get(BulkImportColumnType.ACCESS_TOKEN));
157 156 } else if (CollectionUtils.containsAny(columns, EnumSet.of(BulkImportColumnType.MQTT_CLIENT_ID, BulkImportColumnType.MQTT_USER_NAME, BulkImportColumnType.MQTT_PASSWORD))) {
158 157 credentials.setCredentialsType(DeviceCredentialsType.MQTT_BASIC);
159 158
160 159 BasicMqttCredentials basicMqttCredentials = new BasicMqttCredentials();
161   - basicMqttCredentials.setClientId(getByColumnType(BulkImportColumnType.MQTT_CLIENT_ID, data));
162   - basicMqttCredentials.setUserName(getByColumnType(BulkImportColumnType.MQTT_USER_NAME, data));
163   - basicMqttCredentials.setPassword(getByColumnType(BulkImportColumnType.MQTT_PASSWORD, data));
  160 + basicMqttCredentials.setClientId(fields.get(BulkImportColumnType.MQTT_CLIENT_ID));
  161 + basicMqttCredentials.setUserName(fields.get(BulkImportColumnType.MQTT_USER_NAME));
  162 + basicMqttCredentials.setPassword(fields.get(BulkImportColumnType.MQTT_PASSWORD));
164 163 credentials.setCredentialsValue(JacksonUtil.toString(basicMqttCredentials));
165 164 } else if (columns.contains(BulkImportColumnType.X509)) {
166 165 credentials.setCredentialsType(DeviceCredentialsType.X509_CERTIFICATE);
167   - credentials.setCredentialsValue(getByColumnType(BulkImportColumnType.X509, data));
  166 + credentials.setCredentialsValue(fields.get(BulkImportColumnType.X509));
168 167 } else if (columns.contains(BulkImportColumnType.LWM2M_CLIENT_ENDPOINT)) {
169 168 credentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS);
170 169 ObjectNode lwm2mCredentials = JacksonUtil.newObjectNode();
... ... @@ -173,7 +172,7 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
173 172 Stream.of(BulkImportColumnType.LWM2M_CLIENT_ENDPOINT, BulkImportColumnType.LWM2M_CLIENT_SECURITY_CONFIG_MODE,
174 173 BulkImportColumnType.LWM2M_CLIENT_IDENTITY, BulkImportColumnType.LWM2M_CLIENT_KEY, BulkImportColumnType.LWM2M_CLIENT_CERT)
175 174 .forEach(lwm2mClientProperty -> {
176   - String value = getByColumnType(lwm2mClientProperty, data);
  175 + String value = fields.get(lwm2mClientProperty);
177 176 if (value != null) {
178 177 client.set(lwm2mClientProperty.getKey(), new TextNode(value));
179 178 }
... ... @@ -187,7 +186,7 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
187 186 Stream.of(BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_SECURITY_MODE, BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_PUBLIC_KEY_OR_ID,
188 187 BulkImportColumnType.LWM2M_BOOTSTRAP_SERVER_SECRET_KEY)
189 188 .forEach(lwm2mBootstrapServerProperty -> {
190   - String value = getByColumnType(lwm2mBootstrapServerProperty, data);
  189 + String value = fields.get(lwm2mBootstrapServerProperty);
191 190 if (value != null) {
192 191 bootstrapServer.set(lwm2mBootstrapServerProperty.getKey(), new TextNode(value));
193 192 }
... ... @@ -197,7 +196,7 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
197 196 Stream.of(BulkImportColumnType.LWM2M_SERVER_SECURITY_MODE, BulkImportColumnType.LWM2M_SERVER_CLIENT_PUBLIC_KEY_OR_ID,
198 197 BulkImportColumnType.LWM2M_SERVER_CLIENT_SECRET_KEY)
199 198 .forEach(lwm2mServerProperty -> {
200   - String value = getByColumnType(lwm2mServerProperty, data);
  199 + String value = fields.get(lwm2mServerProperty);
201 200 if (value != null) {
202 201 lwm2mServer.set(lwm2mServerProperty.getKey(), new TextNode(value));
203 202 }
... ...
... ... @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
26 26 import org.thingsboard.server.queue.util.TbCoreComponent;
27 27 import org.thingsboard.server.service.action.EntityActionService;
28 28 import org.thingsboard.server.service.importing.AbstractBulkImportService;
  29 +import org.thingsboard.server.service.importing.BulkImportColumnType;
29 30 import org.thingsboard.server.service.importing.BulkImportRequest;
30 31 import org.thingsboard.server.service.importing.ImportedEntityInfo;
31 32 import org.thingsboard.server.service.security.AccessValidator;
... ... @@ -49,12 +50,12 @@ public class EdgeBulkImportService extends AbstractBulkImportService<Edge> {
49 50 }
50 51
51 52 @Override
52   - protected ImportedEntityInfo<Edge> saveEntity(BulkImportRequest importRequest, Map<BulkImportRequest.ColumnMapping, String> entityData, SecurityUser user) {
  53 + protected ImportedEntityInfo<Edge> saveEntity(BulkImportRequest importRequest, Map<BulkImportColumnType, String> fields, SecurityUser user) {
53 54 ImportedEntityInfo<Edge> importedEntityInfo = new ImportedEntityInfo<>();
54 55
55 56 Edge edge = new Edge();
56 57 edge.setTenantId(user.getTenantId());
57   - setEdgeFields(edge, entityData);
  58 + setEdgeFields(edge, fields);
58 59
59 60 Edge existingEdge = edgeService.findEdgeByTenantIdAndName(user.getTenantId(), edge.getName());
60 61 if (existingEdge != null && importRequest.getMapping().getUpdate()) {
... ... @@ -69,10 +70,10 @@ public class EdgeBulkImportService extends AbstractBulkImportService<Edge> {
69 70 return importedEntityInfo;
70 71 }
71 72
72   - private void setEdgeFields(Edge edge, Map<BulkImportRequest.ColumnMapping, String> data) {
  73 + private void setEdgeFields(Edge edge, Map<BulkImportColumnType, String> fields) {
73 74 ObjectNode additionalInfo = (ObjectNode) Optional.ofNullable(edge.getAdditionalInfo()).orElseGet(JacksonUtil::newObjectNode);
74   - data.forEach((columnMapping, value) -> {
75   - switch (columnMapping.getType()) {
  75 + fields.forEach((columnType, value) -> {
  76 + switch (columnType) {
76 77 case NAME:
77 78 edge.setName(value);
78 79 break;
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.service.importing;
18 18 import com.google.common.util.concurrent.FutureCallback;
19 19 import com.google.gson.JsonObject;
20 20 import com.google.gson.JsonPrimitive;
  21 +import lombok.Data;
21 22 import lombok.RequiredArgsConstructor;
22 23 import lombok.SneakyThrows;
23 24 import org.apache.commons.lang3.StringUtils;
... ... @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.id.EntityId;
29 30 import org.thingsboard.server.common.data.id.UUIDBased;
30 31 import org.thingsboard.server.common.data.kv.AttributeKvEntry;
31 32 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
  33 +import org.thingsboard.server.common.data.kv.DataType;
32 34 import org.thingsboard.server.common.data.kv.TsKvEntry;
33 35 import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
34 36 import org.thingsboard.server.common.transport.adaptor.JsonConverter;
... ... @@ -42,9 +44,12 @@ import org.thingsboard.server.service.security.permission.AccessControlService;
42 44 import org.thingsboard.server.service.security.permission.Operation;
43 45 import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
44 46 import org.thingsboard.server.utils.CsvUtils;
  47 +import org.thingsboard.server.utils.TypeCastUtil;
45 48
46 49 import javax.annotation.Nullable;
47 50 import java.util.ArrayList;
  51 +import java.util.Arrays;
  52 +import java.util.HashMap;
48 53 import java.util.List;
49 54 import java.util.Map;
50 55 import java.util.concurrent.TimeUnit;
... ... @@ -73,12 +78,12 @@ public abstract class AbstractBulkImportService<E extends BaseData<? extends Ent
73 78 parseData(request).forEach(entityData -> {
74 79 i.incrementAndGet();
75 80 try {
76   - ImportedEntityInfo<E> importedEntityInfo = saveEntity(request, entityData, user);
  81 + ImportedEntityInfo<E> importedEntityInfo = saveEntity(request, entityData.getFields(), user);
77 82 onEntityImported.accept(importedEntityInfo);
78 83
79 84 E entity = importedEntityInfo.getEntity();
80 85
81   - saveKvs(user, entity, entityData);
  86 + saveKvs(user, entity, entityData.getKvs());
82 87
83 88 if (importedEntityInfo.getRelatedError() != null) {
84 89 throw new RuntimeException(importedEntityInfo.getRelatedError());
... ... @@ -98,20 +103,21 @@ public abstract class AbstractBulkImportService<E extends BaseData<? extends Ent
98 103 return result;
99 104 }
100 105
101   - protected abstract ImportedEntityInfo<E> saveEntity(BulkImportRequest importRequest, Map<ColumnMapping, String> entityData, SecurityUser user);
  106 + protected abstract ImportedEntityInfo<E> saveEntity(BulkImportRequest importRequest, Map<BulkImportColumnType, String> fields, SecurityUser user);
102 107
103 108 /*
104 109 * Attributes' values are firstly added to JsonObject in order to then make some type cast,
105 110 * because we get all values as strings from CSV
106 111 * */
107   - private void saveKvs(SecurityUser user, E entity, Map<ColumnMapping, String> data) {
108   - Stream.of(BulkImportColumnType.SHARED_ATTRIBUTE, BulkImportColumnType.SERVER_ATTRIBUTE, BulkImportColumnType.TIMESERIES)
  112 + private void saveKvs(SecurityUser user, E entity, Map<ColumnMapping, ParsedValue> data) {
  113 + Arrays.stream(BulkImportColumnType.values())
  114 + .filter(BulkImportColumnType::isKv)
109 115 .map(kvType -> {
110 116 JsonObject kvs = new JsonObject();
111 117 data.entrySet().stream()
112 118 .filter(dataEntry -> dataEntry.getKey().getType() == kvType &&
113 119 StringUtils.isNotEmpty(dataEntry.getKey().getKey()))
114   - .forEach(dataEntry -> kvs.add(dataEntry.getKey().getKey(), new JsonPrimitive(dataEntry.getValue())));
  120 + .forEach(dataEntry -> kvs.add(dataEntry.getKey().getKey(), dataEntry.getValue().toJsonPrimitive()));
115 121 return Map.entry(kvType, kvs);
116 122 })
117 123 .filter(kvsEntry -> kvsEntry.getValue().entrySet().size() > 0)
... ... @@ -127,7 +133,7 @@ public abstract class AbstractBulkImportService<E extends BaseData<? extends Ent
127 133
128 134 @SneakyThrows
129 135 private void saveTelemetry(SecurityUser user, E entity, Map.Entry<BulkImportColumnType, JsonObject> kvsEntry) {
130   - List<TsKvEntry> timeseries = JsonConverter.convertToTelemetry(kvsEntry.getValue(), System.currentTimeMillis(), false, true)
  136 + List<TsKvEntry> timeseries = JsonConverter.convertToTelemetry(kvsEntry.getValue(), System.currentTimeMillis())
131 137 .entrySet().stream()
132 138 .flatMap(entry -> entry.getValue().stream().map(kvEntry -> new BasicTsKvEntry(entry.getKey(), kvEntry)))
133 139 .collect(Collectors.toList());
... ... @@ -155,7 +161,7 @@ public abstract class AbstractBulkImportService<E extends BaseData<? extends Ent
155 161 @SneakyThrows
156 162 private void saveAttributes(SecurityUser user, E entity, Map.Entry<BulkImportColumnType, JsonObject> kvsEntry, BulkImportColumnType kvType) {
157 163 String scope = kvType.getKey();
158   - List<AttributeKvEntry> attributes = new ArrayList<>(JsonConverter.convertToAttributes(kvsEntry.getValue(), true));
  164 + List<AttributeKvEntry> attributes = new ArrayList<>(JsonConverter.convertToAttributes(kvsEntry.getValue()));
159 165
160 166 accessValidator.validateEntityAndCallback(user, Operation.WRITE_ATTRIBUTES, entity.getId(), (result, tenantId, entityId) -> {
161 167 tsSubscriptionService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback<>() {
... ... @@ -178,24 +184,62 @@ public abstract class AbstractBulkImportService<E extends BaseData<? extends Ent
178 184 });
179 185 }
180 186
181   - protected final String getByColumnType(BulkImportColumnType bulkImportColumnType, Map<ColumnMapping, String> data) {
182   - return data.entrySet().stream().filter(entry -> entry.getKey().getType() == bulkImportColumnType).findFirst().map(Map.Entry::getValue).orElse(null);
183   - }
184   -
185   - private List<Map<ColumnMapping, String>> parseData(BulkImportRequest request) throws Exception {
  187 + private List<EntityData> parseData(BulkImportRequest request) throws Exception {
186 188 List<List<String>> records = CsvUtils.parseCsv(request.getFile(), request.getMapping().getDelimiter());
187 189 if (request.getMapping().getHeader()) {
188 190 records.remove(0);
189 191 }
190 192
191 193 List<ColumnMapping> columnsMappings = request.getMapping().getColumns();
192   -
193 194 return records.stream()
194   - .map(record -> Stream.iterate(0, i -> i < record.size(), i -> i + 1)
195   - .map(i -> Map.entry(columnsMappings.get(i), record.get(i)))
196   - .filter(entry -> StringUtils.isNotEmpty(entry.getValue()))
197   - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))
  195 + .map(record -> {
  196 + EntityData entityData = new EntityData();
  197 + Stream.iterate(0, i -> i < record.size(), i -> i + 1)
  198 + .map(i -> Map.entry(columnsMappings.get(i), record.get(i)))
  199 + .filter(entry -> StringUtils.isNotEmpty(entry.getValue()))
  200 + .forEach(entry -> {
  201 + if (!entry.getKey().getType().isKv()) {
  202 + entityData.getFields().put(entry.getKey().getType(), entry.getValue());
  203 + } else {
  204 + Map.Entry<DataType, Object> castResult = TypeCastUtil.castValue(entry.getValue());
  205 + entityData.getKvs().put(entry.getKey(), new ParsedValue(castResult.getValue(), castResult.getKey()));
  206 + }
  207 + });
  208 + return entityData;
  209 + })
198 210 .collect(Collectors.toList());
199 211 }
200 212
  213 + @Data
  214 + protected static class EntityData {
  215 + private final Map<BulkImportColumnType, String> fields = new HashMap<>();
  216 + private final Map<ColumnMapping, ParsedValue> kvs = new HashMap<>();
  217 + }
  218 +
  219 + @Data
  220 + protected static class ParsedValue {
  221 + private final Object value;
  222 + private final DataType dataType;
  223 +
  224 + public JsonPrimitive toJsonPrimitive() {
  225 + switch (dataType) {
  226 + case STRING:
  227 + return new JsonPrimitive((String) value);
  228 + case LONG:
  229 + return new JsonPrimitive((Long) value);
  230 + case DOUBLE:
  231 + return new JsonPrimitive((Double) value);
  232 + case BOOLEAN:
  233 + return new JsonPrimitive((Boolean) value);
  234 + default:
  235 + return null;
  236 + }
  237 + }
  238 +
  239 + public String stringValue() {
  240 + return value.toString();
  241 + }
  242 +
  243 + }
  244 +
201 245 }
... ...
... ... @@ -18,13 +18,14 @@ package org.thingsboard.server.service.importing;
18 18 import lombok.Getter;
19 19 import org.thingsboard.server.common.data.DataConstants;
20 20
  21 +@Getter
21 22 public enum BulkImportColumnType {
22 23 NAME,
23 24 TYPE,
24 25 LABEL,
25   - SHARED_ATTRIBUTE(DataConstants.SHARED_SCOPE),
26   - SERVER_ATTRIBUTE(DataConstants.SERVER_SCOPE),
27   - TIMESERIES,
  26 + SHARED_ATTRIBUTE(DataConstants.SHARED_SCOPE, true),
  27 + SERVER_ATTRIBUTE(DataConstants.SERVER_SCOPE, true),
  28 + TIMESERIES(true),
28 29 ACCESS_TOKEN,
29 30 X509,
30 31 MQTT_CLIENT_ID,
... ... @@ -48,8 +49,8 @@ public enum BulkImportColumnType {
48 49 ROUTING_KEY,
49 50 SECRET;
50 51
51   - @Getter
52 52 private String key;
  53 + private boolean isKv = false;
53 54
54 55 BulkImportColumnType() {
55 56 }
... ... @@ -57,4 +58,13 @@ public enum BulkImportColumnType {
57 58 BulkImportColumnType(String key) {
58 59 this.key = key;
59 60 }
  61 +
  62 + BulkImportColumnType(boolean isKv) {
  63 + this.isKv = isKv;
  64 + }
  65 +
  66 + BulkImportColumnType(String key, boolean isKv) {
  67 + this.key = key;
  68 + this.isKv = isKv;
  69 + }
60 70 }
... ...
  1 +/**
  2 + * Copyright © 2016-2021 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.utils;
  17 +
  18 +import org.apache.commons.lang3.math.NumberUtils;
  19 +import org.thingsboard.server.common.data.kv.DataType;
  20 +
  21 +import java.math.BigDecimal;
  22 +import java.util.Map;
  23 +
  24 +public class TypeCastUtil {
  25 +
  26 + private TypeCastUtil() {}
  27 +
  28 + public static Map.Entry<DataType, Object> castValue(String value) {
  29 + if (isNumber(value)) {
  30 + String formattedValue = value.replace(',', '.');
  31 + try {
  32 + BigDecimal bd = new BigDecimal(formattedValue);
  33 + if (bd.stripTrailingZeros().scale() > 0 || isSimpleDouble(formattedValue)) {
  34 + if (bd.scale() <= 16) {
  35 + return Map.entry(DataType.DOUBLE, bd.doubleValue());
  36 + }
  37 + } else {
  38 + return Map.entry(DataType.LONG, bd.longValueExact());
  39 + }
  40 + } catch (RuntimeException ignored) {}
  41 + } else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) {
  42 + return Map.entry(DataType.BOOLEAN, Boolean.parseBoolean(value));
  43 + }
  44 + return Map.entry(DataType.STRING, value);
  45 + }
  46 +
  47 + private static boolean isNumber(String value) {
  48 + return NumberUtils.isParsable(value.replace(',', '.'));
  49 + }
  50 +
  51 + private static boolean isSimpleDouble(String valueAsString) {
  52 + return valueAsString.contains(".") && !valueAsString.contains("E") && !valueAsString.contains("e");
  53 + }
  54 +
  55 +}
... ...
... ... @@ -76,7 +76,7 @@ public class JsonConverter {
76 76
77 77 public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement, long ts) throws JsonSyntaxException {
78 78 PostTelemetryMsg.Builder builder = PostTelemetryMsg.newBuilder();
79   - convertToTelemetry(jsonElement, ts, null, builder, isTypeCastEnabled);
  79 + convertToTelemetry(jsonElement, ts, null, builder);
80 80 return builder.build();
81 81 }
82 82
... ... @@ -84,13 +84,13 @@ public class JsonConverter {
84 84 return convertToTelemetryProto(jsonElement, System.currentTimeMillis());
85 85 }
86 86
87   - private static void convertToTelemetry(JsonElement jsonElement, long systemTs, Map<Long, List<KvEntry>> result, PostTelemetryMsg.Builder builder, boolean typeCastEnabled) {
  87 + private static void convertToTelemetry(JsonElement jsonElement, long systemTs, Map<Long, List<KvEntry>> result, PostTelemetryMsg.Builder builder) {
88 88 if (jsonElement.isJsonObject()) {
89   - parseObject(systemTs, result, builder, jsonElement.getAsJsonObject(), typeCastEnabled);
  89 + parseObject(systemTs, result, builder, jsonElement.getAsJsonObject());
90 90 } else if (jsonElement.isJsonArray()) {
91 91 jsonElement.getAsJsonArray().forEach(je -> {
92 92 if (je.isJsonObject()) {
93   - parseObject(systemTs, result, builder, je.getAsJsonObject(), typeCastEnabled);
  93 + parseObject(systemTs, result, builder, je.getAsJsonObject());
94 94 } else {
95 95 throw new JsonSyntaxException(CAN_T_PARSE_VALUE + je);
96 96 }
... ... @@ -100,11 +100,11 @@ public class JsonConverter {
100 100 }
101 101 }
102 102
103   - private static void parseObject(long systemTs, Map<Long, List<KvEntry>> result, PostTelemetryMsg.Builder builder, JsonObject jo, boolean typeCastEnabled) {
  103 + private static void parseObject(long systemTs, Map<Long, List<KvEntry>> result, PostTelemetryMsg.Builder builder, JsonObject jo) {
104 104 if (result != null) {
105   - parseObject(result, systemTs, jo, typeCastEnabled);
  105 + parseObject(result, systemTs, jo);
106 106 } else {
107   - parseObject(builder, systemTs, jo, typeCastEnabled);
  107 + parseObject(builder, systemTs, jo);
108 108 }
109 109 }
110 110
... ... @@ -146,7 +146,7 @@ public class JsonConverter {
146 146 public static PostAttributeMsg convertToAttributesProto(JsonElement jsonObject) throws JsonSyntaxException {
147 147 if (jsonObject.isJsonObject()) {
148 148 PostAttributeMsg.Builder result = PostAttributeMsg.newBuilder();
149   - List<KeyValueProto> keyValueList = parseProtoValues(jsonObject.getAsJsonObject(), isTypeCastEnabled);
  149 + List<KeyValueProto> keyValueList = parseProtoValues(jsonObject.getAsJsonObject());
150 150 result.addAllKv(keyValueList);
151 151 return result.build();
152 152 } else {
... ... @@ -164,29 +164,29 @@ public class JsonConverter {
164 164 return result;
165 165 }
166 166
167   - private static void parseObject(PostTelemetryMsg.Builder builder, long systemTs, JsonObject jo, boolean typeCastEnabled) {
  167 + private static void parseObject(PostTelemetryMsg.Builder builder, long systemTs, JsonObject jo) {
168 168 if (jo.has("ts") && jo.has("values")) {
169   - parseWithTs(builder, jo, typeCastEnabled);
  169 + parseWithTs(builder, jo);
170 170 } else {
171   - parseWithoutTs(builder, systemTs, jo, typeCastEnabled);
  171 + parseWithoutTs(builder, systemTs, jo);
172 172 }
173 173 }
174 174
175   - private static void parseWithoutTs(PostTelemetryMsg.Builder request, long systemTs, JsonObject jo, boolean typeCastEnabled) {
  175 + private static void parseWithoutTs(PostTelemetryMsg.Builder request, long systemTs, JsonObject jo) {
176 176 TsKvListProto.Builder builder = TsKvListProto.newBuilder();
177 177 builder.setTs(systemTs);
178   - builder.addAllKv(parseProtoValues(jo, typeCastEnabled));
  178 + builder.addAllKv(parseProtoValues(jo));
179 179 request.addTsKvList(builder.build());
180 180 }
181 181
182   - private static void parseWithTs(PostTelemetryMsg.Builder request, JsonObject jo, boolean typeCastEnabled) {
  182 + private static void parseWithTs(PostTelemetryMsg.Builder request, JsonObject jo) {
183 183 TsKvListProto.Builder builder = TsKvListProto.newBuilder();
184 184 builder.setTs(jo.get("ts").getAsLong());
185   - builder.addAllKv(parseProtoValues(jo.get("values").getAsJsonObject(), typeCastEnabled));
  185 + builder.addAllKv(parseProtoValues(jo.get("values").getAsJsonObject()));
186 186 request.addTsKvList(builder.build());
187 187 }
188 188
189   - private static List<KeyValueProto> parseProtoValues(JsonObject valuesObject, boolean typeCastEnabled) {
  189 + private static List<KeyValueProto> parseProtoValues(JsonObject valuesObject) {
190 190 List<KeyValueProto> result = new ArrayList<>();
191 191 for (Entry<String, JsonElement> valueEntry : valuesObject.entrySet()) {
192 192 JsonElement element = valueEntry.getValue();
... ... @@ -197,9 +197,9 @@ public class JsonConverter {
197 197 String message = String.format("String value length [%d] for key [%s] is greater than maximum allowed [%d]", value.getAsString().length(), valueEntry.getKey(), maxStringValueLength);
198 198 throw new JsonSyntaxException(message);
199 199 }
200   - if (typeCastEnabled && isNumber(value)) {
  200 + if (isTypeCastEnabled && NumberUtils.isParsable(value.getAsString())) {
201 201 try {
202   - result.add(buildNumericKeyValueProto(value, valueEntry.getKey(), typeCastEnabled));
  202 + result.add(buildNumericKeyValueProto(value, valueEntry.getKey()));
203 203 } catch (RuntimeException th) {
204 204 result.add(KeyValueProto.newBuilder().setKey(valueEntry.getKey()).setType(KeyValueType.STRING_V)
205 205 .setStringV(value.getAsString()).build());
... ... @@ -212,7 +212,7 @@ public class JsonConverter {
212 212 result.add(KeyValueProto.newBuilder().setKey(valueEntry.getKey()).setType(KeyValueType.BOOLEAN_V)
213 213 .setBoolV(value.getAsBoolean()).build());
214 214 } else if (value.isNumber()) {
215   - result.add(buildNumericKeyValueProto(value, valueEntry.getKey(), typeCastEnabled));
  215 + result.add(buildNumericKeyValueProto(value, valueEntry.getKey()));
216 216 } else if (!value.isJsonNull()) {
217 217 throw new JsonSyntaxException(CAN_T_PARSE_VALUE + value);
218 218 }
... ... @@ -225,15 +225,15 @@ public class JsonConverter {
225 225 return result;
226 226 }
227 227
228   - private static KeyValueProto buildNumericKeyValueProto(JsonPrimitive value, String key, boolean typeCastEnabled) {
229   - String valueAsString = value.getAsString().replace(',', '.');
  228 + private static KeyValueProto buildNumericKeyValueProto(JsonPrimitive value, String key) {
  229 + String valueAsString = value.getAsString();
230 230 KeyValueProto.Builder builder = KeyValueProto.newBuilder().setKey(key);
231 231 var bd = new BigDecimal(valueAsString);
232 232 if (bd.stripTrailingZeros().scale() <= 0 && !isSimpleDouble(valueAsString)) {
233 233 try {
234 234 return builder.setType(KeyValueType.LONG_V).setLongV(bd.longValueExact()).build();
235 235 } catch (ArithmeticException e) {
236   - if (typeCastEnabled) {
  236 + if (isTypeCastEnabled) {
237 237 return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build();
238 238 } else {
239 239 throw new JsonSyntaxException("Big integer values are not supported!");
... ... @@ -242,7 +242,7 @@ public class JsonConverter {
242 242 } else {
243 243 if (bd.scale() <= 16) {
244 244 return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
245   - } else if (typeCastEnabled) {
  245 + } else if (isTypeCastEnabled) {
246 246 return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build();
247 247 } else {
248 248 throw new JsonSyntaxException("Big integer values are not supported!");
... ... @@ -260,15 +260,15 @@ public class JsonConverter {
260 260 return TransportProtos.ToServerRpcRequestMsg.newBuilder().setRequestId(requestId).setMethodName(object.get("method").getAsString()).setParams(GSON.toJson(object.get("params"))).build();
261 261 }
262 262
263   - private static void parseNumericValue(List<KvEntry> result, Entry<String, JsonElement> valueEntry, JsonPrimitive value, boolean typeCastEnabled) {
264   - String valueAsString = value.getAsString().replace(',', '.');
  263 + private static void parseNumericValue(List<KvEntry> result, Entry<String, JsonElement> valueEntry, JsonPrimitive value) {
  264 + String valueAsString = value.getAsString();
265 265 String key = valueEntry.getKey();
266 266 var bd = new BigDecimal(valueAsString);
267 267 if (bd.stripTrailingZeros().scale() <= 0 && !isSimpleDouble(valueAsString)) {
268 268 try {
269 269 result.add(new LongDataEntry(key, bd.longValueExact()));
270 270 } catch (ArithmeticException e) {
271   - if (typeCastEnabled) {
  271 + if (isTypeCastEnabled) {
272 272 result.add(new StringDataEntry(key, bd.toPlainString()));
273 273 } else {
274 274 throw new JsonSyntaxException("Big integer values are not supported!");
... ... @@ -277,7 +277,7 @@ public class JsonConverter {
277 277 } else {
278 278 if (bd.scale() <= 16) {
279 279 result.add(new DoubleDataEntry(key, bd.doubleValue()));
280   - } else if (typeCastEnabled) {
  280 + } else if (isTypeCastEnabled) {
281 281 result.add(new StringDataEntry(key, bd.toPlainString()));
282 282 } else {
283 283 throw new JsonSyntaxException("Big integer values are not supported!");
... ... @@ -487,17 +487,13 @@ public class JsonConverter {
487 487 }
488 488
489 489 public static Set<AttributeKvEntry> convertToAttributes(JsonElement element) {
490   - return convertToAttributes(element, isTypeCastEnabled);
491   - }
492   -
493   - public static Set<AttributeKvEntry> convertToAttributes(JsonElement element, boolean typeCastEnabled) {
494 490 Set<AttributeKvEntry> result = new HashSet<>();
495 491 long ts = System.currentTimeMillis();
496   - result.addAll(parseValues(element.getAsJsonObject(), typeCastEnabled).stream().map(kv -> new BaseAttributeKvEntry(kv, ts)).collect(Collectors.toList()));
  492 + result.addAll(parseValues(element.getAsJsonObject()).stream().map(kv -> new BaseAttributeKvEntry(kv, ts)).collect(Collectors.toList()));
497 493 return result;
498 494 }
499 495
500   - private static List<KvEntry> parseValues(JsonObject valuesObject, boolean typeCastEnabled) {
  496 + private static List<KvEntry> parseValues(JsonObject valuesObject) {
501 497 List<KvEntry> result = new ArrayList<>();
502 498 for (Entry<String, JsonElement> valueEntry : valuesObject.entrySet()) {
503 499 JsonElement element = valueEntry.getValue();
... ... @@ -508,9 +504,9 @@ public class JsonConverter {
508 504 String message = String.format("String value length [%d] for key [%s] is greater than maximum allowed [%d]", value.getAsString().length(), valueEntry.getKey(), maxStringValueLength);
509 505 throw new JsonSyntaxException(message);
510 506 }
511   - if (typeCastEnabled && isNumber(value)) {
  507 + if (isTypeCastEnabled && NumberUtils.isParsable(value.getAsString())) {
512 508 try {
513   - parseNumericValue(result, valueEntry, value, typeCastEnabled);
  509 + parseNumericValue(result, valueEntry, value);
514 510 } catch (RuntimeException th) {
515 511 result.add(new StringDataEntry(valueEntry.getKey(), value.getAsString()));
516 512 }
... ... @@ -520,7 +516,7 @@ public class JsonConverter {
520 516 } else if (value.isBoolean()) {
521 517 result.add(new BooleanDataEntry(valueEntry.getKey(), value.getAsBoolean()));
522 518 } else if (value.isNumber()) {
523   - parseNumericValue(result, valueEntry, value, typeCastEnabled);
  519 + parseNumericValue(result, valueEntry, value);
524 520 } else {
525 521 throw new JsonSyntaxException(CAN_T_PARSE_VALUE + value);
526 522 }
... ... @@ -545,35 +541,30 @@ public class JsonConverter {
545 541
546 542 public static Map<Long, List<KvEntry>> convertToTelemetry(JsonElement jsonElement, long systemTs, boolean sorted) throws
547 543 JsonSyntaxException {
548   - return convertToTelemetry(jsonElement, systemTs, sorted, isTypeCastEnabled);
549   - }
550   -
551   - public static Map<Long, List<KvEntry>> convertToTelemetry(JsonElement jsonElement, long systemTs, boolean sorted, boolean typeCastEnabled) throws
552   - JsonSyntaxException {
553 544 Map<Long, List<KvEntry>> result = sorted ? new TreeMap<>() : new HashMap<>();
554   - convertToTelemetry(jsonElement, systemTs, result, null, typeCastEnabled);
  545 + convertToTelemetry(jsonElement, systemTs, result, null);
555 546 return result;
556 547 }
557 548
558 549
559   - private static void parseObject(Map<Long, List<KvEntry>> result, long systemTs, JsonObject jo, boolean typeCastEnabled) {
  550 + private static void parseObject(Map<Long, List<KvEntry>> result, long systemTs, JsonObject jo) {
560 551 if (jo.has("ts") && jo.has("values")) {
561   - parseWithTs(result, jo, typeCastEnabled);
  552 + parseWithTs(result, jo);
562 553 } else {
563   - parseWithoutTs(result, systemTs, jo, typeCastEnabled);
  554 + parseWithoutTs(result, systemTs, jo);
564 555 }
565 556 }
566 557
567   - private static void parseWithoutTs(Map<Long, List<KvEntry>> result, long systemTs, JsonObject jo, boolean typeCastEnabled) {
568   - for (KvEntry entry : parseValues(jo, typeCastEnabled)) {
  558 + private static void parseWithoutTs(Map<Long, List<KvEntry>> result, long systemTs, JsonObject jo) {
  559 + for (KvEntry entry : parseValues(jo)) {
569 560 result.computeIfAbsent(systemTs, tmp -> new ArrayList<>()).add(entry);
570 561 }
571 562 }
572 563
573   - public static void parseWithTs(Map<Long, List<KvEntry>> result, JsonObject jo, boolean typeCastEnabled) {
  564 + public static void parseWithTs(Map<Long, List<KvEntry>> result, JsonObject jo) {
574 565 long ts = jo.get("ts").getAsLong();
575 566 JsonObject valuesObject = jo.get("values").getAsJsonObject();
576   - for (KvEntry entry : parseValues(valuesObject, typeCastEnabled)) {
  567 + for (KvEntry entry : parseValues(valuesObject)) {
577 568 result.computeIfAbsent(ts, tmp -> new ArrayList<>()).add(entry);
578 569 }
579 570 }
... ... @@ -643,10 +634,6 @@ public class JsonConverter {
643 634 }
644 635
645 636
646   - private static boolean isNumber(JsonPrimitive value) {
647   - return NumberUtils.isParsable(value.getAsString().replace(',', '.'));
648   - }
649   -
650 637 private static String getStrValue(JsonObject jo, String field, boolean requiredField) {
651 638 if (jo.has(field)) {
652 639 return jo.get(field).getAsString();
... ...
... ... @@ -66,12 +66,6 @@ public class JsonConverterTest {
66 66 }
67 67
68 68 @Test
69   - public void testParseAsDoubleWithCommaDecimalSeparatorAndTypeCast() {
70   - var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": \"-1,1\"}"), 0L);
71   - Assert.assertEquals(-1.1, result.get(0L).get(0).getDoubleValue().get(), 0.0);
72   - }
73   -
74   - @Test
75 69 public void testParseAsLong() {
76 70 var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 11}"), 0L);
77 71 Assert.assertEquals(11L, result.get(0L).get(0).getLongValue().get().longValue());
... ...