Merge with master

@@ -28,6 +28,7 @@ import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg;
 import org.thingsboard.server.actors.ActorSystemContext;
 import org.thingsboard.server.actors.TbActorCtx;
 import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor;
+import org.thingsboard.server.common.data.DataConstants;
 import org.thingsboard.server.common.data.Device;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -79,8 +80,6 @@ import java.util.UUID;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
-import static org.thingsboard.server.common.data.DataConstants.CLIENT_SCOPE;
-import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE;
 
 /**
  * @author Andrew Shvayka
@@ -279,17 +278,17 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
         ListenableFuture<List<AttributeKvEntry>> clientAttributesFuture;
         ListenableFuture<List<AttributeKvEntry>> sharedAttributesFuture;
         if (CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) {
-            clientAttributesFuture = findAllAttributesByScope(CLIENT_SCOPE);
-            sharedAttributesFuture = findAllAttributesByScope(SHARED_SCOPE);
+            clientAttributesFuture = findAllAttributesByScope(DataConstants.CLIENT_SCOPE);
+            sharedAttributesFuture = findAllAttributesByScope(DataConstants.SHARED_SCOPE);
         } else if (!CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && !CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) {
-            clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), CLIENT_SCOPE);
-            sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), SHARED_SCOPE);
+            clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), DataConstants.CLIENT_SCOPE);
+            sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), DataConstants.SHARED_SCOPE);
         } else if (CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && !CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) {
             clientAttributesFuture = Futures.immediateFuture(Collections.emptyList());
-            sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), SHARED_SCOPE);
+            sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), DataConstants.SHARED_SCOPE);
         } else {
             sharedAttributesFuture = Futures.immediateFuture(Collections.emptyList());
-            clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), CLIENT_SCOPE);
+            clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), DataConstants.CLIENT_SCOPE);
         }
         return Futures.allAsList(Arrays.asList(clientAttributesFuture, sharedAttributesFuture));
     }
@@ -316,7 +315,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
             AttributeUpdateNotificationMsg.Builder notification = AttributeUpdateNotificationMsg.newBuilder();
             if (msg.isDeleted()) {
                 List<String> sharedKeys = msg.getDeletedKeys().stream()
-                        .filter(key -> SHARED_SCOPE.equals(key.getScope()))
+                        .filter(key -> DataConstants.SHARED_SCOPE.equals(key.getScope()))
                         .map(AttributeKey::getAttributeKey)
                         .collect(Collectors.toList());
                 if (!sharedKeys.isEmpty()) {
@@ -324,7 +323,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
                     hasNotificationData = true;
                 }
             } else {
-                if (SHARED_SCOPE.equals(msg.getScope())) {
+                if (DataConstants.SHARED_SCOPE.equals(msg.getScope())) {
                     List<AttributeKvEntry> attributes = new ArrayList<>(msg.getValues());
                     if (attributes.size() > 0) {
                         List<TsKvProto> sharedUpdated = msg.getValues().stream().map(this::toTsKvProto)
@@ -334,7 +333,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor {
                             hasNotificationData = true;
                         }
                     } else {
-                        log.debug("[{}] No public server side attributes changed!", deviceId);
+                        log.debug("[{}] No public shared side attributes changed!", deviceId);
                     }
                 }
             }

@@ -163,7 +163,7 @@ public class DefaultTransportApiService implements TransportApiService {
                 return TransportApiResponseMsg.newBuilder()
                         .setGetOrCreateDeviceResponseMsg(GetOrCreateDeviceFromGatewayResponseMsg.newBuilder().setDeviceInfo(getDeviceInfoProto(device)).build()).build();
             } catch (JsonProcessingException e) {
-                log.warn("[{}] Failed to lookup device by gateway id and name", gatewayId, requestMsg.getDeviceName(), e);
+                log.warn("[{}][{}] Failed to lookup device by gateway id and name", gatewayId, requestMsg.getDeviceName(), e);
                 throw new RuntimeException(e);
             } finally {
                 deviceCreationLock.unlock();

@@ -522,7 +522,7 @@ js:
     # Specify thread pool size for JavaScript sandbox resource monitor
     monitor_thread_pool_size: "${LOCAL_JS_SANDBOX_MONITOR_THREAD_POOL_SIZE:4}"
     # Maximum CPU time in milliseconds allowed for script execution
-    max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:10000}"
+    max_cpu_time: "${LOCAL_JS_SANDBOX_MAX_CPU_TIME:8000}"
     # Maximum allowed JavaScript execution errors before JavaScript will be blacklisted
     max_errors: "${LOCAL_JS_SANDBOX_MAX_ERRORS:3}"
     # JS Eval max request timeout. 0 - no timeout

@@ -106,6 +106,25 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>false</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

@@ -75,6 +75,25 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>false</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

@@ -87,6 +87,25 @@
                 <groupId>org.xolstice.maven.plugins</groupId>
                 <artifactId>protobuf-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>false</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

-/**
- * Copyright © 2016-2020 The Thingsboard Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.thingsboard.server.common.msg.kv;
-
-import java.io.Serializable;
-import java.util.List;
-
-import org.thingsboard.server.common.data.kv.AttributeKey;
-import org.thingsboard.server.common.data.kv.AttributeKvEntry;
-
-public interface AttributesKVMsg extends Serializable {
-
-    List<AttributeKvEntry> getClientAttributes();
-    List<AttributeKvEntry> getSharedAttributes();
-    List<AttributeKey> getDeletedAttributes();
-}

-/**
- * Copyright © 2016-2020 The Thingsboard Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.thingsboard.server.common.msg.kv;
-
-import lombok.AccessLevel;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import org.thingsboard.server.common.data.kv.AttributeKey;
-import org.thingsboard.server.common.data.kv.AttributeKvEntry;
-
-import java.util.Collections;
-import java.util.List;
-
-@Data
-@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
-public class BasicAttributeKVMsg implements AttributesKVMsg {
-
-    private static final long serialVersionUID = 1L;
-
-    private final List<AttributeKvEntry> clientAttributes;
-    private final List<AttributeKvEntry> sharedAttributes;
-    private final List<AttributeKey> deletedAttributes;
-
-    public static BasicAttributeKVMsg fromClient(List<AttributeKvEntry> attributes) {
-        return new BasicAttributeKVMsg(attributes, Collections.emptyList(), Collections.emptyList());
-    }
-
-    public static BasicAttributeKVMsg fromShared(List<AttributeKvEntry> attributes) {
-        return new BasicAttributeKVMsg(Collections.emptyList(), attributes, Collections.emptyList());
-    }
-
-    public static BasicAttributeKVMsg from(List<AttributeKvEntry> client, List<AttributeKvEntry> shared) {
-        return new BasicAttributeKVMsg(client, shared, Collections.emptyList());
-    }
-
-    public static AttributesKVMsg fromDeleted(List<AttributeKey> shared) {
-        return new BasicAttributeKVMsg(Collections.emptyList(), Collections.emptyList(), shared);
-    }
-}

@@ -23,16 +23,29 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
 @Slf4j
 public final class InMemoryStorage {
     private static InMemoryStorage instance;
     private final ConcurrentHashMap<String, BlockingQueue<TbQueueMsg>> storage;
+    private static ScheduledExecutorService statExecutor;
 
     private InMemoryStorage() {
         storage = new ConcurrentHashMap<>();
+        statExecutor = Executors.newSingleThreadScheduledExecutor();
+        statExecutor.scheduleAtFixedRate(this::printStats, 60, 60, TimeUnit.SECONDS);
+    }
+
+    private void printStats() {
+        storage.forEach((topic, queue) -> {
+            if (queue.size() > 0) {
+                log.debug("Topic: [{}], Queue size: [{}]", topic, queue.size());
+            }
+        });
     }
 
     public static InMemoryStorage getInstance() {
@@ -77,4 +90,9 @@ public final class InMemoryStorage {
         storage.clear();
     }
 
+    public void destroy() {
+        if (statExecutor != null) {
+            statExecutor.shutdownNow();
+        }
+    }
 }

@@ -53,6 +53,6 @@ public class InMemoryTbQueueProducer<T extends TbQueueMsg> implements TbQueuePro
 
     @Override
     public void stop() {
-
+        storage.destroy();
     }
 }

@@ -127,7 +127,6 @@ message GetAttributeResponseMsg {
   int32 requestId = 1;
   repeated TsKvProto clientAttributeList = 2;
   repeated TsKvProto sharedAttributeList = 3;
-  repeated string deletedAttributeKeys = 4;
   string error = 5;
 }
 

@@ -125,7 +125,7 @@ public class JsonCoapAdaptor implements CoapTransportAdaptor {
 
     @Override
     public Response convertToPublish(CoapTransportResource.CoapSessionListener session, TransportProtos.GetAttributeResponseMsg msg) throws AdaptorException {
-        if (msg.getClientAttributeListCount() == 0 && msg.getSharedAttributeListCount() == 0 && msg.getDeletedAttributeKeysCount() == 0) {
+        if (msg.getClientAttributeListCount() == 0 && msg.getSharedAttributeListCount() == 0) {
             return new Response(CoAP.ResponseCode.NOT_FOUND);
         } else {
             Response response = new Response(CoAP.ResponseCode.CONTENT);

@@ -35,7 +35,6 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry;
 import org.thingsboard.server.common.data.kv.KvEntry;
 import org.thingsboard.server.common.data.kv.LongDataEntry;
 import org.thingsboard.server.common.data.kv.StringDataEntry;
-import org.thingsboard.server.common.msg.kv.AttributesKVMsg;
 import org.thingsboard.server.gen.transport.TransportProtos;
 import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg;
@@ -269,11 +268,6 @@ public class JsonConverter {
             payload.getSharedAttributeListList().forEach(addToObjectFromProto(attrObject));
             result.add("shared", attrObject);
         }
-        if (payload.getDeletedAttributeKeysCount() > 0) {
-            JsonArray attrObject = new JsonArray();
-            payload.getDeletedAttributeKeysList().forEach(attrObject::add);
-            result.add("deleted", attrObject);
-        }
         return result;
     }
 
@@ -290,31 +284,6 @@ public class JsonConverter {
         return result;
     }
 
-    public static JsonObject toJson(AttributesKVMsg payload, boolean asMap) {
-        JsonObject result = new JsonObject();
-        if (asMap) {
-            if (!payload.getClientAttributes().isEmpty()) {
-                JsonObject attrObject = new JsonObject();
-                payload.getClientAttributes().forEach(addToObject(attrObject));
-                result.add("client", attrObject);
-            }
-            if (!payload.getSharedAttributes().isEmpty()) {
-                JsonObject attrObject = new JsonObject();
-                payload.getSharedAttributes().forEach(addToObject(attrObject));
-                result.add("shared", attrObject);
-            }
-        } else {
-            payload.getClientAttributes().forEach(addToObject(result));
-            payload.getSharedAttributes().forEach(addToObject(result));
-        }
-        if (!payload.getDeletedAttributes().isEmpty()) {
-            JsonArray attrObject = new JsonArray();
-            payload.getDeletedAttributes().forEach(addToObject(attrObject));
-            result.add("deleted", attrObject);
-        }
-        return result;
-    }
-
     public static JsonObject getJsonObjectForGateway(String deviceName, TransportProtos.GetAttributeResponseMsg responseMsg) {
         JsonObject result = new JsonObject();
         result.addProperty("id", responseMsg.getRequestId());
@@ -370,10 +339,6 @@ public class JsonConverter {
         }
     }
 
-    private static Consumer<AttributeKey> addToObject(JsonArray result) {
-        return key -> result.add(key.getAttributeKey());
-    }
-
     private static Consumer<TsKvProto> addToObjectFromProto(JsonObject result) {
         return de -> {
             switch (de.getKv().getType()) {

@@ -25,6 +25,7 @@ import org.thingsboard.server.dao.model.sql.AlarmEntity;
 import org.thingsboard.server.dao.model.sql.AlarmInfoEntity;
 
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 
 /**
@@ -70,7 +71,7 @@ public interface AlarmRepository extends CrudRepository<AlarmEntity, UUID> {
                                      @Param("affectedEntityType") String affectedEntityType,
                                      @Param("startTime") Long startTime,
                                      @Param("endTime") Long endTime,
-                                     @Param("alarmStatuses") List<AlarmStatus> alarmStatuses,
+                                     @Param("alarmStatuses") Set<AlarmStatus> alarmStatuses,
                                      @Param("searchText") String searchText,
                                      Pageable pageable);
 

@@ -24,14 +24,12 @@ import org.springframework.stereotype.Component;
 import org.thingsboard.server.common.data.alarm.Alarm;
 import org.thingsboard.server.common.data.alarm.AlarmInfo;
 import org.thingsboard.server.common.data.alarm.AlarmQuery;
-import org.thingsboard.server.common.data.alarm.AlarmSearchStatus;
 import org.thingsboard.server.common.data.alarm.AlarmStatus;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.query.AlarmData;
-import org.thingsboard.server.common.data.query.AlarmDataPageLink;
 import org.thingsboard.server.common.data.query.AlarmDataQuery;
 import org.thingsboard.server.dao.DaoUtil;
 import org.thingsboard.server.dao.alarm.AlarmDao;
@@ -40,7 +38,6 @@ import org.thingsboard.server.dao.relation.RelationDao;
 import org.thingsboard.server.dao.sql.JpaAbstractDao;
 import org.thingsboard.server.dao.sql.query.AlarmQueryRepository;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -102,7 +99,7 @@ public class JpaAlarmDao extends JpaAbstractDao<AlarmEntity, Alarm> implements A
         Set<AlarmStatus> statusSet = null;
         if (query.getSearchStatus() != null) {
             statusSet = query.getSearchStatus().getStatuses();
-        } else if (query.getStatus() != null){
+        } else if (query.getStatus() != null) {
             statusSet = Collections.singleton(query.getStatus());
         }
         return DaoUtil.toPageData(
@@ -112,7 +109,7 @@ public class JpaAlarmDao extends JpaAbstractDao<AlarmEntity, Alarm> implements A
                         affectedEntity.getEntityType().name(),
                         query.getPageLink().getStartTime(),
                         query.getPageLink().getEndTime(),
-                        new ArrayList<>(statusSet),
+                        statusSet,
                         Objects.toString(query.getPageLink().getTextSearch(), ""),
                         DaoUtil.toPageable(query.getPageLink())
                 )

@@ -46,6 +46,7 @@ public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa
         entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null));
         entity.setLongValue(tsKvEntry.getLongValue().orElse(null));
         entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null));
+        entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null));
         log.trace("Saving entity: {}", entity);
         return tsQueue.add(entity);
     }

@@ -41,8 +41,8 @@ public class HsqlLatestInsertTsRepository extends AbstractInsertRepository imple
                     "ON (ts_kv_latest.entity_id=T.entity_id " +
                     "AND ts_kv_latest.key=T.key) " +
                     "WHEN MATCHED THEN UPDATE SET ts_kv_latest.ts = T.ts, ts_kv_latest.bool_v = T.bool_v, ts_kv_latest.str_v = T.str_v, ts_kv_latest.long_v = T.long_v, ts_kv_latest.dbl_v = T.dbl_v, ts_kv_latest.json_v = T.json_v " +
-                    "WHEN NOT MATCHED THEN INSERT (entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " +
-                    "VALUES (T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);";
+                    "WHEN NOT MATCHED THEN INSERT (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " +
+                    "VALUES (T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v, T.json_v);";
 
     @Override
     public void saveOrUpdate(List<TsKvLatestEntity> entities) {

@@ -31,6 +31,7 @@ import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
 @Slf4j
 public abstract class AbstractCassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao {
@@ -85,4 +86,22 @@ public abstract class AbstractCassandraBaseTimeseriesDao extends CassandraAbstra
         return new BasicTsKvEntry(ts, toKvEntry(row, key));
     }
 
+    protected TsKvEntry convertResultToTsKvEntry(String key, Row row) {
+        if (row != null) {
+            Optional<String> foundKeyOpt = getKey(row);
+            long ts = row.getLong(ModelConstants.TS_COLUMN);
+            return new BasicTsKvEntry(ts, toKvEntry(row, foundKeyOpt.orElse(key)));
+        } else {
+            return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null));
+        }
+    }
+
+    private Optional<String> getKey(Row row){
+       try{
+           return Optional.ofNullable(row.getString(ModelConstants.KEY_COLUMN));
+       } catch (IllegalArgumentException e){
+           return Optional.empty();
+       }
+    }
+
 }

@@ -186,15 +186,6 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes
                 rows -> this.convertResultToTsKvEntryList(rows), readResultsProcessingExecutor);
     }
 
-    private TsKvEntry convertResultToTsKvEntry(String key, Row row) {
-        if (row != null) {
-            long ts = row.getLong(ModelConstants.TS_COLUMN);
-            return new BasicTsKvEntry(ts, toKvEntry(row, key));
-        } else {
-            return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null));
-        }
-    }
-
     private PreparedStatement getLatestStmt() {
         if (latestInsertStmt == null) {
             latestInsertStmt = prepare(INSERT_INTO + ModelConstants.TS_KV_LATEST_CF +

@@ -13,6 +13,7 @@ DROP TABLE IF EXISTS relation;
 DROP TABLE IF EXISTS tb_user;
 DROP TABLE IF EXISTS tenant;
 DROP TABLE IF EXISTS ts_kv;
+DROP TABLE IF EXISTS ts_kv_dictionary;
 DROP TABLE IF EXISTS ts_kv_latest;
 DROP TABLE IF EXISTS user_credentials;
 DROP TABLE IF EXISTS widget_type;