Merge branch 'master' of github.com:thingsboard/thingsboard

@@ -18,6 +18,7 @@ package org.thingsboard.server.controller;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import lombok.extern.slf4j.Slf4j;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -422,6 +423,25 @@ public class RuleChainController extends BaseController {
     }
     }
 
 
     private String msgToOutput(TbMsg msg) throws Exception {
     private String msgToOutput(TbMsg msg) throws Exception {
+        JsonNode resultNode = convertMsgToOut(msg);
+        return objectMapper.writeValueAsString(resultNode);
+    }
+
+    private String msgToOutput(List<TbMsg> msgs) throws Exception {
+        JsonNode resultNode;
+        if (msgs.size() > 1) {
+            resultNode = objectMapper.createArrayNode();
+            for (TbMsg msg : msgs) {
+                JsonNode convertedData = convertMsgToOut(msg);
+                ((ArrayNode) resultNode).add(convertedData);
+            }
+        } else {
+            resultNode = convertMsgToOut(msgs.get(0));
+        }
+        return objectMapper.writeValueAsString(resultNode);
+    }
+
+    private JsonNode convertMsgToOut(TbMsg msg) throws Exception{
         ObjectNode msgData = objectMapper.createObjectNode();
         ObjectNode msgData = objectMapper.createObjectNode();
         if (!StringUtils.isEmpty(msg.getData())) {
         if (!StringUtils.isEmpty(msg.getData())) {
             msgData.set("msg", objectMapper.readTree(msg.getData()));
             msgData.set("msg", objectMapper.readTree(msg.getData()));
@@ -429,7 +449,8 @@ public class RuleChainController extends BaseController {
         Map<String, String> metadata = msg.getMetaData().getData();
         Map<String, String> metadata = msg.getMetaData().getData();
         msgData.set("metadata", objectMapper.valueToTree(metadata));
         msgData.set("metadata", objectMapper.valueToTree(metadata));
         msgData.put("msgType", msg.getType());
         msgData.put("msgType", msg.getType());
-        return objectMapper.writeValueAsString(msgData);
+        return msgData;
     }
     }
 
 
+
 }
 }

@@ -108,13 +108,18 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
     }
     }
 
 
     @Override
     @Override
-    public TbMsg executeUpdate(TbMsg msg) throws ScriptException {
+    public List<TbMsg> executeUpdate(TbMsg msg) throws ScriptException {
         JsonNode result = executeScript(msg);
         JsonNode result = executeScript(msg);
-        if (!result.isObject()) {
+        if (result.isObject()) {
+            return Collections.singletonList(unbindMsg(result, msg));
+        } else if (result.isArray()){
+            List<TbMsg> res = new ArrayList<>(result.size());
+            result.forEach(jsonObject -> res.add(unbindMsg(jsonObject, msg)));
+            return res;
+        } else {
             log.warn("Wrong result type: {}", result.getNodeType());
             log.warn("Wrong result type: {}", result.getNodeType());
             throw new ScriptException("Wrong result type: " + result.getNodeType());
             throw new ScriptException("Wrong result type: " + result.getNodeType());
         }
         }
-        return unbindMsg(result, msg);
     }
     }
 
 
     @Override
     @Override

@@ -110,7 +110,7 @@ security:
     # Enable/disable claiming devices, if false -> the device's [claimingAllowed] SERVER_SCOPE attribute must be set to [true] to allow claiming specific device
     # Enable/disable claiming devices, if false -> the device's [claimingAllowed] SERVER_SCOPE attribute must be set to [true] to allow claiming specific device
     allowClaimingByDefault: "${SECURITY_CLAIM_ALLOW_CLAIMING_BY_DEFAULT:true}"
     allowClaimingByDefault: "${SECURITY_CLAIM_ALLOW_CLAIMING_BY_DEFAULT:true}"
     # Time allowed to claim the device in milliseconds
     # Time allowed to claim the device in milliseconds
-    duration: "${SECURITY_CLAIM_DURATION:60000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value
+    duration: "${SECURITY_CLAIM_DURATION:86400000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value
   basic:
   basic:
     enabled: "${SECURITY_BASIC_ENABLED:false}"
     enabled: "${SECURITY_BASIC_ENABLED:false}"
   oauth2:
   oauth2:
@@ -348,8 +348,8 @@ caffeine:
       timeToLiveInMinutes: 1440
       timeToLiveInMinutes: 1440
       maxSize: 0
       maxSize: 0
     claimDevices:
     claimDevices:
-      timeToLiveInMinutes: 1
-      maxSize: 0
+      timeToLiveInMinutes: 1440
+      maxSize: 1000
     securitySettings:
     securitySettings:
       timeToLiveInMinutes: 1440
       timeToLiveInMinutes: 1440
       maxSize: 0
       maxSize: 0

@@ -54,7 +54,8 @@
         <cassandra.version>4.10.0</cassandra.version>
         <cassandra.version>4.10.0</cassandra.version>
         <metrics.version>4.0.5</metrics.version>
         <metrics.version>4.0.5</metrics.version>
         <cassandra-unit.version>4.3.1.0</cassandra-unit.version>
         <cassandra-unit.version>4.3.1.0</cassandra-unit.version>
-        <cassandra-all.version>3.11.9</cassandra-all.version>
+        <cassandra-all.version>3.11.10</cassandra-all.version>
+        <cassandra-driver-core.version>3.11.0</cassandra-driver-core.version>
         <takari-cpsuite.version>1.2.7</takari-cpsuite.version>
         <takari-cpsuite.version>1.2.7</takari-cpsuite.version>
         <guava.version>28.2-jre</guava.version>
         <guava.version>28.2-jre</guava.version>
         <caffeine.version>2.6.1</caffeine.version>
         <caffeine.version>2.6.1</caffeine.version>
@@ -1096,6 +1097,11 @@
                 <version>${cassandra.version}</version>
                 <version>${cassandra.version}</version>
             </dependency>
             </dependency>
             <dependency>
             <dependency>
+                <groupId>com.datastax.cassandra</groupId>
+                <artifactId>cassandra-driver-core</artifactId>
+                <version>${cassandra-driver-core.version}</version>
+            </dependency>
+            <dependency>
                 <groupId>io.dropwizard.metrics</groupId>
                 <groupId>io.dropwizard.metrics</groupId>
                 <artifactId>metrics-jmx</artifactId>
                 <artifactId>metrics-jmx</artifactId>
                 <version>${metrics.version}</version>
                 <version>${metrics.version}</version>

@@ -25,7 +25,7 @@ import java.util.Set;
 
 
 public interface ScriptEngine {
 public interface ScriptEngine {
 
 
-    TbMsg executeUpdate(TbMsg msg) throws ScriptException;
+    List<TbMsg> executeUpdate(TbMsg msg) throws ScriptException;
 
 
     ListenableFuture<List<TbMsg>> executeUpdateAsync(TbMsg msg);
     ListenableFuture<List<TbMsg>> executeUpdateAsync(TbMsg msg);
 
 

@@ -56,8 +56,8 @@
             <artifactId>cassandra-all</artifactId>
             <artifactId>cassandra-all</artifactId>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
-            <groupId>com.datastax.oss</groupId>
-            <artifactId>java-driver-core</artifactId>
+            <groupId>com.datastax.cassandra</groupId>
+            <artifactId>cassandra-driver-core</artifactId>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
             <groupId>commons-io</groupId>
             <groupId>commons-io</groupId>
@@ -70,7 +70,7 @@
             <plugins>
             <plugins>
                 <plugin>
                 <plugin>
                     <artifactId>maven-assembly-plugin</artifactId>
                     <artifactId>maven-assembly-plugin</artifactId>
-                    <configuration>
+                    <configuration combine.self="override">
                         <archive>
                         <archive>
                             <manifest>
                             <manifest>
                                 <mainClass>org.thingsboard.client.tools.migrator.MigratorTool</mainClass>
                                 <mainClass>org.thingsboard.client.tools.migrator.MigratorTool</mainClass>

+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.client.tools.migrator;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DictionaryParser {
+    private Map<String, String> dictionaryParsed = new HashMap<>();
+
+    public DictionaryParser(File sourceFile) throws IOException {
+        parseDictionaryDump(FileUtils.lineIterator(sourceFile));
+    }
+
+    public String getKeyByKeyId(String keyId) {
+        return dictionaryParsed.get(keyId);
+    }
+
+    private boolean isBlockFinished(String line) {
+        return StringUtils.isBlank(line) || line.equals("\\.");
+    }
+
+    private boolean isBlockStarted(String line) {
+        return line.startsWith("COPY public.ts_kv_dictionary (");
+    }
+
+    private void parseDictionaryDump(LineIterator iterator) {
+        try {
+            String tempLine;
+            while (iterator.hasNext()) {
+                tempLine = iterator.nextLine();
+
+                if (isBlockStarted(tempLine)) {
+                    processBlock(iterator);
+                }
+            }
+        } finally {
+            iterator.close();
+        }
+    }
+
+    private void processBlock(LineIterator lineIterator) {
+        String tempLine;
+        String[] lineSplited;
+        while(lineIterator.hasNext()) {
+            tempLine = lineIterator.nextLine();
+            if(isBlockFinished(tempLine)) {
+                return;
+            }
+
+            lineSplited = tempLine.split("\t");
+            dictionaryParsed.put(lineSplited[1], lineSplited[0]);
+        }
+    }
+}

@@ -30,17 +30,26 @@ public class MigratorTool {
     public static void main(String[] args) {
     public static void main(String[] args) {
         CommandLine cmd = parseArgs(args);
         CommandLine cmd = parseArgs(args);
 
 
-
         try {
         try {
-            File latestSource = new File(cmd.getOptionValue("latestTelemetryFrom"));
-            File latestSaveDir = new File(cmd.getOptionValue("latestTelemetryOut"));
-            File tsSource = new File(cmd.getOptionValue("telemetryFrom"));
-            File tsSaveDir = new File(cmd.getOptionValue("telemetryOut"));
-            File partitionsSaveDir = new File(cmd.getOptionValue("partitionsOut"));
             boolean castEnable = Boolean.parseBoolean(cmd.getOptionValue("castEnable"));
             boolean castEnable = Boolean.parseBoolean(cmd.getOptionValue("castEnable"));
+            File allTelemetrySource = new File(cmd.getOptionValue("telemetryFrom"));
+            File tsSaveDir = null;
+            File partitionsSaveDir = null;
+            File latestSaveDir = null;
+
+            RelatedEntitiesParser allEntityIdsAndTypes =
+                    new RelatedEntitiesParser(new File(cmd.getOptionValue("relatedEntities")));
+            DictionaryParser dictionaryParser = new DictionaryParser(allTelemetrySource);
+
+            if(cmd.getOptionValue("latestTelemetryOut") != null) {
+                latestSaveDir = new File(cmd.getOptionValue("latestTelemetryOut"));
+            }
+            if(cmd.getOptionValue("telemetryOut") != null) {
+                tsSaveDir = new File(cmd.getOptionValue("telemetryOut"));
+                partitionsSaveDir = new File(cmd.getOptionValue("partitionsOut"));
+            }
 
 
-            PgCaLatestMigrator.migrateLatest(latestSource, latestSaveDir, castEnable);
-            PostgresToCassandraTelemetryMigrator.migrateTs(tsSource, tsSaveDir, partitionsSaveDir, castEnable);
+            new PgCaMigrator(allTelemetrySource, tsSaveDir, partitionsSaveDir, latestSaveDir, allEntityIdsAndTypes, dictionaryParser, castEnable).migrate();
 
 
         } catch (Throwable th) {
         } catch (Throwable th) {
             th.printStackTrace();
             th.printStackTrace();
@@ -52,30 +61,30 @@ public class MigratorTool {
     private static CommandLine parseArgs(String[] args) {
     private static CommandLine parseArgs(String[] args) {
         Options options = new Options();
         Options options = new Options();
 
 
-        Option latestTsOpt = new Option("latestFrom", "latestTelemetryFrom", true, "latest telemetry source file path");
-        latestTsOpt.setRequired(true);
-        options.addOption(latestTsOpt);
+        Option telemetryAllFrom = new Option("telemetryFrom", "telemetryFrom", true, "telemetry source file");
+        telemetryAllFrom.setRequired(true);
+        options.addOption(telemetryAllFrom);
 
 
         Option latestTsOutOpt = new Option("latestOut", "latestTelemetryOut", true, "latest telemetry save dir");
         Option latestTsOutOpt = new Option("latestOut", "latestTelemetryOut", true, "latest telemetry save dir");
-        latestTsOutOpt.setRequired(true);
+        latestTsOutOpt.setRequired(false);
         options.addOption(latestTsOutOpt);
         options.addOption(latestTsOutOpt);
 
 
-        Option tsOpt = new Option("tsFrom", "telemetryFrom", true, "telemetry source file path");
-        tsOpt.setRequired(true);
-        options.addOption(tsOpt);
-
         Option tsOutOpt = new Option("tsOut", "telemetryOut", true, "sstable save dir");
         Option tsOutOpt = new Option("tsOut", "telemetryOut", true, "sstable save dir");
-        tsOutOpt.setRequired(true);
+        tsOutOpt.setRequired(false);
         options.addOption(tsOutOpt);
         options.addOption(tsOutOpt);
 
 
         Option partitionOutOpt = new Option("partitionsOut", "partitionsOut", true, "partitions save dir");
         Option partitionOutOpt = new Option("partitionsOut", "partitionsOut", true, "partitions save dir");
-        partitionOutOpt.setRequired(true);
+        partitionOutOpt.setRequired(false);
         options.addOption(partitionOutOpt);
         options.addOption(partitionOutOpt);
 
 
         Option castOpt = new Option("castEnable", "castEnable", true, "cast String to Double if possible");
         Option castOpt = new Option("castEnable", "castEnable", true, "cast String to Double if possible");
         castOpt.setRequired(true);
         castOpt.setRequired(true);
         options.addOption(castOpt);
         options.addOption(castOpt);
 
 
+        Option relatedOpt = new Option("relatedEntities", "relatedEntities", true, "related entities source file path");
+        relatedOpt.setRequired(true);
+        options.addOption(relatedOpt);
+
         HelpFormatter formatter = new HelpFormatter();
         HelpFormatter formatter = new HelpFormatter();
         CommandLineParser parser = new BasicParser();
         CommandLineParser parser = new BasicParser();
 
 

-/**
- * Copyright © 2016-2021 The Thingsboard Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.thingsboard.client.tools.migrator;
-
-import com.google.common.collect.Lists;
-import org.apache.cassandra.io.sstable.CQLSSTableWriter;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.LineIterator;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-public class PgCaLatestMigrator {
-
-    private static final long LOG_BATCH = 1000000;
-    private static final long rowPerFile = 1000000;
-
-
-    private static long linesProcessed = 0;
-    private static long linesMigrated = 0;
-    private static long castErrors = 0;
-    private static long castedOk = 0;
-
-    private static long currentWriterCount = 1;
-    private static CQLSSTableWriter currentTsWriter = null;
-
-    public static void migrateLatest(File sourceFile, File outDir, boolean castStringsIfPossible) throws IOException {
-        long startTs = System.currentTimeMillis();
-        long stepLineTs = System.currentTimeMillis();
-        long stepOkLineTs = System.currentTimeMillis();
-        LineIterator iterator = FileUtils.lineIterator(sourceFile);
-        currentTsWriter = WriterBuilder.getTsWriter(outDir);
-
-        boolean isBlockStarted = false;
-        boolean isBlockFinished = false;
-
-        String line;
-        while (iterator.hasNext()) {
-            if (linesProcessed++ % LOG_BATCH == 0) {
-                System.out.println(new Date() + " linesProcessed = " + linesProcessed + " in " + (System.currentTimeMillis() - stepLineTs) + "   castOk " + castedOk + "  castErr " + castErrors);
-                stepLineTs = System.currentTimeMillis();
-            }
-
-            line = iterator.nextLine();
-
-            if (isBlockFinished) {
-                break;
-            }
-
-            if (!isBlockStarted) {
-                if (isBlockStarted(line)) {
-                    System.out.println();
-                    System.out.println();
-                    System.out.println(line);
-                    System.out.println();
-                    System.out.println();
-                    isBlockStarted = true;
-                }
-                continue;
-            }
-
-            if (isBlockFinished(line)) {
-                isBlockFinished = true;
-            } else {
-                try {
-                    List<String> raw = Arrays.stream(line.trim().split("\t"))
-                            .map(String::trim)
-                            .filter(StringUtils::isNotEmpty)
-                            .collect(Collectors.toList());
-                    List<Object> values = toValues(raw);
-
-                    if (currentWriterCount == 0) {
-                        System.out.println(new Date() + " close writer " + new Date());
-                        currentTsWriter.close();
-                        currentTsWriter = WriterBuilder.getLatestWriter(outDir);
-                    }
-
-                    if (castStringsIfPossible) {
-                        currentTsWriter.addRow(castToNumericIfPossible(values));
-                    } else {
-                        currentTsWriter.addRow(values);
-                    }
-                    currentWriterCount++;
-                    if (currentWriterCount >= rowPerFile) {
-                        currentWriterCount = 0;
-                    }
-
-                    if (linesMigrated++ % LOG_BATCH == 0) {
-                        System.out.println(new Date() + " migrated = " + linesMigrated + " in " + (System.currentTimeMillis() - stepOkLineTs));
-                        stepOkLineTs = System.currentTimeMillis();
-                    }
-                } catch (Exception ex) {
-                    System.out.println(ex.getMessage() + " -> " + line);
-                }
-
-            }
-        }
-
-        long endTs = System.currentTimeMillis();
-        System.out.println();
-        System.out.println(new Date() + " Migrated rows " + linesMigrated + " in " + (endTs - startTs));
-
-        currentTsWriter.close();
-        System.out.println();
-        System.out.println("Finished migrate Latest Telemetry");
-    }
-
-
-    private static List<Object> castToNumericIfPossible(List<Object> values) {
-        try {
-            if (values.get(6) != null && NumberUtils.isNumber(values.get(6).toString())) {
-                Double casted = NumberUtils.createDouble(values.get(6).toString());
-                List<Object> numeric = Lists.newArrayList();
-                numeric.addAll(values);
-                numeric.set(6, null);
-                numeric.set(8, casted);
-                castedOk++;
-                return numeric;
-            }
-        } catch (Throwable th) {
-            castErrors++;
-        }
-        return values;
-    }
-
-    private static List<Object> toValues(List<String> raw) {
-        //expected Table structure:
-//        COPY public.ts_kv_latest (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) FROM stdin;
-
-
-        List<Object> result = new ArrayList<>();
-        result.add(raw.get(0));
-        result.add(fromString(raw.get(1)));
-        result.add(raw.get(2));
-
-        long ts = Long.parseLong(raw.get(3));
-        result.add(ts);
-
-        result.add(raw.get(4).equals("\\N") ? null : raw.get(4).equals("t") ? Boolean.TRUE : Boolean.FALSE);
-        result.add(raw.get(5).equals("\\N") ? null : raw.get(5));
-        result.add(raw.get(6).equals("\\N") ? null : Long.parseLong(raw.get(6)));
-        result.add(raw.get(7).equals("\\N") ? null : Double.parseDouble(raw.get(7)));
-        return result;
-    }
-
-    public static UUID fromString(String src) {
-        return UUID.fromString(src.substring(7, 15) + "-" + src.substring(3, 7) + "-1"
-                + src.substring(0, 3) + "-" + src.substring(15, 19) + "-" + src.substring(19));
-    }
-
-    private static boolean isBlockStarted(String line) {
-        return line.startsWith("COPY public.ts_kv_latest");
-    }
-
-    private static boolean isBlockFinished(String line) {
-        return StringUtils.isBlank(line) || line.equals("\\.");
-    }
-}

+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.client.tools.migrator;
+
+import com.google.common.collect.Lists;
+import org.apache.cassandra.io.sstable.CQLSSTableWriter;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class PgCaMigrator {
+
+    private final long LOG_BATCH = 1000000;
+    private final long rowPerFile = 1000000;
+
+    private long linesTsMigrated = 0;
+    private long linesLatestMigrated = 0;
+    private long castErrors = 0;
+    private long castedOk = 0;
+
+    private long currentWriterCount = 1;
+
+    private final File sourceFile;
+    private final boolean castStringIfPossible;
+
+    private final RelatedEntitiesParser entityIdsAndTypes;
+    private final DictionaryParser keyParser;
+    private CQLSSTableWriter currentTsWriter;
+    private CQLSSTableWriter currentPartitionsWriter;
+    private CQLSSTableWriter currentTsLatestWriter;
+    private final Set<String> partitions = new HashSet<>();
+
+    private File outTsDir;
+    private File outTsLatestDir;
+
+    public PgCaMigrator(File sourceFile,
+                                File ourTsDir,
+                                File outTsPartitionDir,
+                                File outTsLatestDir,
+                                RelatedEntitiesParser allEntityIdsAndTypes,
+                                DictionaryParser dictionaryParser,
+                                boolean castStringsIfPossible) {
+        this.sourceFile = sourceFile;
+        this.entityIdsAndTypes = allEntityIdsAndTypes;
+        this.keyParser = dictionaryParser;
+        this.castStringIfPossible = castStringsIfPossible;
+        if(outTsLatestDir != null) {
+            this.currentTsLatestWriter = WriterBuilder.getLatestWriter(outTsLatestDir);
+            this.outTsLatestDir = outTsLatestDir;
+        }
+        if(ourTsDir != null) {
+            this.currentTsWriter = WriterBuilder.getTsWriter(ourTsDir);
+            this.currentPartitionsWriter = WriterBuilder.getPartitionWriter(outTsPartitionDir);
+            this.outTsDir = ourTsDir;
+        }
+    }
+
+    public void migrate() throws IOException {
+        boolean isTsDone = false;
+        boolean isLatestDone = false;
+        String line;
+        LineIterator iterator = FileUtils.lineIterator(this.sourceFile);
+
+        try {
+            while(iterator.hasNext()) {
+                line = iterator.nextLine();
+                if(!isLatestDone && isBlockLatestStarted(line)) {
+                    System.out.println("START TO MIGRATE LATEST");
+                    long start = System.currentTimeMillis();
+                    processBlock(iterator, currentTsLatestWriter, outTsLatestDir, this::toValuesLatest);
+                    System.out.println("TOTAL LINES MIGRATED: " + linesLatestMigrated + ", FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
+                    isLatestDone = true;
+                }
+
+                if(!isTsDone && isBlockTsStarted(line)) {
+                    System.out.println("START TO MIGRATE TS");
+                    long start = System.currentTimeMillis();
+                    processBlock(iterator, currentTsWriter, outTsDir, this::toValuesTs);
+                    System.out.println("TOTAL LINES MIGRATED: " + linesTsMigrated + ", FORMING OF SSL FOR TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
+                    isTsDone = true;
+                }
+            }
+
+            System.out.println("Partitions collected " + partitions.size());
+            long startTs = System.currentTimeMillis();
+            for (String partition : partitions) {
+                String[] split = partition.split("\\|");
+                List<Object> values = Lists.newArrayList();
+                values.add(split[0]);
+                values.add(UUID.fromString(split[1]));
+                values.add(split[2]);
+                values.add(Long.parseLong(split[3]));
+                currentPartitionsWriter.addRow(values);
+            }
+
+            System.out.println(new Date() + " Migrated partitions " + partitions.size() + " in " + (System.currentTimeMillis() - startTs));
+
+            System.out.println();
+            System.out.println("Finished migrate Telemetry");
+
+        } finally {
+            iterator.close();
+            currentTsLatestWriter.close();
+            currentTsWriter.close();
+            currentPartitionsWriter.close();
+        }
+    }
+
+    private void logLinesProcessed(long lines) {
+        if (lines % LOG_BATCH == 0) {
+            System.out.println(new Date() + " lines processed = " + lines + " in, castOk " + castedOk + "  castErr " + castErrors);
+        }
+    }
+
+    private void logLinesMigrated(long lines) {
+        if(lines % LOG_BATCH == 0) {
+            System.out.println(new Date() + " lines migrated = " + lines + " in, castOk " + castedOk + "  castErr " + castErrors);
+        }
+    }
+
+    private void addTypeIdKey(List<Object> result, List<String> raw) {
+        result.add(entityIdsAndTypes.getEntityType(raw.get(0)));
+        result.add(UUID.fromString(raw.get(0)));
+        result.add(keyParser.getKeyByKeyId(raw.get(1)));
+    }
+
+    private void addPartitions(List<Object> result, List<String> raw) {
+        long ts = Long.parseLong(raw.get(2));
+        long partition = toPartitionTs(ts);
+        result.add(partition);
+        result.add(ts);
+    }
+
+    private void addTimeseries(List<Object> result, List<String> raw) {
+        result.add(Long.parseLong(raw.get(2)));
+    }
+
+    private void addValues(List<Object> result, List<String> raw) {
+        result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE);
+        result.add(raw.get(4).equals("\\N") ? null : raw.get(4));
+        result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5)));
+        result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6)));
+        result.add(raw.get(7).equals("\\N") ? null : raw.get(7));
+    }
+
+    private List<Object> toValuesTs(List<String> raw) {
+
+        logLinesMigrated(linesTsMigrated++);
+
+        List<Object> result = new ArrayList<>();
+
+        addTypeIdKey(result, raw);
+        addPartitions(result, raw);
+        addValues(result, raw);
+
+        processPartitions(result);
+
+        return result;
+    }
+
+    private List<Object> toValuesLatest(List<String> raw) {
+        logLinesMigrated(linesLatestMigrated++);
+        List<Object> result = new ArrayList<>();
+
+        addTypeIdKey(result, raw);
+        addTimeseries(result, raw);
+        addValues(result, raw);
+
+        return result;
+    }
+
+    private long toPartitionTs(long ts) {
+        LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
+        return time.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).toInstant(ZoneOffset.UTC).toEpochMilli();
+    }
+
+    private void processPartitions(List<Object> values) {
+        String key = values.get(0) + "|" + values.get(1) + "|" + values.get(2) + "|" + values.get(3);
+        partitions.add(key);
+    }
+
+    private void processBlock(LineIterator iterator, CQLSSTableWriter writer, File outDir, Function<List<String>, List<Object>> function) {
+        String currentLine;
+        long linesProcessed = 0;
+        while(iterator.hasNext()) {
+            logLinesProcessed(linesProcessed++);
+            currentLine = iterator.nextLine();
+            if(isBlockFinished(currentLine)) {
+                return;
+            }
+
+            try {
+                List<String> raw = Arrays.stream(currentLine.trim().split("\t"))
+                        .map(String::trim)
+                        .collect(Collectors.toList());
+                List<Object> values = function.apply(raw);
+
+                if (this.currentWriterCount == 0) {
+                    System.out.println(new Date() + " close writer " + new Date());
+                    writer.close();
+                    writer = WriterBuilder.getLatestWriter(outDir);
+                }
+
+                if (this.castStringIfPossible) {
+                    writer.addRow(castToNumericIfPossible(values));
+                } else {
+                    writer.addRow(values);
+                }
+
+                currentWriterCount++;
+                if (currentWriterCount >= rowPerFile) {
+                    currentWriterCount = 0;
+                }
+            } catch (Exception ex) {
+                System.out.println(ex.getMessage() + " -> " + currentLine);
+            }
+        }
+    }
+
+    private List<Object> castToNumericIfPossible(List<Object> values) {
+        try {
+            if (values.get(6) != null && NumberUtils.isNumber(values.get(6).toString())) {
+                Double casted = NumberUtils.createDouble(values.get(6).toString());
+                List<Object> numeric = Lists.newArrayList();
+                numeric.addAll(values);
+                numeric.set(6, null);
+                numeric.set(8, casted);
+                castedOk++;
+                return numeric;
+            }
+        } catch (Throwable th) {
+            castErrors++;
+        }
+
+        processPartitions(values);
+
+        return values;
+    }
+
+    private boolean isBlockFinished(String line) {
+        return StringUtils.isBlank(line) || line.equals("\\.");
+    }
+
+    private boolean isBlockTsStarted(String line) {
+        return line.startsWith("COPY public.ts_kv (");
+    }
+
+    private boolean isBlockLatestStarted(String line) {
+        return line.startsWith("COPY public.ts_kv_latest (");
+    }
+
+}

-/**
- * Copyright © 2016-2021 The Thingsboard Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.thingsboard.client.tools.migrator;
-
-import com.google.common.collect.Lists;
-import org.apache.cassandra.io.sstable.CQLSSTableWriter;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.LineIterator;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneOffset;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-public class PostgresToCassandraTelemetryMigrator {
-
-    private static final long LOG_BATCH = 1000000;
-    private static final long rowPerFile = 1000000;
-
-
-    private static long linesProcessed = 0;
-    private static long linesMigrated = 0;
-    private static long castErrors = 0;
-    private static long castedOk = 0;
-
-    private static long currentWriterCount = 1;
-    private static CQLSSTableWriter currentTsWriter = null;
-    private static CQLSSTableWriter currentPartitionWriter = null;
-
-    private static Set<String> partitions = new HashSet<>();
-
-
-    public static void migrateTs(File sourceFile, File outTsDir, File outPartitionDir, boolean castStringsIfPossible) throws IOException {
-        long startTs = System.currentTimeMillis();
-        long stepLineTs = System.currentTimeMillis();
-        long stepOkLineTs = System.currentTimeMillis();
-        LineIterator iterator = FileUtils.lineIterator(sourceFile);
-        currentTsWriter = WriterBuilder.getTsWriter(outTsDir);
-        currentPartitionWriter = WriterBuilder.getPartitionWriter(outPartitionDir);
-
-        boolean isBlockStarted = false;
-        boolean isBlockFinished = false;
-
-        String line;
-        while (iterator.hasNext()) {
-            if (linesProcessed++ % LOG_BATCH == 0) {
-                System.out.println(new Date() + " linesProcessed = " + linesProcessed + " in " + (System.currentTimeMillis() - stepLineTs) + "   castOk " + castedOk + "  castErr " + castErrors);
-                stepLineTs = System.currentTimeMillis();
-            }
-
-            line = iterator.nextLine();
-
-            if (isBlockFinished) {
-                break;
-            }
-
-            if (!isBlockStarted) {
-                if (isBlockStarted(line)) {
-                    System.out.println();
-                    System.out.println();
-                    System.out.println(line);
-                    System.out.println();
-                    System.out.println();
-                    isBlockStarted = true;
-                }
-                continue;
-            }
-
-            if (isBlockFinished(line)) {
-                isBlockFinished = true;
-            } else {
-                try {
-                    List<String> raw = Arrays.stream(line.trim().split("\t"))
-                            .map(String::trim)
-                            .filter(StringUtils::isNotEmpty)
-                            .collect(Collectors.toList());
-                    List<Object> values = toValues(raw);
-
-                    if (currentWriterCount == 0) {
-                        System.out.println(new Date() + " close writer " + new Date());
-                        currentTsWriter.close();
-                        currentTsWriter = WriterBuilder.getTsWriter(outTsDir);
-                    }
-
-                    if (castStringsIfPossible) {
-                        currentTsWriter.addRow(castToNumericIfPossible(values));
-                    } else {
-                        currentTsWriter.addRow(values);
-                    }
-                    processPartitions(values);
-                    currentWriterCount++;
-                    if (currentWriterCount >= rowPerFile) {
-                        currentWriterCount = 0;
-                    }
-
-                    if (linesMigrated++ % LOG_BATCH == 0) {
-                        System.out.println(new Date() + " migrated = " + linesMigrated + " in " + (System.currentTimeMillis() - stepOkLineTs) + "   partitions = " + partitions.size());
-                        stepOkLineTs = System.currentTimeMillis();
-                    }
-                } catch (Exception ex) {
-                    System.out.println(ex.getMessage() + " -> " + line);
-                }
-
-            }
-        }
-
-        long endTs = System.currentTimeMillis();
-        System.out.println();
-        System.out.println(new Date() + " Migrated rows " + linesMigrated + " in " + (endTs - startTs));
-        System.out.println("Partitions collected " + partitions.size());
-
-        startTs = System.currentTimeMillis();
-        for (String partition : partitions) {
-            String[] split = partition.split("\\|");
-            List<Object> values = Lists.newArrayList();
-            values.add(split[0]);
-            values.add(UUID.fromString(split[1]));
-            values.add(split[2]);
-            values.add(Long.parseLong(split[3]));
-            currentPartitionWriter.addRow(values);
-        }
-        currentPartitionWriter.close();
-        endTs = System.currentTimeMillis();
-        System.out.println();
-        System.out.println();
-        System.out.println(new Date() + " Migrated partitions " + partitions.size() + " in " + (endTs - startTs));
-
-
-        currentTsWriter.close();
-        System.out.println();
-        System.out.println("Finished migrate Telemetry");
-    }
-
-    private static List<Object> castToNumericIfPossible(List<Object> values) {
-        try {
-            if (values.get(6) != null && NumberUtils.isNumber(values.get(6).toString())) {
-                Double casted = NumberUtils.createDouble(values.get(6).toString());
-                List<Object> numeric = Lists.newArrayList();
-                numeric.addAll(values);
-                numeric.set(6, null);
-                numeric.set(8, casted);
-                castedOk++;
-                return numeric;
-            }
-        } catch (Throwable th) {
-            castErrors++;
-        }
-        return values;
-    }
-
-    private static void processPartitions(List<Object> values) {
-        String key = values.get(0) + "|" + values.get(1) + "|" + values.get(2) + "|" + values.get(3);
-        partitions.add(key);
-    }
-
-    private static List<Object> toValues(List<String> raw) {
-        //expected Table structure:
-//               COPY public.ts_kv (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) FROM stdin;
-
-
-        List<Object> result = new ArrayList<>();
-        result.add(raw.get(0));
-        result.add(fromString(raw.get(1)));
-        result.add(raw.get(2));
-
-        long ts = Long.parseLong(raw.get(3));
-        long partition = toPartitionTs(ts);
-        result.add(partition);
-        result.add(ts);
-
-        result.add(raw.get(4).equals("\\N") ? null : raw.get(4).equals("t") ? Boolean.TRUE : Boolean.FALSE);
-        result.add(raw.get(5).equals("\\N") ? null : raw.get(5));
-        result.add(raw.get(6).equals("\\N") ? null : Long.parseLong(raw.get(6)));
-        result.add(raw.get(7).equals("\\N") ? null : Double.parseDouble(raw.get(7)));
-        return result;
-    }
-
-    public static UUID fromString(String src) {
-        return UUID.fromString(src.substring(7, 15) + "-" + src.substring(3, 7) + "-1"
-                + src.substring(0, 3) + "-" + src.substring(15, 19) + "-" + src.substring(19));
-    }
-
-    private static long toPartitionTs(long ts) {
-        LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
-        return time.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).toInstant(ZoneOffset.UTC).toEpochMilli();
-//        return TsPartitionDate.MONTHS.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli();
-    }
-
-    private static boolean isBlockStarted(String line) {
-        return line.startsWith("COPY public.ts_kv");
-    }
-
-    private static boolean isBlockFinished(String line) {
-        return StringUtils.isBlank(line) || line.equals("\\.");
-    }
-
-}

@@ -21,37 +21,41 @@ It will generate single jar file with all required dependencies inside `target d
 #### Dump data from the source Postgres Database
 #### Dump data from the source Postgres Database
 *Do not use compression if possible because Tool can only work with uncompressed file
 *Do not use compression if possible because Tool can only work with uncompressed file
 
 
-1. Dump table `ts_kv` table:
-
-    `pg_dump -h localhost -U postgres -d thingsboard -t ts_kv > ts_kv.dmp`
-
-2. Dump table `ts_kv_latest` table:
-
-    `pg_dump -h localhost -U postgres -d thingsboard -t ts_kv_latest > ts_kv_latest.dmp`
+1. Dump related tables that need to correct save telemetry
+   
+   `pg_dump -h localhost -U postgres -d thingsboard -T admin_settings -T attribute_kv -T audit_log -T component_discriptor -T device_credentials -T event -T oauth2_client_registration -T oauth2_client_registration_info -T oauth2_client_registration_template -T relation -T rule_node_state tb_schema_settings -T user_credentials > related_entities.dmp`
+   
+2. Dump `ts_kv` and child:
+   
+   `pg_dump -h localhost -U postgres -d thingsboard --load-via-partition-root --data-only -t ts_kv* > ts_kv_all.dmp`
 
 
-3. [Optional] move table dumps to the instance where cassandra will be hosted
+3. [Optional] Move table dumps to the instance where cassandra will be hosted
 
 
 #### Prepare directory structure for SSTables
 #### Prepare directory structure for SSTables
 Tool use 3 different directories for saving SSTables - `ts_kv_cf`, `ts_kv_latest_cf`, `ts_kv_partitions_cf`
 Tool use 3 different directories for saving SSTables - `ts_kv_cf`, `ts_kv_latest_cf`, `ts_kv_partitions_cf`
 
 
 Create 3 empty directories. For example:
 Create 3 empty directories. For example:
 
 
-    /home/ubunut/migration/ts
-    /home/ubunut/migration/ts_latest
-    /home/ubunut/migration/ts_partition
+    /home/user/migration/ts
+    /home/user/migration/ts_latest
+    /home/user/migration/ts_partition
     
     
 #### Run tool
 #### Run tool
-*Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination
+
+**If you want to migrate just `ts_kv` without `ts_kv_latest` or vice versa don't use arguments (paths) for output files*
+
+**Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination*
 
 
 ```
 ```
-java -jar ./tools-2.4.1-SNAPSHOT-jar-with-dependencies.jar 
-        -latestFrom ./source/ts_kv_latest.dmp
-        -latestOut /home/ubunut/migration/ts_latest 
-        -tsFrom ./source/ts_kv.dmp
-        -tsOut /home/ubunut/migration/ts  
-        -partitionsOut /home/ubunut/migration/ts_partition
-        -castEnable false
+java -jar ./tools-3.2.2-SNAPSHOT-jar-with-dependencies.jar 
+        -telemetryFrom /home/user/dump/ts_kv_all.dmp 
+        -relatedEntities /home/user/dump/related_entities.dmp 
+        -latestOut /home/user/migration/ts_latest 
+        -tsOut /home/user/migration/ts 
+        -partitionsOut /home/user/migration/ts_partition 
+        -castEnable false 
 ```  
 ```  
+*Use your paths for program arguments*
 
 
 Tool execution time depends on DB size, CPU resources and Disk throughput
 Tool execution time depends on DB size, CPU resources and Disk throughput
 
 
@@ -59,22 +63,23 @@ Tool execution time depends on DB size, CPU resources and Disk throughput
 * Note that this this part works only for single node Cassandra Cluster. If you have more nodes - it is better to use `sstableloader` tool.
 * Note that this this part works only for single node Cassandra Cluster. If you have more nodes - it is better to use `sstableloader` tool.
 
 
 1. [Optional] install Cassandra on the instance
 1. [Optional] install Cassandra on the instance
-2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this file `schema-ts.cql`
+2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this files `schema-ts.cql` and `schema-ts-latest.cql` using `source` command
 3. Stop Cassandra
 3. Stop Cassandra
-4. Copy generated SSTable files into cassandra data dir:
+4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders
+5. Copy generated SSTable files into cassandra data dir using next command:
 
 
 ```
 ```
-    sudo find /home/ubunut/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \;
-    sudo find /home/ubunut/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \;
-    sudo find /home/ubunut/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \;
+    sudo find /home/user/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \;
+    sudo find /home/user/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \;
+    sudo find /home/user/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \;
 ```   
 ```   
-    
-5. Start Cassandra service and trigger compaction
+  *Pay attention! Data folders have similar name  `ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13`, but you have to use own*  
+6. Start Cassandra service and trigger compaction
+
+    Trigger compactions: `nodetool compact thingsboard`
+   
+    Check compaction status: `nodetool compactionstats`
 
 
-```
-    trigger compactions: nodetool compact thingsboard
-    check compaction status: nodetool compactionstats
-```
     
     
 ## Switch Thignsboard into Hybrid Mode
 ## Switch Thignsboard into Hybrid Mode
 
 

+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.client.tools.migrator;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.apache.commons.lang3.StringUtils;
+import org.thingsboard.server.common.data.EntityType;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RelatedEntitiesParser {
+    private final Map<String, String> allEntityIdsAndTypes = new HashMap<>();
+    
+    private final Map<String, EntityType> tableNameAndEntityType = Map.ofEntries(
+            Map.entry("COPY public.alarm ", EntityType.ALARM),
+            Map.entry("COPY public.asset ", EntityType.ASSET),
+            Map.entry("COPY public.customer ", EntityType.CUSTOMER),
+            Map.entry("COPY public.dashboard ", EntityType.DASHBOARD),
+            Map.entry("COPY public.device ", EntityType.DEVICE),
+            Map.entry("COPY public.rule_chain ", EntityType.RULE_CHAIN),
+            Map.entry("COPY public.rule_node ", EntityType.RULE_NODE),
+            Map.entry("COPY public.tenant ", EntityType.TENANT),
+            Map.entry("COPY public.tb_user ", EntityType.USER),
+            Map.entry("COPY public.entity_view ", EntityType.ENTITY_VIEW),
+            Map.entry("COPY public.widgets_bundle ", EntityType.WIDGETS_BUNDLE),
+            Map.entry("COPY public.widget_type ", EntityType.WIDGET_TYPE),
+            Map.entry("COPY public.tenant_profile ", EntityType.TENANT_PROFILE),
+            Map.entry("COPY public.device_profile ", EntityType.DEVICE_PROFILE),
+            Map.entry("COPY public.api_usage_state ", EntityType.API_USAGE_STATE)
+    );
+
+    public RelatedEntitiesParser(File source) throws IOException {
+        processAllTables(FileUtils.lineIterator(source));
+    }
+
+    public String getEntityType(String uuid) {
+        return this.allEntityIdsAndTypes.get(uuid);
+    }
+
+    private boolean isBlockFinished(String line) {
+        return StringUtils.isBlank(line) || line.equals("\\.");
+    }
+
+    private void processAllTables(LineIterator lineIterator) {
+        String currentLine;
+        try {
+            while (lineIterator.hasNext()) {
+                currentLine = lineIterator.nextLine();
+                for(Map.Entry<String, EntityType> entry : tableNameAndEntityType.entrySet()) {
+                    if(currentLine.startsWith(entry.getKey())) {
+                        processBlock(lineIterator, entry.getValue());
+                    }
+                }
+            }
+        } finally {
+            lineIterator.close();
+        }
+    }
+
+    private void processBlock(LineIterator lineIterator, EntityType entityType) {
+        String currentLine;
+        while(lineIterator.hasNext()) {
+            currentLine = lineIterator.nextLine();
+            if(isBlockFinished(currentLine)) {
+                return;
+            }
+            allEntityIdsAndTypes.put(currentLine.split("\t")[0], entityType.name());
+        }
+    }
+}

@@ -31,6 +31,7 @@ public class WriterBuilder {
             "    str_v text,\n" +
             "    str_v text,\n" +
             "    long_v bigint,\n" +
             "    long_v bigint,\n" +
             "    dbl_v double,\n" +
             "    dbl_v double,\n" +
+            "    json_v text,\n" +
             "    PRIMARY KEY (( entity_type, entity_id, key, partition ), ts)\n" +
             "    PRIMARY KEY (( entity_type, entity_id, key, partition ), ts)\n" +
             ");";
             ");";
 
 
@@ -43,6 +44,7 @@ public class WriterBuilder {
             "    str_v text,\n" +
             "    str_v text,\n" +
             "    long_v bigint,\n" +
             "    long_v bigint,\n" +
             "    dbl_v double,\n" +
             "    dbl_v double,\n" +
+            "    json_v text,\n" +
             "    PRIMARY KEY (( entity_type, entity_id ), key)\n" +
             "    PRIMARY KEY (( entity_type, entity_id ), key)\n" +
             ") WITH compaction = { 'class' :  'LeveledCompactionStrategy'  };";
             ") WITH compaction = { 'class' :  'LeveledCompactionStrategy'  };";
 
 
@@ -59,8 +61,8 @@ public class WriterBuilder {
         return CQLSSTableWriter.builder()
         return CQLSSTableWriter.builder()
                 .inDirectory(dir)
                 .inDirectory(dir)
                 .forTable(tsSchema)
                 .forTable(tsSchema)
-                .using("INSERT INTO thingsboard.ts_kv_cf (entity_type, entity_id, key, partition, ts, bool_v, str_v, long_v, dbl_v) " +
-                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
+                .using("INSERT INTO thingsboard.ts_kv_cf (entity_type, entity_id, key, partition, ts, bool_v, str_v, long_v, dbl_v, json_v) " +
+                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
                 .build();
                 .build();
     }
     }
 
 
@@ -68,8 +70,8 @@ public class WriterBuilder {
         return CQLSSTableWriter.builder()
         return CQLSSTableWriter.builder()
                 .inDirectory(dir)
                 .inDirectory(dir)
                 .forTable(latestSchema)
                 .forTable(latestSchema)
-                .using("INSERT INTO thingsboard.ts_kv_latest_cf (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " +
-                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
+                .using("INSERT INTO thingsboard.ts_kv_latest_cf (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " +
+                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
                 .build();
                 .build();
     }
     }