Code cleaning, improvement of README.md

AndrewVolosytnykhThingsboard · Andrew Shvayka
1 parent 1240099c
Showing 4 changed files with 80 additions and 43 deletions
pom.xml
tools/pom.xml
tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java
tools/src/main/java/org/thingsboard/client/tools/migrator/README.md
--- a/pom.xml
View file @d5658d5
+++ b/pom.xml
View file @d5658d5
@@ -54,7 +54,8 @@
         <cassandra.version>4.10.0</cassandra.version>
         <metrics.version>4.0.5</metrics.version>
         <cassandra-unit.version>4.3.1.0</cassandra-unit.version>
-        <cassandra-all.version>3.11.9</cassandra-all.version>
+        <cassandra-all.version>3.11.10</cassandra-all.version>
+        <cassandra-driver-core.version>3.11.0</cassandra-driver-core.version>
         <takari-cpsuite.version>1.2.7</takari-cpsuite.version>
         <guava.version>28.2-jre</guava.version>
         <caffeine.version>2.6.1</caffeine.version>
@@ -1096,6 +1097,11 @@
                 <version>${cassandra.version}</version>
             </dependency>
             <dependency>
+                <groupId>com.datastax.cassandra</groupId>
+                <artifactId>cassandra-driver-core</artifactId>
+                <version>${cassandra-driver-core.version}</version>
+            </dependency>
+            <dependency>
                 <groupId>io.dropwizard.metrics</groupId>
                 <artifactId>metrics-jmx</artifactId>
                 <version>${metrics.version}</version>
--- a/tools/pom.xml
View file @d5658d5
+++ b/tools/pom.xml
View file @d5658d5
@@ -54,17 +54,14 @@
         <dependency>
             <groupId>org.apache.cassandra</groupId>
             <artifactId>cassandra-all</artifactId>
-            <version>3.11.10</version>
         </dependency>
         <dependency>
             <groupId>com.datastax.cassandra</groupId>
             <artifactId>cassandra-driver-core</artifactId>
-            <version>3.10.1</version>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
-            <version>2.5</version>
         </dependency>
     </dependencies>
--- a/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java
View file @d5658d5
+++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java
View file @d5658d5
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.thingsboard.client.tools.migrator;
 import com.google.common.collect.Lists;
@@ -122,29 +137,52 @@ public class PgCaMigrator {
         }
     }
-    private void logLinesProcessed() {
-        if (linesProcessed++ % LOG_BATCH == 0) {
-            System.out.println(new Date() + " linesProcessed = " + linesProcessed + " in, castOk " + castedOk + "  castErr " + castErrors);
+    private void logLinesProcessed(long lines) {
+        if (lines % LOG_BATCH == 0) {
+            System.out.println(new Date() + " lines processed = " + lines + " in, castOk " + castedOk + "  castErr " + castErrors);
         }
     }
-    private List<Object> toValuesTs(List<String> raw) {
-        linesTsMigrated++;
-        List<Object> result = new ArrayList<>();
+    private void logLinesMigrated(long lines) {
+        if(lines % LOG_BATCH == 0) {
+            System.out.println(new Date() + " lines migrated = " + lines + " in, castOk " + castedOk + "  castErr " + castErrors);
+        }
+    }
+
+    private void addTypeIdKey(List<Object> result, List<String> raw) {
         result.add(entityIdsAndTypes.getEntityType(raw.get(0)));
         result.add(UUID.fromString(raw.get(0)));
         result.add(keyParser.getKeyByKeyId(raw.get(1)));
+    }
+    private void addPartitions(List<Object> result, List<String> raw) {
         long ts = Long.parseLong(raw.get(2));
         long partition = toPartitionTs(ts);
         result.add(partition);
         result.add(ts);
+    }
+
+    private void addTimeseries(List<Object> result, List<String> raw) {
+        result.add(Long.parseLong(raw.get(2)));
+    }
+    private void addValues(List<Object> result, List<String> raw) {
         result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE);
         result.add(raw.get(4).equals("\\N") ? null : raw.get(4));
         result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5)));
         result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6)));
         result.add(raw.get(7).equals("\\N") ? null : raw.get(7));
+    }
+
+    private List<Object> toValuesTs(List<String> raw) {
+
+        logLinesMigrated(linesTsMigrated++);
+
+        List<Object> result = new ArrayList<>();
+
+        addTypeIdKey(result, raw);
+        addPartitions(result, raw);
+        addValues(result, raw);
         processPartitions(result);
@@ -152,20 +190,12 @@ public class PgCaMigrator {
     }
     private List<Object> toValuesLatest(List<String> raw) {
-        linesLatestMigrated++;
+        logLinesMigrated(linesLatestMigrated++);
         List<Object> result = new ArrayList<>();
-        result.add(this.entityIdsAndTypes.getEntityType(raw.get(0)));
-        result.add(UUID.fromString(raw.get(0)));
-        result.add(this.keyParser.getKeyByKeyId(raw.get(1)));
-
-        long ts = Long.parseLong(raw.get(2));
-        result.add(3, ts);
-        result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE);
-        result.add(raw.get(4).equals("\\N") ? null : raw.get(4));
-        result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5)));
-        result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6)));
-        result.add(raw.get(7).equals("\\N") ? null : raw.get(7));
+        addTypeIdKey(result, raw);
+        addTimeseries(result, raw);
+        addValues(result, raw);
         return result;
     }
@@ -184,7 +214,7 @@ public class PgCaMigrator {
         String currentLine;
         linesProcessed = 0;
         while(iterator.hasNext()) {
-            logLinesProcessed();
+            logLinesProcessed(linesProcessed++);
             currentLine = iterator.nextLine();
             if(isBlockFinished(currentLine)) {
                 return;
--- a/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md
View file @d5658d5
+++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md
View file @d5658d5
@@ -21,8 +21,6 @@ It will generate single jar file with all required dependencies inside `target d
 #### Dump data from the source Postgres Database
 *Do not use compression if possible because Tool can only work with uncompressed file
-*If you want to migrate just `ts_kv` without `ts_kv_latest` just don't dump an unnecessary table and when starting the tool don't use arguments (paths) for input dump and output files*
-
 1. Dump related tables that need to correct save telemetry
    `pg_dump -h localhost -U postgres -d thingsboard -t tenant -t customer -t user -t dashboard -t asset -t device -t alarm -t rule_chain -t rule_node -t entity_view -t widgets_bundle -t widget_type -t tenant_profile -t device_profile -t api_usage_state -t tb_user > related_entities.dmp`
@@ -38,21 +36,26 @@ Tool use 3 different directories for saving SSTables - `ts_kv_cf`, `ts_kv_latest
 Create 3 empty directories. For example:
-    /home/ubunut/migration/ts
-    /home/ubunut/migration/ts_latest
-    /home/ubunut/migration/ts_partition
+    /home/user/migration/ts
+    /home/user/migration/ts_latest
+    /home/user/migration/ts_partition
 #### Run tool
-*Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination
+
+**If you want to migrate just `ts_kv` without `ts_kv_latest` or vice versa don't use arguments (paths) for output files*
+
+**Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination*
 ```
 java -jar ./tools-2.4.1-SNAPSHOT-jar-with-dependencies.jar 
-        -telemetryFrom ./source/ts_kv_all.dmp
+        -telemetryFrom /home/user/dump/ts_kv_all.dmp 
+        -relatedEntities /home/user/dump/relatedEntities.dmp 
         -latestOut /home/ubunut/migration/ts_latest 
-        -tsOut /home/ubunut/migration/ts  
-        -partitionsOut /home/ubunut/migration/ts_partition
-        -castEnable false
+        -tsOut /home/ubunut/migration/ts 
+        -partitionsOut /home/ubunut/migration/ts_partition 
+        -castEnable false 
 ```  
+*Use your paths for program arguments*
 Tool execution time depends on DB size, CPU resources and Disk throughput
@@ -62,20 +65,21 @@ Tool execution time depends on DB size, CPU resources and Disk throughput
 1. [Optional] install Cassandra on the instance
 2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this file `schema-ts.cql`
 3. Stop Cassandra
-4. Copy generated SSTable files into cassandra data dir:
+4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders
+5. Copy generated SSTable files into cassandra data dir using next command:
 ```
-    sudo find /home/ubunut/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \;
-    sudo find /home/ubunut/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \;
-    sudo find /home/ubunut/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \;
+    sudo find /home/user/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \;
+    sudo find /home/user/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \;
+    sudo find /home/user/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \;
 ```   
-    
-5. Start Cassandra service and trigger compaction
+  *Pay attention! Data folders have similar name  `ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13`, but you have to use own*  
+6. Start Cassandra service and trigger compaction
+
+    Trigger compactions: `nodetool compact thingsboard`
+   
+    Check compaction status: `nodetool compactionstats`
-```
-    trigger compactions: nodetool compact thingsboard
-    check compaction status: nodetool compactionstats
-```
 ## Switch Thignsboard into Hybrid Mode