简柏林 / thingskit · Commits

Commit f7f06a22978e7515c9657a3f23e93b0d13f85468

Authored by AndrewVolosytnykhThingsboard
Committed by Andrew Shvayka
1 parent d5658d5b

Improvement of solution: README.md cleaned, build plugin fixed, some code cleaning

Inline Side-by-side
Showing 4 changed files with 33 additions and 42 deletions
... ... @@ -70,7 +70,7 @@
70 70 <plugins>
71 71 <plugin>
72 72 <artifactId>maven-assembly-plugin</artifactId>
73   - <configuration>
  73 + <configuration combine.self="override">
74 74 <archive>
75 75 <manifest>
76 76 <mainClass>org.thingsboard.client.tools.migrator.MigratorTool</mainClass>
... ...
... ... @@ -43,7 +43,6 @@ public class PgCaMigrator {
43 43 private final long LOG_BATCH = 1000000;
44 44 private final long rowPerFile = 1000000;
45 45
46   - private long linesProcessed = 0;
47 46 private long linesTsMigrated = 0;
48 47 private long linesLatestMigrated = 0;
49 48 private long castErrors = 0;
... ... @@ -99,7 +98,7 @@ public class PgCaMigrator {
99 98 System.out.println("START TO MIGRATE LATEST");
100 99 long start = System.currentTimeMillis();
101 100 processBlock(iterator, currentTsLatestWriter, outTsLatestDir, this::toValuesLatest);
102   - System.out.println("FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
  101 + System.out.println("TOTAL LINES MIGRATED: " + linesLatestMigrated + ", FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
103 102 isLatestDone = true;
104 103 }
105 104
... ... @@ -107,7 +106,7 @@ public class PgCaMigrator {
107 106 System.out.println("START TO MIGRATE TS");
108 107 long start = System.currentTimeMillis();
109 108 processBlock(iterator, currentTsWriter, outTsDir, this::toValuesTs);
110   - System.out.println("FORMING OF SSL FOR TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
  109 + System.out.println("TOTAL LINES MIGRATED: " + linesTsMigrated + ", FORMING OF SSL FOR TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
111 110 isTsDone = true;
112 111 }
113 112 }
... ... @@ -212,7 +211,7 @@ public class PgCaMigrator {
212 211
213 212 private void processBlock(LineIterator iterator, CQLSSTableWriter writer, File outDir, Function<List<String>, List<Object>> function) {
214 213 String currentLine;
215   - linesProcessed = 0;
  214 + long linesProcessed = 0;
216 215 while(iterator.hasNext()) {
217 216 logLinesProcessed(linesProcessed++);
218 217 currentLine = iterator.nextLine();
... ...
... ... @@ -23,7 +23,7 @@ It will generate single jar file with all required dependencies inside `target d
23 23
24 24 1. Dump related tables that need to correct save telemetry
25 25
26   - `pg_dump -h localhost -U postgres -d thingsboard -t tenant -t customer -t user -t dashboard -t asset -t device -t alarm -t rule_chain -t rule_node -t entity_view -t widgets_bundle -t widget_type -t tenant_profile -t device_profile -t api_usage_state -t tb_user > related_entities.dmp`
  26 + `pg_dump -h localhost -U postgres -d thingsboard -T admin_settings -T attribute_kv -T audit_log -T component_discriptor -T device_credentials -T event -T oauth2_client_registration -T oauth2_client_registration_info -T oauth2_client_registration_template -T relation -T rule_node_state tb_schema_settings -T user_credentials > related_entities.dmp`
27 27
28 28 2. Dump `ts_kv` and child:
29 29
... ... @@ -47,12 +47,12 @@ Create 3 empty directories. For example:
47 47 **Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination*
48 48
49 49 ```
50   -java -jar ./tools-2.4.1-SNAPSHOT-jar-with-dependencies.jar
  50 +java -jar ./tools-3.2.2-SNAPSHOT-jar-with-dependencies.jar
51 51 -telemetryFrom /home/user/dump/ts_kv_all.dmp
52   - -relatedEntities /home/user/dump/relatedEntities.dmp
53   - -latestOut /home/ubunut/migration/ts_latest
54   - -tsOut /home/ubunut/migration/ts
55   - -partitionsOut /home/ubunut/migration/ts_partition
  52 + -relatedEntities /home/user/dump/related_entities.dmp
  53 + -latestOut /home/user/migration/ts_latest
  54 + -tsOut /home/user/migration/ts
  55 + -partitionsOut /home/user/migration/ts_partition
56 56 -castEnable false
57 57 ```
58 58 *Use your paths for program arguments*
... ... @@ -63,7 +63,7 @@ Tool execution time depends on DB size, CPU resources and Disk throughput
63 63 * Note that this this part works only for single node Cassandra Cluster. If you have more nodes - it is better to use `sstableloader` tool.
64 64
65 65 1. [Optional] install Cassandra on the instance
66   -2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this file `schema-ts.cql`
  66 +2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this files `schema-ts.cql` and `schema-ts-latest.cql` using `source` command
67 67 3. Stop Cassandra
68 68 4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders
69 69 5. Copy generated SSTable files into cassandra data dir using next command:
... ...
... ... @@ -27,6 +27,24 @@ import java.util.Map;
27 27
28 28 public class RelatedEntitiesParser {
29 29 private final Map<String, String> allEntityIdsAndTypes = new HashMap<>();
  30 +
  31 + private final Map<String, EntityType> tableNameAndEntityType = Map.ofEntries(
  32 + Map.entry("COPY public.alarm ", EntityType.ALARM),
  33 + Map.entry("COPY public.asset ", EntityType.ASSET),
  34 + Map.entry("COPY public.customer ", EntityType.CUSTOMER),
  35 + Map.entry("COPY public.dashboard ", EntityType.DASHBOARD),
  36 + Map.entry("COPY public.device ", EntityType.DEVICE),
  37 + Map.entry("COPY public.rule_chain ", EntityType.RULE_CHAIN),
  38 + Map.entry("COPY public.rule_node ", EntityType.RULE_NODE),
  39 + Map.entry("COPY public.tenant ", EntityType.TENANT),
  40 + Map.entry("COPY public.tb_user ", EntityType.USER),
  41 + Map.entry("COPY public.entity_view ", EntityType.ENTITY_VIEW),
  42 + Map.entry("COPY public.widgets_bundle ", EntityType.WIDGETS_BUNDLE),
  43 + Map.entry("COPY public.widget_type ", EntityType.WIDGET_TYPE),
  44 + Map.entry("COPY public.tenant_profile ", EntityType.TENANT_PROFILE),
  45 + Map.entry("COPY public.device_profile ", EntityType.DEVICE_PROFILE),
  46 + Map.entry("COPY public.api_usage_state ", EntityType.API_USAGE_STATE)
  47 + );
30 48
31 49 public RelatedEntitiesParser(File source) throws IOException {
32 50 processAllTables(FileUtils.lineIterator(source));
... ... @@ -45,36 +63,10 @@ public class RelatedEntitiesParser {
45 63 try {
46 64 while (lineIterator.hasNext()) {
47 65 currentLine = lineIterator.nextLine();
48   - if (currentLine.startsWith("COPY public.alarm")) {
49   - processBlock(lineIterator, EntityType.ALARM);
50   - } else if (currentLine.startsWith("COPY public.asset")) {
51   - processBlock(lineIterator, EntityType.ASSET);
52   - } else if (currentLine.startsWith("COPY public.customer")) {
53   - processBlock(lineIterator, EntityType.CUSTOMER);
54   - } else if (currentLine.startsWith("COPY public.dashboard")) {
55   - processBlock(lineIterator, EntityType.DASHBOARD);
56   - } else if (currentLine.startsWith("COPY public.device")) {
57   - processBlock(lineIterator, EntityType.DEVICE);
58   - } else if (currentLine.startsWith("COPY public.rule_chain")) {
59   - processBlock(lineIterator, EntityType.RULE_CHAIN);
60   - } else if (currentLine.startsWith("COPY public.rule_node")) {
61   - processBlock(lineIterator, EntityType.RULE_NODE);
62   - } else if (currentLine.startsWith("COPY public.tenant")) {
63   - processBlock(lineIterator, EntityType.TENANT);
64   - } else if (currentLine.startsWith("COPY public.tb_user")) {
65   - processBlock(lineIterator, EntityType.USER);
66   - } else if (currentLine.startsWith("COPY public.entity_view")) {
67   - processBlock(lineIterator, EntityType.ENTITY_VIEW);
68   - } else if (currentLine.startsWith("COPY public.widgets_bundle")) {
69   - processBlock(lineIterator, EntityType.WIDGETS_BUNDLE);
70   - } else if (currentLine.startsWith("COPY public.widget_type")) {
71   - processBlock(lineIterator, EntityType.WIDGET_TYPE);
72   - } else if (currentLine.startsWith("COPY public.tenant_profile")) {
73   - processBlock(lineIterator, EntityType.TENANT_PROFILE);
74   - } else if (currentLine.startsWith("COPY public.device_profile")) {
75   - processBlock(lineIterator, EntityType.DEVICE_PROFILE);
76   - } else if (currentLine.startsWith("COPY public.api_usage_state")) {
77   - processBlock(lineIterator, EntityType.API_USAGE_STATE);
  66 + for(Map.Entry<String, EntityType> entry : tableNameAndEntityType.entrySet()) {
  67 + if(currentLine.startsWith(entry.getKey())) {
  68 + processBlock(lineIterator, entry.getValue());
  69 + }
78 70 }
79 71 }
80 72 } finally {
... ...