简柏林 / thingskit · Commits

Commit b490e4ef4db8ecd6abfa71fd0f343a118a51954c

Authored by Igor Kulikov
1 parent e7d00f4b

Cassandra 2.5.0 to SQL migration

Inline Side-by-side
Showing 8 changed files with 609 additions and 62 deletions
... ... @@ -28,6 +28,7 @@ import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService;
28 28 import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
29 29 import org.thingsboard.server.service.install.SystemDataLoaderService;
30 30 import org.thingsboard.server.service.install.TsDatabaseSchemaService;
  31 +import org.thingsboard.server.service.install.migrate.EntitiesMigrateService;
31 32 import org.thingsboard.server.service.install.update.DataUpdateService;
32 33
33 34 @Service
... ... @@ -68,99 +69,95 @@ public class ThingsboardInstallService {
68 69 @Autowired
69 70 private DataUpdateService dataUpdateService;
70 71
  72 + @Autowired(required = false)
  73 + private EntitiesMigrateService entitiesMigrateService;
  74 +
71 75 public void performInstall() {
72 76 try {
73 77 if (isUpgrade) {
74 78 log.info("Starting ThingsBoard Upgrade from version {} ...", upgradeFromVersion);
75 79
76   - switch (upgradeFromVersion) {
77   - case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
78   - log.info("Upgrading ThingsBoard from version 1.2.3 to 1.3.0 ...");
79   -
80   - databaseEntitiesUpgradeService.upgradeDatabase("1.2.3");
  80 + if ("2.5.0-cassandra".equals(upgradeFromVersion)) {
  81 + log.info("Migrating ThingsBoard entities data from cassandra to SQL database ...");
  82 + entitiesMigrateService.migrate();
  83 + log.info("Updating system data...");
  84 + systemDataLoaderService.updateSystemWidgets();
  85 + } else {
  86 + switch (upgradeFromVersion) {
  87 + case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
  88 + log.info("Upgrading ThingsBoard from version 1.2.3 to 1.3.0 ...");
81 89
82   - case "1.3.0": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
83   - log.info("Upgrading ThingsBoard from version 1.3.0 to 1.3.1 ...");
  90 + databaseEntitiesUpgradeService.upgradeDatabase("1.2.3");
84 91
85   - databaseEntitiesUpgradeService.upgradeDatabase("1.3.0");
  92 + case "1.3.0": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
  93 + log.info("Upgrading ThingsBoard from version 1.3.0 to 1.3.1 ...");
86 94
87   - case "1.3.1": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
88   - log.info("Upgrading ThingsBoard from version 1.3.1 to 1.4.0 ...");
  95 + databaseEntitiesUpgradeService.upgradeDatabase("1.3.0");
89 96
90   - databaseEntitiesUpgradeService.upgradeDatabase("1.3.1");
  97 + case "1.3.1": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion
  98 + log.info("Upgrading ThingsBoard from version 1.3.1 to 1.4.0 ...");
91 99
92   - case "1.4.0":
93   - log.info("Upgrading ThingsBoard from version 1.4.0 to 2.0.0 ...");
  100 + databaseEntitiesUpgradeService.upgradeDatabase("1.3.1");
94 101
95   - databaseEntitiesUpgradeService.upgradeDatabase("1.4.0");
  102 + case "1.4.0":
  103 + log.info("Upgrading ThingsBoard from version 1.4.0 to 2.0.0 ...");
96 104
97   - dataUpdateService.updateData("1.4.0");
  105 + databaseEntitiesUpgradeService.upgradeDatabase("1.4.0");
98 106
99   - case "2.0.0":
100   - log.info("Upgrading ThingsBoard from version 2.0.0 to 2.1.1 ...");
  107 + dataUpdateService.updateData("1.4.0");
101 108
102   - databaseEntitiesUpgradeService.upgradeDatabase("2.0.0");
  109 + case "2.0.0":
  110 + log.info("Upgrading ThingsBoard from version 2.0.0 to 2.1.1 ...");
103 111
104   - case "2.1.1":
105   - log.info("Upgrading ThingsBoard from version 2.1.1 to 2.1.2 ...");
  112 + databaseEntitiesUpgradeService.upgradeDatabase("2.0.0");
106 113
107   - databaseEntitiesUpgradeService.upgradeDatabase("2.1.1");
108   - case "2.1.3":
109   - log.info("Upgrading ThingsBoard from version 2.1.3 to 2.2.0 ...");
  114 + case "2.1.1":
  115 + log.info("Upgrading ThingsBoard from version 2.1.1 to 2.1.2 ...");
110 116
111   - databaseEntitiesUpgradeService.upgradeDatabase("2.1.3");
  117 + databaseEntitiesUpgradeService.upgradeDatabase("2.1.1");
  118 + case "2.1.3":
  119 + log.info("Upgrading ThingsBoard from version 2.1.3 to 2.2.0 ...");
112 120
113   - case "2.3.0":
114   - log.info("Upgrading ThingsBoard from version 2.3.0 to 2.3.1 ...");
  121 + databaseEntitiesUpgradeService.upgradeDatabase("2.1.3");
115 122
116   - databaseEntitiesUpgradeService.upgradeDatabase("2.3.0");
  123 + case "2.3.0":
  124 + log.info("Upgrading ThingsBoard from version 2.3.0 to 2.3.1 ...");
117 125
118   - case "2.3.1":
119   - log.info("Upgrading ThingsBoard from version 2.3.1 to 2.4.0 ...");
  126 + databaseEntitiesUpgradeService.upgradeDatabase("2.3.0");
120 127
121   - databaseEntitiesUpgradeService.upgradeDatabase("2.3.1");
  128 + case "2.3.1":
  129 + log.info("Upgrading ThingsBoard from version 2.3.1 to 2.4.0 ...");
122 130
123   - case "2.4.0":
124   - log.info("Upgrading ThingsBoard from version 2.4.0 to 2.4.1 ...");
  131 + databaseEntitiesUpgradeService.upgradeDatabase("2.3.1");
125 132
126   - case "2.4.1":
127   - log.info("Upgrading ThingsBoard from version 2.4.1 to 2.4.2 ...");
  133 + case "2.4.0":
  134 + log.info("Upgrading ThingsBoard from version 2.4.0 to 2.4.1 ...");
128 135
129   - databaseEntitiesUpgradeService.upgradeDatabase("2.4.1");
130   - case "2.4.2":
131   - log.info("Upgrading ThingsBoard from version 2.4.2 to 2.4.3 ...");
  136 + case "2.4.1":
  137 + log.info("Upgrading ThingsBoard from version 2.4.1 to 2.4.2 ...");
132 138
133   - databaseEntitiesUpgradeService.upgradeDatabase("2.4.2");
  139 + databaseEntitiesUpgradeService.upgradeDatabase("2.4.1");
  140 + case "2.4.2":
  141 + log.info("Upgrading ThingsBoard from version 2.4.2 to 2.4.3 ...");
134 142
135   - case "2.4.3":
136   - log.info("Upgrading ThingsBoard from version 2.4.3 to 2.5 ...");
  143 + databaseEntitiesUpgradeService.upgradeDatabase("2.4.2");
137 144
138   - if (databaseTsUpgradeService != null) {
139   - databaseTsUpgradeService.upgradeDatabase("2.4.3");
140   - }
141   - databaseEntitiesUpgradeService.upgradeDatabase("2.4.3");
  145 + case "2.4.3":
  146 + log.info("Upgrading ThingsBoard from version 2.4.3 to 2.5 ...");
142 147
143   - log.info("Updating system data...");
  148 + if (databaseTsUpgradeService != null) {
  149 + databaseTsUpgradeService.upgradeDatabase("2.4.3");
  150 + }
  151 + databaseEntitiesUpgradeService.upgradeDatabase("2.4.3");
144 152
145   - systemDataLoaderService.deleteSystemWidgetBundle("charts");
146   - systemDataLoaderService.deleteSystemWidgetBundle("cards");
147   - systemDataLoaderService.deleteSystemWidgetBundle("maps");
148   - systemDataLoaderService.deleteSystemWidgetBundle("analogue_gauges");
149   - systemDataLoaderService.deleteSystemWidgetBundle("digital_gauges");
150   - systemDataLoaderService.deleteSystemWidgetBundle("gpio_widgets");
151   - systemDataLoaderService.deleteSystemWidgetBundle("alarm_widgets");
152   - systemDataLoaderService.deleteSystemWidgetBundle("control_widgets");
153   - systemDataLoaderService.deleteSystemWidgetBundle("maps_v2");
154   - systemDataLoaderService.deleteSystemWidgetBundle("gateway_widgets");
155   - systemDataLoaderService.deleteSystemWidgetBundle("input_widgets");
156   - systemDataLoaderService.deleteSystemWidgetBundle("date");
157   - systemDataLoaderService.deleteSystemWidgetBundle("entity_admin_widgets");
  153 + log.info("Updating system data...");
158 154
159   - systemDataLoaderService.loadSystemWidgets();
160   - break;
161   - default:
162   - throw new RuntimeException("Unable to upgrade ThingsBoard, unsupported fromVersion: " + upgradeFromVersion);
  155 + systemDataLoaderService.updateSystemWidgets();
  156 + break;
  157 + default:
  158 + throw new RuntimeException("Unable to upgrade ThingsBoard, unsupported fromVersion: " + upgradeFromVersion);
163 159
  160 + }
164 161 }
165 162 log.info("Upgrade finished successfully!");
166 163
... ...
... ... @@ -168,6 +168,24 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
168 168 installScripts.loadSystemWidgets();
169 169 }
170 170
  171 + @Override
  172 + public void updateSystemWidgets() throws Exception {
  173 + this.deleteSystemWidgetBundle("charts");
  174 + this.deleteSystemWidgetBundle("cards");
  175 + this.deleteSystemWidgetBundle("maps");
  176 + this.deleteSystemWidgetBundle("analogue_gauges");
  177 + this.deleteSystemWidgetBundle("digital_gauges");
  178 + this.deleteSystemWidgetBundle("gpio_widgets");
  179 + this.deleteSystemWidgetBundle("alarm_widgets");
  180 + this.deleteSystemWidgetBundle("control_widgets");
  181 + this.deleteSystemWidgetBundle("maps_v2");
  182 + this.deleteSystemWidgetBundle("gateway_widgets");
  183 + this.deleteSystemWidgetBundle("input_widgets");
  184 + this.deleteSystemWidgetBundle("date");
  185 + this.deleteSystemWidgetBundle("entity_admin_widgets");
  186 + installScripts.loadSystemWidgets();
  187 + }
  188 +
171 189 private User createUser(Authority authority,
172 190 TenantId tenantId,
173 191 CustomerId customerId,
... ...
... ... @@ -23,6 +23,8 @@ public interface SystemDataLoaderService {
23 23
24 24 void loadSystemWidgets() throws Exception;
25 25
  26 + void updateSystemWidgets() throws Exception;
  27 +
26 28 void loadDemoData() throws Exception;
27 29
28 30 void deleteSystemWidgetBundle(String bundleAlias) throws Exception;
... ...
  1 +package org.thingsboard.server.service.install.migrate;
  2 +
  3 +import lombok.extern.slf4j.Slf4j;
  4 +import org.springframework.beans.factory.annotation.Autowired;
  5 +import org.springframework.beans.factory.annotation.Value;
  6 +import org.springframework.context.annotation.Profile;
  7 +import org.springframework.stereotype.Service;
  8 +import org.thingsboard.server.common.data.EntityType;
  9 +import org.thingsboard.server.dao.cassandra.CassandraCluster;
  10 +import org.thingsboard.server.dao.util.SqlDao;
  11 +import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
  12 +
  13 +import java.sql.Connection;
  14 +import java.sql.DriverManager;
  15 +import java.util.Arrays;
  16 +import java.util.List;
  17 +
  18 +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.bigintColumn;
  19 +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.booleanColumn;
  20 +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.doubleColumn;
  21 +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.enumToIntColumn;
  22 +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.idColumn;
  23 +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.stringColumn;
  24 +
  25 +@Service
  26 +@Profile("install")
  27 +@SqlDao
  28 +@Slf4j
  29 +public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateService {
  30 +
  31 + @Autowired
  32 + private EntityDatabaseSchemaService entityDatabaseSchemaService;
  33 +
  34 + @Autowired
  35 + protected CassandraCluster cluster;
  36 +
  37 + @Value("${spring.datasource.url}")
  38 + protected String dbUrl;
  39 +
  40 + @Value("${spring.datasource.username}")
  41 + protected String dbUserName;
  42 +
  43 + @Value("${spring.datasource.password}")
  44 + protected String dbPassword;
  45 +
  46 +
  47 + @Override
  48 + public void migrate() throws Exception {
  49 + log.info("Performing migration of entities data from cassandra to SQL database ...");
  50 + entityDatabaseSchemaService.createDatabaseSchema();
  51 + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
  52 + conn.setAutoCommit(true);
  53 + for (CassandraToSqlTable table: tables) {
  54 + table.migrateToSql(cluster.getSession(), conn);
  55 + }
  56 + } catch (Exception e) {
  57 + log.error("Unexpected error during ThingsBoard entities data migration!", e);
  58 + throw e;
  59 + }
  60 + }
  61 +
  62 + private static List<CassandraToSqlTable> tables = Arrays.asList(
  63 + new CassandraToSqlTable("admin_settings",
  64 + idColumn("id"),
  65 + stringColumn("key"),
  66 + stringColumn("json_value")),
  67 + new CassandraToSqlTable("alarm",
  68 + idColumn("id"),
  69 + idColumn("tenant_id"),
  70 + stringColumn("type"),
  71 + idColumn("originator_id"),
  72 + enumToIntColumn("originator_type", EntityType.class),
  73 + stringColumn("severity"),
  74 + stringColumn("status"),
  75 + bigintColumn("start_ts"),
  76 + bigintColumn("end_ts"),
  77 + bigintColumn("ack_ts"),
  78 + bigintColumn("clear_ts"),
  79 + stringColumn("details", "additional_info"),
  80 + booleanColumn("propagate"),
  81 + stringColumn("propagate_relation_types")),
  82 + new CassandraToSqlTable("asset",
  83 + idColumn("id"),
  84 + idColumn("tenant_id"),
  85 + idColumn("customer_id"),
  86 + stringColumn("name"),
  87 + stringColumn("type"),
  88 + stringColumn("label"),
  89 + stringColumn("search_text"),
  90 + stringColumn("additional_info")),
  91 + new CassandraToSqlTable("audit_log_by_tenant_id", "audit_log",
  92 + idColumn("id"),
  93 + idColumn("tenant_id"),
  94 + idColumn("customer_id"),
  95 + idColumn("entity_id"),
  96 + stringColumn("entity_type"),
  97 + stringColumn("entity_name"),
  98 + idColumn("user_id"),
  99 + stringColumn("user_name"),
  100 + stringColumn("action_type"),
  101 + stringColumn("action_data"),
  102 + stringColumn("action_status"),
  103 + stringColumn("action_failure_details")),
  104 + new CassandraToSqlTable("attributes_kv_cf", "attribute_kv",
  105 + idColumn("entity_id"),
  106 + stringColumn("entity_type"),
  107 + stringColumn("attribute_type"),
  108 + stringColumn("attribute_key"),
  109 + booleanColumn("bool_v"),
  110 + stringColumn("str_v"),
  111 + bigintColumn("long_v"),
  112 + doubleColumn("dbl_v"),
  113 + stringColumn("json_v"),
  114 + bigintColumn("last_update_ts")),
  115 + new CassandraToSqlTable("component_descriptor",
  116 + idColumn("id"),
  117 + stringColumn("type"),
  118 + stringColumn("scope"),
  119 + stringColumn("name"),
  120 + stringColumn("search_text"),
  121 + stringColumn("clazz"),
  122 + stringColumn("configuration_descriptor"),
  123 + stringColumn("actions")),
  124 + new CassandraToSqlTable("customer",
  125 + idColumn("id"),
  126 + idColumn("tenant_id"),
  127 + stringColumn("title"),
  128 + stringColumn("search_text"),
  129 + stringColumn("country"),
  130 + stringColumn("state"),
  131 + stringColumn("city"),
  132 + stringColumn("address"),
  133 + stringColumn("address2"),
  134 + stringColumn("zip"),
  135 + stringColumn("phone"),
  136 + stringColumn("email"),
  137 + stringColumn("additional_info")),
  138 + new CassandraToSqlTable("dashboard",
  139 + idColumn("id"),
  140 + idColumn("tenant_id"),
  141 + stringColumn("title"),
  142 + stringColumn("search_text"),
  143 + stringColumn("assigned_customers"),
  144 + stringColumn("configuration")),
  145 + new CassandraToSqlTable("device",
  146 + idColumn("id"),
  147 + idColumn("tenant_id"),
  148 + idColumn("customer_id"),
  149 + stringColumn("name"),
  150 + stringColumn("type"),
  151 + stringColumn("label"),
  152 + stringColumn("search_text"),
  153 + stringColumn("additional_info")),
  154 + new CassandraToSqlTable("device_credentials",
  155 + idColumn("id"),
  156 + idColumn("device_id"),
  157 + stringColumn("credentials_type"),
  158 + stringColumn("credentials_id"),
  159 + stringColumn("credentials_value")),
  160 + new CassandraToSqlTable("event",
  161 + idColumn("id"),
  162 + idColumn("tenant_id"),
  163 + idColumn("entity_id"),
  164 + stringColumn("entity_type"),
  165 + stringColumn("event_type"),
  166 + stringColumn("event_uid"),
  167 + stringColumn("body")),
  168 + new CassandraToSqlTable("relation",
  169 + idColumn("from_id"),
  170 + stringColumn("from_type"),
  171 + idColumn("to_id"),
  172 + stringColumn("to_type"),
  173 + stringColumn("relation_type_group"),
  174 + stringColumn("relation_type"),
  175 + stringColumn("additional_info")),
  176 + new CassandraToSqlTable("user", "tb_user",
  177 + idColumn("id"),
  178 + idColumn("tenant_id"),
  179 + idColumn("customer_id"),
  180 + stringColumn("email"),
  181 + stringColumn("search_text"),
  182 + stringColumn("authority"),
  183 + stringColumn("first_name"),
  184 + stringColumn("last_name"),
  185 + stringColumn("additional_info")),
  186 + new CassandraToSqlTable("tenant",
  187 + idColumn("id"),
  188 + stringColumn("title"),
  189 + stringColumn("search_text"),
  190 + stringColumn("region"),
  191 + stringColumn("country"),
  192 + stringColumn("state"),
  193 + stringColumn("city"),
  194 + stringColumn("address"),
  195 + stringColumn("address2"),
  196 + stringColumn("zip"),
  197 + stringColumn("phone"),
  198 + stringColumn("email"),
  199 + stringColumn("additional_info")),
  200 + new CassandraToSqlTable("user_credentials",
  201 + idColumn("id"),
  202 + idColumn("user_id"),
  203 + booleanColumn("enabled"),
  204 + stringColumn("password"),
  205 + stringColumn("activate_token"),
  206 + stringColumn("reset_token")),
  207 + new CassandraToSqlTable("widget_type",
  208 + idColumn("id"),
  209 + idColumn("tenant_id"),
  210 + stringColumn("bundle_alias"),
  211 + stringColumn("alias"),
  212 + stringColumn("name"),
  213 + stringColumn("descriptor")),
  214 + new CassandraToSqlTable("widgets_bundle",
  215 + idColumn("id"),
  216 + idColumn("tenant_id"),
  217 + stringColumn("alias"),
  218 + stringColumn("title"),
  219 + stringColumn("search_text")),
  220 + new CassandraToSqlTable("rule_chain",
  221 + idColumn("id"),
  222 + idColumn("tenant_id"),
  223 + stringColumn("name"),
  224 + stringColumn("search_text"),
  225 + idColumn("first_rule_node_id"),
  226 + booleanColumn("root"),
  227 + booleanColumn("debug_mode"),
  228 + stringColumn("configuration"),
  229 + stringColumn("additional_info")),
  230 + new CassandraToSqlTable("rule_node",
  231 + idColumn("id"),
  232 + idColumn("rule_chain_id"),
  233 + stringColumn("type"),
  234 + stringColumn("name"),
  235 + booleanColumn("debug_mode"),
  236 + stringColumn("search_text"),
  237 + stringColumn("configuration"),
  238 + stringColumn("additional_info")),
  239 + new CassandraToSqlTable("entity_view",
  240 + idColumn("id"),
  241 + idColumn("tenant_id"),
  242 + idColumn("customer_id"),
  243 + idColumn("entity_id"),
  244 + stringColumn("entity_type"),
  245 + stringColumn("name"),
  246 + stringColumn("type"),
  247 + stringColumn("keys"),
  248 + bigintColumn("start_ts"),
  249 + bigintColumn("end_ts"),
  250 + stringColumn("search_text"),
  251 + stringColumn("additional_info"))
  252 + );
  253 +}
... ...
  1 +package org.thingsboard.server.service.install.migrate;
  2 +
  3 +import com.datastax.driver.core.Row;
  4 +import lombok.Data;
  5 +import org.thingsboard.server.common.data.UUIDConverter;
  6 +
  7 +import java.sql.PreparedStatement;
  8 +import java.sql.SQLException;
  9 +import java.sql.Types;
  10 +
  11 +@Data
  12 +public class CassandraToSqlColumn {
  13 +
  14 + private String cassandraColumnName;
  15 + private String sqlColumnName;
  16 + private CassandraToSqlColumnType type;
  17 + private int sqlType;
  18 + private Class<? extends Enum> enumClass;
  19 +
  20 + public static CassandraToSqlColumn idColumn(String name) {
  21 + return new CassandraToSqlColumn(name, CassandraToSqlColumnType.ID);
  22 + }
  23 +
  24 + public static CassandraToSqlColumn stringColumn(String name) {
  25 + return new CassandraToSqlColumn(name, CassandraToSqlColumnType.STRING);
  26 + }
  27 +
  28 + public static CassandraToSqlColumn stringColumn(String cassandraColumnName, String sqlColumnName) {
  29 + return new CassandraToSqlColumn(cassandraColumnName, sqlColumnName);
  30 + }
  31 +
  32 + public static CassandraToSqlColumn bigintColumn(String name) {
  33 + return new CassandraToSqlColumn(name, CassandraToSqlColumnType.BIGINT);
  34 + }
  35 +
  36 + public static CassandraToSqlColumn doubleColumn(String name) {
  37 + return new CassandraToSqlColumn(name, CassandraToSqlColumnType.DOUBLE);
  38 + }
  39 +
  40 + public static CassandraToSqlColumn booleanColumn(String name) {
  41 + return new CassandraToSqlColumn(name, CassandraToSqlColumnType.BOOLEAN);
  42 + }
  43 +
  44 + public static CassandraToSqlColumn enumToIntColumn(String name, Class<? extends Enum> enumClass) {
  45 + return new CassandraToSqlColumn(name, CassandraToSqlColumnType.ENUM_TO_INT, enumClass);
  46 + }
  47 +
  48 + public CassandraToSqlColumn(String columnName) {
  49 + this(columnName, columnName, CassandraToSqlColumnType.STRING, null);
  50 + }
  51 +
  52 + public CassandraToSqlColumn(String columnName, CassandraToSqlColumnType type) {
  53 + this(columnName, columnName, type, null);
  54 + }
  55 +
  56 + public CassandraToSqlColumn(String columnName, CassandraToSqlColumnType type, Class<? extends Enum> enumClass) {
  57 + this(columnName, columnName, type, enumClass);
  58 + }
  59 +
  60 + public CassandraToSqlColumn(String cassandraColumnName, String sqlColumnName) {
  61 + this(cassandraColumnName, sqlColumnName, CassandraToSqlColumnType.STRING, null);
  62 + }
  63 +
  64 + public CassandraToSqlColumn(String cassandraColumnName, String sqlColumnName, CassandraToSqlColumnType type,
  65 + Class<? extends Enum> enumClass) {
  66 + this.cassandraColumnName = cassandraColumnName;
  67 + this.sqlColumnName = sqlColumnName;
  68 + this.type = type;
  69 + this.enumClass = enumClass;
  70 + switch (this.type) {
  71 + case ID:
  72 + case STRING:
  73 + this.sqlType = Types.VARCHAR;
  74 + break;
  75 + case DOUBLE:
  76 + this.sqlType = Types.DOUBLE;
  77 + break;
  78 + case INTEGER:
  79 + case ENUM_TO_INT:
  80 + this.sqlType = Types.INTEGER;
  81 + break;
  82 + case FLOAT:
  83 + this.sqlType = Types.FLOAT;
  84 + break;
  85 + case BIGINT:
  86 + this.sqlType = Types.BIGINT;
  87 + break;
  88 + case BOOLEAN:
  89 + this.sqlType = Types.BOOLEAN;
  90 + break;
  91 + }
  92 + }
  93 +
  94 + public void prepareColumnValue(Row row, PreparedStatement sqlInsertStatement, int index) throws SQLException {
  95 + String value = this.getColumnValue(row, index);
  96 + this.setColumnValue(sqlInsertStatement, index, value);
  97 + }
  98 +
  99 + private String getColumnValue(Row row, int index) {
  100 + if (row.isNull(index)) {
  101 + return null;
  102 + } else {
  103 + switch (this.type) {
  104 + case ID:
  105 + return UUIDConverter.fromTimeUUID(row.getUUID(index));
  106 + case DOUBLE:
  107 + return Double.toString(row.getDouble(index));
  108 + case INTEGER:
  109 + return Integer.toString(row.getInt(index));
  110 + case FLOAT:
  111 + return Float.toString(row.getFloat(index));
  112 + case BIGINT:
  113 + return Long.toString(row.getLong(index));
  114 + case BOOLEAN:
  115 + return Boolean.toString(row.getBool(index));
  116 + case STRING:
  117 + case ENUM_TO_INT:
  118 + default:
  119 + return row.getString(index);
  120 + }
  121 + }
  122 + }
  123 +
  124 + private void setColumnValue(PreparedStatement sqlInsertStatement, int index, String value) throws SQLException {
  125 + if (value == null) {
  126 + sqlInsertStatement.setNull(index, this.sqlType);
  127 + } else {
  128 + switch (this.type) {
  129 + case DOUBLE:
  130 + sqlInsertStatement.setDouble(index, Double.parseDouble(value));
  131 + break;
  132 + case INTEGER:
  133 + sqlInsertStatement.setInt(index, Integer.parseInt(value));
  134 + break;
  135 + case FLOAT:
  136 + sqlInsertStatement.setFloat(index, Float.parseFloat(value));
  137 + break;
  138 + case BIGINT:
  139 + sqlInsertStatement.setLong(index, Long.parseLong(value));
  140 + break;
  141 + case BOOLEAN:
  142 + sqlInsertStatement.setBoolean(index, Boolean.parseBoolean(value));
  143 + break;
  144 + case ENUM_TO_INT:
  145 + Enum enumVal = Enum.valueOf(this.enumClass, value);
  146 + int intValue = enumVal.ordinal();
  147 + sqlInsertStatement.setInt(index, intValue);
  148 + break;
  149 + case STRING:
  150 + case ID:
  151 + default:
  152 + sqlInsertStatement.setString(index, value);
  153 + break;
  154 + }
  155 + }
  156 + }
  157 +
  158 +}
  159 +
... ...
  1 +package org.thingsboard.server.service.install.migrate;
  2 +
  3 +public enum CassandraToSqlColumnType {
  4 + ID,
  5 + DOUBLE,
  6 + INTEGER,
  7 + FLOAT,
  8 + BIGINT,
  9 + BOOLEAN,
  10 + STRING,
  11 + ENUM_TO_INT
  12 +}
... ...
  1 +package org.thingsboard.server.service.install.migrate;
  2 +
  3 +import com.datastax.driver.core.ResultSet;
  4 +import com.datastax.driver.core.Row;
  5 +import com.datastax.driver.core.Session;
  6 +import com.datastax.driver.core.SimpleStatement;
  7 +import com.datastax.driver.core.Statement;
  8 +import lombok.Data;
  9 +import lombok.extern.slf4j.Slf4j;
  10 +
  11 +import java.sql.Connection;
  12 +import java.sql.PreparedStatement;
  13 +import java.sql.SQLException;
  14 +import java.util.Arrays;
  15 +import java.util.Iterator;
  16 +import java.util.List;
  17 +
  18 +@Data
  19 +@Slf4j
  20 +public class CassandraToSqlTable {
  21 +
  22 + private String cassandraCf;
  23 + private String sqlTableName;
  24 +
  25 + private List<CassandraToSqlColumn> columns;
  26 +
  27 + public CassandraToSqlTable(String tableName, CassandraToSqlColumn... columns) {
  28 + this(tableName, tableName, columns);
  29 + }
  30 +
  31 + public CassandraToSqlTable(String cassandraCf, String sqlTableName, CassandraToSqlColumn... columns) {
  32 + this.cassandraCf = cassandraCf;
  33 + this.sqlTableName = sqlTableName;
  34 + this.columns = Arrays.asList(columns);
  35 + }
  36 +
  37 + public void migrateToSql(Session session, Connection conn) throws SQLException {
  38 + log.info("Migrating data from cassandra '{}' Column Family to '{}' SQL table...", this.cassandraCf, this.sqlTableName);
  39 + PreparedStatement sqlInsertStatement = createSqlInsertStatement(conn);
  40 + Statement cassandraSelectStatement = createCassandraSelectStatement();
  41 + cassandraSelectStatement.setFetchSize(100);
  42 + ResultSet rs = session.execute(cassandraSelectStatement);
  43 + Iterator<Row> iter = rs.iterator();
  44 + int rowCounter = 0;
  45 + while (iter.hasNext()) {
  46 + Row row = iter.next();
  47 + if (row != null) {
  48 + this.migrateRowToSql(row, sqlInsertStatement);
  49 + rowCounter++;
  50 + if (rowCounter % 100 == 0) {
  51 + sqlInsertStatement.executeBatch();
  52 + log.info("{} records migrated so far...", rowCounter);
  53 + }
  54 + }
  55 + }
  56 + if (rowCounter % 100 > 0) {
  57 + sqlInsertStatement.executeBatch();
  58 + }
  59 + sqlInsertStatement.close();
  60 + log.info("{} total records migrated.", rowCounter);
  61 + log.info("Finished migration data from cassandra '{}' Column Family to '{}' SQL table.", this.cassandraCf, this.sqlTableName);
  62 + }
  63 +
  64 + private void migrateRowToSql(Row row, PreparedStatement sqlInsertStatement) throws SQLException {
  65 + for (int i=0; i<this.columns.size();i++) {
  66 + CassandraToSqlColumn column = this.columns.get(i);
  67 + column.prepareColumnValue(row, sqlInsertStatement, i);
  68 + }
  69 + sqlInsertStatement.addBatch();
  70 + }
  71 +
  72 + private Statement createCassandraSelectStatement() {
  73 + StringBuilder selectStatementBuilder = new StringBuilder();
  74 + selectStatementBuilder.append("SELECT ");
  75 + for (CassandraToSqlColumn column : columns) {
  76 + selectStatementBuilder.append(column.getCassandraColumnName()).append(",");
  77 + }
  78 + selectStatementBuilder.deleteCharAt(selectStatementBuilder.length() - 1);
  79 + selectStatementBuilder.append(" FROM ").append(cassandraCf);
  80 + return new SimpleStatement(selectStatementBuilder.toString());
  81 + }
  82 +
  83 + private PreparedStatement createSqlInsertStatement(Connection conn) throws SQLException {
  84 + StringBuilder insertStatementBuilder = new StringBuilder();
  85 + insertStatementBuilder.append("INSERT INTO ").append(this.sqlTableName).append(" (");
  86 + for (CassandraToSqlColumn column : columns) {
  87 + insertStatementBuilder.append(column.getSqlColumnName()).append(",");
  88 + }
  89 + insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1);
  90 + insertStatementBuilder.append(") VALUES (");
  91 + for (CassandraToSqlColumn ignored : columns) {
  92 + insertStatementBuilder.append("?").append(",");
  93 + }
  94 + insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1);
  95 + insertStatementBuilder.append(")");
  96 + return conn.prepareStatement(insertStatementBuilder.toString());
  97 + }
  98 +
  99 +}
... ...
  1 +package org.thingsboard.server.service.install.migrate;
  2 +
  3 +public interface EntitiesMigrateService {
  4 +
  5 + void migrate() throws Exception;
  6 +
  7 +}
... ...