简柏林 / thingskit · Commits

Commit 6bab8bd6d36549230ec3e45bba1447dff5264946

Authored by Igor Kulikov
2 parents 2d235cf9 b9bc280d

Merge branch 'master' into develop/3.0

Inline Side-by-side
Showing 37 changed files with 980 additions and 328 deletions
@@ -164,25 +164,24 @@ BEGIN @@ -164,25 +164,24 @@ BEGIN
164 END; 164 END;
165 $$; 165 $$;
166 166
167 --- call insert_into_ts_kv(); 167 +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
  168 +$$
  169 +BEGIN
  170 + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
  171 + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
  172 +END;
  173 +$$ LANGUAGE plpgsql;
168 174
169 -CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$  
170 -DECLARE  
171 - insert_size CONSTANT integer := 10000;  
172 - insert_counter integer DEFAULT 0;  
173 - insert_record RECORD;  
174 - insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id, 175 +CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$
  176 +BEGIN
  177 + EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
175 ts_kv_records.key AS key, 178 ts_kv_records.key AS key,
176 ts_kv_records.ts AS ts, 179 ts_kv_records.ts AS ts,
177 ts_kv_records.bool_v AS bool_v, 180 ts_kv_records.bool_v AS bool_v,
178 ts_kv_records.str_v AS str_v, 181 ts_kv_records.str_v AS str_v,
179 ts_kv_records.long_v AS long_v, 182 ts_kv_records.long_v AS long_v,
180 ts_kv_records.dbl_v AS dbl_v 183 ts_kv_records.dbl_v AS dbl_v
181 - FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,  
182 - SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,  
183 - SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,  
184 - SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,  
185 - SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part, 184 + FROM (SELECT entity_id AS entity_id,
186 key_id AS key, 185 key_id AS key,
187 ts, 186 ts,
188 bool_v, 187 bool_v,
@@ -190,46 +189,23 @@ DECLARE @@ -190,46 +189,23 @@ DECLARE
190 long_v, 189 long_v,
191 dbl_v 190 dbl_v
192 FROM ts_kv_old 191 FROM ts_kv_old
193 - INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records;  
194 -BEGIN  
195 - OPEN insert_cursor;  
196 - LOOP  
197 - insert_counter := insert_counter + 1;  
198 - FETCH insert_cursor INTO insert_record;  
199 - IF NOT FOUND THEN  
200 - RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1;  
201 - EXIT;  
202 - END IF;  
203 - INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)  
204 - VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,  
205 - insert_record.long_v, insert_record.dbl_v);  
206 - IF MOD(insert_counter, insert_size) = 0 THEN  
207 - RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter;  
208 - END IF;  
209 - END LOOP;  
210 - CLOSE insert_cursor;  
211 -END; 192 + INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records) TO %L;', path_to_file);
  193 + EXECUTE format ('COPY ts_kv FROM %L', path_to_file);
  194 +END
212 $$; 195 $$;
213 196
214 -- call insert_into_ts_kv_latest(); 197 -- call insert_into_ts_kv_latest();
215 198
216 -CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$  
217 -DECLARE  
218 - insert_size CONSTANT integer := 10000;  
219 - insert_counter integer DEFAULT 0;  
220 - insert_record RECORD;  
221 - insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id, 199 +CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest(IN path_to_file varchar) LANGUAGE plpgsql AS $$
  200 +BEGIN
  201 + EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
222 ts_kv_latest_records.key AS key, 202 ts_kv_latest_records.key AS key,
223 ts_kv_latest_records.ts AS ts, 203 ts_kv_latest_records.ts AS ts,
224 ts_kv_latest_records.bool_v AS bool_v, 204 ts_kv_latest_records.bool_v AS bool_v,
225 ts_kv_latest_records.str_v AS str_v, 205 ts_kv_latest_records.str_v AS str_v,
226 ts_kv_latest_records.long_v AS long_v, 206 ts_kv_latest_records.long_v AS long_v,
227 ts_kv_latest_records.dbl_v AS dbl_v 207 ts_kv_latest_records.dbl_v AS dbl_v
228 - FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,  
229 - SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,  
230 - SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,  
231 - SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,  
232 - SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part, 208 + FROM (SELECT entity_id AS entity_id,
233 key_id AS key, 209 key_id AS key,
234 ts, 210 ts,
235 bool_v, 211 bool_v,
@@ -237,24 +213,8 @@ DECLARE @@ -237,24 +213,8 @@ DECLARE
237 long_v, 213 long_v,
238 dbl_v 214 dbl_v
239 FROM ts_kv_latest_old 215 FROM ts_kv_latest_old
240 - INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records;  
241 -BEGIN  
242 - OPEN insert_cursor;  
243 - LOOP  
244 - insert_counter := insert_counter + 1;  
245 - FETCH insert_cursor INTO insert_record;  
246 - IF NOT FOUND THEN  
247 - RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter - 1;  
248 - EXIT;  
249 - END IF;  
250 - INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)  
251 - VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,  
252 - insert_record.long_v, insert_record.dbl_v);  
253 - IF MOD(insert_counter, insert_size) = 0 THEN  
254 - RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter;  
255 - END IF;  
256 - END LOOP;  
257 - CLOSE insert_cursor; 216 + INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records) TO %L;', path_to_file);
  217 + EXECUTE format ('COPY ts_kv_latest FROM %L', path_to_file);
258 END; 218 END;
259 $$; 219 $$;
260 220
@@ -96,51 +96,36 @@ BEGIN @@ -96,51 +96,36 @@ BEGIN
96 END; 96 END;
97 $$; 97 $$;
98 98
99 --- call insert_into_ts_kv(); 99 +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
  100 +$$
  101 +BEGIN
  102 + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
  103 + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
  104 +END;
  105 +$$ LANGUAGE plpgsql;
100 106
101 -CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$ 107 +-- call insert_into_ts_kv();
102 108
103 -DECLARE  
104 - insert_size CONSTANT integer := 10000;  
105 - insert_counter integer DEFAULT 0;  
106 - insert_record RECORD;  
107 - insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,  
108 - new_ts_kv_records.key AS key,  
109 - new_ts_kv_records.ts AS ts,  
110 - new_ts_kv_records.bool_v AS bool_v,  
111 - new_ts_kv_records.str_v AS str_v,  
112 - new_ts_kv_records.long_v AS long_v,  
113 - new_ts_kv_records.dbl_v AS dbl_v  
114 - FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,  
115 - SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,  
116 - SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,  
117 - SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,  
118 - SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,  
119 - key_id AS key,  
120 - ts,  
121 - bool_v,  
122 - str_v,  
123 - long_v,  
124 - dbl_v  
125 - FROM tenant_ts_kv_old  
126 - INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records; 109 +CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$
127 BEGIN 110 BEGIN
128 - OPEN insert_cursor;  
129 - LOOP  
130 - insert_counter := insert_counter + 1;  
131 - FETCH insert_cursor INTO insert_record;  
132 - IF NOT FOUND THEN  
133 - RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter - 1;  
134 - EXIT;  
135 - END IF;  
136 - INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)  
137 - VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,  
138 - insert_record.long_v, insert_record.dbl_v);  
139 - IF MOD(insert_counter, insert_size) = 0 THEN  
140 - RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter;  
141 - END IF;  
142 - END LOOP;  
143 - CLOSE insert_cursor; 111 +
  112 + EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
  113 + new_ts_kv_records.key AS key,
  114 + new_ts_kv_records.ts AS ts,
  115 + new_ts_kv_records.bool_v AS bool_v,
  116 + new_ts_kv_records.str_v AS str_v,
  117 + new_ts_kv_records.long_v AS long_v,
  118 + new_ts_kv_records.dbl_v AS dbl_v
  119 + FROM (SELECT entity_id AS entity_id,
  120 + key_id AS key,
  121 + ts,
  122 + bool_v,
  123 + str_v,
  124 + long_v,
  125 + dbl_v
  126 + FROM tenant_ts_kv_old
  127 + INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records) TO %L;', path_to_file);
  128 + EXECUTE format ('COPY ts_kv FROM %L', path_to_file);
144 END; 129 END;
145 $$; 130 $$;
146 131
@@ -34,6 +34,9 @@ public abstract class AbstractSqlTsDatabaseUpgradeService { @@ -34,6 +34,9 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
34 protected static final String CALL_REGEX = "call "; 34 protected static final String CALL_REGEX = "call ";
35 protected static final String DROP_TABLE = "DROP TABLE "; 35 protected static final String DROP_TABLE = "DROP TABLE ";
36 protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS "; 36 protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS ";
  37 + protected static final String TS_KV_SQL = "ts_kv.sql";
  38 + protected static final String PATH_TO_USERS_PUBLIC_FOLDER = "C:\\Users\\Public";
  39 + protected static final String THINGSBOARD_WINDOWS_UPGRADE_DIR = "THINGSBOARD_WINDOWS_UPGRADE_DIR";
37 40
38 @Value("${spring.datasource.url}") 41 @Value("${spring.datasource.url}")
39 protected String dbUrl; 42 protected String dbUrl;
@@ -16,12 +16,17 @@ @@ -16,12 +16,17 @@
16 package org.thingsboard.server.service.install; 16 package org.thingsboard.server.service.install;
17 17
18 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
  19 +import org.apache.commons.lang3.StringUtils;
  20 +import org.apache.commons.lang3.SystemUtils;
19 import org.springframework.beans.factory.annotation.Value; 21 import org.springframework.beans.factory.annotation.Value;
20 import org.springframework.context.annotation.Profile; 22 import org.springframework.context.annotation.Profile;
21 import org.springframework.stereotype.Service; 23 import org.springframework.stereotype.Service;
22 import org.thingsboard.server.dao.util.PsqlDao; 24 import org.thingsboard.server.dao.util.PsqlDao;
23 import org.thingsboard.server.dao.util.SqlTsDao; 25 import org.thingsboard.server.dao.util.SqlTsDao;
24 26
  27 +import java.io.File;
  28 +import java.io.IOException;
  29 +import java.nio.file.Files;
25 import java.nio.file.Path; 30 import java.nio.file.Path;
26 import java.nio.file.Paths; 31 import java.nio.file.Paths;
27 import java.sql.Connection; 32 import java.sql.Connection;
@@ -37,6 +42,7 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe @@ -37,6 +42,7 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
37 @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") 42 @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}")
38 private String partitionType; 43 private String partitionType;
39 44
  45 + private static final String TS_KV_LATEST_SQL = "ts_kv_latest.sql";
40 private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; 46 private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql";
41 private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql"; 47 private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql";
42 private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql"; 48 private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql";
@@ -49,15 +55,13 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe @@ -49,15 +55,13 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
49 private static final String CREATE_PARTITIONS = "create_partitions(IN partition_type varchar)"; 55 private static final String CREATE_PARTITIONS = "create_partitions(IN partition_type varchar)";
50 private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; 56 private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
51 private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; 57 private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
52 - private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()";  
53 - private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; 58 + private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)";
  59 + private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest(IN path_to_file varchar)";
54 60
55 private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE; 61 private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE;
56 private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE; 62 private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE;
57 private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; 63 private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
58 private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; 64 private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
59 - private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV;  
60 - private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;  
61 65
62 private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD; 66 private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD;
63 private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD; 67 private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD;
@@ -94,9 +98,58 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe @@ -94,9 +98,58 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
94 } 98 }
95 executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); 99 executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
96 executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); 100 executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
97 - executeQuery(conn, CALL_INSERT_INTO_TS_KV);  
98 - executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);  
99 - executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); 101 +
  102 + Path pathToTempTsKvFile;
  103 + Path pathToTempTsKvLatestFile;
  104 + if (SystemUtils.IS_OS_WINDOWS) {
  105 + log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR);
  106 + Path pathToDir;
  107 + String thingsboardWindowsUpgradeDir = System.getenv("THINGSBOARD_WINDOWS_UPGRADE_DIR");
  108 + if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) {
  109 + log.info("Environment variable: {} was found!", THINGSBOARD_WINDOWS_UPGRADE_DIR);
  110 + pathToDir = Paths.get(thingsboardWindowsUpgradeDir);
  111 + } else {
  112 + log.info("Failed to lookup environment variable: {}", THINGSBOARD_WINDOWS_UPGRADE_DIR);
  113 + pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER);
  114 + }
  115 + log.info("Directory: {} will be used for creation temporary upgrade files!", pathToDir);
  116 + try {
  117 + Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql");
  118 + Path tsKvLatestFile = Files.createTempFile(pathToDir, "ts_kv_latest", ".sql");
  119 + pathToTempTsKvFile = tsKvFile.toAbsolutePath();
  120 + pathToTempTsKvLatestFile = tsKvLatestFile.toAbsolutePath();
  121 + executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
  122 + executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
  123 + executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');");
  124 + } catch (IOException | SecurityException e) {
  125 + throw new RuntimeException("Failed to create time-series upgrade files due to: " + e);
  126 + }
  127 + } else {
  128 + Path tempDirPath = Files.createTempDirectory("ts_kv");
  129 + File tempDirAsFile = tempDirPath.toFile();
  130 + boolean writable = tempDirAsFile.setWritable(true, false);
  131 + boolean readable = tempDirAsFile.setReadable(true, false);
  132 + boolean executable = tempDirAsFile.setExecutable(true, false);
  133 + if (writable && readable && executable) {
  134 + pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath();
  135 + pathToTempTsKvLatestFile = tempDirPath.resolve(TS_KV_LATEST_SQL).toAbsolutePath();
  136 + executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
  137 + executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
  138 + executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');");
  139 + } else {
  140 + throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
  141 + }
  142 + }
  143 + if (pathToTempTsKvFile.toFile().exists() && pathToTempTsKvLatestFile.toFile().exists()) {
  144 + boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
  145 + if (deleteTsKvFile) {
  146 + log.info("Successfully deleted the temp file for ts_kv table upgrade!");
  147 + }
  148 + boolean deleteTsKvLatestFile = pathToTempTsKvLatestFile.toFile().delete();
  149 + if (deleteTsKvLatestFile) {
  150 + log.info("Successfully deleted the temp file for ts_kv_latest table upgrade!");
  151 + }
  152 + }
100 153
101 executeQuery(conn, DROP_TABLE_TS_KV_OLD); 154 executeQuery(conn, DROP_TABLE_TS_KV_OLD);
102 executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD); 155 executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD);
@@ -16,6 +16,8 @@ @@ -16,6 +16,8 @@
16 package org.thingsboard.server.service.install; 16 package org.thingsboard.server.service.install;
17 17
18 import lombok.extern.slf4j.Slf4j; 18 import lombok.extern.slf4j.Slf4j;
  19 +import org.apache.commons.lang3.StringUtils;
  20 +import org.apache.commons.lang3.SystemUtils;
19 import org.springframework.beans.factory.annotation.Autowired; 21 import org.springframework.beans.factory.annotation.Autowired;
20 import org.springframework.beans.factory.annotation.Value; 22 import org.springframework.beans.factory.annotation.Value;
21 import org.springframework.context.annotation.Profile; 23 import org.springframework.context.annotation.Profile;
@@ -23,6 +25,9 @@ import org.springframework.stereotype.Service; @@ -23,6 +25,9 @@ import org.springframework.stereotype.Service;
23 import org.thingsboard.server.dao.util.PsqlDao; 25 import org.thingsboard.server.dao.util.PsqlDao;
24 import org.thingsboard.server.dao.util.TimescaleDBTsDao; 26 import org.thingsboard.server.dao.util.TimescaleDBTsDao;
25 27
  28 +import java.io.File;
  29 +import java.io.IOException;
  30 +import java.nio.file.Files;
26 import java.nio.file.Path; 31 import java.nio.file.Path;
27 import java.nio.file.Paths; 32 import java.nio.file.Paths;
28 import java.sql.Connection; 33 import java.sql.Connection;
@@ -47,14 +52,13 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr @@ -47,14 +52,13 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
47 private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()"; 52 private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()";
48 private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; 53 private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
49 private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; 54 private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
50 - private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; 55 + private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)";
51 private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; 56 private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()";
52 57
53 private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE; 58 private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE;
54 private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE; 59 private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE;
55 private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; 60 private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
56 private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; 61 private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
57 - private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV;  
58 private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST; 62 private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;
59 63
60 private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE; 64 private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE;
@@ -63,7 +67,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr @@ -63,7 +67,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
63 private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE; 67 private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE;
64 private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE; 68 private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
65 private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY; 69 private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
66 - private static final String DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV; 70 + private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
67 private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST; 71 private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
68 72
69 @Autowired 73 @Autowired
@@ -91,7 +95,49 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr @@ -91,7 +95,49 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
91 95
92 executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); 96 executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
93 executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); 97 executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
94 - executeQuery(conn, CALL_INSERT_INTO_TS_KV); 98 +
  99 + Path pathToTempTsKvFile;
  100 + if (SystemUtils.IS_OS_WINDOWS) {
  101 + Path pathToDir;
  102 + log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR);
  103 + String thingsboardWindowsUpgradeDir = System.getenv(THINGSBOARD_WINDOWS_UPGRADE_DIR);
  104 + if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) {
  105 + log.info("Environment variable: {} was found!", THINGSBOARD_WINDOWS_UPGRADE_DIR);
  106 + pathToDir = Paths.get(thingsboardWindowsUpgradeDir);
  107 + } else {
  108 + log.info("Failed to lookup environment variable: {}", THINGSBOARD_WINDOWS_UPGRADE_DIR);
  109 + pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER);
  110 + }
  111 + log.info("Directory: {} will be used for creation temporary upgrade file!", pathToDir);
  112 + try {
  113 + Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql");
  114 + pathToTempTsKvFile = tsKvFile.toAbsolutePath();
  115 + executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
  116 + pathToTempTsKvFile.toFile().deleteOnExit();
  117 + } catch (IOException | SecurityException e) {
  118 + throw new RuntimeException("Failed to create time-series upgrade files due to: " + e);
  119 + }
  120 + } else {
  121 + Path tempDirPath = Files.createTempDirectory("ts_kv");
  122 + File tempDirAsFile = tempDirPath.toFile();
  123 + boolean writable = tempDirAsFile.setWritable(true, false);
  124 + boolean readable = tempDirAsFile.setReadable(true, false);
  125 + boolean executable = tempDirAsFile.setExecutable(true, false);
  126 + if (writable && readable && executable) {
  127 + pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath();
  128 + executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
  129 + } else {
  130 + throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
  131 + }
  132 + }
  133 +
  134 + if (pathToTempTsKvFile.toFile().exists()) {
  135 + boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
  136 + if (deleteTsKvFile) {
  137 + log.info("Successfully deleted the temp file for ts_kv table upgrade!");
  138 + }
  139 + }
  140 +
95 executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); 141 executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
96 142
97 executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE); 143 executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE);
@@ -100,7 +146,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr @@ -100,7 +146,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
100 executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY); 146 executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY);
101 executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); 147 executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE);
102 executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); 148 executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY);
103 - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV); 149 + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV);
104 executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); 150 executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
105 151
106 executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); 152 executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");
  1 +TB_QUEUE_TYPE=kafka
1 2
2 DOCKER_REPO=thingsboard 3 DOCKER_REPO=thingsboard
3 4
@@ -32,6 +32,32 @@ function additionalComposeArgs() { @@ -32,6 +32,32 @@ function additionalComposeArgs() {
32 echo $ADDITIONAL_COMPOSE_ARGS 32 echo $ADDITIONAL_COMPOSE_ARGS
33 } 33 }
34 34
  35 +function additionalComposeQueueArgs() {
  36 + source .env
  37 + ADDITIONAL_COMPOSE_QUEUE_ARGS=""
  38 + case $TB_QUEUE_TYPE in
  39 + kafka)
  40 + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.kafka.yml"
  41 + ;;
  42 + aws-sqs)
  43 + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.aws-sqs.yml"
  44 + ;;
  45 + pubsub)
  46 + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.pubsub.yml"
  47 + ;;
  48 + rabbitmq)
  49 + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.rabbitmq.yml"
  50 + ;;
  51 + service-bus)
  52 + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml"
  53 + ;;
  54 + *)
  55 + echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or aws-sqs or pubsub or rabbitmq or service-bus." >&2
  56 + exit 1
  57 + esac
  58 + echo $ADDITIONAL_COMPOSE_QUEUE_ARGS
  59 +}
  60 +
35 function additionalStartupServices() { 61 function additionalStartupServices() {
36 source .env 62 source .env
37 ADDITIONAL_STARTUP_SERVICES="" 63 ADDITIONAL_STARTUP_SERVICES=""
  1 +#
  2 +# Copyright © 2016-2020 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +version: '2.2'
  18 +
  19 +services:
  20 + tb-js-executor:
  21 + env_file:
  22 + - queue-aws-sqs.env
  23 + tb-core1:
  24 + env_file:
  25 + - queue-aws-sqs.env
  26 + depends_on:
  27 + - zookeeper
  28 + - redis
  29 + tb-core2:
  30 + env_file:
  31 + - queue-aws-sqs.env
  32 + depends_on:
  33 + - zookeeper
  34 + - redis
  35 + tb-rule-engine1:
  36 + env_file:
  37 + - queue-aws-sqs.env
  38 + depends_on:
  39 + - zookeeper
  40 + - redis
  41 + tb-rule-engine2:
  42 + env_file:
  43 + - queue-aws-sqs.env
  44 + depends_on:
  45 + - zookeeper
  46 + - redis
  47 + tb-mqtt-transport1:
  48 + env_file:
  49 + - queue-aws-sqs.env
  50 + depends_on:
  51 + - zookeeper
  52 + tb-mqtt-transport2:
  53 + env_file:
  54 + - queue-aws-sqs.env
  55 + depends_on:
  56 + - zookeeper
  57 + tb-http-transport1:
  58 + env_file:
  59 + - queue-aws-sqs.env
  60 + depends_on:
  61 + - zookeeper
  62 + tb-http-transport2:
  63 + env_file:
  64 + - queue-aws-sqs.env
  65 + depends_on:
  66 + - zookeeper
  67 + tb-coap-transport:
  68 + env_file:
  69 + - queue-aws-sqs.env
  70 + depends_on:
  71 + - zookeeper
@@ -38,7 +38,7 @@ services: @@ -38,7 +38,7 @@ services:
38 env_file: 38 env_file:
39 - tb-node.hybrid.env 39 - tb-node.hybrid.env
40 depends_on: 40 depends_on:
41 - - kafka 41 + - zookeeper
42 - redis 42 - redis
43 - postgres 43 - postgres
44 - cassandra 44 - cassandra
@@ -46,7 +46,7 @@ services: @@ -46,7 +46,7 @@ services:
46 env_file: 46 env_file:
47 - tb-node.hybrid.env 47 - tb-node.hybrid.env
48 depends_on: 48 depends_on:
49 - - kafka 49 + - zookeeper
50 - redis 50 - redis
51 - postgres 51 - postgres
52 - cassandra 52 - cassandra
@@ -54,7 +54,7 @@ services: @@ -54,7 +54,7 @@ services:
54 env_file: 54 env_file:
55 - tb-node.hybrid.env 55 - tb-node.hybrid.env
56 depends_on: 56 depends_on:
57 - - kafka 57 + - zookeeper
58 - redis 58 - redis
59 - postgres 59 - postgres
60 - cassandra 60 - cassandra
@@ -62,7 +62,7 @@ services: @@ -62,7 +62,7 @@ services:
62 env_file: 62 env_file:
63 - tb-node.hybrid.env 63 - tb-node.hybrid.env
64 depends_on: 64 depends_on:
65 - - kafka 65 + - zookeeper
66 - redis 66 - redis
67 - postgres 67 - postgres
68 - cassandra 68 - cassandra
  1 +#
  2 +# Copyright © 2016-2020 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +version: '2.2'
  18 +
  19 +services:
  20 + kafka:
  21 + restart: always
  22 + image: "wurstmeister/kafka:2.12-2.3.0"
  23 + ports:
  24 + - "9092:9092"
  25 + env_file:
  26 + - kafka.env
  27 + depends_on:
  28 + - zookeeper
  29 + tb-js-executor:
  30 + env_file:
  31 + - queue-kafka.env
  32 + depends_on:
  33 + - kafka
  34 + tb-core1:
  35 + env_file:
  36 + - queue-kafka.env
  37 + depends_on:
  38 + - kafka
  39 + - redis
  40 + tb-core2:
  41 + env_file:
  42 + - queue-kafka.env
  43 + depends_on:
  44 + - kafka
  45 + - redis
  46 + tb-rule-engine1:
  47 + env_file:
  48 + - queue-kafka.env
  49 + depends_on:
  50 + - kafka
  51 + - redis
  52 + tb-rule-engine2:
  53 + env_file:
  54 + - queue-kafka.env
  55 + depends_on:
  56 + - kafka
  57 + - redis
  58 + tb-mqtt-transport1:
  59 + env_file:
  60 + - queue-kafka.env
  61 + depends_on:
  62 + - kafka
  63 + tb-mqtt-transport2:
  64 + env_file:
  65 + - queue-kafka.env
  66 + depends_on:
  67 + - kafka
  68 + tb-http-transport1:
  69 + env_file:
  70 + - queue-kafka.env
  71 + depends_on:
  72 + - kafka
  73 + tb-http-transport2:
  74 + env_file:
  75 + - queue-kafka.env
  76 + depends_on:
  77 + - kafka
  78 + tb-coap-transport:
  79 + env_file:
  80 + - queue-kafka.env
  81 + depends_on:
  82 + - kafka
@@ -31,27 +31,27 @@ services: @@ -31,27 +31,27 @@ services:
31 env_file: 31 env_file:
32 - tb-node.postgres.env 32 - tb-node.postgres.env
33 depends_on: 33 depends_on:
34 - - kafka 34 + - zookeeper
35 - redis 35 - redis
36 - postgres 36 - postgres
37 tb-core2: 37 tb-core2:
38 env_file: 38 env_file:
39 - tb-node.postgres.env 39 - tb-node.postgres.env
40 depends_on: 40 depends_on:
41 - - kafka 41 + - zookeeper
42 - redis 42 - redis
43 - postgres 43 - postgres
44 tb-rule-engine1: 44 tb-rule-engine1:
45 env_file: 45 env_file:
46 - tb-node.postgres.env 46 - tb-node.postgres.env
47 depends_on: 47 depends_on:
48 - - kafka 48 + - zookeeper
49 - redis 49 - redis
50 - postgres 50 - postgres
51 tb-rule-engine2: 51 tb-rule-engine2:
52 env_file: 52 env_file:
53 - tb-node.postgres.env 53 - tb-node.postgres.env
54 depends_on: 54 depends_on:
55 - - kafka 55 + - zookeeper
56 - redis 56 - redis
57 - postgres 57 - postgres
  1 +#
  2 +# Copyright © 2016-2020 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +version: '2.2'
  18 +
  19 +services:
  20 + tb-js-executor:
  21 + env_file:
  22 + - queue-pubsub.env.env
  23 + tb-core1:
  24 + env_file:
  25 + - queue-pubsub.env.env
  26 + depends_on:
  27 + - zookeeper
  28 + - redis
  29 + tb-core2:
  30 + env_file:
  31 + - queue-pubsub.env
  32 + depends_on:
  33 + - zookeeper
  34 + - redis
  35 + tb-rule-engine1:
  36 + env_file:
  37 + - queue-pubsub.env
  38 + depends_on:
  39 + - zookeeper
  40 + - redis
  41 + tb-rule-engine2:
  42 + env_file:
  43 + - queue-pubsub.env
  44 + depends_on:
  45 + - zookeeper
  46 + - redis
  47 + tb-mqtt-transport1:
  48 + env_file:
  49 + - queue-pubsub.env
  50 + depends_on:
  51 + - zookeeper
  52 + tb-mqtt-transport2:
  53 + env_file:
  54 + - queue-pubsub.env
  55 + depends_on:
  56 + - zookeeper
  57 + tb-http-transport1:
  58 + env_file:
  59 + - queue-pubsub.env
  60 + depends_on:
  61 + - zookeeper
  62 + tb-http-transport2:
  63 + env_file:
  64 + - queue-pubsub.env
  65 + depends_on:
  66 + - zookeeper
  67 + tb-coap-transport:
  68 + env_file:
  69 + - queue-pubsub.env
  70 + depends_on:
  71 + - zookeeper
  1 +#
  2 +# Copyright © 2016-2020 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +version: '2.2'
  18 +
  19 +services:
  20 + tb-js-executor:
  21 + env_file:
  22 + - queue-rabbitmq.env
  23 + tb-core1:
  24 + env_file:
  25 + - queue-rabbitmq.env
  26 + depends_on:
  27 + - zookeeper
  28 + - redis
  29 + tb-core2:
  30 + env_file:
  31 + - queue-rabbitmq.env
  32 + depends_on:
  33 + - zookeeper
  34 + - redis
  35 + tb-rule-engine1:
  36 + env_file:
  37 + - queue-rabbitmq.env
  38 + depends_on:
  39 + - zookeeper
  40 + - redis
  41 + tb-rule-engine2:
  42 + env_file:
  43 + - queue-rabbitmq.env
  44 + depends_on:
  45 + - zookeeper
  46 + - redis
  47 + tb-mqtt-transport1:
  48 + env_file:
  49 + - queue-rabbitmq.env
  50 + depends_on:
  51 + - zookeeper
  52 + tb-mqtt-transport2:
  53 + env_file:
  54 + - queue-rabbitmq.env
  55 + depends_on:
  56 + - zookeeper
  57 + tb-http-transport1:
  58 + env_file:
  59 + - queue-rabbitmq.env
  60 + depends_on:
  61 + - zookeeper
  62 + tb-http-transport2:
  63 + env_file:
  64 + - queue-rabbitmq.env
  65 + depends_on:
  66 + - zookeeper
  67 + tb-coap-transport:
  68 + env_file:
  69 + - queue-rabbitmq.env
  70 + depends_on:
  71 + - zookeeper
  1 +#
  2 +# Copyright © 2016-2020 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +version: '2.2'
  18 +
  19 +services:
  20 + tb-js-executor:
  21 + env_file:
  22 + - queue-service-bus.env
  23 + tb-core1:
  24 + env_file:
  25 + - queue-service-bus.env
  26 + depends_on:
  27 + - zookeeper
  28 + - redis
  29 + tb-core2:
  30 + env_file:
  31 + - queue-service-bus.env
  32 + depends_on:
  33 + - zookeeper
  34 + - redis
  35 + tb-rule-engine1:
  36 + env_file:
  37 + - queue-service-bus.env
  38 + depends_on:
  39 + - zookeeper
  40 + - redis
  41 + tb-rule-engine2:
  42 + env_file:
  43 + - queue-service-bus.env
  44 + depends_on:
  45 + - zookeeper
  46 + - redis
  47 + tb-mqtt-transport1:
  48 + env_file:
  49 + - queue-service-bus.env
  50 + depends_on:
  51 + - zookeeper
  52 + tb-mqtt-transport2:
  53 + env_file:
  54 + - queue-service-bus.env
  55 + depends_on:
  56 + - zookeeper
  57 + tb-http-transport1:
  58 + env_file:
  59 + - queue-service-bus.env
  60 + depends_on:
  61 + - zookeeper
  62 + tb-http-transport2:
  63 + env_file:
  64 + - queue-service-bus.env
  65 + depends_on:
  66 + - zookeeper
  67 + tb-coap-transport:
  68 + env_file:
  69 + - queue-service-bus.env
  70 + depends_on:
  71 + - zookeeper
@@ -26,15 +26,6 @@ services: @@ -26,15 +26,6 @@ services:
26 environment: 26 environment:
27 ZOO_MY_ID: 1 27 ZOO_MY_ID: 1
28 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 28 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181
29 - kafka:  
30 - restart: always  
31 - image: "wurstmeister/kafka:2.12-2.3.0"  
32 - ports:  
33 - - "9092:9092"  
34 - env_file:  
35 - - kafka.env  
36 - depends_on:  
37 - - zookeeper  
38 redis: 29 redis:
39 restart: always 30 restart: always
40 image: redis:4.0 31 image: redis:4.0
@@ -46,8 +37,6 @@ services: @@ -46,8 +37,6 @@ services:
46 scale: 20 37 scale: 20
47 env_file: 38 env_file:
48 - tb-js-executor.env 39 - tb-js-executor.env
49 - depends_on:  
50 - - kafka  
51 tb-core1: 40 tb-core1:
52 restart: always 41 restart: always
53 image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" 42 image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}"
@@ -67,7 +56,7 @@ services: @@ -67,7 +56,7 @@ services:
67 - ./tb-node/conf:/config 56 - ./tb-node/conf:/config
68 - ./tb-node/log:/var/log/thingsboard 57 - ./tb-node/log:/var/log/thingsboard
69 depends_on: 58 depends_on:
70 - - kafka 59 + - zookeeper
71 - redis 60 - redis
72 - tb-js-executor 61 - tb-js-executor
73 - tb-rule-engine1 62 - tb-rule-engine1
@@ -91,7 +80,7 @@ services: @@ -91,7 +80,7 @@ services:
91 - ./tb-node/conf:/config 80 - ./tb-node/conf:/config
92 - ./tb-node/log:/var/log/thingsboard 81 - ./tb-node/log:/var/log/thingsboard
93 depends_on: 82 depends_on:
94 - - kafka 83 + - zookeeper
95 - redis 84 - redis
96 - tb-js-executor 85 - tb-js-executor
97 - tb-rule-engine1 86 - tb-rule-engine1
@@ -115,7 +104,7 @@ services: @@ -115,7 +104,7 @@ services:
115 - ./tb-node/conf:/config 104 - ./tb-node/conf:/config
116 - ./tb-node/log:/var/log/thingsboard 105 - ./tb-node/log:/var/log/thingsboard
117 depends_on: 106 depends_on:
118 - - kafka 107 + - zookeeper
119 - redis 108 - redis
120 - tb-js-executor 109 - tb-js-executor
121 tb-rule-engine2: 110 tb-rule-engine2:
@@ -137,7 +126,7 @@ services: @@ -137,7 +126,7 @@ services:
137 - ./tb-node/conf:/config 126 - ./tb-node/conf:/config
138 - ./tb-node/log:/var/log/thingsboard 127 - ./tb-node/log:/var/log/thingsboard
139 depends_on: 128 depends_on:
140 - - kafka 129 + - zookeeper
141 - redis 130 - redis
142 - tb-js-executor 131 - tb-js-executor
143 tb-mqtt-transport1: 132 tb-mqtt-transport1:
@@ -153,7 +142,7 @@ services: @@ -153,7 +142,7 @@ services:
153 - ./tb-transports/mqtt/conf:/config 142 - ./tb-transports/mqtt/conf:/config
154 - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport 143 - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport
155 depends_on: 144 depends_on:
156 - - kafka 145 + - zookeeper
157 tb-mqtt-transport2: 146 tb-mqtt-transport2:
158 restart: always 147 restart: always
159 image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" 148 image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
@@ -167,7 +156,7 @@ services: @@ -167,7 +156,7 @@ services:
167 - ./tb-transports/mqtt/conf:/config 156 - ./tb-transports/mqtt/conf:/config
168 - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport 157 - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport
169 depends_on: 158 depends_on:
170 - - kafka 159 + - zookeeper
171 tb-http-transport1: 160 tb-http-transport1:
172 restart: always 161 restart: always
173 image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" 162 image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
@@ -181,7 +170,7 @@ services: @@ -181,7 +170,7 @@ services:
181 - ./tb-transports/http/conf:/config 170 - ./tb-transports/http/conf:/config
182 - ./tb-transports/http/log:/var/log/tb-http-transport 171 - ./tb-transports/http/log:/var/log/tb-http-transport
183 depends_on: 172 depends_on:
184 - - kafka 173 + - zookeeper
185 tb-http-transport2: 174 tb-http-transport2:
186 restart: always 175 restart: always
187 image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" 176 image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
@@ -195,7 +184,7 @@ services: @@ -195,7 +184,7 @@ services:
195 - ./tb-transports/http/conf:/config 184 - ./tb-transports/http/conf:/config
196 - ./tb-transports/http/log:/var/log/tb-http-transport 185 - ./tb-transports/http/log:/var/log/tb-http-transport
197 depends_on: 186 depends_on:
198 - - kafka 187 + - zookeeper
199 tb-coap-transport: 188 tb-coap-transport:
200 restart: always 189 restart: always
201 image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" 190 image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
@@ -209,7 +198,7 @@ services: @@ -209,7 +198,7 @@ services:
209 - ./tb-transports/coap/conf:/config 198 - ./tb-transports/coap/conf:/config
210 - ./tb-transports/coap/log:/var/log/tb-coap-transport 199 - ./tb-transports/coap/log:/var/log/tb-coap-transport
211 depends_on: 200 depends_on:
212 - - kafka 201 + - zookeeper
213 tb-web-ui1: 202 tb-web-ui1:
214 restart: always 203 restart: always
215 image: "${DOCKER_REPO}/${WEB_UI_DOCKER_NAME}:${TB_VERSION}" 204 image: "${DOCKER_REPO}/${WEB_UI_DOCKER_NAME}:${TB_VERSION}"
@@ -41,14 +41,16 @@ set -e @@ -41,14 +41,16 @@ set -e
41 41
42 source compose-utils.sh 42 source compose-utils.sh
43 43
  44 +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
  45 +
44 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? 46 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
45 47
46 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? 48 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
47 49
48 if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then 50 if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
49 - docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES 51 + docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
50 fi 52 fi
51 53
52 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 54 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1
53 55
54 56
@@ -19,6 +19,8 @@ set -e @@ -19,6 +19,8 @@ set -e
19 19
20 source compose-utils.sh 20 source compose-utils.sh
21 21
  22 +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
  23 +
22 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? 24 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
23 25
24 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS down -v 26 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS down -v
@@ -19,6 +19,8 @@ set -e @@ -19,6 +19,8 @@ set -e
19 19
20 source compose-utils.sh 20 source compose-utils.sh
21 21
  22 +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
  23 +
22 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? 24 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
23 25
24 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d 26 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d
@@ -19,6 +19,8 @@ set -e @@ -19,6 +19,8 @@ set -e
19 19
20 source compose-utils.sh 20 source compose-utils.sh
21 21
  22 +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
  23 +
22 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? 24 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
23 25
24 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS stop 26 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS stop
@@ -19,7 +19,9 @@ set -e @@ -19,7 +19,9 @@ set -e
19 19
20 source compose-utils.sh 20 source compose-utils.sh
21 21
  22 +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
  23 +
22 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? 24 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
23 25
24 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull $@  
25 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d --no-deps --build $@ 26 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@
  27 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@
@@ -40,12 +40,14 @@ set -e @@ -40,12 +40,14 @@ set -e
40 40
41 source compose-utils.sh 41 source compose-utils.sh
42 42
  43 +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
  44 +
43 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? 45 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
44 46
45 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? 47 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
46 48
47 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb-core1 49 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1
48 50
49 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES 51 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
50 52
51 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 53 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1
  1 +TB_QUEUE_TYPE=aws-sqs
  2 +TB_QUEUE_AWS_SQS_ACCESS_KEY_ID=YOUR_KEY
  3 +TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY=YOUR_SECRET
  4 +TB_QUEUE_AWS_SQS_REGION=YOUR_REGION
  1 +TB_QUEUE_TYPE=kafka
  2 +TB_KAFKA_SERVERS=kafka:9092
  1 +TB_QUEUE_TYPE=pubsub
  2 +TB_QUEUE_PUBSUB_PROJECT_ID=YOUR_PROJECT_ID
  3 +TB_QUEUE_PUBSUB_SERVICE_ACCOUNT=YOUR_SERVICE_ACCOUNT
  1 +TB_QUEUE_TYPE=rabbitmq
  2 +TB_QUEUE_RABBIT_MQ_HOST=localhost
  3 +TB_QUEUE_RABBIT_MQ_PORT=5672
  4 +TB_QUEUE_RABBIT_MQ_USERNAME=YOUR_USERNAME
  5 +TB_QUEUE_RABBIT_MQ_PASSWORD=YOUR_PASSWORD
  1 +TB_QUEUE_TYPE=service-bus
  2 +TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME=YOUR_NAMESPACE_NAME
  3 +TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME=YOUR_SAS_KEY_NAME
  4 +TB_QUEUE_SERVICE_BUS_SAS_KEY=YOUR_SAS_KEY
@@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181 @@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181
4 COAP_BIND_ADDRESS=0.0.0.0 4 COAP_BIND_ADDRESS=0.0.0.0
5 COAP_BIND_PORT=5683 5 COAP_BIND_PORT=5683
6 COAP_TIMEOUT=10000 6 COAP_TIMEOUT=10000
7 -  
8 -TB_QUEUE_TYPE=kafka  
9 -TB_KAFKA_SERVERS=kafka:9092  
@@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181 @@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181
4 HTTP_BIND_ADDRESS=0.0.0.0 4 HTTP_BIND_ADDRESS=0.0.0.0
5 HTTP_BIND_PORT=8081 5 HTTP_BIND_PORT=8081
6 HTTP_REQUEST_TIMEOUT=60000 6 HTTP_REQUEST_TIMEOUT=60000
7 -  
8 -TB_QUEUE_TYPE=kafka  
9 -TB_KAFKA_SERVERS=kafka:9092  
1 -TB_QUEUE_TYPE=kafka  
2 REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests 1 REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests
3 -TB_KAFKA_SERVERS=kafka:9092  
4 LOGGER_LEVEL=info 2 LOGGER_LEVEL=info
5 LOG_FOLDER=logs 3 LOG_FOLDER=logs
6 LOGGER_FILENAME=tb-js-executor-%DATE%.log 4 LOGGER_FILENAME=tb-js-executor-%DATE%.log
@@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181 @@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181
4 MQTT_BIND_ADDRESS=0.0.0.0 4 MQTT_BIND_ADDRESS=0.0.0.0
5 MQTT_BIND_PORT=1883 5 MQTT_BIND_PORT=1883
6 MQTT_TIMEOUT=10000 6 MQTT_TIMEOUT=10000
7 -  
8 -TB_QUEUE_TYPE=kafka  
9 -TB_KAFKA_SERVERS=kafka:9092  
@@ -2,8 +2,6 @@ @@ -2,8 +2,6 @@
2 2
3 ZOOKEEPER_ENABLED=true 3 ZOOKEEPER_ENABLED=true
4 ZOOKEEPER_URL=zookeeper:2181 4 ZOOKEEPER_URL=zookeeper:2181
5 -TB_QUEUE_TYPE=kafka  
6 -TB_KAFKA_SERVERS=kafka:9092  
7 JS_EVALUATOR=remote 5 JS_EVALUATOR=remote
8 TRANSPORT_TYPE=remote 6 TRANSPORT_TYPE=remote
9 CACHE_TYPE=redis 7 CACHE_TYPE=redis
@@ -40,6 +40,26 @@ Where: @@ -40,6 +40,26 @@ Where:
40 40
41 ## Running 41 ## Running
42 42
  43 +Execute the following command to deploy thirdparty resources:
  44 +
  45 +`
  46 +$ ./k8s-deploy-thirdparty.sh
  47 +`
  48 +
  49 +Get list of the running tb-redis pods and verify that all of them are in running state:
  50 +
  51 +`
  52 +$ kubectl get pods -l app=tb-redis
  53 +`
  54 +
  55 +Execute the following command to create redis cluster:
  56 +
  57 +`
  58 +$ kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ')
  59 +`
  60 +
  61 +Type **'yes'** when prompted.
  62 +
43 Execute the following command to deploy resources: 63 Execute the following command to deploy resources:
44 64
45 ` 65 `
@@ -19,3 +19,4 @@ set -e @@ -19,3 +19,4 @@ set -e
19 19
20 kubectl config set-context $(kubectl config current-context) --namespace=thingsboard 20 kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
21 kubectl delete -f thingsboard.yml 21 kubectl delete -f thingsboard.yml
  22 +kubectl delete -f thirdparty.yml
  1 +#!/bin/bash
  2 +#
  3 +# Copyright © 2016-2020 The Thingsboard Authors
  4 +#
  5 +# Licensed under the Apache License, Version 2.0 (the "License");
  6 +# you may not use this file except in compliance with the License.
  7 +# You may obtain a copy of the License at
  8 +#
  9 +# http://www.apache.org/licenses/LICENSE-2.0
  10 +#
  11 +# Unless required by applicable law or agreed to in writing, software
  12 +# distributed under the License is distributed on an "AS IS" BASIS,
  13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 +# See the License for the specific language governing permissions and
  15 +# limitations under the License.
  16 +#
  17 +
  18 +set -e
  19 +
  20 +kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
  21 +kubectl delete -f thirdparty.yml
  1 +#!/bin/bash
  2 +#
  3 +# Copyright © 2016-2020 The Thingsboard Authors
  4 +#
  5 +# Licensed under the Apache License, Version 2.0 (the "License");
  6 +# you may not use this file except in compliance with the License.
  7 +# You may obtain a copy of the License at
  8 +#
  9 +# http://www.apache.org/licenses/LICENSE-2.0
  10 +#
  11 +# Unless required by applicable law or agreed to in writing, software
  12 +# distributed under the License is distributed on an "AS IS" BASIS,
  13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 +# See the License for the specific language governing permissions and
  15 +# limitations under the License.
  16 +#
  17 +
  18 +set -e
  19 +
  20 +kubectl apply -f tb-namespace.yml
  21 +kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
  22 +kubectl apply -f thirdparty.yml
@@ -17,169 +17,6 @@ @@ -17,169 +17,6 @@
17 apiVersion: apps/v1 17 apiVersion: apps/v1
18 kind: Deployment 18 kind: Deployment
19 metadata: 19 metadata:
20 - name: zookeeper  
21 - namespace: thingsboard  
22 -spec:  
23 - selector:  
24 - matchLabels:  
25 - app: zookeeper  
26 - template:  
27 - metadata:  
28 - labels:  
29 - app: zookeeper  
30 - spec:  
31 - containers:  
32 - - name: server  
33 - imagePullPolicy: Always  
34 - image: zookeeper:3.5  
35 - ports:  
36 - - containerPort: 2181  
37 - readinessProbe:  
38 - periodSeconds: 5  
39 - tcpSocket:  
40 - port: 2181  
41 - livenessProbe:  
42 - periodSeconds: 5  
43 - tcpSocket:  
44 - port: 2181  
45 - env:  
46 - - name: ZOO_MY_ID  
47 - value: "1"  
48 - - name: ZOO_SERVERS  
49 - value: "server.1=0.0.0.0:2888:3888;0.0.0.0:2181"  
50 - restartPolicy: Always  
51 ----  
52 -apiVersion: v1  
53 -kind: Service  
54 -metadata:  
55 - name: zookeeper  
56 - namespace: thingsboard  
57 -spec:  
58 - type: ClusterIP  
59 - selector:  
60 - app: zookeeper  
61 - ports:  
62 - - name: zk-port  
63 - port: 2181  
64 ----  
65 -apiVersion: apps/v1  
66 -kind: Deployment  
67 -metadata:  
68 - name: tb-kafka  
69 - namespace: thingsboard  
70 -spec:  
71 - selector:  
72 - matchLabels:  
73 - app: tb-kafka  
74 - template:  
75 - metadata:  
76 - labels:  
77 - app: tb-kafka  
78 - spec:  
79 - containers:  
80 - - name: server  
81 - imagePullPolicy: Always  
82 - image: wurstmeister/kafka:2.12-2.2.1  
83 - ports:  
84 - - containerPort: 9092  
85 - readinessProbe:  
86 - periodSeconds: 20  
87 - tcpSocket:  
88 - port: 9092  
89 - livenessProbe:  
90 - periodSeconds: 5  
91 - tcpSocket:  
92 - port: 9092  
93 - env:  
94 - - name: KAFKA_ZOOKEEPER_CONNECT  
95 - value: "zookeeper:2181"  
96 - - name: KAFKA_LISTENERS  
97 - value: "INSIDE://:9093,OUTSIDE://:9092"  
98 - - name: KAFKA_ADVERTISED_LISTENERS  
99 - value: "INSIDE://:9093,OUTSIDE://tb-kafka:9092"  
100 - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP  
101 - value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"  
102 - - name: KAFKA_INTER_BROKER_LISTENER_NAME  
103 - value: "INSIDE"  
104 - - name: KAFKA_CREATE_TOPICS  
105 - value: "js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"  
106 - - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE  
107 - value: "false"  
108 - - name: KAFKA_LOG_RETENTION_BYTES  
109 - value: "1073741824"  
110 - - name: KAFKA_LOG_SEGMENT_BYTES  
111 - value: "268435456"  
112 - - name: KAFKA_LOG_RETENTION_MS  
113 - value: "300000"  
114 - - name: KAFKA_LOG_CLEANUP_POLICY  
115 - value: "delete"  
116 - restartPolicy: Always  
117 ----  
118 -apiVersion: v1  
119 -kind: Service  
120 -metadata:  
121 - name: tb-kafka  
122 - namespace: thingsboard  
123 -spec:  
124 - type: ClusterIP  
125 - selector:  
126 - app: tb-kafka  
127 - ports:  
128 - - name: tb-kafka-port  
129 - port: 9092  
130 ----  
131 -apiVersion: apps/v1  
132 -kind: Deployment  
133 -metadata:  
134 - name: tb-redis  
135 - namespace: thingsboard  
136 -spec:  
137 - selector:  
138 - matchLabels:  
139 - app: tb-redis  
140 - template:  
141 - metadata:  
142 - labels:  
143 - app: tb-redis  
144 - spec:  
145 - containers:  
146 - - name: server  
147 - imagePullPolicy: Always  
148 - image: redis:4.0  
149 - ports:  
150 - - containerPort: 6379  
151 - readinessProbe:  
152 - periodSeconds: 5  
153 - tcpSocket:  
154 - port: 6379  
155 - livenessProbe:  
156 - periodSeconds: 5  
157 - tcpSocket:  
158 - port: 6379  
159 - volumeMounts:  
160 - - mountPath: /data  
161 - name: redis-data  
162 - volumes:  
163 - - name: redis-data  
164 - emptyDir: {}  
165 - restartPolicy: Always  
166 ----  
167 -apiVersion: v1  
168 -kind: Service  
169 -metadata:  
170 - name: tb-redis  
171 - namespace: thingsboard  
172 -spec:  
173 - type: ClusterIP  
174 - selector:  
175 - app: tb-redis  
176 - ports:  
177 - - name: tb-redis-port  
178 - port: 6379  
179 ----  
180 -apiVersion: apps/v1  
181 -kind: Deployment  
182 -metadata:  
183 name: tb-js-executor 20 name: tb-js-executor
184 namespace: thingsboard 21 namespace: thingsboard
185 spec: 22 spec:
@@ -267,6 +104,10 @@ spec: @@ -267,6 +104,10 @@ spec:
267 value: "redis" 104 value: "redis"
268 - name: REDIS_HOST 105 - name: REDIS_HOST
269 value: "tb-redis" 106 value: "tb-redis"
  107 + - name: REDIS_CONNECTION_TYPE
  108 + value: "cluster"
  109 + - name: REDIS_NODES
  110 + value: "tb-redis:6379"
270 - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE 111 - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE
271 value: "false" 112 value: "false"
272 envFrom: 113 envFrom:
  1 +#
  2 +# Copyright © 2016-2020 The Thingsboard Authors
  3 +#
  4 +# Licensed under the Apache License, Version 2.0 (the "License");
  5 +# you may not use this file except in compliance with the License.
  6 +# You may obtain a copy of the License at
  7 +#
  8 +# http://www.apache.org/licenses/LICENSE-2.0
  9 +#
  10 +# Unless required by applicable law or agreed to in writing, software
  11 +# distributed under the License is distributed on an "AS IS" BASIS,
  12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +# See the License for the specific language governing permissions and
  14 +# limitations under the License.
  15 +#
  16 +
  17 +apiVersion: apps/v1
  18 +kind: StatefulSet
  19 +metadata:
  20 + name: zookeeper
  21 + namespace: thingsboard
  22 +spec:
  23 + serviceName: "zookeeper"
  24 + replicas: 3
  25 + podManagementPolicy: Parallel
  26 + selector:
  27 + matchLabels:
  28 + app: zookeeper
  29 + template:
  30 + metadata:
  31 + labels:
  32 + app: zookeeper
  33 + spec:
  34 + containers:
  35 + - name: zookeeper
  36 + imagePullPolicy: Always
  37 + image: zookeeper:3.5
  38 + ports:
  39 + - containerPort: 2181
  40 + name: client
  41 + - containerPort: 2888
  42 + name: server
  43 + - containerPort: 3888
  44 + name: election
  45 + readinessProbe:
  46 + periodSeconds: 60
  47 + tcpSocket:
  48 + port: 2181
  49 + livenessProbe:
  50 + periodSeconds: 60
  51 + tcpSocket:
  52 + port: 2181
  53 + env:
  54 + - name: ZOO_SERVERS
  55 + value: "server.0=zookeeper-0.zookeeper:2888:3888;2181 server.1=zookeeper-1.zookeeper:2888:3888;2181 server.2=zookeeper-2.zookeeper:2888:3888;2181"
  56 + - name: JVMFLAGS
  57 + value: "-Dzookeeper.electionPortBindRetry=0"
  58 + volumeMounts:
  59 + - name: data
  60 + mountPath: /data
  61 + readOnly: false
  62 + initContainers:
  63 + - command:
  64 + - /bin/bash
  65 + - -c
  66 + - |-
  67 + set -ex;
  68 + mkdir -p "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR";
  69 + chown "$ZOO_USER:$ZOO_USER" "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR"
  70 + if [[ ! -f "$ZOO_DATA_DIR/myid" ]]; then
  71 + echo $HOSTNAME| rev | cut -d "-" -f1 | rev > "$ZOO_DATA_DIR/myid"
  72 + fi
  73 + env:
  74 + - name: HOSTNAME
  75 + valueFrom:
  76 + fieldRef:
  77 + fieldPath: metadata.name
  78 + image: zookeeper:3.5
  79 + imagePullPolicy: IfNotPresent
  80 + name: zookeeper-init
  81 + securityContext:
  82 + runAsUser: 0
  83 + volumeMounts:
  84 + - name: data
  85 + mountPath: /data
  86 + readOnly: false
  87 + volumeClaimTemplates:
  88 + - metadata:
  89 + name: data
  90 + spec:
  91 + accessModes: [ "ReadWriteOnce" ]
  92 + resources:
  93 + requests:
  94 + storage: 100Mi
  95 +---
  96 +apiVersion: v1
  97 +kind: Service
  98 +metadata:
  99 + name: zookeeper
  100 + namespace: thingsboard
  101 +spec:
  102 + type: ClusterIP
  103 + ports:
  104 + - port: 2181
  105 + targetPort: 2181
  106 + name: client
  107 + - port: 2888
  108 + targetPort: 2888
  109 + name: server
  110 + - port: 3888
  111 + targetPort: 3888
  112 + name: election
  113 + selector:
  114 + app: zookeeper
  115 +---
  116 +apiVersion: apps/v1
  117 +kind: StatefulSet
  118 +metadata:
  119 + name: tb-kafka
  120 + namespace: thingsboard
  121 +spec:
  122 + serviceName: "tb-kafka"
  123 + replicas: 3
  124 + podManagementPolicy: Parallel
  125 + selector:
  126 + matchLabels:
  127 + app: tb-kafka
  128 + template:
  129 + metadata:
  130 + labels:
  131 + app: tb-kafka
  132 + spec:
  133 + containers:
  134 + - name: tb-kafka
  135 + imagePullPolicy: Always
  136 + image: wurstmeister/kafka:2.12-2.2.1
  137 + ports:
  138 + - containerPort: 9092
  139 + name: kafka-int
  140 + readinessProbe:
  141 + periodSeconds: 5
  142 + timeoutSeconds: 5
  143 + tcpSocket:
  144 + port: 9092
  145 + initialDelaySeconds: 60
  146 + livenessProbe:
  147 + timeoutSeconds: 5
  148 + periodSeconds: 5
  149 + tcpSocket:
  150 + port: 9092
  151 + initialDelaySeconds: 80
  152 + env:
  153 + - name: BROKER_ID_COMMAND
  154 + value: "hostname | cut -d'-' -f3"
  155 + - name: KAFKA_ZOOKEEPER_CONNECT
  156 + value: "zookeeper:2181"
  157 + - name: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS
  158 + value: "60000"
  159 + - name: KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE
  160 + value: "true"
  161 + - name: KAFKA_LISTENERS
  162 + value: "INSIDE://:9092"
  163 + - name: KAFKA_ADVERTISED_LISTENERS
  164 + value: "INSIDE://:9092"
  165 + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
  166 + value: "INSIDE:PLAINTEXT"
  167 + - name: KAFKA_INTER_BROKER_LISTENER_NAME
  168 + value: "INSIDE"
  169 + - name: KAFKA_CONTROLLER_SHUTDOWN_ENABLE
  170 + value: "true"
  171 + - name: KAFKA_CREATE_TOPICS
  172 + value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
  173 + - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
  174 + value: "false"
  175 + - name: KAFKA_LOG_RETENTION_BYTES
  176 + value: "1073741824"
  177 + - name: KAFKA_LOG_SEGMENT_BYTES
  178 + value: "268435456"
  179 + - name: KAFKA_LOG_RETENTION_MS
  180 + value: "300000"
  181 + - name: KAFKA_LOG_CLEANUP_POLICY
  182 + value: "delete"
  183 + - name: KAFKA_PORT
  184 + value: "9092"
  185 + - name: KAFKA_LOG_DIRS
  186 + value: "/kafka-logs"
  187 + volumeMounts:
  188 + - name: logs
  189 + mountPath: /kafka-logs
  190 + subPath: logs
  191 + volumeClaimTemplates:
  192 + - metadata:
  193 + name: logs
  194 + spec:
  195 + accessModes:
  196 + - ReadWriteOnce
  197 + resources:
  198 + requests:
  199 + storage: 1Gi
  200 +---
  201 +apiVersion: v1
  202 +kind: Service
  203 +metadata:
  204 + name: tb-kafka
  205 + namespace: thingsboard
  206 +spec:
  207 + type: ClusterIP
  208 + ports:
  209 + - port: 9092
  210 + targetPort: 9092
  211 + name: kafka-int
  212 + selector:
  213 + app: tb-kafka
  214 +---
  215 +apiVersion: v1
  216 +kind: ConfigMap
  217 +metadata:
  218 + name: tb-redis
  219 + namespace: thingsboard
  220 +data:
  221 + update-node.sh: |
  222 + #!/bin/sh
  223 + REDIS_NODES="/data/nodes.conf"
  224 + sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES}
  225 + exec "$@"
  226 + redis.conf: |+
  227 + cluster-enabled yes
  228 + cluster-require-full-coverage no
  229 + cluster-node-timeout 15000
  230 + cluster-config-file /data/nodes.conf
  231 + cluster-migration-barrier 1
  232 + appendonly yes
  233 + protected-mode no
  234 +---
  235 +apiVersion: apps/v1
  236 +kind: StatefulSet
  237 +metadata:
  238 + name: tb-redis
  239 + namespace: thingsboard
  240 +spec:
  241 + serviceName: server
  242 + replicas: 6
  243 + selector:
  244 + matchLabels:
  245 + app: tb-redis
  246 + template:
  247 + metadata:
  248 + labels:
  249 + app: tb-redis
  250 + spec:
  251 + containers:
  252 + - name: redis
  253 + image: redis:5.0.1-alpine
  254 + ports:
  255 + - containerPort: 6379
  256 + name: client
  257 + - containerPort: 16379
  258 + name: gossip
  259 + command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
  260 + env:
  261 + - name: POD_IP
  262 + valueFrom:
  263 + fieldRef:
  264 + fieldPath: status.podIP
  265 + volumeMounts:
  266 + - name: conf
  267 + mountPath: /conf
  268 + readOnly: false
  269 + - name: data
  270 + mountPath: /data
  271 + readOnly: false
  272 + volumes:
  273 + - name: conf
  274 + configMap:
  275 + name: tb-redis
  276 + defaultMode: 0755
  277 + volumeClaimTemplates:
  278 + - metadata:
  279 + name: data
  280 + spec:
  281 + accessModes: [ "ReadWriteOnce" ]
  282 + resources:
  283 + requests:
  284 + storage: 100Mi
  285 +---
  286 +apiVersion: v1
  287 +kind: Service
  288 +metadata:
  289 + name: tb-redis
  290 + namespace: thingsboard
  291 +spec:
  292 + type: ClusterIP
  293 + ports:
  294 + - port: 6379
  295 + targetPort: 6379
  296 + name: client
  297 + - port: 16379
  298 + targetPort: 16379
  299 + name: gossip
  300 + selector:
  301 + app: tb-redis