简柏林 / thingskit · Commits

Commit 2f6c6ad1fe03e403d6f0378c9a39d4d2ae757e22

Authored by Andrii Shvaika
2 parents c7b428fb 11479935

Merge with master

Inline Side-by-side
Showing 100 changed files with 1421 additions and 644 deletions

Too many changes to show.
Reload with full diff Plain diff Email patch

To preserve performance only 100 of 145 files are displayed.

  1 +--
  2 +-- Copyright © 2016-2020 The Thingsboard Authors
  3 +--
  4 +-- Licensed under the Apache License, Version 2.0 (the "License");
  5 +-- you may not use this file except in compliance with the License.
  6 +-- You may obtain a copy of the License at
  7 +--
  8 +-- http://www.apache.org/licenses/LICENSE-2.0
  9 +--
  10 +-- Unless required by applicable law or agreed to in writing, software
  11 +-- distributed under the License is distributed on an "AS IS" BASIS,
  12 +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +-- See the License for the specific language governing permissions and
  14 +-- limitations under the License.
  15 +--
  16 +
  17 +CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint)
  18 + LANGUAGE plpgsql AS
  19 +$$
  20 +DECLARE
  21 + max_tenant_ttl bigint;
  22 + max_customer_ttl bigint;
  23 + max_ttl bigint;
  24 + date timestamp;
  25 + partition_by_max_ttl_date varchar;
  26 + partition_month varchar;
  27 + partition_day varchar;
  28 + partition_year varchar;
  29 + partition varchar;
  30 + partition_to_delete varchar;
  31 +
  32 +
  33 +BEGIN
  34 + SELECT max(attribute_kv.long_v)
  35 + FROM tenant
  36 + INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id
  37 + WHERE attribute_kv.attribute_key = 'TTL'
  38 + into max_tenant_ttl;
  39 + SELECT max(attribute_kv.long_v)
  40 + FROM customer
  41 + INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id
  42 + WHERE attribute_kv.attribute_key = 'TTL'
  43 + into max_customer_ttl;
  44 + max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
  45 + if max_ttl IS NOT NULL AND max_ttl > 0 THEN
  46 + date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
  47 + partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
  48 + RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
  49 + IF partition_by_max_ttl_date IS NOT NULL THEN
  50 + CASE
  51 + WHEN partition_type = 'DAYS' THEN
  52 + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
  53 + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
  54 + partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5);
  55 + WHEN partition_type = 'MONTHS' THEN
  56 + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
  57 + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
  58 + ELSE
  59 + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
  60 + END CASE;
  61 + FOR partition IN SELECT tablename
  62 + FROM pg_tables
  63 + WHERE schemaname = 'public'
  64 + AND tablename like 'ts_kv_' || '%'
  65 + AND tablename != 'ts_kv_latest'
  66 + AND tablename != 'ts_kv_dictionary'
  67 + LOOP
  68 + IF partition != partition_by_max_ttl_date THEN
  69 + IF partition_year IS NOT NULL THEN
  70 + IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN
  71 + partition_to_delete := partition;
  72 + ELSE
  73 + IF partition_month IS NOT NULL THEN
  74 + IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN
  75 + partition_to_delete := partition;
  76 + ELSE
  77 + IF partition_day IS NOT NULL THEN
  78 + IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN
  79 + partition_to_delete := partition;
  80 + END IF;
  81 + END IF;
  82 + END IF;
  83 + END IF;
  84 + END IF;
  85 + END IF;
  86 + END IF;
  87 + IF partition_to_delete IS NOT NULL THEN
  88 + RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
  89 + EXECUTE format('DROP TABLE %I', partition_to_delete);
  90 + deleted := deleted + 1;
  91 + END IF;
  92 + END LOOP;
  93 + END IF;
  94 + END IF;
  95 +END
  96 +$$;
  97 +
  98 +CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS
  99 +$$
  100 +BEGIN
  101 + CASE
  102 + WHEN partition_type = 'DAYS' THEN
  103 + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd');
  104 + WHEN partition_type = 'MONTHS' THEN
  105 + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM');
  106 + WHEN partition_type = 'YEARS' THEN
  107 + partition := 'ts_kv_' || to_char(date, 'yyyy');
  108 + WHEN partition_type = 'INDEFINITE' THEN
  109 + partition := NULL;
  110 + ELSE
  111 + partition := NULL;
  112 + END CASE;
  113 + IF partition IS NOT NULL THEN
  114 + IF NOT EXISTS(SELECT
  115 + FROM pg_tables
  116 + WHERE schemaname = 'public'
  117 + AND tablename = partition) THEN
  118 + partition := NULL;
  119 + RAISE NOTICE 'Failed to found partition by ttl';
  120 + END IF;
  121 + END IF;
  122 +END;
  123 +$$ LANGUAGE plpgsql;
... ...
... ... @@ -14,33 +14,9 @@
14 14 -- limitations under the License.
15 15 --
16 16
17   --- select check_version();
  17 +-- call create_partition_ts_kv_table();
18 18
19   -CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$
20   -DECLARE
21   - current_version integer;
22   - valid_version boolean;
23   -BEGIN
24   - RAISE NOTICE 'Check the current installed PostgreSQL version...';
25   - SELECT current_setting('server_version_num') INTO current_version;
26   - IF current_version < 100000 THEN
27   - valid_version := FALSE;
28   - ELSE
29   - valid_version := TRUE;
30   - END IF;
31   - IF valid_version = FALSE THEN
32   - RAISE NOTICE 'Postgres version should be at least more than 10!';
33   - ELSE
34   - RAISE NOTICE 'PostgreSQL version is valid!';
35   - RAISE NOTICE 'Schema update started...';
36   - END IF;
37   - RETURN valid_version;
38   -END;
39   -$$ LANGUAGE 'plpgsql';
40   -
41   --- select create_partition_ts_kv_table();
42   -
43   -CREATE OR REPLACE FUNCTION create_partition_ts_kv_table() RETURNS VOID AS $$
  19 +CREATE OR REPLACE PROCEDURE create_partition_ts_kv_table() LANGUAGE plpgsql AS $$
44 20
45 21 BEGIN
46 22 ALTER TABLE ts_kv
... ... @@ -57,11 +33,11 @@ BEGIN
57 33 ALTER TABLE ts_kv
58 34 ALTER COLUMN key TYPE integer USING key::integer;
59 35 END;
60   -$$ LANGUAGE 'plpgsql';
  36 +$$;
61 37
62   --- select create_new_ts_kv_latest_table();
  38 +-- call create_new_ts_kv_latest_table();
63 39
64   -CREATE OR REPLACE FUNCTION create_new_ts_kv_latest_table() RETURNS VOID AS $$
  40 +CREATE OR REPLACE PROCEDURE create_new_ts_kv_latest_table() LANGUAGE plpgsql AS $$
65 41
66 42 BEGIN
67 43 ALTER TABLE ts_kv_latest
... ... @@ -81,13 +57,13 @@ BEGIN
81 57 ALTER TABLE ts_kv_latest
82 58 ADD CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key);
83 59 END;
84   -$$ LANGUAGE 'plpgsql';
  60 +$$;
  61 +
85 62
  63 +-- call create_partitions();
86 64
87   --- select create_partitions();
  65 +CREATE OR REPLACE PROCEDURE create_partitions() LANGUAGE plpgsql AS $$
88 66
89   -CREATE OR REPLACE FUNCTION create_partitions() RETURNS VOID AS
90   -$$
91 67 DECLARE
92 68 partition_date varchar;
93 69 from_ts bigint;
... ... @@ -111,11 +87,11 @@ BEGIN
111 87
112 88 CLOSE key_cursor;
113 89 END;
114   -$$ language 'plpgsql';
  90 +$$;
115 91
116   --- select create_ts_kv_dictionary_table();
  92 +-- call create_ts_kv_dictionary_table();
117 93
118   -CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$
  94 +CREATE OR REPLACE PROCEDURE create_ts_kv_dictionary_table() LANGUAGE plpgsql AS $$
119 95
120 96 BEGIN
121 97 CREATE TABLE IF NOT EXISTS ts_kv_dictionary
... ... @@ -125,12 +101,12 @@ BEGIN
125 101 CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
126 102 );
127 103 END;
128   -$$ LANGUAGE 'plpgsql';
  104 +$$;
  105 +
  106 +-- call insert_into_dictionary();
129 107
130   --- select insert_into_dictionary();
  108 +CREATE OR REPLACE PROCEDURE insert_into_dictionary() LANGUAGE plpgsql AS $$
131 109
132   -CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS
133   -$$
134 110 DECLARE
135 111 insert_record RECORD;
136 112 key_cursor CURSOR FOR SELECT DISTINCT key
... ... @@ -150,28 +126,27 @@ BEGIN
150 126 END LOOP;
151 127 CLOSE key_cursor;
152 128 END;
153   -$$ language 'plpgsql';
  129 +$$;
154 130
155   --- select insert_into_ts_kv();
  131 +-- call insert_into_ts_kv();
156 132
157   -CREATE OR REPLACE FUNCTION insert_into_ts_kv() RETURNS void AS
158   -$$
  133 +CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$
159 134 DECLARE
160 135 insert_size CONSTANT integer := 10000;
161 136 insert_counter integer DEFAULT 0;
162 137 insert_record RECORD;
163   - insert_cursor CURSOR FOR SELECT CONCAT(first_part_uuid, '-', second_part_uuid, '-1', third_part_uuid, '-', fourth_part_uuid, '-', fifth_part_uuid)::uuid AS entity_id,
  138 + insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
164 139 ts_kv_records.key AS key,
165 140 ts_kv_records.ts AS ts,
166 141 ts_kv_records.bool_v AS bool_v,
167 142 ts_kv_records.str_v AS str_v,
168 143 ts_kv_records.long_v AS long_v,
169 144 ts_kv_records.dbl_v AS dbl_v
170   - FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first_part_uuid,
171   - SUBSTRING(entity_id, 4, 4) AS second_part_uuid,
172   - SUBSTRING(entity_id, 1, 3) AS third_part_uuid,
173   - SUBSTRING(entity_id, 16, 4) AS fourth_part_uuid,
174   - SUBSTRING(entity_id, 20) AS fifth_part_uuid,
  145 + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
  146 + SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
  147 + SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
  148 + SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
  149 + SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
175 150 key_id AS key,
176 151 ts,
177 152 bool_v,
... ... @@ -198,28 +173,27 @@ BEGIN
198 173 END LOOP;
199 174 CLOSE insert_cursor;
200 175 END;
201   -$$ LANGUAGE 'plpgsql';
  176 +$$;
202 177
203   --- select insert_into_ts_kv_latest();
  178 +-- call insert_into_ts_kv_latest();
204 179
205   -CREATE OR REPLACE FUNCTION insert_into_ts_kv_latest() RETURNS void AS
206   -$$
  180 +CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$
207 181 DECLARE
208 182 insert_size CONSTANT integer := 10000;
209 183 insert_counter integer DEFAULT 0;
210 184 insert_record RECORD;
211   - insert_cursor CURSOR FOR SELECT CONCAT(first_part_uuid, '-', second_part_uuid, '-1', third_part_uuid, '-', fourth_part_uuid, '-', fifth_part_uuid)::uuid AS entity_id,
  185 + insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
212 186 ts_kv_latest_records.key AS key,
213 187 ts_kv_latest_records.ts AS ts,
214 188 ts_kv_latest_records.bool_v AS bool_v,
215 189 ts_kv_latest_records.str_v AS str_v,
216 190 ts_kv_latest_records.long_v AS long_v,
217 191 ts_kv_latest_records.dbl_v AS dbl_v
218   - FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first_part_uuid,
219   - SUBSTRING(entity_id, 4, 4) AS second_part_uuid,
220   - SUBSTRING(entity_id, 1, 3) AS third_part_uuid,
221   - SUBSTRING(entity_id, 16, 4) AS fourth_part_uuid,
222   - SUBSTRING(entity_id, 20) AS fifth_part_uuid,
  192 + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
  193 + SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
  194 + SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
  195 + SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
  196 + SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
223 197 key_id AS key,
224 198 ts,
225 199 bool_v,
... ... @@ -246,6 +220,6 @@ BEGIN
246 220 END LOOP;
247 221 CLOSE insert_cursor;
248 222 END;
249   -$$ LANGUAGE 'plpgsql';
  223 +$$;
250 224
251 225
... ...
... ... @@ -14,60 +14,32 @@
14 14 -- limitations under the License.
15 15 --
16 16
17   --- select check_version();
  17 +-- call create_new_ts_kv_table();
18 18
19   -CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$
20   -DECLARE
21   - current_version integer;
22   - valid_version boolean;
23   -BEGIN
24   - RAISE NOTICE 'Check the current installed PostgreSQL version...';
25   - SELECT current_setting('server_version_num') INTO current_version;
26   - IF current_version < 90600 THEN
27   - valid_version := FALSE;
28   - ELSE
29   - valid_version := TRUE;
30   - END IF;
31   - IF valid_version = FALSE THEN
32   - RAISE NOTICE 'Postgres version should be at least more than 9.6!';
33   - ELSE
34   - RAISE NOTICE 'PostgreSQL version is valid!';
35   - RAISE NOTICE 'Schema update started...';
36   - END IF;
37   - RETURN valid_version;
38   -END;
39   -$$ LANGUAGE 'plpgsql';
40   -
41   --- select create_new_tenant_ts_kv_table();
42   -
43   -CREATE OR REPLACE FUNCTION create_new_tenant_ts_kv_table() RETURNS VOID AS $$
  19 +CREATE OR REPLACE PROCEDURE create_new_ts_kv_table() LANGUAGE plpgsql AS $$
44 20
45 21 BEGIN
46 22 ALTER TABLE tenant_ts_kv
47 23 RENAME TO tenant_ts_kv_old;
48   - CREATE TABLE IF NOT EXISTS tenant_ts_kv
  24 + CREATE TABLE IF NOT EXISTS ts_kv
49 25 (
50 26 LIKE tenant_ts_kv_old
51 27 );
52   - ALTER TABLE tenant_ts_kv
53   - ALTER COLUMN tenant_id TYPE uuid USING tenant_id::uuid;
54   - ALTER TABLE tenant_ts_kv
55   - ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
56   - ALTER TABLE tenant_ts_kv
57   - ALTER COLUMN key TYPE integer USING key::integer;
58   - ALTER TABLE tenant_ts_kv
59   - ADD CONSTRAINT tenant_ts_kv_pkey PRIMARY KEY(tenant_id, entity_id, key, ts);
  28 + ALTER TABLE ts_kv ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
  29 + ALTER TABLE ts_kv ALTER COLUMN key TYPE integer USING key::integer;
  30 + ALTER INDEX ts_kv_pkey RENAME TO tenant_ts_kv_pkey_old;
60 31 ALTER INDEX idx_tenant_ts_kv RENAME TO idx_tenant_ts_kv_old;
61 32 ALTER INDEX tenant_ts_kv_ts_idx RENAME TO tenant_ts_kv_ts_idx_old;
62   --- PERFORM create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => 86400000, if_not_exists => true);
63   - CREATE INDEX IF NOT EXISTS idx_tenant_ts_kv ON tenant_ts_kv(tenant_id, entity_id, key, ts);
  33 + ALTER TABLE ts_kv ADD CONSTRAINT ts_kv_pkey PRIMARY KEY(entity_id, key, ts);
  34 +-- CREATE INDEX IF NOT EXISTS ts_kv_ts_idx ON ts_kv(ts DESC);
  35 + ALTER TABLE ts_kv DROP COLUMN IF EXISTS tenant_id;
64 36 END;
65   -$$ LANGUAGE 'plpgsql';
  37 +$$;
66 38
67 39
68   --- select create_ts_kv_latest_table();
  40 +-- call create_ts_kv_latest_table();
69 41
70   -CREATE OR REPLACE FUNCTION create_ts_kv_latest_table() RETURNS VOID AS $$
  42 +CREATE OR REPLACE PROCEDURE create_ts_kv_latest_table() LANGUAGE plpgsql AS $$
71 43
72 44 BEGIN
73 45 CREATE TABLE IF NOT EXISTS ts_kv_latest
... ... @@ -82,12 +54,12 @@ BEGIN
82 54 CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key)
83 55 );
84 56 END;
85   -$$ LANGUAGE 'plpgsql';
  57 +$$;
86 58
87 59
88   --- select create_ts_kv_dictionary_table();
  60 +-- call create_ts_kv_dictionary_table();
89 61
90   -CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$
  62 +CREATE OR REPLACE PROCEDURE create_ts_kv_dictionary_table() LANGUAGE plpgsql AS $$
91 63
92 64 BEGIN
93 65 CREATE TABLE IF NOT EXISTS ts_kv_dictionary
... ... @@ -97,12 +69,12 @@ BEGIN
97 69 CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
98 70 );
99 71 END;
100   -$$ LANGUAGE 'plpgsql';
  72 +$$;
101 73
102   --- select insert_into_dictionary();
  74 +-- call insert_into_dictionary();
  75 +
  76 +CREATE OR REPLACE PROCEDURE insert_into_dictionary() LANGUAGE plpgsql AS $$
103 77
104   -CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS
105   -$$
106 78 DECLARE
107 79 insert_record RECORD;
108 80 key_cursor CURSOR FOR SELECT DISTINCT key
... ... @@ -122,34 +94,28 @@ BEGIN
122 94 END LOOP;
123 95 CLOSE key_cursor;
124 96 END;
125   -$$ language 'plpgsql';
  97 +$$;
  98 +
  99 +-- call insert_into_ts_kv();
126 100
127   --- select insert_into_tenant_ts_kv();
  101 +CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$
128 102
129   -CREATE OR REPLACE FUNCTION insert_into_tenant_ts_kv() RETURNS void AS
130   -$$
131 103 DECLARE
132 104 insert_size CONSTANT integer := 10000;
133 105 insert_counter integer DEFAULT 0;
134 106 insert_record RECORD;
135   - insert_cursor CURSOR FOR SELECT CONCAT(tenant_id_first_part_uuid, '-', tenant_id_second_part_uuid, '-1', tenant_id_third_part_uuid, '-', tenant_id_fourth_part_uuid, '-', tenant_id_fifth_part_uuid)::uuid AS tenant_id,
136   - CONCAT(entity_id_first_part_uuid, '-', entity_id_second_part_uuid, '-1', entity_id_third_part_uuid, '-', entity_id_fourth_part_uuid, '-', entity_id_fifth_part_uuid)::uuid AS entity_id,
137   - tenant_ts_kv_records.key AS key,
138   - tenant_ts_kv_records.ts AS ts,
139   - tenant_ts_kv_records.bool_v AS bool_v,
140   - tenant_ts_kv_records.str_v AS str_v,
141   - tenant_ts_kv_records.long_v AS long_v,
142   - tenant_ts_kv_records.dbl_v AS dbl_v
143   - FROM (SELECT SUBSTRING(tenant_id, 8, 8) AS tenant_id_first_part_uuid,
144   - SUBSTRING(tenant_id, 4, 4) AS tenant_id_second_part_uuid,
145   - SUBSTRING(tenant_id, 1, 3) AS tenant_id_third_part_uuid,
146   - SUBSTRING(tenant_id, 16, 4) AS tenant_id_fourth_part_uuid,
147   - SUBSTRING(tenant_id, 20) AS tenant_id_fifth_part_uuid,
148   - SUBSTRING(entity_id, 8, 8) AS entity_id_first_part_uuid,
149   - SUBSTRING(entity_id, 4, 4) AS entity_id_second_part_uuid,
150   - SUBSTRING(entity_id, 1, 3) AS entity_id_third_part_uuid,
151   - SUBSTRING(entity_id, 16, 4) AS entity_id_fourth_part_uuid,
152   - SUBSTRING(entity_id, 20) AS entity_id_fifth_part_uuid,
  107 + insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
  108 + new_ts_kv_records.key AS key,
  109 + new_ts_kv_records.ts AS ts,
  110 + new_ts_kv_records.bool_v AS bool_v,
  111 + new_ts_kv_records.str_v AS str_v,
  112 + new_ts_kv_records.long_v AS long_v,
  113 + new_ts_kv_records.dbl_v AS dbl_v
  114 + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
  115 + SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
  116 + SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
  117 + SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
  118 + SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
153 119 key_id AS key,
154 120 ts,
155 121 bool_v,
... ... @@ -157,31 +123,31 @@ DECLARE
157 123 long_v,
158 124 dbl_v
159 125 FROM tenant_ts_kv_old
160   - INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS tenant_ts_kv_records;
  126 + INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records;
161 127 BEGIN
162 128 OPEN insert_cursor;
163 129 LOOP
164 130 insert_counter := insert_counter + 1;
165 131 FETCH insert_cursor INTO insert_record;
166 132 IF NOT FOUND THEN
167   - RAISE NOTICE '% records have been inserted into the new tenant_ts_kv table!',insert_counter - 1;
  133 + RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter - 1;
168 134 EXIT;
169 135 END IF;
170   - INSERT INTO tenant_ts_kv(tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
171   - VALUES (insert_record.tenant_id, insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
  136 + INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
  137 + VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
172 138 insert_record.long_v, insert_record.dbl_v);
173 139 IF MOD(insert_counter, insert_size) = 0 THEN
174   - RAISE NOTICE '% records have been inserted into the new tenant_ts_kv table!',insert_counter;
  140 + RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter;
175 141 END IF;
176 142 END LOOP;
177 143 CLOSE insert_cursor;
178 144 END;
179   -$$ LANGUAGE 'plpgsql';
  145 +$$;
  146 +
  147 +-- call insert_into_ts_kv_latest();
180 148
181   --- select insert_into_ts_kv_latest();
  149 +CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$
182 150
183   -CREATE OR REPLACE FUNCTION insert_into_ts_kv_latest() RETURNS void AS
184   -$$
185 151 DECLARE
186 152 insert_size CONSTANT integer := 10000;
187 153 insert_counter integer DEFAULT 0;
... ... @@ -191,7 +157,7 @@ DECLARE
191 157 latest_records.key AS key,
192 158 latest_records.entity_id AS entity_id,
193 159 latest_records.ts AS ts
194   - FROM (SELECT DISTINCT key AS key, entity_id AS entity_id, MAX(ts) AS ts FROM tenant_ts_kv GROUP BY key, entity_id) AS latest_records;
  160 + FROM (SELECT DISTINCT key AS key, entity_id AS entity_id, MAX(ts) AS ts FROM ts_kv GROUP BY key, entity_id) AS latest_records;
195 161 BEGIN
196 162 OPEN insert_cursor;
197 163 LOOP
... ... @@ -201,7 +167,7 @@ BEGIN
201 167 RAISE NOTICE '% records have been inserted into the ts_kv_latest table!',insert_counter - 1;
202 168 EXIT;
203 169 END IF;
204   - SELECT entity_id AS entity_id, key AS key, ts AS ts, bool_v AS bool_v, str_v AS str_v, long_v AS long_v, dbl_v AS dbl_v INTO insert_record FROM tenant_ts_kv WHERE entity_id = latest_record.entity_id AND key = latest_record.key AND ts = latest_record.ts;
  170 + SELECT entity_id AS entity_id, key AS key, ts AS ts, bool_v AS bool_v, str_v AS str_v, long_v AS long_v, dbl_v AS dbl_v INTO insert_record FROM ts_kv WHERE entity_id = latest_record.entity_id AND key = latest_record.key AND ts = latest_record.ts;
205 171 INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
206 172 VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, insert_record.long_v, insert_record.dbl_v);
207 173 IF MOD(insert_counter, insert_size) = 0 THEN
... ... @@ -210,4 +176,4 @@ BEGIN
210 176 END LOOP;
211 177 CLOSE insert_cursor;
212 178 END;
213   -$$ LANGUAGE 'plpgsql';
  179 +$$;
... ...
  1 +--
  2 +-- Copyright © 2016-2020 The Thingsboard Authors
  3 +--
  4 +-- Licensed under the Apache License, Version 2.0 (the "License");
  5 +-- you may not use this file except in compliance with the License.
  6 +-- You may obtain a copy of the License at
  7 +--
  8 +-- http://www.apache.org/licenses/LICENSE-2.0
  9 +--
  10 +-- Unless required by applicable law or agreed to in writing, software
  11 +-- distributed under the License is distributed on an "AS IS" BASIS,
  12 +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 +-- See the License for the specific language governing permissions and
  14 +-- limitations under the License.
  15 +--
  16 +
  17 +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
  18 +$$
  19 +BEGIN
  20 + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
  21 + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
  22 +END;
  23 +$$ LANGUAGE plpgsql;
  24 +
  25 +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  26 + OUT deleted bigint) AS
  27 +$$
  28 +BEGIN
  29 + EXECUTE format(
  30 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  31 + tenant_id, customer_id, ttl) into deleted;
  32 +END;
  33 +$$ LANGUAGE plpgsql;
  34 +
  35 +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  36 + OUT deleted bigint) AS
  37 +$$
  38 +BEGIN
  39 + EXECUTE format(
  40 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  41 + tenant_id, customer_id, ttl) into deleted;
  42 +END;
  43 +$$ LANGUAGE plpgsql;
  44 +
  45 +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  46 + OUT deleted bigint) AS
  47 +$$
  48 +BEGIN
  49 + EXECUTE format(
  50 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  51 + tenant_id, customer_id, ttl) into deleted;
  52 +END;
  53 +$$ LANGUAGE plpgsql;
  54 +
  55 +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31),
  56 + IN system_ttl bigint, INOUT deleted bigint)
  57 + LANGUAGE plpgsql AS
  58 +$$
  59 +DECLARE
  60 + tenant_cursor CURSOR FOR select tenant.id as tenant_id
  61 + from tenant;
  62 + tenant_id_record varchar;
  63 + customer_id_record varchar;
  64 + tenant_ttl bigint;
  65 + customer_ttl bigint;
  66 + deleted_for_entities bigint;
  67 + tenant_ttl_ts bigint;
  68 + customer_ttl_ts bigint;
  69 +BEGIN
  70 + OPEN tenant_cursor;
  71 + FETCH tenant_cursor INTO tenant_id_record;
  72 + WHILE FOUND
  73 + LOOP
  74 + EXECUTE format(
  75 + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
  76 + tenant_id_record, 'TTL') INTO tenant_ttl;
  77 + if tenant_ttl IS NULL THEN
  78 + tenant_ttl := system_ttl;
  79 + END IF;
  80 + IF tenant_ttl > 0 THEN
  81 + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint;
  82 + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
  83 + deleted := deleted + deleted_for_entities;
  84 + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record;
  85 + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
  86 + deleted := deleted + deleted_for_entities;
  87 + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record;
  88 + END IF;
  89 + FOR customer_id_record IN
  90 + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record
  91 + LOOP
  92 + EXECUTE format(
  93 + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
  94 + customer_id_record, 'TTL') INTO customer_ttl;
  95 + IF customer_ttl IS NULL THEN
  96 + customer_ttl_ts := tenant_ttl_ts;
  97 + ELSE
  98 + IF customer_ttl > 0 THEN
  99 + customer_ttl_ts :=
  100 + (EXTRACT(EPOCH FROM current_timestamp) * 1000 -
  101 + customer_ttl::bigint * 1000)::bigint;
  102 + END IF;
  103 + END IF;
  104 + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN
  105 + deleted_for_entities :=
  106 + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record,
  107 + customer_ttl_ts);
  108 + deleted := deleted + deleted_for_entities;
  109 + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record;
  110 + deleted_for_entities :=
  111 + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record,
  112 + customer_ttl_ts);
  113 + deleted := deleted + deleted_for_entities;
  114 + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
  115 + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record,
  116 + customer_id_record,
  117 + customer_ttl_ts);
  118 + deleted := deleted + deleted_for_entities;
  119 + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
  120 + END IF;
  121 + END LOOP;
  122 + FETCH tenant_cursor INTO tenant_id_record;
  123 + END LOOP;
  124 +END
  125 +$$;
... ...
... ... @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
24 24 import com.google.common.util.concurrent.FutureCallback;
25 25 import com.google.common.util.concurrent.Futures;
26 26 import com.google.common.util.concurrent.ListenableFuture;
  27 +import com.google.common.util.concurrent.MoreExecutors;
27 28 import com.typesafe.config.Config;
28 29 import com.typesafe.config.ConfigFactory;
29 30 import lombok.Getter;
... ... @@ -474,7 +475,7 @@ public class ActorSystemContext {
474 475 public void onFailure(Throwable th) {
475 476 log.error("Could not save debug Event for Node", th);
476 477 }
477   - });
  478 + }, MoreExecutors.directExecutor());
478 479 } catch (IOException ex) {
479 480 log.warn("Failed to persist rule node debug message", ex);
480 481 }
... ... @@ -527,7 +528,7 @@ public class ActorSystemContext {
527 528 public void onFailure(Throwable th) {
528 529 log.error("Could not save debug Event for Rule Chain", th);
529 530 }
530   - });
  531 + }, MoreExecutors.directExecutor());
531 532 }
532 533
533 534 public static Exception toException(Throwable error) {
... ...
... ... @@ -107,7 +107,6 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt
107 107 return filter;
108 108 }
109 109
110   - @Bean
111 110 protected JwtTokenAuthenticationProcessingFilter buildJwtTokenAuthenticationProcessingFilter() throws Exception {
112 111 List<String> pathsToSkip = new ArrayList(Arrays.asList(NON_TOKEN_BASED_AUTH_ENTRY_POINTS));
113 112 pathsToSkip.addAll(Arrays.asList(WS_TOKEN_BASED_AUTH_ENTRY_POINT, TOKEN_REFRESH_ENTRY_POINT, FORM_BASED_LOGIN_ENTRY_POINT,
... ...
... ... @@ -120,7 +120,9 @@ public class AlarmController extends BaseController {
120 120 try {
121 121 AlarmId alarmId = new AlarmId(toUUID(strAlarmId));
122 122 Alarm alarm = checkAlarmId(alarmId, Operation.WRITE);
123   - alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, System.currentTimeMillis()).get();
  123 + long ackTs = System.currentTimeMillis();
  124 + alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get();
  125 + alarm.setAckTs(ackTs);
124 126 logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null);
125 127 } catch (Exception e) {
126 128 throw handleException(e);
... ... @@ -135,7 +137,9 @@ public class AlarmController extends BaseController {
135 137 try {
136 138 AlarmId alarmId = new AlarmId(toUUID(strAlarmId));
137 139 Alarm alarm = checkAlarmId(alarmId, Operation.WRITE);
138   - alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, System.currentTimeMillis()).get();
  140 + long clearTs = System.currentTimeMillis();
  141 + alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get();
  142 + alarm.setClearTs(clearTs);
139 143 logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null);
140 144 } catch (Exception e) {
141 145 throw handleException(e);
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.controller;
18 18 import com.google.common.util.concurrent.FutureCallback;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22 import org.springframework.http.HttpStatus;
22 23 import org.springframework.http.ResponseEntity;
23 24 import org.springframework.security.access.prepost.PreAuthorize;
... ... @@ -32,6 +33,7 @@ import org.springframework.web.bind.annotation.RestController;
32 33 import org.springframework.web.context.request.async.DeferredResult;
33 34 import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg;
34 35 import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg;
  36 +import org.thingsboard.server.common.data.ClaimRequest;
35 37 import org.thingsboard.server.common.data.Customer;
36 38 import org.thingsboard.server.common.data.DataConstants;
37 39 import org.thingsboard.server.common.data.Device;
... ... @@ -46,7 +48,6 @@ import org.thingsboard.server.common.data.id.TenantId;
46 48 import org.thingsboard.server.common.data.page.TextPageData;
47 49 import org.thingsboard.server.common.data.page.TextPageLink;
48 50 import org.thingsboard.server.common.data.security.DeviceCredentials;
49   -import org.thingsboard.server.common.data.ClaimRequest;
50 51 import org.thingsboard.server.dao.device.claim.ClaimResponse;
51 52 import org.thingsboard.server.dao.device.claim.ClaimResult;
52 53 import org.thingsboard.server.dao.exception.IncorrectParameterException;
... ... @@ -432,7 +433,7 @@ public class DeviceController extends BaseController {
432 433 public void onFailure(Throwable t) {
433 434 deferredResult.setErrorResult(t);
434 435 }
435   - });
  436 + }, MoreExecutors.directExecutor());
436 437 return deferredResult;
437 438 } catch (Exception e) {
438 439 throw handleException(e);
... ... @@ -469,7 +470,7 @@ public class DeviceController extends BaseController {
469 470 public void onFailure(Throwable t) {
470 471 deferredResult.setErrorResult(t);
471 472 }
472   - });
  473 + }, MoreExecutors.directExecutor());
473 474 return deferredResult;
474 475 } catch (Exception e) {
475 476 throw handleException(e);
... ...
... ... @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent;
37 37 import org.thingsboard.server.service.security.permission.Operation;
38 38
39 39 import java.util.List;
  40 +import java.util.stream.Collectors;
40 41
41 42
42 43 @RestController
... ... @@ -167,7 +168,7 @@ public class EntityRelationController extends BaseController {
167 168 checkEntityId(entityId, Operation.READ);
168 169 RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
169 170 try {
170   - return checkNotNull(relationService.findByFrom(getTenantId(), entityId, typeGroup));
  171 + return checkNotNull(filterRelationsByReadPermission(relationService.findByFrom(getTenantId(), entityId, typeGroup)));
171 172 } catch (Exception e) {
172 173 throw handleException(e);
173 174 }
... ... @@ -185,7 +186,7 @@ public class EntityRelationController extends BaseController {
185 186 checkEntityId(entityId, Operation.READ);
186 187 RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
187 188 try {
188   - return checkNotNull(relationService.findInfoByFrom(getTenantId(), entityId, typeGroup).get());
  189 + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByFrom(getTenantId(), entityId, typeGroup).get()));
189 190 } catch (Exception e) {
190 191 throw handleException(e);
191 192 }
... ... @@ -205,7 +206,7 @@ public class EntityRelationController extends BaseController {
205 206 checkEntityId(entityId, Operation.READ);
206 207 RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
207 208 try {
208   - return checkNotNull(relationService.findByFromAndType(getTenantId(), entityId, strRelationType, typeGroup));
  209 + return checkNotNull(filterRelationsByReadPermission(relationService.findByFromAndType(getTenantId(), entityId, strRelationType, typeGroup)));
209 210 } catch (Exception e) {
210 211 throw handleException(e);
211 212 }
... ... @@ -223,7 +224,7 @@ public class EntityRelationController extends BaseController {
223 224 checkEntityId(entityId, Operation.READ);
224 225 RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
225 226 try {
226   - return checkNotNull(relationService.findByTo(getTenantId(), entityId, typeGroup));
  227 + return checkNotNull(filterRelationsByReadPermission(relationService.findByTo(getTenantId(), entityId, typeGroup)));
227 228 } catch (Exception e) {
228 229 throw handleException(e);
229 230 }
... ... @@ -241,7 +242,7 @@ public class EntityRelationController extends BaseController {
241 242 checkEntityId(entityId, Operation.READ);
242 243 RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
243 244 try {
244   - return checkNotNull(relationService.findInfoByTo(getTenantId(), entityId, typeGroup).get());
  245 + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByTo(getTenantId(), entityId, typeGroup).get()));
245 246 } catch (Exception e) {
246 247 throw handleException(e);
247 248 }
... ... @@ -261,7 +262,7 @@ public class EntityRelationController extends BaseController {
261 262 checkEntityId(entityId, Operation.READ);
262 263 RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
263 264 try {
264   - return checkNotNull(relationService.findByToAndType(getTenantId(), entityId, strRelationType, typeGroup));
  265 + return checkNotNull(filterRelationsByReadPermission(relationService.findByToAndType(getTenantId(), entityId, strRelationType, typeGroup)));
265 266 } catch (Exception e) {
266 267 throw handleException(e);
267 268 }
... ... @@ -276,7 +277,7 @@ public class EntityRelationController extends BaseController {
276 277 checkNotNull(query.getFilters());
277 278 checkEntityId(query.getParameters().getEntityId(), Operation.READ);
278 279 try {
279   - return checkNotNull(relationService.findByQuery(getTenantId(), query).get());
  280 + return checkNotNull(filterRelationsByReadPermission(relationService.findByQuery(getTenantId(), query).get()));
280 281 } catch (Exception e) {
281 282 throw handleException(e);
282 283 }
... ... @@ -291,12 +292,28 @@ public class EntityRelationController extends BaseController {
291 292 checkNotNull(query.getFilters());
292 293 checkEntityId(query.getParameters().getEntityId(), Operation.READ);
293 294 try {
294   - return checkNotNull(relationService.findInfoByQuery(getTenantId(), query).get());
  295 + return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByQuery(getTenantId(), query).get()));
295 296 } catch (Exception e) {
296 297 throw handleException(e);
297 298 }
298 299 }
299 300
  301 + private <T extends EntityRelation> List<T> filterRelationsByReadPermission(List<T> relationsByQuery) {
  302 + return relationsByQuery.stream().filter(relationByQuery -> {
  303 + try {
  304 + checkEntityId(relationByQuery.getTo(), Operation.READ);
  305 + } catch (ThingsboardException e) {
  306 + return false;
  307 + }
  308 + try {
  309 + checkEntityId(relationByQuery.getFrom(), Operation.READ);
  310 + } catch (ThingsboardException e) {
  311 + return false;
  312 + }
  313 + return true;
  314 + }).collect(Collectors.toList());
  315 + }
  316 +
300 317 private RelationTypeGroup parseRelationTypeGroup(String strRelationTypeGroup, RelationTypeGroup defaultValue) {
301 318 RelationTypeGroup result = defaultValue;
302 319 if (strRelationTypeGroup != null && strRelationTypeGroup.trim().length() > 0) {
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.controller;
18 18 import com.google.common.util.concurrent.FutureCallback;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22 import lombok.extern.slf4j.Slf4j;
22 23 import org.springframework.http.HttpStatus;
23 24 import org.springframework.security.access.prepost.PreAuthorize;
... ... @@ -160,7 +161,7 @@ public class EntityViewController extends BaseController {
160 161 });
161 162 }
162 163 return null;
163   - });
  164 + }, MoreExecutors.directExecutor());
164 165 } else {
165 166 return Futures.immediateFuture(null);
166 167 }
... ...
... ... @@ -22,6 +22,7 @@ import com.google.common.base.Function;
22 22 import com.google.common.util.concurrent.FutureCallback;
23 23 import com.google.common.util.concurrent.Futures;
24 24 import com.google.common.util.concurrent.ListenableFuture;
  25 +import com.google.common.util.concurrent.MoreExecutors;
25 26 import com.google.gson.JsonElement;
26 27 import com.google.gson.JsonParseException;
27 28 import com.google.gson.JsonParser;
... ... @@ -175,7 +176,7 @@ public class TelemetryController extends BaseController {
175 176 public DeferredResult<ResponseEntity> getTimeseriesKeys(
176 177 @PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr) throws ThingsboardException {
177 178 return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr,
178   - (result, tenantId, entityId) -> Futures.addCallback(tsService.findAllLatest(tenantId, entityId), getTsKeysToResponseCallback(result)));
  179 + (result, tenantId, entityId) -> Futures.addCallback(tsService.findAllLatest(tenantId, entityId), getTsKeysToResponseCallback(result), MoreExecutors.directExecutor()));
179 180 }
180 181
181 182 @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
... ... @@ -211,7 +212,7 @@ public class TelemetryController extends BaseController {
211 212 List<ReadTsKvQuery> queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg))
212 213 .collect(Collectors.toList());
213 214
214   - Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes));
  215 + Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor());
215 216 });
216 217 }
217 218
... ... @@ -461,7 +462,7 @@ public class TelemetryController extends BaseController {
461 462 } else {
462 463 future = tsService.findLatest(user.getTenantId(), entityId, toKeysList(keys));
463 464 }
464   - Futures.addCallback(future, getTsKvListCallback(result, useStrictDataTypes));
  465 + Futures.addCallback(future, getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor());
465 466 }
466 467
467 468 private void getAttributeValuesCallback(@Nullable DeferredResult<ResponseEntity> result, SecurityUser user, EntityId entityId, String scope, String keys) {
... ... @@ -469,9 +470,9 @@ public class TelemetryController extends BaseController {
469 470 FutureCallback<List<AttributeKvEntry>> callback = getAttributeValuesToResponseCallback(result, user, scope, entityId, keyList);
470 471 if (!StringUtils.isEmpty(scope)) {
471 472 if (keyList != null && !keyList.isEmpty()) {
472   - Futures.addCallback(attributesService.find(user.getTenantId(), entityId, scope, keyList), callback);
  473 + Futures.addCallback(attributesService.find(user.getTenantId(), entityId, scope, keyList), callback, MoreExecutors.directExecutor());
473 474 } else {
474   - Futures.addCallback(attributesService.findAll(user.getTenantId(), entityId, scope), callback);
  475 + Futures.addCallback(attributesService.findAll(user.getTenantId(), entityId, scope), callback, MoreExecutors.directExecutor());
475 476 }
476 477 } else {
477 478 List<ListenableFuture<List<AttributeKvEntry>>> futures = new ArrayList<>();
... ... @@ -485,12 +486,12 @@ public class TelemetryController extends BaseController {
485 486
486 487 ListenableFuture<List<AttributeKvEntry>> future = mergeAllAttributesFutures(futures);
487 488
488   - Futures.addCallback(future, callback);
  489 + Futures.addCallback(future, callback, MoreExecutors.directExecutor());
489 490 }
490 491 }
491 492
492 493 private void getAttributeKeysCallback(@Nullable DeferredResult<ResponseEntity> result, TenantId tenantId, EntityId entityId, String scope) {
493   - Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), getAttributeKeysToResponseCallback(result));
  494 + Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), getAttributeKeysToResponseCallback(result), MoreExecutors.directExecutor());
494 495 }
495 496
496 497 private void getAttributeKeysCallback(@Nullable DeferredResult<ResponseEntity> result, TenantId tenantId, EntityId entityId) {
... ... @@ -501,7 +502,7 @@ public class TelemetryController extends BaseController {
501 502
502 503 ListenableFuture<List<AttributeKvEntry>> future = mergeAllAttributesFutures(futures);
503 504
504   - Futures.addCallback(future, getAttributeKeysToResponseCallback(result));
  505 + Futures.addCallback(future, getAttributeKeysToResponseCallback(result), MoreExecutors.directExecutor());
505 506 }
506 507
507 508 private FutureCallback<List<TsKvEntry>> getTsKeysToResponseCallback(final DeferredResult<ResponseEntity> response) {
... ...
... ... @@ -22,38 +22,18 @@ import org.springframework.beans.factory.annotation.Value;
22 22 import java.nio.charset.StandardCharsets;
23 23 import java.nio.file.Files;
24 24 import java.nio.file.Path;
25   -import java.sql.CallableStatement;
26 25 import java.sql.Connection;
  26 +import java.sql.ResultSet;
27 27 import java.sql.SQLException;
28 28 import java.sql.SQLWarning;
29   -import java.sql.Types;
  29 +import java.sql.Statement;
30 30
31 31 @Slf4j
32 32 public abstract class AbstractSqlTsDatabaseUpgradeService {
33 33
34 34 protected static final String CALL_REGEX = "call ";
35   - protected static final String CHECK_VERSION = "check_version()";
36 35 protected static final String DROP_TABLE = "DROP TABLE ";
37   - protected static final String DROP_FUNCTION_IF_EXISTS = "DROP FUNCTION IF EXISTS ";
38   -
39   - private static final String CALL_CHECK_VERSION = CALL_REGEX + CHECK_VERSION;
40   -
41   -
42   - private static final String FUNCTION = "function: {}";
43   - private static final String DROP_STATEMENT = "drop statement: {}";
44   - private static final String QUERY = "query: {}";
45   - private static final String SUCCESSFULLY_EXECUTED = "Successfully executed ";
46   - private static final String FAILED_TO_EXECUTE = "Failed to execute ";
47   - private static final String FAILED_DUE_TO = " due to: {}";
48   -
49   - protected static final String SUCCESSFULLY_EXECUTED_FUNCTION = SUCCESSFULLY_EXECUTED + FUNCTION;
50   - protected static final String FAILED_TO_EXECUTE_FUNCTION_DUE_TO = FAILED_TO_EXECUTE + FUNCTION + FAILED_DUE_TO;
51   -
52   - protected static final String SUCCESSFULLY_EXECUTED_DROP_STATEMENT = SUCCESSFULLY_EXECUTED + DROP_STATEMENT;
53   - protected static final String FAILED_TO_EXECUTE_DROP_STATEMENT = FAILED_TO_EXECUTE + DROP_STATEMENT + FAILED_DUE_TO;
54   -
55   - protected static final String SUCCESSFULLY_EXECUTED_QUERY = SUCCESSFULLY_EXECUTED + QUERY;
56   - protected static final String FAILED_TO_EXECUTE_QUERY = FAILED_TO_EXECUTE + QUERY + FAILED_DUE_TO;
  36 + protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS ";
57 37
58 38 @Value("${spring.datasource.url}")
59 39 protected String dbUrl;
... ... @@ -67,7 +47,7 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
67 47 @Autowired
68 48 protected InstallScripts installScripts;
69 49
70   - protected abstract void loadSql(Connection conn);
  50 + protected abstract void loadSql(Connection conn, String fileName);
71 51
72 52 protected void loadFunctions(Path sqlFile, Connection conn) throws Exception {
73 53 String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8);
... ... @@ -75,26 +55,46 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
75 55 }
76 56
77 57 protected boolean checkVersion(Connection conn) {
78   - log.info("Check the current PostgreSQL version...");
79 58 boolean versionValid = false;
80 59 try {
81   - CallableStatement callableStatement = conn.prepareCall("{? = " + CALL_CHECK_VERSION + " }");
82   - callableStatement.registerOutParameter(1, Types.BOOLEAN);
83   - callableStatement.execute();
84   - versionValid = callableStatement.getBoolean(1);
85   - callableStatement.close();
  60 + Statement statement = conn.createStatement();
  61 + ResultSet resultSet = statement.executeQuery("SELECT current_setting('server_version_num')");
  62 + resultSet.next();
  63 + if(resultSet.getLong(1) > 110000) {
  64 + versionValid = true;
  65 + }
  66 + statement.close();
86 67 } catch (Exception e) {
87 68 log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage());
88 69 }
89 70 return versionValid;
90 71 }
91 72
92   - protected void executeFunction(Connection conn, String query) {
93   - log.info("{} ... ", query);
  73 + protected boolean isOldSchema(Connection conn, long fromVersion) {
  74 + boolean isOldSchema = true;
94 75 try {
95   - CallableStatement callableStatement = conn.prepareCall("{" + query + "}");
96   - callableStatement.execute();
97   - SQLWarning warnings = callableStatement.getWarnings();
  76 + Statement statement = conn.createStatement();
  77 + statement.execute("CREATE TABLE IF NOT EXISTS tb_schema_settings ( schema_version bigint NOT NULL, CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version));");
  78 + Thread.sleep(1000);
  79 + ResultSet resultSet = statement.executeQuery("SELECT schema_version FROM tb_schema_settings;");
  80 + if (resultSet.next()) {
  81 + isOldSchema = resultSet.getLong(1) <= fromVersion;
  82 + } else {
  83 + resultSet.close();
  84 + statement.execute("INSERT INTO tb_schema_settings (schema_version) VALUES (" + fromVersion + ")");
  85 + }
  86 + statement.close();
  87 + } catch (InterruptedException | SQLException e) {
  88 + log.info("Failed to check current PostgreSQL schema due to: {}", e.getMessage());
  89 + }
  90 + return isOldSchema;
  91 + }
  92 +
  93 + protected void executeQuery(Connection conn, String query) {
  94 + try {
  95 + Statement statement = conn.createStatement();
  96 + statement.execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
  97 + SQLWarning warnings = statement.getWarnings();
98 98 if (warnings != null) {
99 99 log.info("{}", warnings.getMessage());
100 100 SQLWarning nextWarning = warnings.getNextWarning();
... ... @@ -103,31 +103,10 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
103 103 nextWarning = nextWarning.getNextWarning();
104 104 }
105 105 }
106   - callableStatement.close();
107   - log.info(SUCCESSFULLY_EXECUTED_FUNCTION, query.replace(CALL_REGEX, ""));
108 106 Thread.sleep(2000);
109   - } catch (Exception e) {
110   - log.info(FAILED_TO_EXECUTE_FUNCTION_DUE_TO, query, e.getMessage());
111   - }
112   - }
113   -
114   - protected void executeDropStatement(Connection conn, String query) {
115   - try {
116   - conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
117   - log.info(SUCCESSFULLY_EXECUTED_DROP_STATEMENT, query);
118   - Thread.sleep(5000);
119   - } catch (InterruptedException | SQLException e) {
120   - log.info(FAILED_TO_EXECUTE_DROP_STATEMENT, query, e.getMessage());
121   - }
122   - }
123   -
124   - protected void executeQuery(Connection conn, String query) {
125   - try {
126   - conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
127   - log.info(SUCCESSFULLY_EXECUTED_QUERY, query);
128   - Thread.sleep(5000);
  107 + log.info("Successfully executed query: {}", query);
129 108 } catch (InterruptedException | SQLException e) {
130   - log.info(FAILED_TO_EXECUTE_QUERY, query, e.getMessage());
  109 + log.info("Failed to execute query: {} due to: {}", query, e.getMessage());
131 110 }
132 111 }
133 112
... ...
... ... @@ -34,6 +34,8 @@ import java.sql.DriverManager;
34 34 public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService {
35 35
36 36 private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql";
  37 + private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql";
  38 + private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql";
37 39
38 40 private static final String TS_KV_OLD = "ts_kv_old;";
39 41 private static final String TS_KV_LATEST_OLD = "ts_kv_latest_old;";
... ... @@ -57,52 +59,58 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
57 59 private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD;
58 60 private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD;
59 61
60   - private static final String DROP_FUNCTION_CHECK_VERSION = DROP_FUNCTION_IF_EXISTS + CHECK_VERSION;
61   - private static final String DROP_FUNCTION_CREATE_PARTITION_TS_KV_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_PARTITION_TS_KV_TABLE;
62   - private static final String DROP_FUNCTION_CREATE_NEW_TS_KV_LATEST_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_NEW_TS_KV_LATEST_TABLE;
63   - private static final String DROP_FUNCTION_CREATE_PARTITIONS = DROP_FUNCTION_IF_EXISTS + CREATE_PARTITIONS;
64   - private static final String DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
65   - private static final String DROP_FUNCTION_INSERT_INTO_DICTIONARY = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_DICTIONARY;
66   - private static final String DROP_FUNCTION_INSERT_INTO_TS_KV = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV;
67   - private static final String DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
  62 + private static final String DROP_PROCEDURE_CREATE_PARTITION_TS_KV_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_PARTITION_TS_KV_TABLE;
  63 + private static final String DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_LATEST_TABLE;
  64 + private static final String DROP_PROCEDURE_CREATE_PARTITIONS = DROP_PROCEDURE_IF_EXISTS + CREATE_PARTITIONS;
  65 + private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
  66 + private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
  67 + private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
  68 + private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
68 69
69 70 @Override
70 71 public void upgradeDatabase(String fromVersion) throws Exception {
71 72 switch (fromVersion) {
72 73 case "2.4.3":
73 74 try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
74   - log.info("Updating timeseries schema ...");
75   - log.info("Load upgrade functions ...");
76   - loadSql(conn);
  75 + log.info("Check the current PostgreSQL version...");
77 76 boolean versionValid = checkVersion(conn);
78 77 if (!versionValid) {
79   - log.info("PostgreSQL version should be at least more than 10!");
80   - log.info("Please upgrade your PostgreSQL and restart the script!");
  78 + throw new RuntimeException("PostgreSQL version should be at least more than 11, please upgrade your PostgreSQL and restart the script!");
81 79 } else {
82 80 log.info("PostgreSQL version is valid!");
83   - log.info("Updating schema ...");
84   - executeFunction(conn, CALL_CREATE_PARTITION_TS_KV_TABLE);
85   - executeFunction(conn, CALL_CREATE_PARTITIONS);
86   - executeFunction(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
87   - executeFunction(conn, CALL_INSERT_INTO_DICTIONARY);
88   - executeFunction(conn, CALL_INSERT_INTO_TS_KV);
89   - executeFunction(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
90   - executeFunction(conn, CALL_INSERT_INTO_TS_KV_LATEST);
91   -
92   - executeDropStatement(conn, DROP_TABLE_TS_KV_OLD);
93   - executeDropStatement(conn, DROP_TABLE_TS_KV_LATEST_OLD);
94   -
95   - executeDropStatement(conn, DROP_FUNCTION_CHECK_VERSION);
96   - executeDropStatement(conn, DROP_FUNCTION_CREATE_PARTITION_TS_KV_TABLE);
97   - executeDropStatement(conn, DROP_FUNCTION_CREATE_PARTITIONS);
98   - executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE);
99   - executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_DICTIONARY);
100   - executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV);
101   - executeDropStatement(conn, DROP_FUNCTION_CREATE_NEW_TS_KV_LATEST_TABLE);
102   - executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST);
103   -
104   - executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN json_v json;");
105   - executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN json_v json;");
  81 + if (isOldSchema(conn, 2004003)) {
  82 + log.info("Load upgrade functions ...");
  83 + loadSql(conn, LOAD_FUNCTIONS_SQL);
  84 + log.info("Updating timeseries schema ...");
  85 + executeQuery(conn, CALL_CREATE_PARTITION_TS_KV_TABLE);
  86 + executeQuery(conn, CALL_CREATE_PARTITIONS);
  87 + executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
  88 + executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
  89 + executeQuery(conn, CALL_INSERT_INTO_TS_KV);
  90 + executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
  91 + executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
  92 +
  93 + executeQuery(conn, DROP_TABLE_TS_KV_OLD);
  94 + executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD);
  95 +
  96 + executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITION_TS_KV_TABLE);
  97 + executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITIONS);
  98 + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE);
  99 + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY);
  100 + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV);
  101 + executeQuery(conn, DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE);
  102 + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
  103 +
  104 + executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");
  105 + executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;");
  106 + }
  107 +
  108 + log.info("Load TTL functions ...");
  109 + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL);
  110 + log.info("Load Drop Partitions functions ...");
  111 + loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL);
  112 +
  113 + executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000");
106 114
107 115 log.info("schema timeseries updated!");
108 116 }
... ... @@ -113,11 +121,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
113 121 }
114 122 }
115 123
116   - protected void loadSql(Connection conn) {
117   - Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL);
  124 + @Override
  125 + protected void loadSql(Connection conn, String fileName) {
  126 + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName);
118 127 try {
119 128 loadFunctions(schemaUpdateFile, conn);
120   - log.info("Upgrade functions successfully loaded!");
  129 + log.info("Functions successfully loaded!");
121 130 } catch (Exception e) {
122 131 log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage());
123 132 }
... ...
... ... @@ -45,13 +45,13 @@ public class TimescaleTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaS
45 45 private long chunkTimeInterval;
46 46
47 47 public TimescaleTsDatabaseSchemaService() {
48   - super("schema-timescale.sql", "schema-timescale-idx.sql");
  48 + super("schema-timescale.sql", null);
49 49 }
50 50
51 51 @Override
52 52 public void createDatabaseSchema() throws Exception {
53 53 super.createDatabaseSchema();
54   - executeQuery("SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);");
  54 + executeQuery("SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);");
55 55 }
56 56
57 57 private void executeQuery(String query) {
... ...
... ... @@ -39,31 +39,32 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
39 39 private long chunkTimeInterval;
40 40
41 41 private static final String LOAD_FUNCTIONS_SQL = "schema_update_timescale_ts.sql";
  42 + private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql";
42 43
43 44 private static final String TENANT_TS_KV_OLD_TABLE = "tenant_ts_kv_old;";
44 45
45 46 private static final String CREATE_TS_KV_LATEST_TABLE = "create_ts_kv_latest_table()";
46   - private static final String CREATE_NEW_TENANT_TS_KV_TABLE = "create_new_tenant_ts_kv_table()";
  47 + private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()";
47 48 private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
48 49 private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
49   - private static final String INSERT_INTO_TENANT_TS_KV = "insert_into_tenant_ts_kv()";
  50 + private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()";
50 51 private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()";
51 52
52 53 private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE;
53   - private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TENANT_TS_KV_TABLE;
  54 + private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE;
54 55 private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
55 56 private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
56   - private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TENANT_TS_KV;
  57 + private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV;
57 58 private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;
58 59
59 60 private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE;
60 61
61   - private static final String DROP_FUNCTION_CREATE_TS_KV_LATEST_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_LATEST_TABLE;
62   - private static final String DROP_FUNCTION_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_FUNCTION_IF_EXISTS + CREATE_NEW_TENANT_TS_KV_TABLE;
63   - private static final String DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
64   - private static final String DROP_FUNCTION_INSERT_INTO_DICTIONARY = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_DICTIONARY;
65   - private static final String DROP_FUNCTION_INSERT_INTO_TENANT_TS_KV = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TENANT_TS_KV;
66   - private static final String DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
  62 + private static final String DROP_PROCEDURE_CREATE_TS_KV_LATEST_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_LATEST_TABLE;
  63 + private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE;
  64 + private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
  65 + private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
  66 + private static final String DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
  67 + private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
67 68
68 69 @Autowired
69 70 private InstallScripts installScripts;
... ... @@ -73,41 +74,44 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
73 74 switch (fromVersion) {
74 75 case "2.4.3":
75 76 try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
76   - log.info("Updating timescale schema ...");
77   - log.info("Load upgrade functions ...");
78   - loadSql(conn);
  77 + log.info("Check the current PostgreSQL version...");
79 78 boolean versionValid = checkVersion(conn);
80 79 if (!versionValid) {
81   - log.info("PostgreSQL version should be at least more than 9.6!");
82   - log.info("Please upgrade your PostgreSQL and restart the script!");
  80 + throw new RuntimeException("PostgreSQL version should be at least more than 11, please upgrade your PostgreSQL and restart the script!");
83 81 } else {
84 82 log.info("PostgreSQL version is valid!");
85   - log.info("Updating schema ...");
86   - executeFunction(conn, CALL_CREATE_TS_KV_LATEST_TABLE);
87   - executeFunction(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE);
88   -
89   - executeQuery(conn, "SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);");
90   -
91   - executeFunction(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
92   - executeFunction(conn, CALL_INSERT_INTO_DICTIONARY);
93   - executeFunction(conn, CALL_INSERT_INTO_TS_KV);
94   - executeFunction(conn, CALL_INSERT_INTO_TS_KV_LATEST);
95   -
96   - //executeQuery(conn, "SELECT set_chunk_time_interval('tenant_ts_kv', " + chunkTimeInterval +");");
97   -
98   - executeDropStatement(conn, DROP_OLD_TENANT_TS_KV_TABLE);
99   -
100   - executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_LATEST_TABLE);
101   - executeDropStatement(conn, DROP_FUNCTION_CREATE_TENANT_TS_KV_TABLE_COPY);
102   - executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE);
103   - executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_DICTIONARY);
104   - executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TENANT_TS_KV);
105   - executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST);
106   -
107   - executeQuery(conn, "ALTER TABLE tenant_ts_kv ADD COLUMN json_v json;");
108   - executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN json_v json;");
109   -
110   - log.info("schema timeseries updated!");
  83 + if (isOldSchema(conn, 2004003)) {
  84 + log.info("Load upgrade functions ...");
  85 + loadSql(conn, LOAD_FUNCTIONS_SQL);
  86 + log.info("Updating timescale schema ...");
  87 + executeQuery(conn, CALL_CREATE_TS_KV_LATEST_TABLE);
  88 + executeQuery(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE);
  89 +
  90 + executeQuery(conn, "SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);");
  91 +
  92 + executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
  93 + executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
  94 + executeQuery(conn, CALL_INSERT_INTO_TS_KV);
  95 + executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
  96 +
  97 + executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE);
  98 +
  99 + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_LATEST_TABLE);
  100 + executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY);
  101 + executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE);
  102 + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY);
  103 + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV);
  104 + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
  105 +
  106 + executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");
  107 + executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;");
  108 + }
  109 +
  110 + log.info("Load TTL functions ...");
  111 + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL);
  112 +
  113 + executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000");
  114 + log.info("schema timescale updated!");
111 115 }
112 116 }
113 117 break;
... ... @@ -116,13 +120,14 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
116 120 }
117 121 }
118 122
119   - protected void loadSql(Connection conn) {
120   - Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL);
  123 + @Override
  124 + protected void loadSql(Connection conn, String fileName) {
  125 + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName);
121 126 try {
122 127 loadFunctions(schemaUpdateFile, conn);
123   - log.info("Upgrade functions successfully loaded!");
  128 + log.info("Functions successfully loaded!");
124 129 } catch (Exception e) {
125   - log.info("Failed to load Timescale upgrade functions due to: {}", e.getMessage());
  130 + log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage());
126 131 }
127 132 }
128 133 }
\ No newline at end of file
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.service.script;
18 18 import com.google.common.util.concurrent.FutureCallback;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22 import delight.nashornsandbox.NashornSandbox;
22 23 import delight.nashornsandbox.NashornSandboxes;
23 24 import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
... ... @@ -28,20 +29,17 @@ import org.springframework.beans.factory.annotation.Value;
28 29 import org.springframework.scheduling.annotation.Scheduled;
29 30 import org.thingsboard.common.util.ThingsBoardThreadFactory;
30 31
31   -import javax.annotation.Nullable;
32 32 import javax.annotation.PostConstruct;
33 33 import javax.annotation.PreDestroy;
34 34 import javax.script.Invocable;
35 35 import javax.script.ScriptEngine;
36 36 import javax.script.ScriptException;
37 37 import java.util.UUID;
38   -import java.util.concurrent.Callable;
39 38 import java.util.concurrent.ExecutionException;
40 39 import java.util.concurrent.ExecutorService;
41 40 import java.util.concurrent.Executors;
42 41 import java.util.concurrent.ScheduledExecutorService;
43 42 import java.util.concurrent.TimeUnit;
44   -import java.util.concurrent.TimeoutException;
45 43 import java.util.concurrent.atomic.AtomicInteger;
46 44
47 45 @Slf4j
... ... @@ -140,7 +138,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
140 138 if (maxRequestsTimeout > 0) {
141 139 result = Futures.withTimeout(result, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
142 140 }
143   - Futures.addCallback(result, evalCallback);
  141 + Futures.addCallback(result, evalCallback, MoreExecutors.directExecutor());
144 142 return result;
145 143 }
146 144
... ... @@ -163,7 +161,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
163 161 if (maxRequestsTimeout > 0) {
164 162 result = Futures.withTimeout(result, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
165 163 }
166   - Futures.addCallback(result, invokeCallback);
  164 + Futures.addCallback(result, invokeCallback, MoreExecutors.directExecutor());
167 165 return result;
168 166 }
169 167
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.service.script;
18 18 import com.google.common.util.concurrent.FutureCallback;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22 import lombok.Getter;
22 23 import lombok.extern.slf4j.Slf4j;
23 24 import org.springframework.beans.factory.annotation.Value;
... ... @@ -158,7 +159,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
158 159 }
159 160 kafkaFailedMsgs.incrementAndGet();
160 161 }
161   - });
  162 + }, MoreExecutors.directExecutor());
162 163 return Futures.transform(future, response -> {
163 164 JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse();
164 165 UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB());
... ... @@ -170,7 +171,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
170 171 log.debug("[{}] Failed to compile script due to [{}]: {}", compiledScriptId, compilationResult.getErrorCode().name(), compilationResult.getErrorDetails());
171 172 throw new RuntimeException(compilationResult.getErrorDetails());
172 173 }
173   - });
  174 + }, MoreExecutors.directExecutor());
174 175 }
175 176
176 177 @Override
... ... @@ -209,7 +210,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
209 210 }
210 211 kafkaFailedMsgs.incrementAndGet();
211 212 }
212   - });
  213 + }, MoreExecutors.directExecutor());
213 214 return Futures.transform(future, response -> {
214 215 JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse();
215 216 if (invokeResult.getSuccess()) {
... ... @@ -218,7 +219,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
218 219 log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails());
219 220 throw new RuntimeException(invokeResult.getErrorDetails());
220 221 }
221   - });
  222 + }, MoreExecutors.directExecutor());
222 223 }
223 224
224 225 @Override
... ...
... ... @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
21 21 import com.google.common.collect.Sets;
22 22 import com.google.common.util.concurrent.Futures;
23 23 import com.google.common.util.concurrent.ListenableFuture;
  24 +import com.google.common.util.concurrent.MoreExecutors;
24 25 import lombok.extern.slf4j.Slf4j;
25 26 import org.apache.commons.lang3.StringUtils;
26 27 import org.thingsboard.server.common.data.id.EntityId;
... ... @@ -121,7 +122,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
121 122 } else {
122 123 return Futures.immediateFuture(unbindMsg(json, msg));
123 124 }
124   - });
  125 + }, MoreExecutors.directExecutor());
125 126 }
126 127
127 128 @Override
... ... @@ -174,7 +175,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
174 175 } else {
175 176 return Futures.immediateFuture(json.asBoolean());
176 177 }
177   - });
  178 + }, MoreExecutors.directExecutor());
178 179 }
179 180
180 181 @Override
... ... @@ -232,7 +233,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
232 233 return Futures.immediateFailedFuture(new ScriptException(e));
233 234 }
234 235 }
235   - });
  236 + }, MoreExecutors.directExecutor());
236 237 }
237 238
238 239 public void destroy() {
... ...
... ... @@ -272,7 +272,7 @@ public class DefaultDeviceStateService implements DeviceStateService {
272 272 log.warn("Failed to register device to the state service", t);
273 273 callback.onFailure(t);
274 274 }
275   - });
  275 + }, MoreExecutors.directExecutor());
276 276 } else if (proto.getUpdated()) {
277 277 DeviceStateData stateData = getOrFetchDeviceStateData(device.getId());
278 278 if (stateData != null) {
... ... @@ -347,7 +347,7 @@ public class DefaultDeviceStateService implements DeviceStateService {
347 347 }
348 348 return null;
349 349 }
350   - });
  350 + }, MoreExecutors.directExecutor());
351 351 fetchFutures.add(future);
352 352 }
353 353 }
... ... @@ -440,10 +440,10 @@ public class DefaultDeviceStateService implements DeviceStateService {
440 440 private ListenableFuture<DeviceStateData> fetchDeviceState(Device device) {
441 441 if (persistToTelemetry) {
442 442 ListenableFuture<List<TsKvEntry>> tsData = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), PERSISTENT_ATTRIBUTES);
443   - return Futures.transform(tsData, extractDeviceStateData(device));
  443 + return Futures.transform(tsData, extractDeviceStateData(device), MoreExecutors.directExecutor());
444 444 } else {
445 445 ListenableFuture<List<AttributeKvEntry>> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), DataConstants.SERVER_SCOPE, PERSISTENT_ATTRIBUTES);
446   - return Futures.transform(attrData, extractDeviceStateData(device));
  446 + return Futures.transform(attrData, extractDeviceStateData(device), MoreExecutors.directExecutor());
447 447 }
448 448 }
449 449
... ...
... ... @@ -21,6 +21,7 @@ import com.google.common.base.Function;
21 21 import com.google.common.util.concurrent.FutureCallback;
22 22 import com.google.common.util.concurrent.Futures;
23 23 import com.google.common.util.concurrent.ListenableFuture;
  24 +import com.google.common.util.concurrent.MoreExecutors;
24 25 import lombok.extern.slf4j.Slf4j;
25 26 import org.springframework.beans.factory.annotation.Autowired;
26 27 import org.springframework.beans.factory.annotation.Value;
... ... @@ -76,7 +77,10 @@ import java.util.List;
76 77 import java.util.Map;
77 78 import java.util.Optional;
78 79 import java.util.Set;
79   -import java.util.concurrent.*;
  80 +import java.util.concurrent.ConcurrentHashMap;
  81 +import java.util.concurrent.ConcurrentMap;
  82 +import java.util.concurrent.ExecutorService;
  83 +import java.util.concurrent.Executors;
80 84 import java.util.function.Consumer;
81 85 import java.util.stream.Collectors;
82 86
... ... @@ -654,7 +658,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
654 658 }
655 659
656 660 ListenableFuture<List<AttributeKvEntry>> future = mergeAllAttributesFutures(futures);
657   - Futures.addCallback(future, callback);
  661 + Futures.addCallback(future, callback, MoreExecutors.directExecutor());
658 662 }
659 663
660 664 @Override
... ... @@ -668,7 +672,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
668 672 return new FutureCallback<ValidationResult>() {
669 673 @Override
670 674 public void onSuccess(@Nullable ValidationResult result) {
671   - Futures.addCallback(attributesService.find(tenantId, entityId, scope, keys), callback);
  675 + Futures.addCallback(attributesService.find(tenantId, entityId, scope, keys), callback, MoreExecutors.directExecutor());
672 676 }
673 677
674 678 @Override
... ... @@ -688,7 +692,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
688 692 }
689 693
690 694 ListenableFuture<List<AttributeKvEntry>> future = mergeAllAttributesFutures(futures);
691   - Futures.addCallback(future, callback);
  695 + Futures.addCallback(future, callback, MoreExecutors.directExecutor());
692 696 }
693 697
694 698 @Override
... ... @@ -702,7 +706,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
702 706 return new FutureCallback<ValidationResult>() {
703 707 @Override
704 708 public void onSuccess(@Nullable ValidationResult result) {
705   - Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), callback);
  709 + Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), callback, MoreExecutors.directExecutor());
706 710 }
707 711
708 712 @Override
... ...
... ... @@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
19 19 import com.fasterxml.jackson.databind.ObjectMapper;
20 20 import com.google.common.util.concurrent.Futures;
21 21 import com.google.common.util.concurrent.ListenableFuture;
  22 +import com.google.common.util.concurrent.MoreExecutors;
22 23 import lombok.extern.slf4j.Slf4j;
23 24 import org.springframework.beans.factory.annotation.Autowired;
24 25 import org.springframework.stereotype.Service;
... ... @@ -80,14 +81,14 @@ public class DefaultTransportApiService implements TransportApiService {
80 81 TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue();
81 82 if (transportApiRequestMsg.hasValidateTokenRequestMsg()) {
82 83 ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg();
83   - return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()));
  84 + return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
84 85 } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) {
85 86 ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg();
86   - return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()));
  87 + return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
87 88 } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) {
88   - return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()));
  89 + return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
89 90 }
90   - return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()));
  91 + return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
91 92 }
92 93
93 94 private ListenableFuture<TransportApiResponseMsg> validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) {
... ... @@ -147,7 +148,7 @@ public class DefaultTransportApiService implements TransportApiService {
147 148 log.warn("[{}] Failed to lookup device by id", deviceId, e);
148 149 return getEmptyTransportApiResponse();
149 150 }
150   - });
  151 + }, MoreExecutors.directExecutor());
151 152 }
152 153
153 154 private DeviceInfoProto getDeviceInfoProto(Device device) throws JsonProcessingException {
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.ttl;
  17 +
  18 +import lombok.extern.slf4j.Slf4j;
  19 +import org.springframework.beans.factory.annotation.Value;
  20 +import org.springframework.scheduling.annotation.Scheduled;
  21 +import org.thingsboard.server.dao.util.PsqlTsAnyDao;
  22 +
  23 +import java.sql.Connection;
  24 +import java.sql.DriverManager;
  25 +import java.sql.ResultSet;
  26 +import java.sql.SQLException;
  27 +import java.sql.SQLWarning;
  28 +import java.sql.Statement;
  29 +
  30 +@PsqlTsAnyDao
  31 +@Slf4j
  32 +public abstract class AbstractTimeseriesCleanUpService {
  33 +
  34 + @Value("${sql.ttl.ts_key_value_ttl}")
  35 + protected long systemTtl;
  36 +
  37 + @Value("${sql.ttl.enabled}")
  38 + private boolean ttlTaskExecutionEnabled;
  39 +
  40 + @Value("${spring.datasource.url}")
  41 + private String dbUrl;
  42 +
  43 + @Value("${spring.datasource.username}")
  44 + private String dbUserName;
  45 +
  46 + @Value("${spring.datasource.password}")
  47 + private String dbPassword;
  48 +
  49 + @Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}")
  50 + public void cleanUp() {
  51 + if (ttlTaskExecutionEnabled) {
  52 + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
  53 + doCleanUp(conn);
  54 + } catch (SQLException e) {
  55 + log.error("SQLException occurred during TTL task execution ", e);
  56 + }
  57 + }
  58 + }
  59 +
  60 + protected abstract void doCleanUp(Connection connection);
  61 +
  62 + protected long executeQuery(Connection conn, String query) {
  63 + long removed = 0L;
  64 + try {
  65 + Statement statement = conn.createStatement();
  66 + ResultSet resultSet = statement.executeQuery(query);
  67 + getWarnings(statement);
  68 + resultSet.next();
  69 + removed = resultSet.getLong(1);
  70 + log.debug("Successfully executed query: {}", query);
  71 + } catch (SQLException e) {
  72 + log.debug("Failed to execute query: {} due to: {}", query, e.getMessage());
  73 + }
  74 + return removed;
  75 + }
  76 +
  77 + private void getWarnings(Statement statement) throws SQLException {
  78 + SQLWarning warnings = statement.getWarnings();
  79 + if (warnings != null) {
  80 + log.debug("{}", warnings.getMessage());
  81 + SQLWarning nextWarning = warnings.getNextWarning();
  82 + while (nextWarning != null) {
  83 + log.debug("{}", nextWarning.getMessage());
  84 + nextWarning = nextWarning.getNextWarning();
  85 + }
  86 + }
  87 + }
  88 +
  89 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.ttl;
  17 +
  18 +import lombok.extern.slf4j.Slf4j;
  19 +import org.springframework.beans.factory.annotation.Value;
  20 +import org.springframework.stereotype.Service;
  21 +import org.thingsboard.server.dao.model.ModelConstants;
  22 +import org.thingsboard.server.dao.util.PsqlTsDao;
  23 +
  24 +import java.sql.Connection;
  25 +
  26 +@PsqlTsDao
  27 +@Service
  28 +@Slf4j
  29 +public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService {
  30 +
  31 + @Value("${sql.postgres.ts_key_value_partitioning}")
  32 + private String partitionType;
  33 +
  34 + @Override
  35 + protected void doCleanUp(Connection connection) {
  36 + long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);");
  37 + log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved);
  38 + long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);");
  39 + log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
  40 + }
  41 +}
\ No newline at end of file
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.service.ttl;
  17 +
  18 +import lombok.extern.slf4j.Slf4j;
  19 +import org.springframework.stereotype.Service;
  20 +import org.thingsboard.server.dao.model.ModelConstants;
  21 +import org.thingsboard.server.dao.util.TimescaleDBTsDao;
  22 +
  23 +import java.sql.Connection;
  24 +
  25 +@TimescaleDBTsDao
  26 +@Service
  27 +@Slf4j
  28 +public class TimescaleTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService {
  29 +
  30 + @Override
  31 + protected void doCleanUp(Connection connection) {
  32 + long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);");
  33 + log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
  34 + }
  35 +}
\ No newline at end of file
... ...
... ... @@ -202,6 +202,10 @@ sql:
202 202 timescale:
203 203 # Specify Interval size for new data chunks storage.
204 204 chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}"
  205 + ttl:
  206 + enabled: "${SQL_TTL_ENABLED:true}"
  207 + execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds
  208 + ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds
205 209
206 210 # Actor system parameters
207 211 actors:
... ...
... ... @@ -2,8 +2,8 @@
2 2
3 3 set -e
4 4
5   -chown -R ${pkg.name}: ${pkg.logFolder}
6   -chown -R ${pkg.name}: ${pkg.installFolder}
  5 +chown -R ${pkg.user}: ${pkg.logFolder}
  6 +chown -R ${pkg.user}: ${pkg.installFolder}
7 7 systemctl --no-reload enable ${pkg.name}.service >/dev/null 2>&1 || :
8 8
9 9 exit 0
... ...
... ... @@ -2,21 +2,21 @@
2 2
3 3 set -e
4 4
5   -if ! getent group ${pkg.name} >/dev/null; then
6   - addgroup --system ${pkg.name}
  5 +if ! getent group ${pkg.user} >/dev/null; then
  6 + addgroup --system ${pkg.user}
7 7 fi
8 8
9   -if ! getent passwd ${pkg.name} >/dev/null; then
  9 +if ! getent passwd ${pkg.user} >/dev/null; then
10 10 adduser --quiet \
11 11 --system \
12   - --ingroup ${pkg.name} \
  12 + --ingroup ${pkg.user} \
13 13 --quiet \
14 14 --disabled-login \
15 15 --disabled-password \
16 16 --home ${pkg.installFolder} \
17 17 --no-create-home \
18 18 -gecos "Thingsboard application" \
19   - ${pkg.name}
  19 + ${pkg.user}
20 20 fi
21 21
22 22 exit 0
\ No newline at end of file
... ...
1 1 #!/bin/sh
2 2
3   -chown -R ${pkg.name}: ${pkg.logFolder}
4   -chown -R ${pkg.name}: ${pkg.installFolder}
  3 +chown -R ${pkg.user}: ${pkg.logFolder}
  4 +chown -R ${pkg.user}: ${pkg.installFolder}
5 5
6 6 if [ $1 -eq 1 ] ; then
7 7 # Initial installation
... ...
1 1 #!/bin/sh
2 2
3   -getent group ${pkg.name} >/dev/null || groupadd -r ${pkg.name}
4   -getent passwd ${pkg.name} >/dev/null || \
5   -useradd -d ${pkg.installFolder} -g ${pkg.name} -M -r ${pkg.name} -s /sbin/nologin \
  3 +getent group ${pkg.user} >/dev/null || groupadd -r ${pkg.user}
  4 +getent passwd ${pkg.user} >/dev/null || \
  5 +useradd -d ${pkg.installFolder} -g ${pkg.user} -M -r ${pkg.user} -s /sbin/nologin \
6 6 -c "Thingsboard application"
... ...
... ... @@ -3,7 +3,7 @@ Description=${pkg.name}
3 3 After=syslog.target
4 4
5 5 [Service]
6   -User=${pkg.name}
  6 +User=${pkg.user}
7 7 ExecStart=${pkg.installFolder}/bin/${pkg.name}.jar
8 8 SuccessExitStatus=143
9 9
... ...
... ... @@ -44,7 +44,7 @@ installDir=${pkg.installFolder}/data
44 44
45 45 source "${CONF_FOLDER}/${configfile}"
46 46
47   -run_user=${pkg.name}
  47 +run_user=${pkg.user}
48 48
49 49 su -s /bin/sh -c "java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \
50 50 -Dinstall.data_dir=${installDir} \
... ...
... ... @@ -43,7 +43,7 @@ installDir=${pkg.installFolder}/data
43 43
44 44 source "${CONF_FOLDER}/${configfile}"
45 45
46   -run_user=${pkg.name}
  46 +run_user=${pkg.user}
47 47
48 48 su -s /bin/sh -c "java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \
49 49 -Dinstall.data_dir=${installDir} \
... ...
... ... @@ -39,7 +39,7 @@ public interface AlarmService {
39 39
40 40 ListenableFuture<Boolean> ackAlarm(TenantId tenantId, AlarmId alarmId, long ackTs);
41 41
42   - ListenableFuture<Boolean> clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long ackTs);
  42 + ListenableFuture<Boolean> clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long clearTs);
43 43
44 44 ListenableFuture<Alarm> findAlarmByIdAsync(TenantId tenantId, AlarmId alarmId);
45 45
... ...
  1 +/**
  2 + * Copyright © 2016-2020 The Thingsboard Authors
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package org.thingsboard.server.dao.util;
  17 +
  18 +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
  19 +
  20 +@ConditionalOnExpression("'${database.ts.type}'=='sql' && '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'")
  21 +public @interface PsqlTsDao { }
\ No newline at end of file
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.common;
18 18 import com.google.common.util.concurrent.FutureCallback;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22
22 23 import java.util.concurrent.Executor;
23 24 import java.util.concurrent.ScheduledExecutorService;
... ... @@ -59,7 +60,7 @@ public class AsyncCallbackTemplate {
59 60 if (executor != null) {
60 61 Futures.addCallback(future, callback, executor);
61 62 } else {
62   - Futures.addCallback(future, callback);
  63 + Futures.addCallback(future, callback, MoreExecutors.directExecutor());
63 64 }
64 65 }
65 66
... ...
... ... @@ -97,7 +97,7 @@ public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
97 97 callback.onFailure(t);
98 98 }
99 99 }
100   - });
  100 + }, producerExecutor);
101 101 }
102 102
103 103 @Override
... ...
... ... @@ -18,19 +18,20 @@ package org.thingsboard.common.util;
18 18 import com.google.common.util.concurrent.FutureCallback;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22
22 23 import java.util.concurrent.Executor;
23 24 import java.util.function.Consumer;
24 25
25 26 public class DonAsynchron {
26 27
27   - public static <T> void withCallback(ListenableFuture<T> future, Consumer<T> onSuccess,
28   - Consumer<Throwable> onFailure) {
  28 + public static <T> void withCallback(ListenableFuture<T> future, Consumer<T> onSuccess,
  29 + Consumer<Throwable> onFailure) {
29 30 withCallback(future, onSuccess, onFailure, null);
30 31 }
31 32
32   - public static <T> void withCallback(ListenableFuture<T> future, Consumer<T> onSuccess,
33   - Consumer<Throwable> onFailure, Executor executor) {
  33 + public static <T> void withCallback(ListenableFuture<T> future, Consumer<T> onSuccess,
  34 + Consumer<Throwable> onFailure, Executor executor) {
34 35 FutureCallback<T> callback = new FutureCallback<T>() {
35 36 @Override
36 37 public void onSuccess(T result) {
... ... @@ -49,7 +50,7 @@ public class DonAsynchron {
49 50 if (executor != null) {
50 51 Futures.addCallback(future, callback, executor);
51 52 } else {
52   - Futures.addCallback(future, callback);
  53 + Futures.addCallback(future, callback, MoreExecutors.directExecutor());
53 54 }
54 55 }
55 56 }
... ...
... ... @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode;
20 20 import com.google.common.base.Function;
21 21 import com.google.common.util.concurrent.Futures;
22 22 import com.google.common.util.concurrent.ListenableFuture;
  23 +import com.google.common.util.concurrent.MoreExecutors;
23 24 import lombok.extern.slf4j.Slf4j;
24 25 import org.springframework.beans.factory.annotation.Autowired;
25 26 import org.springframework.stereotype.Service;
... ... @@ -53,7 +54,6 @@ import javax.annotation.Nullable;
53 54 import javax.annotation.PostConstruct;
54 55 import javax.annotation.PreDestroy;
55 56 import java.util.ArrayList;
56   -import java.util.Collections;
57 57 import java.util.Comparator;
58 58 import java.util.List;
59 59 import java.util.Set;
... ... @@ -264,9 +264,8 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
264 264 entityService.fetchEntityNameAsync(tenantId, alarmInfo.getOriginator()), originatorName -> {
265 265 alarmInfo.setOriginatorName(originatorName);
266 266 return alarmInfo;
267   - }
268   - );
269   - });
  267 + }, MoreExecutors.directExecutor());
  268 + }, MoreExecutors.directExecutor());
270 269 }
271 270
272 271 @Override
... ... @@ -283,11 +282,11 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
283 282 }
284 283 alarmInfo.setOriginatorName(originatorName);
285 284 return alarmInfo;
286   - }
  285 + }, MoreExecutors.directExecutor()
287 286 ));
288 287 }
289 288 return Futures.successfulAsList(alarmFutures);
290   - });
  289 + }, MoreExecutors.directExecutor());
291 290 }
292 291 return Futures.transform(alarms, new Function<List<AlarmInfo>, TimePageData<AlarmInfo>>() {
293 292 @Nullable
... ... @@ -295,7 +294,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
295 294 public TimePageData<AlarmInfo> apply(@Nullable List<AlarmInfo> alarms) {
296 295 return new TimePageData<>(alarms, query.getPageLink());
297 296 }
298   - });
  297 + }, MoreExecutors.directExecutor());
299 298 }
300 299
301 300 @Override
... ...
... ... @@ -20,6 +20,7 @@ import com.datastax.driver.core.querybuilder.QueryBuilder;
20 20 import com.datastax.driver.core.querybuilder.Select;
21 21 import com.google.common.util.concurrent.Futures;
22 22 import com.google.common.util.concurrent.ListenableFuture;
  23 +import com.google.common.util.concurrent.MoreExecutors;
23 24 import lombok.extern.slf4j.Slf4j;
24 25 import org.springframework.beans.factory.annotation.Autowired;
25 26 import org.springframework.stereotype.Component;
... ... @@ -82,10 +83,10 @@ public class CassandraAlarmDao extends CassandraAbstractModelDao<AlarmEntity, Al
82 83 @Override
83 84 public Boolean deleteAlarm(TenantId tenantId, Alarm alarm) {
84 85 Statement delete = QueryBuilder.delete().all().from(getColumnFamilyName()).where(eq(ModelConstants.ID_PROPERTY, alarm.getId().getId()))
85   - .and(eq(ALARM_TENANT_ID_PROPERTY, tenantId.getId()))
86   - .and(eq(ALARM_ORIGINATOR_ID_PROPERTY, alarm.getOriginator().getId()))
87   - .and(eq(ALARM_ORIGINATOR_TYPE_PROPERTY, alarm.getOriginator().getEntityType()))
88   - .and(eq(ALARM_TYPE_PROPERTY, alarm.getType()));
  86 + .and(eq(ALARM_TENANT_ID_PROPERTY, tenantId.getId()))
  87 + .and(eq(ALARM_ORIGINATOR_ID_PROPERTY, alarm.getOriginator().getId()))
  88 + .and(eq(ALARM_ORIGINATOR_TYPE_PROPERTY, alarm.getOriginator().getEntityType()))
  89 + .and(eq(ALARM_TYPE_PROPERTY, alarm.getType()));
89 90 log.debug("Remove request: {}", delete.toString());
90 91 return executeWrite(tenantId, delete).wasApplied();
91 92 }
... ... @@ -122,10 +123,10 @@ public class CassandraAlarmDao extends CassandraAbstractModelDao<AlarmEntity, Al
122 123 for (EntityRelation relation : input) {
123 124 alarmFutures.add(Futures.transform(
124 125 findAlarmByIdAsync(tenantId, relation.getTo().getId()),
125   - AlarmInfo::new));
  126 + AlarmInfo::new, MoreExecutors.directExecutor()));
126 127 }
127 128 return Futures.successfulAsList(alarmFutures);
128   - });
  129 + }, MoreExecutors.directExecutor());
129 130 }
130 131
131 132 @Override
... ...
... ... @@ -16,9 +16,9 @@
16 16 package org.thingsboard.server.dao.asset;
17 17
18 18
19   -import com.google.common.base.Function;
20 19 import com.google.common.util.concurrent.Futures;
21 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
22 22 import lombok.extern.slf4j.Slf4j;
23 23 import org.hibernate.exception.ConstraintViolationException;
24 24 import org.springframework.beans.factory.annotation.Autowired;
... ... @@ -29,7 +29,6 @@ import org.springframework.cache.annotation.Cacheable;
29 29 import org.springframework.stereotype.Service;
30 30 import org.springframework.util.StringUtils;
31 31 import org.thingsboard.server.common.data.Customer;
32   -import org.thingsboard.server.common.data.Device;
33 32 import org.thingsboard.server.common.data.EntitySubtype;
34 33 import org.thingsboard.server.common.data.EntityType;
35 34 import org.thingsboard.server.common.data.EntityView;
... ... @@ -62,7 +61,10 @@ import java.util.stream.Collectors;
62 61 import static org.thingsboard.server.common.data.CacheConstants.ASSET_CACHE;
63 62 import static org.thingsboard.server.dao.DaoUtil.toUUIDs;
64 63 import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
65   -import static org.thingsboard.server.dao.service.Validator.*;
  64 +import static org.thingsboard.server.dao.service.Validator.validateId;
  65 +import static org.thingsboard.server.dao.service.Validator.validateIds;
  66 +import static org.thingsboard.server.dao.service.Validator.validatePageLink;
  67 +import static org.thingsboard.server.dao.service.Validator.validateString;
66 68
67 69 @Service
68 70 @Slf4j
... ... @@ -258,9 +260,9 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ
258 260 }
259 261 }
260 262 return Futures.successfulAsList(futures);
261   - });
  263 + }, MoreExecutors.directExecutor());
262 264 assets = Futures.transform(assets, assetList ->
263   - assetList == null ? Collections.emptyList() : assetList.stream().filter(asset -> query.getAssetTypes().contains(asset.getType())).collect(Collectors.toList())
  265 + assetList == null ? Collections.emptyList() : assetList.stream().filter(asset -> query.getAssetTypes().contains(asset.getType())).collect(Collectors.toList()), MoreExecutors.directExecutor()
264 266 );
265 267 return assets;
266 268 }
... ... @@ -274,7 +276,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ
274 276 assetTypes -> {
275 277 assetTypes.sort(Comparator.comparing(EntitySubtype::getType));
276 278 return assetTypes;
277   - });
  279 + }, MoreExecutors.directExecutor());
278 280 }
279 281
280 282 private DataValidator<Asset> assetValidator =
... ... @@ -335,18 +337,18 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ
335 337 };
336 338
337 339 private PaginatedRemover<TenantId, Asset> tenantAssetsRemover =
338   - new PaginatedRemover<TenantId, Asset>() {
  340 + new PaginatedRemover<TenantId, Asset>() {
339 341
340   - @Override
341   - protected List<Asset> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
342   - return assetDao.findAssetsByTenantId(id.getId(), pageLink);
343   - }
  342 + @Override
  343 + protected List<Asset> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
  344 + return assetDao.findAssetsByTenantId(id.getId(), pageLink);
  345 + }
344 346
345   - @Override
346   - protected void removeEntity(TenantId tenantId, Asset entity) {
347   - deleteAsset(tenantId, new AssetId(entity.getId().getId()));
348   - }
349   - };
  347 + @Override
  348 + protected void removeEntity(TenantId tenantId, Asset entity) {
  349 + deleteAsset(tenantId, new AssetId(entity.getId().getId()));
  350 + }
  351 + };
350 352
351 353 private PaginatedRemover<CustomerId, Asset> customerAssetsUnasigner = new PaginatedRemover<CustomerId, Asset>() {
352 354
... ...
... ... @@ -23,6 +23,7 @@ import com.datastax.driver.mapping.Result;
23 23 import com.google.common.base.Function;
24 24 import com.google.common.util.concurrent.Futures;
25 25 import com.google.common.util.concurrent.ListenableFuture;
  26 +import com.google.common.util.concurrent.MoreExecutors;
26 27 import lombok.extern.slf4j.Slf4j;
27 28 import org.springframework.stereotype.Component;
28 29 import org.thingsboard.server.common.data.EntitySubtype;
... ... @@ -185,7 +186,7 @@ public class CassandraAssetDao extends CassandraAbstractSearchTextDao<AssetEntit
185 186 return Collections.emptyList();
186 187 }
187 188 }
188   - });
  189 + }, MoreExecutors.directExecutor());
189 190 }
190 191
191 192 }
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.dashboard;
17 17
18 18 import com.google.common.util.concurrent.Futures;
19 19 import com.google.common.util.concurrent.ListenableFuture;
  20 +import com.google.common.util.concurrent.MoreExecutors;
20 21 import lombok.extern.slf4j.Slf4j;
21 22 import org.springframework.beans.factory.annotation.Autowired;
22 23 import org.springframework.stereotype.Component;
... ... @@ -85,7 +86,7 @@ public class CassandraDashboardInfoDao extends CassandraAbstractSearchTextDao<Da
85 86 dashboardFutures.add(findByIdAsync(new TenantId(tenantId), relation.getTo().getId()));
86 87 }
87 88 return Futures.successfulAsList(dashboardFutures);
88   - });
  89 + }, MoreExecutors.directExecutor());
89 90 }
90 91
91 92 }
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.dashboard;
18 18 import com.google.common.base.Function;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22 import lombok.extern.slf4j.Slf4j;
22 23 import org.apache.commons.lang3.StringUtils;
23 24 import org.springframework.beans.factory.annotation.Autowired;
... ... @@ -64,7 +65,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
64 65
65 66 @Autowired
66 67 private TenantDao tenantDao;
67   -
  68 +
68 69 @Autowired
69 70 private CustomerDao customerDao;
70 71
... ... @@ -102,7 +103,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
102 103 dashboardValidator.validate(dashboard, DashboardInfo::getTenantId);
103 104 return dashboardDao.save(dashboard.getTenantId(), dashboard);
104 105 }
105   -
  106 +
106 107 @Override
107 108 public Dashboard assignDashboardToCustomer(TenantId tenantId, DashboardId dashboardId, CustomerId customerId) {
108 109 Dashboard dashboard = findDashboardById(tenantId, dashboardId);
... ... @@ -203,7 +204,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
203 204 public TimePageData<DashboardInfo> apply(@Nullable List<DashboardInfo> dashboards) {
204 205 return new TimePageData<>(dashboards, pageLink);
205 206 }
206   - });
  207 + }, MoreExecutors.directExecutor());
207 208 }
208 209
209 210 @Override
... ... @@ -244,24 +245,24 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
244 245 }
245 246 }
246 247 }
247   - };
248   -
  248 + };
  249 +
249 250 private PaginatedRemover<TenantId, DashboardInfo> tenantDashboardsRemover =
250 251 new PaginatedRemover<TenantId, DashboardInfo>() {
251   -
252   - @Override
253   - protected List<DashboardInfo> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
254   - return dashboardInfoDao.findDashboardsByTenantId(id.getId(), pageLink);
255   - }
256 252
257   - @Override
258   - protected void removeEntity(TenantId tenantId, DashboardInfo entity) {
259   - deleteDashboard(tenantId, new DashboardId(entity.getUuidId()));
260   - }
261   - };
262   -
  253 + @Override
  254 + protected List<DashboardInfo> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
  255 + return dashboardInfoDao.findDashboardsByTenantId(id.getId(), pageLink);
  256 + }
  257 +
  258 + @Override
  259 + protected void removeEntity(TenantId tenantId, DashboardInfo entity) {
  260 + deleteDashboard(tenantId, new DashboardId(entity.getUuidId()));
  261 + }
  262 + };
  263 +
263 264 private class CustomerDashboardsUnassigner extends TimePaginatedRemover<Customer, DashboardInfo> {
264   -
  265 +
265 266 private Customer customer;
266 267
267 268 CustomerDashboardsUnassigner(Customer customer) {
... ... @@ -282,7 +283,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
282 283 protected void removeEntity(TenantId tenantId, DashboardInfo entity) {
283 284 unassignDashboardFromCustomer(customer.getTenantId(), new DashboardId(entity.getUuidId()), this.customer.getId());
284 285 }
285   -
  286 +
286 287 }
287 288
288 289 private class CustomerDashboardsUpdater extends TimePaginatedRemover<Customer, DashboardInfo> {
... ...
... ... @@ -23,6 +23,7 @@ import com.datastax.driver.mapping.Result;
23 23 import com.google.common.base.Function;
24 24 import com.google.common.util.concurrent.Futures;
25 25 import com.google.common.util.concurrent.ListenableFuture;
  26 +import com.google.common.util.concurrent.MoreExecutors;
26 27 import lombok.extern.slf4j.Slf4j;
27 28 import org.springframework.stereotype.Component;
28 29 import org.thingsboard.server.common.data.Device;
... ... @@ -178,14 +179,14 @@ public class CassandraDeviceDao extends CassandraAbstractSearchTextDao<DeviceEnt
178 179 if (result != null) {
179 180 List<EntitySubtype> entitySubtypes = new ArrayList<>();
180 181 result.all().forEach((entitySubtypeEntity) ->
181   - entitySubtypes.add(entitySubtypeEntity.toEntitySubtype())
  182 + entitySubtypes.add(entitySubtypeEntity.toEntitySubtype())
182 183 );
183 184 return entitySubtypes;
184 185 } else {
185 186 return Collections.emptyList();
186 187 }
187 188 }
188   - });
  189 + }, MoreExecutors.directExecutor());
189 190 }
190 191
191 192 }
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device;
18 18 import com.fasterxml.jackson.databind.ObjectMapper;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22 import lombok.extern.slf4j.Slf4j;
22 23 import org.springframework.beans.factory.annotation.Autowired;
23 24 import org.springframework.beans.factory.annotation.Value;
... ... @@ -97,9 +98,9 @@ public class ClaimDevicesServiceImpl implements ClaimDevicesService {
97 98 }
98 99 log.warn("Failed to find claimingAllowed attribute for device or it is already claimed![{}]", device.getName());
99 100 throw new IllegalArgumentException();
100   - });
  101 + }, MoreExecutors.directExecutor());
101 102 }
102   - });
  103 + }, MoreExecutors.directExecutor());
103 104 }
104 105
105 106 private ClaimDataInfo getClaimData(Cache cache, Device device) throws ExecutionException, InterruptedException {
... ... @@ -138,9 +139,9 @@ public class ClaimDevicesServiceImpl implements ClaimDevicesService {
138 139 if (device.getCustomerId().getId().equals(ModelConstants.NULL_UUID)) {
139 140 device.setCustomerId(customerId);
140 141 Device savedDevice = deviceService.saveDevice(device);
141   - return Futures.transform(removeClaimingSavedData(cache, claimData, device), result -> new ClaimResult(savedDevice, ClaimResponse.SUCCESS));
  142 + return Futures.transform(removeClaimingSavedData(cache, claimData, device), result -> new ClaimResult(savedDevice, ClaimResponse.SUCCESS), MoreExecutors.directExecutor());
142 143 }
143   - return Futures.transform(removeClaimingSavedData(cache, claimData, device), result -> new ClaimResult(null, ClaimResponse.CLAIMED));
  144 + return Futures.transform(removeClaimingSavedData(cache, claimData, device), result -> new ClaimResult(null, ClaimResponse.CLAIMED), MoreExecutors.directExecutor());
144 145 }
145 146 } else {
146 147 log.warn("Failed to find the device's claiming message![{}]", device.getName());
... ...
... ... @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device;
18 18 import com.google.common.base.Function;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22 import lombok.extern.slf4j.Slf4j;
22 23 import org.apache.commons.lang3.RandomStringUtils;
23 24 import org.hibernate.exception.ConstraintViolationException;
... ... @@ -291,7 +292,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
291 292 }
292 293 }
293 294 return Futures.successfulAsList(futures);
294   - });
  295 + }, MoreExecutors.directExecutor());
295 296
296 297 devices = Futures.transform(devices, new Function<List<Device>, List<Device>>() {
297 298 @Nullable
... ... @@ -299,7 +300,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
299 300 public List<Device> apply(@Nullable List<Device> deviceList) {
300 301 return deviceList == null ? Collections.emptyList() : deviceList.stream().filter(device -> query.getDeviceTypes().contains(device.getType())).collect(Collectors.toList());
301 302 }
302   - });
  303 + }, MoreExecutors.directExecutor());
303 304
304 305 return devices;
305 306 }
... ... @@ -313,7 +314,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
313 314 deviceTypes -> {
314 315 deviceTypes.sort(Comparator.comparing(EntitySubtype::getType));
315 316 return deviceTypes;
316   - });
  317 + }, MoreExecutors.directExecutor());
317 318 }
318 319
319 320 private DataValidator<Device> deviceValidator =
... ... @@ -374,18 +375,18 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
374 375 };
375 376
376 377 private PaginatedRemover<TenantId, Device> tenantDevicesRemover =
377   - new PaginatedRemover<TenantId, Device>() {
  378 + new PaginatedRemover<TenantId, Device>() {
378 379
379   - @Override
380   - protected List<Device> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
381   - return deviceDao.findDevicesByTenantId(id.getId(), pageLink);
382   - }
  380 + @Override
  381 + protected List<Device> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
  382 + return deviceDao.findDevicesByTenantId(id.getId(), pageLink);
  383 + }
383 384
384   - @Override
385   - protected void removeEntity(TenantId tenantId, Device entity) {
386   - deleteDevice(tenantId, new DeviceId(entity.getUuidId()));
387   - }
388   - };
  385 + @Override
  386 + protected void removeEntity(TenantId tenantId, Device entity) {
  387 + deleteDevice(tenantId, new DeviceId(entity.getUuidId()));
  388 + }
  389 + };
389 390
390 391 private PaginatedRemover<CustomerId, Device> customerDeviceUnasigner = new PaginatedRemover<CustomerId, Device>() {
391 392
... ...
... ... @@ -23,6 +23,7 @@ import com.datastax.driver.mapping.Result;
23 23 import com.google.common.base.Function;
24 24 import com.google.common.util.concurrent.Futures;
25 25 import com.google.common.util.concurrent.ListenableFuture;
  26 +import com.google.common.util.concurrent.MoreExecutors;
26 27 import lombok.extern.slf4j.Slf4j;
27 28 import org.springframework.stereotype.Component;
28 29 import org.thingsboard.server.common.data.EntitySubtype;
... ... @@ -97,7 +98,7 @@ public class CassandraEntityViewDao extends CassandraAbstractSearchTextDao<Entit
97 98 log.debug("Try to find entity views by tenantId [{}] and pageLink [{}]", tenantId, pageLink);
98 99 List<EntityViewEntity> entityViewEntities =
99 100 findPageWithTextSearch(new TenantId(tenantId), ENTITY_VIEW_BY_TENANT_AND_SEARCH_TEXT_CF,
100   - Collections.singletonList(eq(TENANT_ID_PROPERTY, tenantId)), pageLink);
  101 + Collections.singletonList(eq(TENANT_ID_PROPERTY, tenantId)), pageLink);
101 102 log.trace("Found entity views [{}] by tenantId [{}] and pageLink [{}]",
102 103 entityViewEntities, tenantId, pageLink);
103 104 return DaoUtil.convertDataList(entityViewEntities);
... ... @@ -181,6 +182,6 @@ public class CassandraEntityViewDao extends CassandraAbstractSearchTextDao<Entit
181 182 return Collections.emptyList();
182 183 }
183 184 }
184   - });
  185 + }, MoreExecutors.directExecutor());
185 186 }
186 187 }
... ...
... ... @@ -19,6 +19,7 @@ import com.google.common.base.Function;
19 19 import com.google.common.util.concurrent.FutureCallback;
20 20 import com.google.common.util.concurrent.Futures;
21 21 import com.google.common.util.concurrent.ListenableFuture;
  22 +import com.google.common.util.concurrent.MoreExecutors;
22 23 import lombok.extern.slf4j.Slf4j;
23 24 import org.apache.commons.lang3.StringUtils;
24 25 import org.springframework.beans.factory.annotation.Autowired;
... ... @@ -199,7 +200,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
199 200 }
200 201 }
201 202 return Futures.successfulAsList(futures);
202   - });
  203 + }, MoreExecutors.directExecutor());
203 204
204 205 entityViews = Futures.transform(entityViews, new Function<List<EntityView>, List<EntityView>>() {
205 206 @Nullable
... ... @@ -207,7 +208,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
207 208 public List<EntityView> apply(@Nullable List<EntityView> entityViewList) {
208 209 return entityViewList == null ? Collections.emptyList() : entityViewList.stream().filter(entityView -> query.getEntityViewTypes().contains(entityView.getType())).collect(Collectors.toList());
209 210 }
210   - });
  211 + }, MoreExecutors.directExecutor());
211 212
212 213 return entityViews;
213 214 }
... ... @@ -246,7 +247,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
246 247 public void onFailure(Throwable t) {
247 248 log.error("Error while finding entity views by tenantId and entityId", t);
248 249 }
249   - });
  250 + }, MoreExecutors.directExecutor());
250 251 return entityViewsFuture;
251 252 }
252 253 }
... ... @@ -279,7 +280,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
279 280 entityViewTypes -> {
280 281 entityViewTypes.sort(Comparator.comparing(EntitySubtype::getType));
281 282 return entityViewTypes;
282   - });
  283 + }, MoreExecutors.directExecutor());
283 284 }
284 285
285 286 private DataValidator<EntityView> entityViewValidator =
... ...
... ... @@ -22,6 +22,7 @@ import com.datastax.driver.core.querybuilder.Select;
22 22 import com.datastax.driver.core.utils.UUIDs;
23 23 import com.google.common.util.concurrent.Futures;
24 24 import com.google.common.util.concurrent.ListenableFuture;
  25 +import com.google.common.util.concurrent.MoreExecutors;
25 26 import lombok.extern.slf4j.Slf4j;
26 27 import org.apache.commons.lang3.StringUtils;
27 28 import org.springframework.beans.factory.annotation.Value;
... ... @@ -45,10 +46,12 @@ import java.util.UUID;
45 46 import java.util.concurrent.ExecutionException;
46 47
47 48 import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
48   -import static com.datastax.driver.core.querybuilder.QueryBuilder.in;
49 49 import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
50 50 import static com.datastax.driver.core.querybuilder.QueryBuilder.ttl;
51   -import static org.thingsboard.server.dao.model.ModelConstants.*;
  51 +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BY_ID_VIEW_NAME;
  52 +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BY_TYPE_AND_ID_VIEW_NAME;
  53 +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME;
  54 +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
52 55
53 56 @Component
54 57 @Slf4j
... ... @@ -96,7 +99,7 @@ public class CassandraBaseEventDao extends CassandraAbstractSearchTimeDao<EventE
96 99 event.setUid(event.getId().toString());
97 100 }
98 101 ListenableFuture<Optional<Event>> optionalSave = saveAsync(event.getTenantId(), new EventEntity(event), false, eventsTtl);
99   - return Futures.transform(optionalSave, opt -> opt.orElse(null));
  102 + return Futures.transform(optionalSave, opt -> opt.orElse(null), MoreExecutors.directExecutor());
100 103 }
101 104
102 105 @Override
... ... @@ -210,6 +213,6 @@ public class CassandraBaseEventDao extends CassandraAbstractSearchTimeDao<EventE
210 213 } else {
211 214 return Optional.empty();
212 215 }
213   - });
  216 + }, MoreExecutors.directExecutor());
214 217 }
215 218 }
... ...
... ... @@ -36,6 +36,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.BOOLEAN_VALUE_COLU
36 36 import static org.thingsboard.server.dao.model.ModelConstants.DOUBLE_VALUE_COLUMN;
37 37 import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN;
38 38 import static org.thingsboard.server.dao.model.ModelConstants.JSON_VALUE_COLUMN;
  39 +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
39 40 import static org.thingsboard.server.dao.model.ModelConstants.LONG_VALUE_COLUMN;
40 41 import static org.thingsboard.server.dao.model.ModelConstants.STRING_VALUE_COLUMN;
41 42 import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
... ... @@ -54,6 +55,10 @@ public abstract class AbstractTsKvEntity implements ToData<TsKvEntry> {
54 55 protected UUID entityId;
55 56
56 57 @Id
  58 + @Column(name = KEY_COLUMN)
  59 + protected int key;
  60 +
  61 + @Id
57 62 @Column(name = TS_COLUMN)
58 63 protected Long ts;
59 64
... ...
... ... @@ -69,10 +69,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
69 69 })
70 70 public final class TsKvLatestEntity extends AbstractTsKvEntity {
71 71
72   - @Id
73   - @Column(name = KEY_COLUMN)
74   - private int key;
75   -
76 72 @Override
77 73 public boolean isNotEmpty() {
78 74 return strValue != null || longValue != null || doubleValue != null || booleanValue != null || jsonValue != null;
... ...
... ... @@ -31,7 +31,6 @@ public class TimescaleTsKvCompositeKey implements Serializable {
31 31 @Transient
32 32 private static final long serialVersionUID = -4089175869616037523L;
33 33
34   - private UUID tenantId;
35 34 private UUID entityId;
36 35 private int key;
37 36 private long ts;
... ...
... ... @@ -18,25 +18,18 @@ package org.thingsboard.server.dao.model.sqlts.timescale.ts;
18 18 import lombok.Data;
19 19 import lombok.EqualsAndHashCode;
20 20 import org.springframework.util.StringUtils;
21   -import org.thingsboard.server.common.data.kv.TsKvEntry;
22   -import org.thingsboard.server.dao.model.ToData;
23 21 import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
24 22
25   -import javax.persistence.Column;
26 23 import javax.persistence.ColumnResult;
27 24 import javax.persistence.ConstructorResult;
28 25 import javax.persistence.Entity;
29   -import javax.persistence.Id;
30 26 import javax.persistence.IdClass;
31 27 import javax.persistence.NamedNativeQueries;
32 28 import javax.persistence.NamedNativeQuery;
33 29 import javax.persistence.SqlResultSetMapping;
34 30 import javax.persistence.SqlResultSetMappings;
35 31 import javax.persistence.Table;
36   -import java.util.UUID;
37 32
38   -import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
39   -import static org.thingsboard.server.dao.model.ModelConstants.TENANT_ID_COLUMN;
40 33 import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG;
41 34 import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG_QUERY;
42 35 import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_COUNT;
... ... @@ -52,7 +45,7 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F
52 45 @Data
53 46 @EqualsAndHashCode(callSuper = true)
54 47 @Entity
55   -@Table(name = "tenant_ts_kv")
  48 +@Table(name = "ts_kv")
56 49 @IdClass(TimescaleTsKvCompositeKey.class)
57 50 @SqlResultSetMappings({
58 51 @SqlResultSetMapping(
... ... @@ -116,15 +109,7 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F
116 109 resultSetMapping = "timescaleCountMapping"
117 110 )
118 111 })
119   -public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToData<TsKvEntry> {
120   -
121   - @Id
122   - @Column(name = TENANT_ID_COLUMN, columnDefinition = "uuid")
123   - private UUID tenantId;
124   -
125   - @Id
126   - @Column(name = KEY_COLUMN)
127   - private int key;
  112 +public final class TimescaleTsKvEntity extends AbstractTsKvEntity {
128 113
129 114 public TimescaleTsKvEntity() {
130 115 }
... ...
... ... @@ -32,10 +32,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
32 32 @IdClass(TsKvCompositeKey.class)
33 33 public final class TsKvEntity extends AbstractTsKvEntity {
34 34
35   - @Id
36   - @Column(name = KEY_COLUMN)
37   - private int key;
38   -
39 35 public TsKvEntity() {
40 36 }
41 37
... ...
... ... @@ -26,6 +26,7 @@ import com.datastax.driver.mapping.Result;
26 26 import com.google.common.base.Function;
27 27 import com.google.common.util.concurrent.Futures;
28 28 import com.google.common.util.concurrent.ListenableFuture;
  29 +import com.google.common.util.concurrent.MoreExecutors;
29 30 import lombok.extern.slf4j.Slf4j;
30 31 import org.thingsboard.server.common.data.id.TenantId;
31 32 import org.thingsboard.server.dao.Dao;
... ... @@ -86,7 +87,7 @@ public abstract class CassandraAbstractModelDao<E extends BaseEntity<D>, D> exte
86 87 return Collections.emptyList();
87 88 }
88 89 }
89   - });
  90 + }, MoreExecutors.directExecutor());
90 91 }
91 92 return Futures.immediateFuture(Collections.emptyList());
92 93 }
... ... @@ -120,7 +121,7 @@ public abstract class CassandraAbstractModelDao<E extends BaseEntity<D>, D> exte
120 121 return null;
121 122 }
122 123 }
123   - });
  124 + }, MoreExecutors.directExecutor());
124 125 }
125 126 return Futures.immediateFuture(null);
126 127 }
... ... @@ -191,5 +192,5 @@ public abstract class CassandraAbstractModelDao<E extends BaseEntity<D>, D> exte
191 192 List<E> entities = findListByStatement(tenantId, QueryBuilder.select().all().from(getColumnFamilyName()).setConsistencyLevel(cluster.getDefaultReadConsistencyLevel()));
192 193 return DaoUtil.convertDataList(entities);
193 194 }
194   -
  195 +
195 196 }
... ...
... ... @@ -22,6 +22,7 @@ import com.datastax.driver.core.Statement;
22 22 import com.google.common.util.concurrent.FutureCallback;
23 23 import com.google.common.util.concurrent.Futures;
24 24 import com.google.common.util.concurrent.ListenableFuture;
  25 +import com.google.common.util.concurrent.MoreExecutors;
25 26 import com.google.common.util.concurrent.Uninterruptibles;
26 27 import org.thingsboard.server.dao.exception.BufferLimitException;
27 28 import org.thingsboard.server.dao.util.AsyncRateLimiter;
... ... @@ -44,9 +45,9 @@ public class RateLimitedResultSetFuture implements ResultSetFuture {
44 45 rateLimiter.release();
45 46 }
46 47 return Futures.immediateFailedFuture(t);
47   - });
  48 + }, MoreExecutors.directExecutor());
48 49 this.originalFuture = Futures.transform(rateLimitFuture,
49   - i -> executeAsyncWithRelease(rateLimiter, session, statement));
  50 + i -> executeAsyncWithRelease(rateLimiter, session, statement), MoreExecutors.directExecutor());
50 51
51 52 }
52 53
... ... @@ -145,7 +146,7 @@ public class RateLimitedResultSetFuture implements ResultSetFuture {
145 146 public void onFailure(Throwable t) {
146 147 rateLimiter.release();
147 148 }
148   - });
  149 + }, MoreExecutors.directExecutor());
149 150 return resultSetFuture;
150 151 } catch (RuntimeException re) {
151 152 rateLimiter.release();
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.alarm;
17 17
18 18 import com.google.common.util.concurrent.Futures;
19 19 import com.google.common.util.concurrent.ListenableFuture;
  20 +import com.google.common.util.concurrent.MoreExecutors;
20 21 import lombok.extern.slf4j.Slf4j;
21 22 import org.springframework.beans.factory.annotation.Autowired;
22 23 import org.springframework.data.domain.PageRequest;
... ... @@ -108,9 +109,9 @@ public class JpaAlarmDao extends JpaAbstractDao<AlarmEntity, Alarm> implements A
108 109 for (EntityRelation relation : input) {
109 110 alarmFutures.add(Futures.transform(
110 111 findAlarmByIdAsync(tenantId, relation.getTo().getId()),
111   - AlarmInfo::new));
  112 + AlarmInfo::new, MoreExecutors.directExecutor()));
112 113 }
113 114 return Futures.successfulAsList(alarmFutures);
114   - });
  115 + }, MoreExecutors.directExecutor());
115 116 }
116 117 }
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.dashboard;
17 17
18 18 import com.google.common.util.concurrent.Futures;
19 19 import com.google.common.util.concurrent.ListenableFuture;
  20 +import com.google.common.util.concurrent.MoreExecutors;
20 21 import lombok.extern.slf4j.Slf4j;
21 22 import org.springframework.beans.factory.annotation.Autowired;
22 23 import org.springframework.data.domain.PageRequest;
... ... @@ -91,6 +92,6 @@ public class JpaDashboardInfoDao extends JpaAbstractSearchTextDao<DashboardInfoE
91 92 dashboardFutures.add(findByIdAsync(new TenantId(tenantId), relation.getTo().getId()));
92 93 }
93 94 return Futures.successfulAsList(dashboardFutures);
94   - });
  95 + }, MoreExecutors.directExecutor());
95 96 }
96 97 }
... ...
... ... @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sqlts;
17 17
18 18 import com.google.common.util.concurrent.Futures;
19 19 import com.google.common.util.concurrent.ListenableFuture;
  20 +import com.google.common.util.concurrent.MoreExecutors;
20 21 import com.google.common.util.concurrent.SettableFuture;
21 22 import lombok.extern.slf4j.Slf4j;
22 23 import org.springframework.beans.factory.annotation.Autowired;
... ... @@ -95,7 +96,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
95 96
96 97 @Override
97 98 public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
98   - return getRemoveLatestFuture(tenantId, entityId, query);
  99 + return getRemoveLatestFuture(entityId, query);
99 100 }
100 101
101 102 @Override
... ... @@ -124,9 +125,9 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
124 125 }
125 126
126 127 @Override
127   - protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
  128 + protected ListenableFuture<List<TsKvEntry>> findAllAsync(EntityId entityId, ReadTsKvQuery query) {
128 129 if (query.getAggregation() == Aggregation.NONE) {
129   - return findAllAsyncWithLimit(tenantId, entityId, query);
  130 + return findAllAsyncWithLimit(entityId, query);
130 131 } else {
131 132 long stepTs = query.getStartTs();
132 133 List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
... ... @@ -134,7 +135,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
134 135 long startTs = stepTs;
135 136 long endTs = stepTs + query.getInterval();
136 137 long ts = startTs + (endTs - startTs) / 2;
137   - futures.add(findAndAggregateAsync(tenantId, entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()));
  138 + futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()));
138 139 stepTs = endTs;
139 140 }
140 141 return getTskvEntriesFuture(Futures.allAsList(futures));
... ... @@ -142,7 +143,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
142 143 }
143 144
144 145 @Override
145   - protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
  146 + protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) {
146 147 Integer keyId = getOrSaveKeyId(query.getKey());
147 148 List<TsKvEntity> tsKvEntities = tsKvRepository.findAllWithLimit(
148 149 entityId.getId(),
... ... @@ -156,9 +157,9 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
156 157 return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities));
157 158 }
158 159
159   - protected ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
  160 + private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
160 161 List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>();
161   - switchAggregation(tenantId, entityId, key, startTs, endTs, aggregation, entitiesFutures);
  162 + switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures);
162 163 return Futures.transform(setFutures(entitiesFutures), entity -> {
163 164 if (entity != null && entity.isNotEmpty()) {
164 165 entity.setEntityId(entityId.getId());
... ... @@ -168,32 +169,32 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
168 169 } else {
169 170 return Optional.empty();
170 171 }
171   - });
  172 + }, MoreExecutors.directExecutor());
172 173 }
173 174
174   - protected void switchAggregation(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  175 + protected void switchAggregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
175 176 switch (aggregation) {
176 177 case AVG:
177   - findAvg(tenantId, entityId, key, startTs, endTs, entitiesFutures);
  178 + findAvg(entityId, key, startTs, endTs, entitiesFutures);
178 179 break;
179 180 case MAX:
180   - findMax(tenantId, entityId, key, startTs, endTs, entitiesFutures);
  181 + findMax(entityId, key, startTs, endTs, entitiesFutures);
181 182 break;
182 183 case MIN:
183   - findMin(tenantId, entityId, key, startTs, endTs, entitiesFutures);
  184 + findMin(entityId, key, startTs, endTs, entitiesFutures);
184 185 break;
185 186 case SUM:
186   - findSum(tenantId, entityId, key, startTs, endTs, entitiesFutures);
  187 + findSum(entityId, key, startTs, endTs, entitiesFutures);
187 188 break;
188 189 case COUNT:
189   - findCount(tenantId, entityId, key, startTs, endTs, entitiesFutures);
  190 + findCount(entityId, key, startTs, endTs, entitiesFutures);
190 191 break;
191 192 default:
192 193 throw new IllegalArgumentException("Not supported aggregation type: " + aggregation);
193 194 }
194 195 }
195 196
196   - protected void findCount(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  197 + protected void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
197 198 Integer keyId = getOrSaveKeyId(key);
198 199 entitiesFutures.add(tsKvRepository.findCount(
199 200 entityId.getId(),
... ... @@ -202,7 +203,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
202 203 endTs));
203 204 }
204 205
205   - protected void findSum(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  206 + protected void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
206 207 Integer keyId = getOrSaveKeyId(key);
207 208 entitiesFutures.add(tsKvRepository.findSum(
208 209 entityId.getId(),
... ... @@ -211,7 +212,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
211 212 endTs));
212 213 }
213 214
214   - protected void findMin(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  215 + protected void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
215 216 Integer keyId = getOrSaveKeyId(key);
216 217 entitiesFutures.add(tsKvRepository.findStringMin(
217 218 entityId.getId(),
... ... @@ -225,7 +226,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
225 226 endTs));
226 227 }
227 228
228   - protected void findMax(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  229 + protected void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
229 230 Integer keyId = getOrSaveKeyId(key);
230 231 entitiesFutures.add(tsKvRepository.findStringMax(
231 232 entityId.getId(),
... ... @@ -239,7 +240,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
239 240 endTs));
240 241 }
241 242
242   - protected void findAvg(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
  243 + protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
243 244 Integer keyId = getOrSaveKeyId(key);
244 245 entitiesFutures.add(tsKvRepository.findAvg(
245 246 entityId.getId(),
... ...
... ... @@ -20,6 +20,7 @@ import com.google.common.collect.Lists;
20 20 import com.google.common.util.concurrent.FutureCallback;
21 21 import com.google.common.util.concurrent.Futures;
22 22 import com.google.common.util.concurrent.ListenableFuture;
  23 +import com.google.common.util.concurrent.MoreExecutors;
23 24 import lombok.extern.slf4j.Slf4j;
24 25 import org.hibernate.exception.ConstraintViolationException;
25 26 import org.springframework.beans.factory.annotation.Autowired;
... ... @@ -126,7 +127,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
126 127 protected ListenableFuture<List<TsKvEntry>> processFindAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
127 128 List<ListenableFuture<List<TsKvEntry>>> futures = queries
128 129 .stream()
129   - .map(query -> findAllAsync(tenantId, entityId, query))
  130 + .map(query -> findAllAsync(entityId, query))
130 131 .collect(Collectors.toList());
131 132 return Futures.transform(Futures.allAsList(futures), new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() {
132 133 @Nullable
... ... @@ -143,9 +144,9 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
143 144 }, service);
144 145 }
145 146
146   - protected abstract ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query);
  147 + protected abstract ListenableFuture<List<TsKvEntry>> findAllAsync(EntityId entityId, ReadTsKvQuery query);
147 148
148   - protected abstract ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query);
  149 + protected abstract ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query);
149 150
150 151 protected ListenableFuture<List<TsKvEntry>> getTskvEntriesFuture(ListenableFuture<List<Optional<TsKvEntry>>> future) {
151 152 return Futures.transform(future, new Function<List<Optional<TsKvEntry>>, List<TsKvEntry>>() {
... ... @@ -163,12 +164,12 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
163 164 }, service);
164 165 }
165 166
166   - protected ListenableFuture<List<TsKvEntry>> findNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  167 + protected ListenableFuture<List<TsKvEntry>> findNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) {
167 168 long startTs = 0;
168 169 long endTs = query.getStartTs() - 1;
169 170 ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1,
170 171 Aggregation.NONE, DESC_ORDER);
171   - return findAllAsync(tenantId, entityId, findNewLatestQuery);
  172 + return findAllAsync(entityId, findNewLatestQuery);
172 173 }
173 174
174 175 protected ListenableFuture<TsKvEntry> getFindLatestFuture(EntityId entityId, String key) {
... ... @@ -188,7 +189,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
188 189 return Futures.immediateFuture(result);
189 190 }
190 191
191   - protected ListenableFuture<Void> getRemoveLatestFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
  192 + protected ListenableFuture<Void> getRemoveLatestFuture(EntityId entityId, DeleteTsKvQuery query) {
192 193 ListenableFuture<TsKvEntry> latestFuture = getFindLatestFuture(entityId, query.getKey());
193 194
194 195 ListenableFuture<Boolean> booleanFuture = Futures.transform(latestFuture, tsKvEntry -> {
... ... @@ -216,7 +217,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
216 217 if (query.getRewriteLatestIfDeleted()) {
217 218 ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
218 219 if (isRemove) {
219   - return getNewLatestEntryFuture(tenantId, entityId, query);
  220 + return getNewLatestEntryFuture(entityId, query);
220 221 }
221 222 return Futures.immediateFuture(null);
222 223 }, service);
... ... @@ -235,7 +236,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
235 236 public void onFailure(Throwable t) {
236 237 log.warn("[{}] Failed to process remove of the latest value", entityId, t);
237 238 }
238   - });
  239 + }, MoreExecutors.directExecutor());
239 240 return resultFuture;
240 241 }
241 242
... ... @@ -295,8 +296,8 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
295 296 return keyId;
296 297 }
297 298
298   - private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
299   - ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
  299 + private ListenableFuture<Void> getNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) {
  300 + ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(entityId, query);
300 301 return Futures.transformAsync(future, entryList -> {
301 302 if (entryList.size() == 1) {
302 303 return getSaveLatestFuture(entityId, entryList.get(0));
... ...
... ... @@ -37,8 +37,8 @@ import java.util.List;
37 37 public class TimescaleInsertTsRepository extends AbstractInsertRepository implements InsertTsRepository<TimescaleTsKvEntity> {
38 38
39 39 private static final String INSERT_OR_UPDATE =
40   - "INSERT INTO tenant_ts_kv (tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " +
41   - "ON CONFLICT (tenant_id, entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json);";
  40 + "INSERT INTO ts_kv (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " +
  41 + "ON CONFLICT (entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json);";
42 42
43 43 @Override
44 44 public void saveOrUpdate(List<EntityContainer<TimescaleTsKvEntity>> entities) {
... ... @@ -46,41 +46,40 @@ public class TimescaleInsertTsRepository extends AbstractInsertRepository implem
46 46 @Override
47 47 public void setValues(PreparedStatement ps, int i) throws SQLException {
48 48 TimescaleTsKvEntity tsKvEntity = entities.get(i).getEntity();
49   - ps.setObject(1, tsKvEntity.getTenantId());
50   - ps.setObject(2, tsKvEntity.getEntityId());
51   - ps.setInt(3, tsKvEntity.getKey());
52   - ps.setLong(4, tsKvEntity.getTs());
  49 + ps.setObject(1, tsKvEntity.getEntityId());
  50 + ps.setInt(2, tsKvEntity.getKey());
  51 + ps.setLong(3, tsKvEntity.getTs());
53 52
54 53 if (tsKvEntity.getBooleanValue() != null) {
55   - ps.setBoolean(5, tsKvEntity.getBooleanValue());
56   - ps.setBoolean(10, tsKvEntity.getBooleanValue());
  54 + ps.setBoolean(4, tsKvEntity.getBooleanValue());
  55 + ps.setBoolean(9, tsKvEntity.getBooleanValue());
57 56 } else {
58   - ps.setNull(5, Types.BOOLEAN);
59   - ps.setNull(10, Types.BOOLEAN);
  57 + ps.setNull(4, Types.BOOLEAN);
  58 + ps.setNull(9, Types.BOOLEAN);
60 59 }
61 60
62   - ps.setString(6, replaceNullChars(tsKvEntity.getStrValue()));
63   - ps.setString(11, replaceNullChars(tsKvEntity.getStrValue()));
  61 + ps.setString(5, replaceNullChars(tsKvEntity.getStrValue()));
  62 + ps.setString(10, replaceNullChars(tsKvEntity.getStrValue()));
64 63
65 64
66 65 if (tsKvEntity.getLongValue() != null) {
67   - ps.setLong(7, tsKvEntity.getLongValue());
68   - ps.setLong(12, tsKvEntity.getLongValue());
  66 + ps.setLong(6, tsKvEntity.getLongValue());
  67 + ps.setLong(11, tsKvEntity.getLongValue());
69 68 } else {
70   - ps.setNull(7, Types.BIGINT);
71   - ps.setNull(12, Types.BIGINT);
  69 + ps.setNull(6, Types.BIGINT);
  70 + ps.setNull(11, Types.BIGINT);
72 71 }
73 72
74 73 if (tsKvEntity.getDoubleValue() != null) {
75   - ps.setDouble(8, tsKvEntity.getDoubleValue());
76   - ps.setDouble(13, tsKvEntity.getDoubleValue());
  74 + ps.setDouble(7, tsKvEntity.getDoubleValue());
  75 + ps.setDouble(12, tsKvEntity.getDoubleValue());
77 76 } else {
78   - ps.setNull(8, Types.DOUBLE);
79   - ps.setNull(13, Types.DOUBLE);
  77 + ps.setNull(7, Types.DOUBLE);
  78 + ps.setNull(12, Types.DOUBLE);
80 79 }
81 80
82   - ps.setString(9, replaceNullChars(tsKvEntity.getJsonValue()));
83   - ps.setString(14, replaceNullChars(tsKvEntity.getJsonValue()));
  81 + ps.setString(8, replaceNullChars(tsKvEntity.getJsonValue()));
  82 + ps.setString(13, replaceNullChars(tsKvEntity.getJsonValue()));
84 83 }
85 84
86 85 @Override
... ...
... ... @@ -36,7 +36,7 @@ public class AggregationRepository {
36 36 public static final String FIND_SUM = "findSum";
37 37 public static final String FIND_COUNT = "findCount";
38 38
39   - public static final String FROM_WHERE_CLAUSE = "FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS uuid) AND tskv.entity_id = cast(:entityId AS uuid) AND tskv.key= cast(:entityKey AS int) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket";
  39 + public static final String FROM_WHERE_CLAUSE = "FROM ts_kv tskv WHERE tskv.entity_id = cast(:entityId AS uuid) AND tskv.key= cast(:entityKey AS int) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.entity_id, tskv.key, tsBucket";
40 40
41 41 public static final String FIND_AVG_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts) AS tsBucket, :timeBucket AS interval, SUM(COALESCE(tskv.long_v, 0)) AS longValue, SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, null AS strValue, 'AVG' AS aggType ";
42 42
... ... @@ -52,43 +52,42 @@ public class AggregationRepository {
52 52 private EntityManager entityManager;
53 53
54 54 @Async
55   - public CompletableFuture<List<TimescaleTsKvEntity>> findAvg(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
  55 + public CompletableFuture<List<TimescaleTsKvEntity>> findAvg(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
56 56 @SuppressWarnings("unchecked")
57   - List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG);
  57 + List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG);
58 58 return CompletableFuture.supplyAsync(() -> resultList);
59 59 }
60 60
61 61 @Async
62   - public CompletableFuture<List<TimescaleTsKvEntity>> findMax(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
  62 + public CompletableFuture<List<TimescaleTsKvEntity>> findMax(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
63 63 @SuppressWarnings("unchecked")
64   - List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX);
  64 + List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX);
65 65 return CompletableFuture.supplyAsync(() -> resultList);
66 66 }
67 67
68 68 @Async
69   - public CompletableFuture<List<TimescaleTsKvEntity>> findMin(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
  69 + public CompletableFuture<List<TimescaleTsKvEntity>> findMin(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
70 70 @SuppressWarnings("unchecked")
71   - List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN);
  71 + List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN);
72 72 return CompletableFuture.supplyAsync(() -> resultList);
73 73 }
74 74
75 75 @Async
76   - public CompletableFuture<List<TimescaleTsKvEntity>> findSum(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
  76 + public CompletableFuture<List<TimescaleTsKvEntity>> findSum(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
77 77 @SuppressWarnings("unchecked")
78   - List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM);
  78 + List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM);
79 79 return CompletableFuture.supplyAsync(() -> resultList);
80 80 }
81 81
82 82 @Async
83   - public CompletableFuture<List<TimescaleTsKvEntity>> findCount(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
  83 + public CompletableFuture<List<TimescaleTsKvEntity>> findCount(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
84 84 @SuppressWarnings("unchecked")
85   - List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT);
  85 + List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT);
86 86 return CompletableFuture.supplyAsync(() -> resultList);
87 87 }
88 88
89   - private List getResultList(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs, String query) {
  89 + private List getResultList(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs, String query) {
90 90 return entityManager.createNamedQuery(query)
91   - .setParameter("tenantId", tenantId)
92 91 .setParameter("entityId", entityId)
93 92 .setParameter("entityKey", entityKey)
94 93 .setParameter("timeBucket", timeBucket)
... ...
... ... @@ -88,24 +88,23 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
88 88 }
89 89
90 90 @Override
91   - protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
  91 + protected ListenableFuture<List<TsKvEntry>> findAllAsync(EntityId entityId, ReadTsKvQuery query) {
92 92 if (query.getAggregation() == Aggregation.NONE) {
93   - return findAllAsyncWithLimit(tenantId, entityId, query);
  93 + return findAllAsyncWithLimit(entityId, query);
94 94 } else {
95 95 long startTs = query.getStartTs();
96 96 long endTs = query.getEndTs();
97 97 long timeBucket = query.getInterval();
98   - ListenableFuture<List<Optional<TsKvEntry>>> future = findAllAndAggregateAsync(tenantId, entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation());
  98 + ListenableFuture<List<Optional<TsKvEntry>>> future = findAllAndAggregateAsync(entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation());
99 99 return getTskvEntriesFuture(future);
100 100 }
101 101 }
102 102
103 103 @Override
104   - protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
  104 + protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) {
105 105 String strKey = query.getKey();
106 106 Integer keyId = getOrSaveKeyId(strKey);
107 107 List<TimescaleTsKvEntity> timescaleTsKvEntities = tsKvRepository.findAllWithLimit(
108   - tenantId.getId(),
109 108 entityId.getId(),
110 109 keyId,
111 110 query.getStartTs(),
... ... @@ -117,8 +116,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
117 116 return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities));
118 117 }
119 118
120   - private ListenableFuture<List<Optional<TsKvEntry>>> findAllAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) {
121   - CompletableFuture<List<TimescaleTsKvEntity>> listCompletableFuture = switchAggregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId(), tenantId.getId());
  119 + private ListenableFuture<List<Optional<TsKvEntry>>> findAllAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) {
  120 + CompletableFuture<List<TimescaleTsKvEntity>> listCompletableFuture = switchAggregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId());
122 121 SettableFuture<List<TimescaleTsKvEntity>> listenableFuture = SettableFuture.create();
123 122 listCompletableFuture.whenComplete((timescaleTsKvEntities, throwable) -> {
124 123 if (throwable != null) {
... ... @@ -133,7 +132,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
133 132 timescaleTsKvEntities.forEach(entity -> {
134 133 if (entity != null && entity.isNotEmpty()) {
135 134 entity.setEntityId(entityId.getId());
136   - entity.setTenantId(tenantId.getId());
137 135 entity.setStrKey(key);
138 136 result.add(Optional.of(DaoUtil.getData(entity)));
139 137 } else {
... ... @@ -167,7 +165,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
167 165 String strKey = tsKvEntry.getKey();
168 166 Integer keyId = getOrSaveKeyId(strKey);
169 167 TimescaleTsKvEntity entity = new TimescaleTsKvEntity();
170   - entity.setTenantId(tenantId.getId());
171 168 entity.setEntityId(entityId.getId());
172 169 entity.setTs(tsKvEntry.getTs());
173 170 entity.setKey(keyId);
... ... @@ -197,7 +194,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
197 194 Integer keyId = getOrSaveKeyId(strKey);
198 195 return service.submit(() -> {
199 196 tsKvRepository.delete(
200   - tenantId.getId(),
201 197 entityId.getId(),
202 198 keyId,
203 199 query.getStartTs(),
... ... @@ -208,7 +204,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
208 204
209 205 @Override
210 206 public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
211   - return getRemoveLatestFuture(tenantId, entityId, query);
  207 + return getRemoveLatestFuture(entityId, query);
212 208 }
213 209
214 210 @Override
... ... @@ -216,27 +212,26 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
216 212 return service.submit(() -> null);
217 213 }
218 214
219   - private CompletableFuture<List<TimescaleTsKvEntity>> switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId, UUID tenantId) {
  215 + private CompletableFuture<List<TimescaleTsKvEntity>> switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId) {
220 216 switch (aggregation) {
221 217 case AVG:
222   - return findAvg(key, startTs, endTs, timeBucket, entityId, tenantId);
  218 + return findAvg(key, startTs, endTs, timeBucket, entityId);
223 219 case MAX:
224   - return findMax(key, startTs, endTs, timeBucket, entityId, tenantId);
  220 + return findMax(key, startTs, endTs, timeBucket, entityId);
225 221 case MIN:
226   - return findMin(key, startTs, endTs, timeBucket, entityId, tenantId);
  222 + return findMin(key, startTs, endTs, timeBucket, entityId);
227 223 case SUM:
228   - return findSum(key, startTs, endTs, timeBucket, entityId, tenantId);
  224 + return findSum(key, startTs, endTs, timeBucket, entityId);
229 225 case COUNT:
230   - return findCount(key, startTs, endTs, timeBucket, entityId, tenantId);
  226 + return findCount(key, startTs, endTs, timeBucket, entityId);
231 227 default:
232 228 throw new IllegalArgumentException("Not supported aggregation type: " + aggregation);
233 229 }
234 230 }
235 231
236   - private CompletableFuture<List<TimescaleTsKvEntity>> findCount(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  232 + private CompletableFuture<List<TimescaleTsKvEntity>> findCount(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
237 233 Integer keyId = getOrSaveKeyId(key);
238 234 return aggregationRepository.findCount(
239   - tenantId,
240 235 entityId,
241 236 keyId,
242 237 timeBucket,
... ... @@ -244,10 +239,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
244 239 endTs);
245 240 }
246 241
247   - private CompletableFuture<List<TimescaleTsKvEntity>> findSum(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  242 + private CompletableFuture<List<TimescaleTsKvEntity>> findSum(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
248 243 Integer keyId = getOrSaveKeyId(key);
249 244 return aggregationRepository.findSum(
250   - tenantId,
251 245 entityId,
252 246 keyId,
253 247 timeBucket,
... ... @@ -255,10 +249,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
255 249 endTs);
256 250 }
257 251
258   - private CompletableFuture<List<TimescaleTsKvEntity>> findMin(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  252 + private CompletableFuture<List<TimescaleTsKvEntity>> findMin(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
259 253 Integer keyId = getOrSaveKeyId(key);
260 254 return aggregationRepository.findMin(
261   - tenantId,
262 255 entityId,
263 256 keyId,
264 257 timeBucket,
... ... @@ -266,10 +259,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
266 259 endTs);
267 260 }
268 261
269   - private CompletableFuture<List<TimescaleTsKvEntity>> findMax(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  262 + private CompletableFuture<List<TimescaleTsKvEntity>> findMax(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
270 263 Integer keyId = getOrSaveKeyId(key);
271 264 return aggregationRepository.findMax(
272   - tenantId,
273 265 entityId,
274 266 keyId,
275 267 timeBucket,
... ... @@ -277,10 +269,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
277 269 endTs);
278 270 }
279 271
280   - private CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
  272 + private CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
281 273 Integer keyId = getOrSaveKeyId(key);
282 274 return aggregationRepository.findAvg(
283   - tenantId,
284 275 entityId,
285 276 keyId,
286 277 timeBucket,
... ...
... ... @@ -31,12 +31,10 @@ import java.util.UUID;
31 31 @TimescaleDBTsDao
32 32 public interface TsKvTimescaleRepository extends CrudRepository<TimescaleTsKvEntity, TimescaleTsKvCompositeKey> {
33 33
34   - @Query("SELECT tskv FROM TimescaleTsKvEntity tskv WHERE tskv.tenantId = :tenantId " +
35   - "AND tskv.entityId = :entityId " +
  34 + @Query("SELECT tskv FROM TimescaleTsKvEntity tskv WHERE tskv.entityId = :entityId " +
36 35 "AND tskv.key = :entityKey " +
37 36 "AND tskv.ts > :startTs AND tskv.ts <= :endTs")
38 37 List<TimescaleTsKvEntity> findAllWithLimit(
39   - @Param("tenantId") UUID tenantId,
40 38 @Param("entityId") UUID entityId,
41 39 @Param("entityKey") int key,
42 40 @Param("startTs") long startTs,
... ... @@ -44,12 +42,10 @@ public interface TsKvTimescaleRepository extends CrudRepository<TimescaleTsKvEnt
44 42
45 43 @Transactional
46 44 @Modifying
47   - @Query("DELETE FROM TimescaleTsKvEntity tskv WHERE tskv.tenantId = :tenantId " +
48   - "AND tskv.entityId = :entityId " +
  45 + @Query("DELETE FROM TimescaleTsKvEntity tskv WHERE tskv.entityId = :entityId " +
49 46 "AND tskv.key = :entityKey " +
50 47 "AND tskv.ts > :startTs AND tskv.ts <= :endTs")
51   - void delete(@Param("tenantId") UUID tenantId,
52   - @Param("entityId") UUID entityId,
  48 + void delete(@Param("entityId") UUID entityId,
53 49 @Param("entityKey") int key,
54 50 @Param("startTs") long startTs,
55 51 @Param("endTs") long endTs);
... ...
1   ---
2   --- Copyright © 2016-2020 The Thingsboard Authors
3   ---
4   --- Licensed under the Apache License, Version 2.0 (the "License");
5   --- you may not use this file except in compliance with the License.
6   --- You may obtain a copy of the License at
7   ---
8   --- http://www.apache.org/licenses/LICENSE-2.0
9   ---
10   --- Unless required by applicable law or agreed to in writing, software
11   --- distributed under the License is distributed on an "AS IS" BASIS,
12   --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   --- See the License for the specific language governing permissions and
14   --- limitations under the License.
15   ---
16   -
17   -CREATE INDEX IF NOT EXISTS idx_tenant_ts_kv ON tenant_ts_kv(tenant_id, entity_id, key, ts);
\ No newline at end of file
... ... @@ -16,8 +16,7 @@
16 16
17 17 CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
18 18
19   -CREATE TABLE IF NOT EXISTS tenant_ts_kv (
20   - tenant_id uuid NOT NULL,
  19 +CREATE TABLE IF NOT EXISTS ts_kv (
21 20 entity_id uuid NOT NULL,
22 21 key int NOT NULL,
23 22 ts bigint NOT NULL,
... ... @@ -26,7 +25,7 @@ CREATE TABLE IF NOT EXISTS tenant_ts_kv (
26 25 long_v bigint,
27 26 dbl_v double precision,
28 27 json_v json,
29   - CONSTRAINT tenant_ts_kv_pkey PRIMARY KEY (tenant_id, entity_id, key, ts)
  28 + CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts)
30 29 );
31 30
32 31 CREATE TABLE IF NOT EXISTS ts_kv_dictionary (
... ... @@ -45,4 +44,122 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest (
45 44 dbl_v double precision,
46 45 json_v json,
47 46 CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key)
48   -);
\ No newline at end of file
  47 +);
  48 +
  49 +CREATE TABLE IF NOT EXISTS tb_schema_settings
  50 +(
  51 + schema_version bigint NOT NULL,
  52 + CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
  53 +);
  54 +
  55 +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000);
  56 +
  57 +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
  58 +$$
  59 +BEGIN
  60 + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
  61 + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
  62 +END;
  63 +$$ LANGUAGE plpgsql;
  64 +
  65 +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  66 + OUT deleted bigint) AS
  67 +$$
  68 +BEGIN
  69 + EXECUTE format(
  70 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  71 + tenant_id, customer_id, ttl) into deleted;
  72 +END;
  73 +$$ LANGUAGE plpgsql;
  74 +
  75 +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  76 + OUT deleted bigint) AS
  77 +$$
  78 +BEGIN
  79 + EXECUTE format(
  80 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  81 + tenant_id, customer_id, ttl) into deleted;
  82 +END;
  83 +$$ LANGUAGE plpgsql;
  84 +
  85 +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  86 + OUT deleted bigint) AS
  87 +$$
  88 +BEGIN
  89 + EXECUTE format(
  90 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  91 + tenant_id, customer_id, ttl) into deleted;
  92 +END;
  93 +$$ LANGUAGE plpgsql;
  94 +
  95 +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31),
  96 + IN system_ttl bigint, INOUT deleted bigint)
  97 + LANGUAGE plpgsql AS
  98 +$$
  99 +DECLARE
  100 + tenant_cursor CURSOR FOR select tenant.id as tenant_id
  101 + from tenant;
  102 + tenant_id_record varchar;
  103 + customer_id_record varchar;
  104 + tenant_ttl bigint;
  105 + customer_ttl bigint;
  106 + deleted_for_entities bigint;
  107 + tenant_ttl_ts bigint;
  108 + customer_ttl_ts bigint;
  109 +BEGIN
  110 + OPEN tenant_cursor;
  111 + FETCH tenant_cursor INTO tenant_id_record;
  112 + WHILE FOUND
  113 + LOOP
  114 + EXECUTE format(
  115 + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
  116 + tenant_id_record, 'TTL') INTO tenant_ttl;
  117 + if tenant_ttl IS NULL THEN
  118 + tenant_ttl := system_ttl;
  119 + END IF;
  120 + IF tenant_ttl > 0 THEN
  121 + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint;
  122 + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
  123 + deleted := deleted + deleted_for_entities;
  124 + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record;
  125 + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
  126 + deleted := deleted + deleted_for_entities;
  127 + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record;
  128 + END IF;
  129 + FOR customer_id_record IN
  130 + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record
  131 + LOOP
  132 + EXECUTE format(
  133 + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
  134 + customer_id_record, 'TTL') INTO customer_ttl;
  135 + IF customer_ttl IS NULL THEN
  136 + customer_ttl_ts := tenant_ttl_ts;
  137 + ELSE
  138 + IF customer_ttl > 0 THEN
  139 + customer_ttl_ts :=
  140 + (EXTRACT(EPOCH FROM current_timestamp) * 1000 -
  141 + customer_ttl::bigint * 1000)::bigint;
  142 + END IF;
  143 + END IF;
  144 + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN
  145 + deleted_for_entities :=
  146 + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record,
  147 + customer_ttl_ts);
  148 + deleted := deleted + deleted_for_entities;
  149 + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record;
  150 + deleted_for_entities :=
  151 + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record,
  152 + customer_ttl_ts);
  153 + deleted := deleted + deleted_for_entities;
  154 + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
  155 + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record,
  156 + customer_id_record,
  157 + customer_ttl_ts);
  158 + deleted := deleted + deleted_for_entities;
  159 + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
  160 + END IF;
  161 + END LOOP;
  162 + FETCH tenant_cursor INTO tenant_id_record;
  163 + END LOOP;
  164 +END
  165 +$$;
... ...
... ... @@ -14,31 +14,260 @@
14 14 -- limitations under the License.
15 15 --
16 16
17   -CREATE TABLE IF NOT EXISTS ts_kv (
18   - entity_id uuid NOT NULL,
19   - key int NOT NULL,
20   - ts bigint NOT NULL,
21   - bool_v boolean,
22   - str_v varchar(10000000),
23   - long_v bigint,
24   - dbl_v double precision,
25   - json_v json
  17 +CREATE TABLE IF NOT EXISTS ts_kv
  18 +(
  19 + entity_id uuid NOT NULL,
  20 + key int NOT NULL,
  21 + ts bigint NOT NULL,
  22 + bool_v boolean,
  23 + str_v varchar(10000000),
  24 + long_v bigint,
  25 + dbl_v double precision,
  26 + json_v json
26 27 ) PARTITION BY RANGE (ts);
27 28
28   -CREATE TABLE IF NOT EXISTS ts_kv_latest (
29   - entity_id uuid NOT NULL,
30   - key int NOT NULL,
31   - ts bigint NOT NULL,
32   - bool_v boolean,
33   - str_v varchar(10000000),
34   - long_v bigint,
35   - dbl_v double precision,
36   - json_v json,
  29 +CREATE TABLE IF NOT EXISTS ts_kv_latest
  30 +(
  31 + entity_id uuid NOT NULL,
  32 + key int NOT NULL,
  33 + ts bigint NOT NULL,
  34 + bool_v boolean,
  35 + str_v varchar(10000000),
  36 + long_v bigint,
  37 + dbl_v double precision,
  38 + json_v json,
37 39 CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key)
38 40 );
39 41
40   -CREATE TABLE IF NOT EXISTS ts_kv_dictionary (
41   - key varchar(255) NOT NULL,
  42 +CREATE TABLE IF NOT EXISTS ts_kv_dictionary
  43 +(
  44 + key varchar(255) NOT NULL,
42 45 key_id serial UNIQUE,
43 46 CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
44   -);
\ No newline at end of file
  47 +);
  48 +
  49 +CREATE TABLE IF NOT EXISTS tb_schema_settings
  50 +(
  51 + schema_version bigint NOT NULL,
  52 + CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
  53 +);
  54 +
  55 +INSERT INTO tb_schema_settings (schema_version) VALUES (2005000);
  56 +
  57 +CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint)
  58 + LANGUAGE plpgsql AS
  59 +$$
  60 +DECLARE
  61 + max_tenant_ttl bigint;
  62 + max_customer_ttl bigint;
  63 + max_ttl bigint;
  64 + date timestamp;
  65 + partition_by_max_ttl_date varchar;
  66 + partition_month varchar;
  67 + partition_day varchar;
  68 + partition_year varchar;
  69 + partition varchar;
  70 + partition_to_delete varchar;
  71 +
  72 +
  73 +BEGIN
  74 + SELECT max(attribute_kv.long_v)
  75 + FROM tenant
  76 + INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id
  77 + WHERE attribute_kv.attribute_key = 'TTL'
  78 + into max_tenant_ttl;
  79 + SELECT max(attribute_kv.long_v)
  80 + FROM customer
  81 + INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id
  82 + WHERE attribute_kv.attribute_key = 'TTL'
  83 + into max_customer_ttl;
  84 + max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
  85 + if max_ttl IS NOT NULL AND max_ttl > 0 THEN
  86 + date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
  87 + partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
  88 + RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
  89 + IF partition_by_max_ttl_date IS NOT NULL THEN
  90 + CASE
  91 + WHEN partition_type = 'DAYS' THEN
  92 + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
  93 + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
  94 + partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5);
  95 + WHEN partition_type = 'MONTHS' THEN
  96 + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
  97 + partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
  98 + ELSE
  99 + partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
  100 + END CASE;
  101 + FOR partition IN SELECT tablename
  102 + FROM pg_tables
  103 + WHERE schemaname = 'public'
  104 + AND tablename like 'ts_kv_' || '%'
  105 + AND tablename != 'ts_kv_latest'
  106 + AND tablename != 'ts_kv_dictionary'
  107 + LOOP
  108 + IF partition != partition_by_max_ttl_date THEN
  109 + IF partition_year IS NOT NULL THEN
  110 + IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN
  111 + partition_to_delete := partition;
  112 + ELSE
  113 + IF partition_month IS NOT NULL THEN
  114 + IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN
  115 + partition_to_delete := partition;
  116 + ELSE
  117 + IF partition_day IS NOT NULL THEN
  118 + IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN
  119 + partition_to_delete := partition;
  120 + END IF;
  121 + END IF;
  122 + END IF;
  123 + END IF;
  124 + END IF;
  125 + END IF;
  126 + END IF;
  127 + IF partition_to_delete IS NOT NULL THEN
  128 + RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
  129 + EXECUTE format('DROP TABLE %I', partition_to_delete);
  130 + deleted := deleted + 1;
  131 + END IF;
  132 + END LOOP;
  133 + END IF;
  134 + END IF;
  135 +END
  136 +$$;
  137 +
  138 +CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS
  139 +$$
  140 +BEGIN
  141 + CASE
  142 + WHEN partition_type = 'DAYS' THEN
  143 + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd');
  144 + WHEN partition_type = 'MONTHS' THEN
  145 + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM');
  146 + WHEN partition_type = 'YEARS' THEN
  147 + partition := 'ts_kv_' || to_char(date, 'yyyy');
  148 + WHEN partition_type = 'INDEFINITE' THEN
  149 + partition := NULL;
  150 + ELSE
  151 + partition := NULL;
  152 + END CASE;
  153 + IF partition IS NOT NULL THEN
  154 + IF NOT EXISTS(SELECT
  155 + FROM pg_tables
  156 + WHERE schemaname = 'public'
  157 + AND tablename = partition) THEN
  158 + partition := NULL;
  159 + RAISE NOTICE 'Failed to found partition by ttl';
  160 + END IF;
  161 + END IF;
  162 +END;
  163 +$$ LANGUAGE plpgsql;
  164 +
  165 +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
  166 +$$
  167 +BEGIN
  168 + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
  169 + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
  170 +END;
  171 +$$ LANGUAGE plpgsql;
  172 +
  173 +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  174 + OUT deleted bigint) AS
  175 +$$
  176 +BEGIN
  177 + EXECUTE format(
  178 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  179 + tenant_id, customer_id, ttl) into deleted;
  180 +END;
  181 +$$ LANGUAGE plpgsql;
  182 +
  183 +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  184 + OUT deleted bigint) AS
  185 +$$
  186 +BEGIN
  187 + EXECUTE format(
  188 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  189 + tenant_id, customer_id, ttl) into deleted;
  190 +END;
  191 +$$ LANGUAGE plpgsql;
  192 +
  193 +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
  194 + OUT deleted bigint) AS
  195 +$$
  196 +BEGIN
  197 + EXECUTE format(
  198 + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
  199 + tenant_id, customer_id, ttl) into deleted;
  200 +END;
  201 +$$ LANGUAGE plpgsql;
  202 +
  203 +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31),
  204 + IN system_ttl bigint, INOUT deleted bigint)
  205 + LANGUAGE plpgsql AS
  206 +$$
  207 +DECLARE
  208 + tenant_cursor CURSOR FOR select tenant.id as tenant_id
  209 + from tenant;
  210 + tenant_id_record varchar;
  211 + customer_id_record varchar;
  212 + tenant_ttl bigint;
  213 + customer_ttl bigint;
  214 + deleted_for_entities bigint;
  215 + tenant_ttl_ts bigint;
  216 + customer_ttl_ts bigint;
  217 +BEGIN
  218 + OPEN tenant_cursor;
  219 + FETCH tenant_cursor INTO tenant_id_record;
  220 + WHILE FOUND
  221 + LOOP
  222 + EXECUTE format(
  223 + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
  224 + tenant_id_record, 'TTL') INTO tenant_ttl;
  225 + if tenant_ttl IS NULL THEN
  226 + tenant_ttl := system_ttl;
  227 + END IF;
  228 + IF tenant_ttl > 0 THEN
  229 + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint;
  230 + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
  231 + deleted := deleted + deleted_for_entities;
  232 + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record;
  233 + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
  234 + deleted := deleted + deleted_for_entities;
  235 + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record;
  236 + END IF;
  237 + FOR customer_id_record IN
  238 + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record
  239 + LOOP
  240 + EXECUTE format(
  241 + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
  242 + customer_id_record, 'TTL') INTO customer_ttl;
  243 + IF customer_ttl IS NULL THEN
  244 + customer_ttl_ts := tenant_ttl_ts;
  245 + ELSE
  246 + IF customer_ttl > 0 THEN
  247 + customer_ttl_ts :=
  248 + (EXTRACT(EPOCH FROM current_timestamp) * 1000 -
  249 + customer_ttl::bigint * 1000)::bigint;
  250 + END IF;
  251 + END IF;
  252 + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN
  253 + deleted_for_entities :=
  254 + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record,
  255 + customer_ttl_ts);
  256 + deleted := deleted + deleted_for_entities;
  257 + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record;
  258 + deleted_for_entities :=
  259 + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record,
  260 + customer_ttl_ts);
  261 + deleted := deleted + deleted_for_entities;
  262 + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
  263 + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record,
  264 + customer_id_record,
  265 + customer_ttl_ts);
  266 + deleted := deleted + deleted_for_entities;
  267 + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
  268 + END IF;
  269 + END LOOP;
  270 + FETCH tenant_cursor INTO tenant_id_record;
  271 + END LOOP;
  272 +END
  273 +$$;
... ...
... ... @@ -44,7 +44,7 @@ public class SqlDaoServiceTestSuite {
44 44
45 45 // @ClassRule
46 46 // public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
47   -// Arrays.asList("sql/schema-timescale.sql", "sql/schema-timescale-idx.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
  47 +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
48 48 // "sql/timescale/drop-all-tables.sql",
49 49 // "sql-test.properties"
50 50 // );
... ...
... ... @@ -119,7 +119,7 @@ public class RateLimitedResultSetFutureTest {
119 119
120 120 resultSetFuture = new RateLimitedResultSetFuture(session, rateLimiter, statement);
121 121
122   - ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one);
  122 + ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one, MoreExecutors.directExecutor());
123 123 Row actualRow = transform.get();
124 124
125 125 assertSame(row, actualRow);
... ... @@ -132,7 +132,7 @@ public class RateLimitedResultSetFutureTest {
132 132 when(rateLimiter.acquireAsync()).thenReturn(Futures.immediateFuture(null));
133 133 when(session.executeAsync(statement)).thenThrow(new UnsupportedFeatureException(ProtocolVersion.V3, "hjg"));
134 134 resultSetFuture = new RateLimitedResultSetFuture(session, rateLimiter, statement);
135   - ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one);
  135 + ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one, MoreExecutors.directExecutor());
136 136 try {
137 137 transform.get();
138 138 fail();
... ... @@ -156,7 +156,7 @@ public class RateLimitedResultSetFutureTest {
156 156
157 157 when(realFuture.get()).thenThrow(new ExecutionException("Fail", new TimeoutException("timeout")));
158 158 resultSetFuture = new RateLimitedResultSetFuture(session, rateLimiter, statement);
159   - ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one);
  159 + ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one, MoreExecutors.directExecutor());
160 160 try {
161 161 transform.get();
162 162 fail();
... ... @@ -177,7 +177,7 @@ public class RateLimitedResultSetFutureTest {
177 177 when(rateLimiter.acquireAsync()).thenReturn(future);
178 178 resultSetFuture = new RateLimitedResultSetFuture(session, rateLimiter, statement);
179 179
180   - ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one);
  180 + ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one, MoreExecutors.directExecutor());
181 181 // TimeUnit.MILLISECONDS.sleep(200);
182 182 future.cancel(false);
183 183 latch.countDown();
... ...
... ... @@ -12,7 +12,7 @@ DROP TABLE IF EXISTS event;
12 12 DROP TABLE IF EXISTS relation;
13 13 DROP TABLE IF EXISTS tb_user;
14 14 DROP TABLE IF EXISTS tenant;
15   -DROP TABLE IF EXISTS tenant_ts_kv;
  15 +DROP TABLE IF EXISTS ts_kv;
16 16 DROP TABLE IF EXISTS ts_kv_latest;
17 17 DROP TABLE IF EXISTS user_credentials;
18 18 DROP TABLE IF EXISTS widget_type;
... ...
... ... @@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env`
17 17
18 18 **NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.cassandra.yml` for details).
19 19
  20 +Execute the following command to create log folders for the services and chown of these folders to the docker container users.
  21 +To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access):
  22 +
  23 +`
  24 +$ ./docker-create-log-folders.sh
  25 +`
  26 +
20 27 Execute the following command to run installation:
21 28
22 29 `
... ...
  1 +#!/bin/bash
  2 +#
  3 +# Copyright © 2016-2020 The Thingsboard Authors
  4 +#
  5 +# Licensed under the Apache License, Version 2.0 (the "License");
  6 +# you may not use this file except in compliance with the License.
  7 +# You may obtain a copy of the License at
  8 +#
  9 +# http://www.apache.org/licenses/LICENSE-2.0
  10 +#
  11 +# Unless required by applicable law or agreed to in writing, software
  12 +# distributed under the License is distributed on an "AS IS" BASIS,
  13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 +# See the License for the specific language governing permissions and
  15 +# limitations under the License.
  16 +#
  17 +
  18 +mkdir -p tb-node/log/ && sudo chown -R 799:799 tb-node/log/
  19 +
  20 +mkdir -p tb-transports/coap/log && sudo chown -R 799:799 tb-transports/coap/log
  21 +
  22 +mkdir -p tb-transports/http/log && sudo chown -R 799:799 tb-transports/http/log
  23 +
  24 +mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log
\ No newline at end of file
... ...
... ... @@ -39,5 +39,5 @@ spec:
39 39 volumeMounts:
40 40 - mountPath: /config
41 41 name: tb-node-config
42   - command: ['sh', '-c', 'while [ ! -f /install-finished ]; do sleep 2; done;']
  42 + command: ['sh', '-c', 'while [ ! -f /tmp/install-finished ]; do sleep 2; done;']
43 43 restartPolicy: Never
... ...
... ... @@ -15,4 +15,6 @@
15 15 # limitations under the License.
16 16 #
17 17
18   -kubectl -n thingsboard delete svc,sts,deploy,pv,pvc,cm,po,ing --all
  18 +kubectl -n thingsboard delete svc,sts,deploy,cm,po,ing --all
  19 +
  20 +kubectl -n thingsboard get pvc --no-headers=true | awk '//{print $1}' | xargs kubectl -n thingsboard delete --ignore-not-found=true pvc
\ No newline at end of file
... ...
... ... @@ -22,7 +22,7 @@ function installTb() {
22 22 kubectl apply -f tb-node-configmap.yml
23 23 kubectl apply -f database-setup.yml &&
24 24 kubectl wait --for=condition=Ready pod/tb-db-setup --timeout=120s &&
25   - kubectl exec tb-db-setup -- sh -c 'export INSTALL_TB=true; export LOAD_DEMO='"$loadDemo"'; start-tb-node.sh; touch /install-finished;'
  25 + kubectl exec tb-db-setup -- sh -c 'export INSTALL_TB=true; export LOAD_DEMO='"$loadDemo"'; start-tb-node.sh; touch /tmp/install-finished;'
26 26
27 27 kubectl delete pod tb-db-setup
28 28
... ...
... ... @@ -38,6 +38,6 @@ fi
38 38
39 39 kubectl apply -f database-setup.yml &&
40 40 kubectl wait --for=condition=Ready pod/tb-db-setup --timeout=120s &&
41   -kubectl exec tb-db-setup -- sh -c 'export UPGRADE_TB=true; export FROM_VERSION='"$fromVersion"'; start-tb-node.sh; touch /install-finished;'
  41 +kubectl exec tb-db-setup -- sh -c 'export UPGRADE_TB=true; export FROM_VERSION='"$fromVersion"'; start-tb-node.sh; touch /tmp/install-finished;'
42 42
43 43 kubectl delete pod tb-db-setup
... ...
... ... @@ -58,6 +58,8 @@ spec:
58 58 env:
59 59 - name: POSTGRES_DB
60 60 value: "thingsboard"
  61 + - name: POSTGRES_PASSWORD
  62 + value: "postgres"
61 63 - name: PGDATA
62 64 value: /var/lib/postgresql/data/pgdata
63 65 volumeMounts:
... ...
... ... @@ -14,7 +14,7 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -FROM debian:stretch
  17 +FROM thingsboard/base
18 18
19 19 COPY start-js-executor.sh ${pkg.name}.deb /tmp/
20 20
... ... @@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
25 25
26 26 RUN update-rc.d ${pkg.name} disable
27 27
  28 +USER ${pkg.user}
  29 +
28 30 CMD ["start-js-executor.sh"]
... ...
... ... @@ -26,4 +26,6 @@ identity=${pkg.name}
26 26
27 27 source "${CONF_FOLDER}/${configfile}"
28 28
29   -su -s /bin/sh -c "$mainfile"
  29 +cd ${pkg.installFolder}/bin
  30 +
  31 +exec /bin/sh -c "$mainfile"
... ...
... ... @@ -36,7 +36,6 @@
36 36 <main.dir>${basedir}/../..</main.dir>
37 37 <pkg.name>tb-js-executor</pkg.name>
38 38 <docker.name>tb-js-executor</docker.name>
39   - <pkg.user>thingsboard</pkg.user>
40 39 <pkg.unixLogFolder>/var/log/${pkg.name}</pkg.unixLogFolder>
41 40 <pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
42 41 <pkg.linux.dist>${project.build.directory}/package/linux</pkg.linux.dist>
... ...
... ... @@ -25,4 +25,8 @@ RUN dpkg -i /tmp/${pkg.name}.deb
25 25
26 26 RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :
27 27
  28 +RUN chown -R ${pkg.user}:${pkg.user} /tmp
  29 +
  30 +USER ${pkg.user}
  31 +
28 32 CMD ["start-tb-node.sh"]
... ...
... ... @@ -18,12 +18,14 @@
18 18 CONF_FOLDER="/config"
19 19 jarfile=${pkg.installFolder}/bin/${pkg.name}.jar
20 20 configfile=${pkg.name}.conf
21   -run_user=${pkg.name}
  21 +run_user=${pkg.user}
22 22
23 23 source "${CONF_FOLDER}/${configfile}"
24 24
25 25 export LOADER_PATH=/config,${LOADER_PATH}
26 26
  27 +cd ${pkg.installFolder}/bin
  28 +
27 29 if [ "$INSTALL_TB" == "true" ]; then
28 30
29 31 if [ "$LOAD_DEMO" == "true" ]; then
... ...
... ... @@ -36,7 +36,6 @@
36 36 <main.dir>${basedir}/../..</main.dir>
37 37 <pkg.name>thingsboard</pkg.name>
38 38 <docker.name>tb-node</docker.name>
39   - <pkg.user>thingsboard</pkg.user>
40 39 <pkg.unixLogFolder>/var/log/${pkg.name}</pkg.unixLogFolder>
41 40 <pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
42 41 </properties>
... ...
... ... @@ -38,7 +38,6 @@
38 38 <tb.docker.name>tb</tb.docker.name>
39 39 <tb-postgres.docker.name>tb-postgres</tb-postgres.docker.name>
40 40 <tb-cassandra.docker.name>tb-cassandra</tb-cassandra.docker.name>
41   - <pkg.user>thingsboard</pkg.user>
42 41 <pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
43 42 <pkg.upgradeVersion>2.4.2</pkg.upgradeVersion>
44 43 </properties>
... ...
... ... @@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
25 25
26 26 RUN update-rc.d ${pkg.name} disable
27 27
  28 +USER ${pkg.user}
  29 +
28 30 CMD ["start-tb-coap-transport.sh"]
... ...
... ... @@ -25,6 +25,8 @@ export LOADER_PATH=/config,${LOADER_PATH}
25 25
26 26 echo "Starting '${project.name}' ..."
27 27
  28 +cd ${pkg.installFolder}/bin
  29 +
28 30 exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.coap.ThingsboardCoapTransportApplication \
29 31 -Dspring.jpa.hibernate.ddl-auto=none \
30 32 -Dlogging.config=/config/logback.xml \
... ...
... ... @@ -36,7 +36,6 @@
36 36 <main.dir>${basedir}/../../..</main.dir>
37 37 <pkg.name>tb-coap-transport</pkg.name>
38 38 <docker.name>tb-coap-transport</docker.name>
39   - <pkg.user>thingsboard</pkg.user>
40 39 <pkg.logFolder>/var/log/${pkg.name}</pkg.logFolder>
41 40 <pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
42 41 </properties>
... ...
... ... @@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
25 25
26 26 RUN update-rc.d ${pkg.name} disable
27 27
  28 +USER ${pkg.user}
  29 +
28 30 CMD ["start-tb-http-transport.sh"]
... ...
... ... @@ -25,6 +25,8 @@ export LOADER_PATH=/config,${LOADER_PATH}
25 25
26 26 echo "Starting '${project.name}' ..."
27 27
  28 +cd ${pkg.installFolder}/bin
  29 +
28 30 exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.http.ThingsboardHttpTransportApplication \
29 31 -Dspring.jpa.hibernate.ddl-auto=none \
30 32 -Dlogging.config=/config/logback.xml \
... ...
... ... @@ -36,7 +36,6 @@
36 36 <main.dir>${basedir}/../../..</main.dir>
37 37 <pkg.name>tb-http-transport</pkg.name>
38 38 <docker.name>tb-http-transport</docker.name>
39   - <pkg.user>thingsboard</pkg.user>
40 39 <pkg.logFolder>/var/log/${pkg.name}</pkg.logFolder>
41 40 <pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
42 41 </properties>
... ...
... ... @@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
25 25
26 26 RUN update-rc.d ${pkg.name} disable
27 27
  28 +USER ${pkg.user}
  29 +
28 30 CMD ["start-tb-mqtt-transport.sh"]
... ...
... ... @@ -25,6 +25,8 @@ export LOADER_PATH=/config,${LOADER_PATH}
25 25
26 26 echo "Starting '${project.name}' ..."
27 27
  28 +cd ${pkg.installFolder}/bin
  29 +
28 30 exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.mqtt.ThingsboardMqttTransportApplication \
29 31 -Dspring.jpa.hibernate.ddl-auto=none \
30 32 -Dlogging.config=/config/logback.xml \
... ...
... ... @@ -36,7 +36,6 @@
36 36 <main.dir>${basedir}/../../..</main.dir>
37 37 <pkg.name>tb-mqtt-transport</pkg.name>
38 38 <docker.name>tb-mqtt-transport</docker.name>
39   - <pkg.user>thingsboard</pkg.user>
40 39 <pkg.logFolder>/var/log/${pkg.name}</pkg.logFolder>
41 40 <pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
42 41 </properties>
... ...
... ... @@ -14,7 +14,7 @@
14 14 # limitations under the License.
15 15 #
16 16
17   -FROM debian:stretch
  17 +FROM thingsboard/base
18 18
19 19 COPY start-web-ui.sh ${pkg.name}.deb /tmp/
20 20
... ... @@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
25 25
26 26 RUN update-rc.d ${pkg.name} disable
27 27
  28 +USER ${pkg.user}
  29 +
28 30 CMD ["start-web-ui.sh"]
... ...
... ... @@ -26,4 +26,6 @@ identity=${pkg.name}
26 26
27 27 source "${CONF_FOLDER}/${configfile}"
28 28
29   -su -s /bin/sh -c "$mainfile"
  29 +cd ${pkg.installFolder}/bin
  30 +
  31 +exec /bin/sh -c "$mainfile"
... ...
... ... @@ -36,7 +36,6 @@
36 36 <main.dir>${basedir}/../..</main.dir>
37 37 <pkg.name>tb-web-ui</pkg.name>
38 38 <docker.name>tb-web-ui</docker.name>
39   - <pkg.user>thingsboard</pkg.user>
40 39 <pkg.unixLogFolder>/var/log/${pkg.name}</pkg.unixLogFolder>
41 40 <pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
42 41 <pkg.linux.dist>${project.build.directory}/package/linux</pkg.linux.dist>
... ...
... ... @@ -407,8 +407,12 @@ final class MqttClientImpl implements MqttClient {
407 407 }
408 408
409 409 private MqttMessageIdVariableHeader getNewMessageId() {
410   - this.nextMessageId.compareAndSet(0xffff, 1);
411   - return MqttMessageIdVariableHeader.from(this.nextMessageId.getAndIncrement());
  410 + int messageId;
  411 + synchronized (this.nextMessageId) {
  412 + this.nextMessageId.compareAndSet(0xffff, 1);
  413 + messageId = this.nextMessageId.getAndIncrement();
  414 + }
  415 + return MqttMessageIdVariableHeader.from(messageId);
412 416 }
413 417
414 418 private Future<Void> createSubscription(String topic, MqttHandler handler, boolean once, MqttQoS qos) {
... ...
... ... @@ -29,6 +29,7 @@
29 29
30 30 <properties>
31 31 <main.dir>${basedir}</main.dir>
  32 + <pkg.user>thingsboard</pkg.user>
32 33 <spring-boot.version>2.1.3.RELEASE</spring-boot.version>
33 34 <spring.version>5.1.5.RELEASE</spring.version>
34 35 <spring-security.version>5.1.4.RELEASE</spring-security.version>
... ... @@ -44,7 +45,7 @@
44 45 <cassandra.version>3.6.0</cassandra.version>
45 46 <cassandra-unit.version>3.5.0.1</cassandra-unit.version>
46 47 <takari-cpsuite.version>1.2.7</takari-cpsuite.version>
47   - <guava.version>21.0</guava.version>
  48 + <guava.version>28.2-jre</guava.version>
48 49 <caffeine.version>2.6.1</caffeine.version>
49 50 <commons-lang3.version>3.4</commons-lang3.version>
50 51 <commons-validator.version>1.6</commons-validator.version>
... ... @@ -63,7 +64,7 @@
63 64 <mail.version>1.4.3</mail.version>
64 65 <curator.version>4.2.0</curator.version>
65 66 <zookeeper.version>3.5.5</zookeeper.version>
66   - <protobuf.version>3.9.1</protobuf.version>
  67 + <protobuf.version>3.11.4</protobuf.version>
67 68 <grpc.version>1.22.1</grpc.version>
68 69 <lombok.version>1.16.18</lombok.version>
69 70 <paho.client.version>1.1.0</paho.client.version>
... ...
... ... @@ -20,6 +20,7 @@ import com.google.common.cache.CacheLoader;
20 20 import com.google.common.cache.LoadingCache;
21 21 import com.google.common.util.concurrent.Futures;
22 22 import com.google.common.util.concurrent.ListenableFuture;
  23 +import com.google.common.util.concurrent.MoreExecutors;
23 24 import lombok.AllArgsConstructor;
24 25 import lombok.Data;
25 26 import lombok.NoArgsConstructor;
... ... @@ -54,9 +55,9 @@ import java.util.List;
54 55 import java.util.Optional;
55 56 import java.util.concurrent.TimeUnit;
56 57
  58 +import static org.thingsboard.common.util.DonAsynchron.withCallback;
57 59 import static org.thingsboard.rule.engine.api.TbRelationTypes.FAILURE;
58 60 import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS;
59   -import static org.thingsboard.common.util.DonAsynchron.withCallback;
60 61
61 62 @Slf4j
62 63 public abstract class TbAbstractRelationActionNode<C extends TbAbstractRelationActionNodeConfiguration> implements TbNode {
... ... @@ -86,7 +87,7 @@ public abstract class TbAbstractRelationActionNode<C extends TbAbstractRelationA
86 87 }
87 88
88 89 protected ListenableFuture<RelationContainer> processEntityRelationAction(TbContext ctx, TbMsg msg) {
89   - return Futures.transformAsync(getEntity(ctx, msg), entityContainer -> doProcessEntityRelationAction(ctx, msg, entityContainer));
  90 + return Futures.transformAsync(getEntity(ctx, msg), entityContainer -> doProcessEntityRelationAction(ctx, msg, entityContainer), MoreExecutors.directExecutor());
90 91 }
91 92
92 93 protected abstract boolean createEntityIfNotExists();
... ...
... ... @@ -18,12 +18,13 @@ package org.thingsboard.rule.engine.action;
18 18 import com.fasterxml.jackson.databind.JsonNode;
19 19 import com.google.common.util.concurrent.Futures;
20 20 import com.google.common.util.concurrent.ListenableFuture;
  21 +import com.google.common.util.concurrent.MoreExecutors;
21 22 import lombok.extern.slf4j.Slf4j;
22   -import org.thingsboard.rule.engine.api.util.TbNodeUtils;
23 23 import org.thingsboard.rule.engine.api.RuleNode;
24 24 import org.thingsboard.rule.engine.api.TbContext;
25 25 import org.thingsboard.rule.engine.api.TbNodeConfiguration;
26 26 import org.thingsboard.rule.engine.api.TbNodeException;
  27 +import org.thingsboard.rule.engine.api.util.TbNodeUtils;
27 28 import org.thingsboard.server.common.data.alarm.Alarm;
28 29 import org.thingsboard.server.common.data.alarm.AlarmStatus;
29 30 import org.thingsboard.server.common.data.plugin.ComponentType;
... ... @@ -80,8 +81,8 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode<TbClearAlarmNodeConfig
80 81 }
81 82 alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK);
82 83 return Futures.immediateFuture(new AlarmResult(false, false, true, alarm));
83   - });
84   - });
  84 + }, MoreExecutors.directExecutor());
  85 + }, MoreExecutors.directExecutor());
85 86 }, ctx.getDbCallbackExecutor());
86 87 }
87 88 }
... ...