Commit 84cb471e0d7e2e49807a232b87c949d44e377fb2
Committed by
Andrew Shvayka
1 parent
cc0e5417
Sql timeseries improvements (#2033)
* init commit * cleaned code and add test-properties * cleaned code * psql-update * timescale-update * code-refactoring * fix typo * renamed dao * revert indents * refactored code * fix typo * init-partitioning * code updated * cleaned code * fixed license * fix typo * fixed code after review * add annotation to repository * update psql version for docker * postgres-10 * postgres-10 * update docker compose config * fixed partition saving * change key_id to serial column definition * upgrade psql added * add separate upgrade service * added upgrade script * change image on k8s * change logs * resolve conflict after merge with master * revert datasource url in yml * fix typo * license header fix * remove old methods for the timeseries inserts * clean up code * fix saveOrUpdate for PostgreSQL * refactoring & revert Timescale to use latest table * added PsqlTsAnyDao * duplicated code method removed * remove unused invert dictionary map * change the upgrade directory from 2.4.1 to 2.4.3 * refactor JpaPsqlTimeseriesDao
Showing
72 changed files
with
2497 additions
and
1245 deletions
1 | +-- | |
2 | +-- Copyright © 2016-2020 The Thingsboard Authors | |
3 | +-- | |
4 | +-- Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | +-- you may not use this file except in compliance with the License. | |
6 | +-- You may obtain a copy of the License at | |
7 | +-- | |
8 | +-- http://www.apache.org/licenses/LICENSE-2.0 | |
9 | +-- | |
10 | +-- Unless required by applicable law or agreed to in writing, software | |
11 | +-- distributed under the License is distributed on an "AS IS" BASIS, | |
12 | +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | +-- See the License for the specific language governing permissions and | |
14 | +-- limitations under the License. | |
15 | +-- | |
16 | + | |
17 | +-- load function check_version() | |
18 | + | |
19 | +CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$ | |
20 | +DECLARE | |
21 | + current_version integer; | |
22 | + valid_version boolean; | |
23 | +BEGIN | |
24 | + RAISE NOTICE 'Check the current installed PostgreSQL version...'; | |
25 | + SELECT current_setting('server_version_num') INTO current_version; | |
26 | + IF current_version < 100000 THEN | |
27 | + valid_version := FALSE; | |
28 | + ELSE | |
29 | + valid_version := TRUE; | |
30 | + END IF; | |
31 | + IF valid_version = FALSE THEN | |
32 | + RAISE NOTICE 'Postgres version should be at least more than 10!'; | |
33 | + ELSE | |
34 | + RAISE NOTICE 'PostgreSQL version is valid!'; | |
35 | + RAISE NOTICE 'Schema update started...'; | |
36 | + END IF; | |
37 | + RETURN valid_version; | |
38 | +END; | |
39 | +$$ LANGUAGE 'plpgsql'; | |
40 | + | |
41 | +-- load function create_partition_table() | |
42 | + | |
43 | +CREATE OR REPLACE FUNCTION create_partition_table() RETURNS VOID AS $$ | |
44 | + | |
45 | +BEGIN | |
46 | + ALTER TABLE ts_kv | |
47 | + RENAME TO ts_kv_old; | |
48 | + CREATE TABLE IF NOT EXISTS ts_kv | |
49 | + ( | |
50 | + LIKE ts_kv_old | |
51 | + ) | |
52 | + PARTITION BY RANGE (ts); | |
53 | + ALTER TABLE ts_kv | |
54 | + DROP COLUMN entity_type; | |
55 | + ALTER TABLE ts_kv | |
56 | + ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; | |
57 | + ALTER TABLE ts_kv | |
58 | + ALTER COLUMN key TYPE integer USING key::integer; | |
59 | +END; | |
60 | +$$ LANGUAGE 'plpgsql'; | |
61 | + | |
62 | + | |
63 | +-- load function create_partitions() | |
64 | + | |
65 | +CREATE OR REPLACE FUNCTION create_partitions() RETURNS VOID AS | |
66 | +$$ | |
67 | +DECLARE | |
68 | + partition_date varchar; | |
69 | + from_ts bigint; | |
70 | + to_ts bigint; | |
71 | + key_cursor CURSOR FOR select SUBSTRING(month_date.first_date, 1, 7) AS partition_date, | |
72 | + extract(epoch from (month_date.first_date)::timestamp) * 1000 as from_ts, | |
73 | + extract(epoch from (month_date.first_date::date + INTERVAL '1 MONTH')::timestamp) * | |
74 | + 1000 as to_ts | |
75 | + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_01') AS first_date | |
76 | + FROM ts_kv_old) AS month_date; | |
77 | +BEGIN | |
78 | + OPEN key_cursor; | |
79 | + LOOP | |
80 | + FETCH key_cursor INTO partition_date, from_ts, to_ts; | |
81 | + EXIT WHEN NOT FOUND; | |
82 | + EXECUTE 'CREATE TABLE IF NOT EXISTS ts_kv_' || partition_date || | |
83 | + ' PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (' || from_ts || | |
84 | + ') TO (' || to_ts || ');'; | |
85 | + RAISE NOTICE 'A partition % has been created!',CONCAT('ts_kv_', partition_date); | |
86 | + END LOOP; | |
87 | + | |
88 | + CLOSE key_cursor; | |
89 | +END; | |
90 | +$$ language 'plpgsql'; | |
91 | + | |
92 | +-- load function create_ts_kv_dictionary_table() | |
93 | + | |
94 | +CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$ | |
95 | + | |
96 | +BEGIN | |
97 | + CREATE TABLE IF NOT EXISTS ts_kv_dictionary | |
98 | + ( | |
99 | + key varchar(255) NOT NULL, | |
100 | + key_id serial UNIQUE, | |
101 | + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) | |
102 | + ); | |
103 | +END; | |
104 | +$$ LANGUAGE 'plpgsql'; | |
105 | + | |
106 | +-- load function insert_into_dictionary() | |
107 | + | |
108 | +CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS | |
109 | +$$ | |
110 | +DECLARE | |
111 | + insert_record RECORD; | |
112 | + key_cursor CURSOR FOR SELECT DISTINCT key | |
113 | + FROM ts_kv_old | |
114 | + ORDER BY key; | |
115 | +BEGIN | |
116 | + OPEN key_cursor; | |
117 | + LOOP | |
118 | + FETCH key_cursor INTO insert_record; | |
119 | + EXIT WHEN NOT FOUND; | |
120 | + IF NOT EXISTS(SELECT key FROM ts_kv_dictionary WHERE key = insert_record.key) THEN | |
121 | + INSERT INTO ts_kv_dictionary(key) VALUES (insert_record.key); | |
122 | + RAISE NOTICE 'Key: % has been inserted into the dictionary!',insert_record.key; | |
123 | + ELSE | |
124 | + RAISE NOTICE 'Key: % already exists in the dictionary!',insert_record.key; | |
125 | + END IF; | |
126 | + END LOOP; | |
127 | + CLOSE key_cursor; | |
128 | +END; | |
129 | +$$ language 'plpgsql'; | |
130 | + | |
131 | +-- load function insert_into_ts_kv() | |
132 | + | |
133 | +CREATE OR REPLACE FUNCTION insert_into_ts_kv() RETURNS void AS | |
134 | +$$ | |
135 | +DECLARE | |
136 | + insert_size CONSTANT integer := 10000; | |
137 | + insert_counter integer DEFAULT 0; | |
138 | + insert_record RECORD; | |
139 | + insert_cursor CURSOR FOR SELECT CONCAT(first, '-', second, '-1', third, '-', fourth, '-', fifth)::uuid AS entity_id, | |
140 | + substrings.key AS key, | |
141 | + substrings.ts AS ts, | |
142 | + substrings.bool_v AS bool_v, | |
143 | + substrings.str_v AS str_v, | |
144 | + substrings.long_v AS long_v, | |
145 | + substrings.dbl_v AS dbl_v | |
146 | + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first, | |
147 | + SUBSTRING(entity_id, 4, 4) AS second, | |
148 | + SUBSTRING(entity_id, 1, 3) AS third, | |
149 | + SUBSTRING(entity_id, 16, 4) AS fourth, | |
150 | + SUBSTRING(entity_id, 20) AS fifth, | |
151 | + key_id AS key, | |
152 | + ts, | |
153 | + bool_v, | |
154 | + str_v, | |
155 | + long_v, | |
156 | + dbl_v | |
157 | + FROM ts_kv_old | |
158 | + INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS substrings; | |
159 | +BEGIN | |
160 | + OPEN insert_cursor; | |
161 | + LOOP | |
162 | + insert_counter := insert_counter + 1; | |
163 | + FETCH insert_cursor INTO insert_record; | |
164 | + IF NOT FOUND THEN | |
165 | + RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1; | |
166 | + EXIT; | |
167 | + END IF; | |
168 | + INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) | |
169 | + VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, | |
170 | + insert_record.long_v, insert_record.dbl_v); | |
171 | + IF MOD(insert_counter, insert_size) = 0 THEN | |
172 | + RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter; | |
173 | + END IF; | |
174 | + END LOOP; | |
175 | + CLOSE insert_cursor; | |
176 | +END; | |
177 | +$$ LANGUAGE 'plpgsql'; | |
178 | + | |
179 | + | ... | ... |
... | ... | @@ -23,7 +23,8 @@ import org.springframework.context.ApplicationContext; |
23 | 23 | import org.springframework.context.annotation.Profile; |
24 | 24 | import org.springframework.stereotype.Service; |
25 | 25 | import org.thingsboard.server.service.component.ComponentDiscoveryService; |
26 | -import org.thingsboard.server.service.install.DatabaseUpgradeService; | |
26 | +import org.thingsboard.server.service.install.DatabaseTsUpgradeService; | |
27 | +import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; | |
27 | 28 | import org.thingsboard.server.service.install.EntityDatabaseSchemaService; |
28 | 29 | import org.thingsboard.server.service.install.SystemDataLoaderService; |
29 | 30 | import org.thingsboard.server.service.install.TsDatabaseSchemaService; |
... | ... | @@ -50,7 +51,10 @@ public class ThingsboardInstallService { |
50 | 51 | private TsDatabaseSchemaService tsDatabaseSchemaService; |
51 | 52 | |
52 | 53 | @Autowired |
53 | - private DatabaseUpgradeService databaseUpgradeService; | |
54 | + private DatabaseEntitiesUpgradeService databaseEntitiesUpgradeService; | |
55 | + | |
56 | + @Autowired | |
57 | + private DatabaseTsUpgradeService databaseTsUpgradeService; | |
54 | 58 | |
55 | 59 | @Autowired |
56 | 60 | private ComponentDiscoveryService componentDiscoveryService; |
... | ... | @@ -73,48 +77,48 @@ public class ThingsboardInstallService { |
73 | 77 | case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion |
74 | 78 | log.info("Upgrading ThingsBoard from version 1.2.3 to 1.3.0 ..."); |
75 | 79 | |
76 | - databaseUpgradeService.upgradeDatabase("1.2.3"); | |
80 | + databaseEntitiesUpgradeService.upgradeDatabase("1.2.3"); | |
77 | 81 | |
78 | 82 | case "1.3.0": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion |
79 | 83 | log.info("Upgrading ThingsBoard from version 1.3.0 to 1.3.1 ..."); |
80 | 84 | |
81 | - databaseUpgradeService.upgradeDatabase("1.3.0"); | |
85 | + databaseEntitiesUpgradeService.upgradeDatabase("1.3.0"); | |
82 | 86 | |
83 | 87 | case "1.3.1": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion |
84 | 88 | log.info("Upgrading ThingsBoard from version 1.3.1 to 1.4.0 ..."); |
85 | 89 | |
86 | - databaseUpgradeService.upgradeDatabase("1.3.1"); | |
90 | + databaseEntitiesUpgradeService.upgradeDatabase("1.3.1"); | |
87 | 91 | |
88 | 92 | case "1.4.0": |
89 | 93 | log.info("Upgrading ThingsBoard from version 1.4.0 to 2.0.0 ..."); |
90 | 94 | |
91 | - databaseUpgradeService.upgradeDatabase("1.4.0"); | |
95 | + databaseEntitiesUpgradeService.upgradeDatabase("1.4.0"); | |
92 | 96 | |
93 | 97 | dataUpdateService.updateData("1.4.0"); |
94 | 98 | |
95 | 99 | case "2.0.0": |
96 | 100 | log.info("Upgrading ThingsBoard from version 2.0.0 to 2.1.1 ..."); |
97 | 101 | |
98 | - databaseUpgradeService.upgradeDatabase("2.0.0"); | |
102 | + databaseEntitiesUpgradeService.upgradeDatabase("2.0.0"); | |
99 | 103 | |
100 | 104 | case "2.1.1": |
101 | 105 | log.info("Upgrading ThingsBoard from version 2.1.1 to 2.1.2 ..."); |
102 | 106 | |
103 | - databaseUpgradeService.upgradeDatabase("2.1.1"); | |
107 | + databaseEntitiesUpgradeService.upgradeDatabase("2.1.1"); | |
104 | 108 | case "2.1.3": |
105 | 109 | log.info("Upgrading ThingsBoard from version 2.1.3 to 2.2.0 ..."); |
106 | 110 | |
107 | - databaseUpgradeService.upgradeDatabase("2.1.3"); | |
111 | + databaseEntitiesUpgradeService.upgradeDatabase("2.1.3"); | |
108 | 112 | |
109 | 113 | case "2.3.0": |
110 | 114 | log.info("Upgrading ThingsBoard from version 2.3.0 to 2.3.1 ..."); |
111 | 115 | |
112 | - databaseUpgradeService.upgradeDatabase("2.3.0"); | |
116 | + databaseEntitiesUpgradeService.upgradeDatabase("2.3.0"); | |
113 | 117 | |
114 | 118 | case "2.3.1": |
115 | 119 | log.info("Upgrading ThingsBoard from version 2.3.1 to 2.4.0 ..."); |
116 | 120 | |
117 | - databaseUpgradeService.upgradeDatabase("2.3.1"); | |
121 | + databaseEntitiesUpgradeService.upgradeDatabase("2.3.1"); | |
118 | 122 | |
119 | 123 | case "2.4.0": |
120 | 124 | log.info("Upgrading ThingsBoard from version 2.4.0 to 2.4.1 ..."); |
... | ... | @@ -122,11 +126,16 @@ public class ThingsboardInstallService { |
122 | 126 | case "2.4.1": |
123 | 127 | log.info("Upgrading ThingsBoard from version 2.4.1 to 2.4.2 ..."); |
124 | 128 | |
125 | - databaseUpgradeService.upgradeDatabase("2.4.1"); | |
129 | + databaseEntitiesUpgradeService.upgradeDatabase("2.4.1"); | |
126 | 130 | case "2.4.2": |
127 | 131 | log.info("Upgrading ThingsBoard from version 2.4.2 to 2.4.3 ..."); |
128 | 132 | |
129 | - databaseUpgradeService.upgradeDatabase("2.4.2"); | |
133 | + databaseEntitiesUpgradeService.upgradeDatabase("2.4.2"); | |
134 | + | |
135 | + case "2.4.3": | |
136 | + log.info("Upgrading ThingsBoard from version 2.4.3 to 2.5 ..."); | |
137 | + | |
138 | + databaseTsUpgradeService.upgradeDatabase("2.4.3"); | |
130 | 139 | |
131 | 140 | log.info("Updating system data..."); |
132 | 141 | ... | ... |
... | ... | @@ -59,7 +59,7 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; |
59 | 59 | @NoSqlDao |
60 | 60 | @Profile("install") |
61 | 61 | @Slf4j |
62 | -public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService { | |
62 | +public class CassandraDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { | |
63 | 63 | |
64 | 64 | private static final String SCHEMA_UPDATE_CQL = "schema_update.cql"; |
65 | 65 | ... | ... |
application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java
renamed from
application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java
application/src/main/java/org/thingsboard/server/service/install/DatabaseTsUpgradeService.java
0 → 100644
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.service.install; | |
17 | + | |
18 | +public interface DatabaseTsUpgradeService { | |
19 | + | |
20 | + void upgradeDatabase(String fromVersion) throws Exception; | |
21 | + | |
22 | +} | |
\ No newline at end of file | ... | ... |
application/src/main/java/org/thingsboard/server/service/install/HsqlTsDatabaseSchemaService.java
renamed from
application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java
... | ... | @@ -17,14 +17,16 @@ package org.thingsboard.server.service.install; |
17 | 17 | |
18 | 18 | import org.springframework.context.annotation.Profile; |
19 | 19 | import org.springframework.stereotype.Service; |
20 | +import org.thingsboard.server.dao.util.HsqlDao; | |
20 | 21 | import org.thingsboard.server.dao.util.SqlTsDao; |
21 | 22 | |
22 | 23 | @Service |
23 | 24 | @SqlTsDao |
25 | +@HsqlDao | |
24 | 26 | @Profile("install") |
25 | -public class SqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService | |
27 | +public class HsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService | |
26 | 28 | implements TsDatabaseSchemaService { |
27 | - public SqlTsDatabaseSchemaService() { | |
28 | - super("schema-ts.sql", null); | |
29 | + public HsqlTsDatabaseSchemaService() { | |
30 | + super("schema-ts-hsql.sql", null); | |
29 | 31 | } |
30 | 32 | } |
\ No newline at end of file | ... | ... |
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java
0 → 100644
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.service.install; | |
17 | + | |
18 | +import org.springframework.context.annotation.Profile; | |
19 | +import org.springframework.stereotype.Service; | |
20 | +import org.thingsboard.server.dao.util.PsqlDao; | |
21 | +import org.thingsboard.server.dao.util.SqlTsDao; | |
22 | + | |
23 | +@Service | |
24 | +@SqlTsDao | |
25 | +@PsqlDao | |
26 | +@Profile("install") | |
27 | +public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService | |
28 | + implements TsDatabaseSchemaService { | |
29 | + public PsqlTsDatabaseSchemaService() { | |
30 | + super("schema-ts-psql.sql", null); | |
31 | + } | |
32 | +} | |
\ No newline at end of file | ... | ... |
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
0 → 100644
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.service.install; | |
17 | + | |
18 | +import lombok.extern.slf4j.Slf4j; | |
19 | +import org.springframework.beans.factory.annotation.Autowired; | |
20 | +import org.springframework.beans.factory.annotation.Value; | |
21 | +import org.springframework.context.annotation.Profile; | |
22 | +import org.springframework.stereotype.Service; | |
23 | +import org.thingsboard.server.dao.util.PsqlDao; | |
24 | +import org.thingsboard.server.dao.util.SqlTsDao; | |
25 | + | |
26 | +import java.nio.charset.StandardCharsets; | |
27 | +import java.nio.file.Files; | |
28 | +import java.nio.file.Path; | |
29 | +import java.nio.file.Paths; | |
30 | +import java.sql.CallableStatement; | |
31 | +import java.sql.Connection; | |
32 | +import java.sql.DriverManager; | |
33 | +import java.sql.SQLException; | |
34 | +import java.sql.Types; | |
35 | + | |
36 | +@Service | |
37 | +@Profile("install") | |
38 | +@Slf4j | |
39 | +@SqlTsDao | |
40 | +@PsqlDao | |
41 | +public class PsqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { | |
42 | + | |
43 | + private static final String CALL_REGEX = "call "; | |
44 | + private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; | |
45 | + private static final String CHECK_VERSION = CALL_REGEX + "check_version()"; | |
46 | + private static final String CREATE_PARTITION_TABLE = CALL_REGEX + "create_partition_table()"; | |
47 | + private static final String CREATE_PARTITIONS = CALL_REGEX + "create_partitions()"; | |
48 | + private static final String CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + "create_ts_kv_dictionary_table()"; | |
49 | + private static final String INSERT_INTO_DICTIONARY = CALL_REGEX + "insert_into_dictionary()"; | |
50 | + private static final String INSERT_INTO_TS_KV = CALL_REGEX + "insert_into_ts_kv()"; | |
51 | + private static final String DROP_OLD_TABLE = "DROP TABLE ts_kv_old;"; | |
52 | + | |
53 | + @Value("${spring.datasource.url}") | |
54 | + private String dbUrl; | |
55 | + | |
56 | + @Value("${spring.datasource.username}") | |
57 | + private String dbUserName; | |
58 | + | |
59 | + @Value("${spring.datasource.password}") | |
60 | + private String dbPassword; | |
61 | + | |
62 | + @Autowired | |
63 | + private InstallScripts installScripts; | |
64 | + | |
65 | + @Override | |
66 | + public void upgradeDatabase(String fromVersion) throws Exception { | |
67 | + switch (fromVersion) { | |
68 | + case "2.4.3": | |
69 | + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { | |
70 | + log.info("Updating timeseries schema ..."); | |
71 | + log.info("Load upgrade functions ..."); | |
72 | + loadSql(conn); | |
73 | + log.info("Upgrade functions successfully loaded!"); | |
74 | + boolean versionValid = checkVersion(conn); | |
75 | + if (!versionValid) { | |
76 | + log.info("PostgreSQL version should be at least more than 10!"); | |
77 | + log.info("Please upgrade your PostgreSQL and restart the script!"); | |
78 | + } else { | |
79 | + log.info("PostgreSQL version is valid!"); | |
80 | + log.info("Updating schema ..."); | |
81 | + executeFunction(conn, CREATE_PARTITION_TABLE); | |
82 | + executeFunction(conn, CREATE_PARTITIONS); | |
83 | + executeFunction(conn, CREATE_TS_KV_DICTIONARY_TABLE); | |
84 | + executeFunction(conn, INSERT_INTO_DICTIONARY); | |
85 | + executeFunction(conn, INSERT_INTO_TS_KV); | |
86 | + dropOldTable(conn, DROP_OLD_TABLE); | |
87 | + log.info("schema timeseries updated!"); | |
88 | + } | |
89 | + } | |
90 | + break; | |
91 | + default: | |
92 | + throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); | |
93 | + } | |
94 | + } | |
95 | + | |
96 | + private void loadSql(Connection conn) { | |
97 | + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL); | |
98 | + try { | |
99 | + loadFunctions(schemaUpdateFile, conn); | |
100 | + } catch (Exception e) { | |
101 | + log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); | |
102 | + } | |
103 | + } | |
104 | + | |
105 | + private void loadFunctions(Path sqlFile, Connection conn) throws Exception { | |
106 | + String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8); | |
107 | + conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script | |
108 | + } | |
109 | + | |
110 | + private boolean checkVersion(Connection conn) { | |
111 | + log.info("Check the current PostgreSQL version..."); | |
112 | + boolean versionValid = false; | |
113 | + try { | |
114 | + CallableStatement callableStatement = conn.prepareCall("{? = " + CHECK_VERSION + " }"); | |
115 | + callableStatement.registerOutParameter(1, Types.BOOLEAN); | |
116 | + callableStatement.execute(); | |
117 | + versionValid = callableStatement.getBoolean(1); | |
118 | + callableStatement.close(); | |
119 | + } catch (Exception e) { | |
120 | + log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage()); | |
121 | + } | |
122 | + return versionValid; | |
123 | + } | |
124 | + | |
125 | + private void executeFunction(Connection conn, String query) { | |
126 | + log.info("{} ... ", query); | |
127 | + try { | |
128 | + CallableStatement callableStatement = conn.prepareCall("{" + query + "}"); | |
129 | + callableStatement.execute(); | |
130 | + callableStatement.close(); | |
131 | + log.info("Successfully executed: {}", query.replace(CALL_REGEX, "")); | |
132 | + } catch (Exception e) { | |
133 | + log.info("Failed to execute {} due to: {}", query, e.getMessage()); | |
134 | + } | |
135 | + } | |
136 | + | |
137 | + private void dropOldTable(Connection conn, String query) { | |
138 | + try { | |
139 | + conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script | |
140 | + Thread.sleep(5000); | |
141 | + } catch (InterruptedException | SQLException e) { | |
142 | + log.info("Failed to drop table {} due to: {}", query.replace("DROP TABLE ", ""), e.getMessage()); | |
143 | + } | |
144 | + } | |
145 | +} | |
\ No newline at end of file | ... | ... |
... | ... | @@ -54,7 +54,7 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; |
54 | 54 | @Profile("install") |
55 | 55 | @Slf4j |
56 | 56 | @SqlDao |
57 | -public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { | |
57 | +public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { | |
58 | 58 | |
59 | 59 | private static final String SCHEMA_UPDATE_SQL = "schema_update.sql"; |
60 | 60 | |
... | ... | @@ -172,7 +172,8 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { |
172 | 172 | loadSql(schemaUpdateFile, conn); |
173 | 173 | try { |
174 | 174 | conn.createStatement().execute("ALTER TABLE device ADD COLUMN label varchar(255)"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script |
175 | - } catch (Exception e) {} | |
175 | + } catch (Exception e) { | |
176 | + } | |
176 | 177 | log.info("Schema updated."); |
177 | 178 | } |
178 | 179 | break; |
... | ... | @@ -201,7 +202,8 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { |
201 | 202 | log.info("Updating schema ..."); |
202 | 203 | try { |
203 | 204 | conn.createStatement().execute("ALTER TABLE alarm ADD COLUMN propagate_relation_types varchar"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script |
204 | - } catch (Exception e) {} | |
205 | + } catch (Exception e) { | |
206 | + } | |
205 | 207 | log.info("Schema updated."); |
206 | 208 | } |
207 | 209 | break; | ... | ... |
... | ... | @@ -171,7 +171,7 @@ cassandra: |
171 | 171 | read_consistency_level: "${CASSANDRA_READ_CONSISTENCY_LEVEL:ONE}" |
172 | 172 | write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}" |
173 | 173 | default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}" |
174 | - # Specify partitioning size for timestamp key-value storage. Example MINUTES, HOURS, DAYS, MONTHS,INDEFINITE | |
174 | + # Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS,INDEFINITE | |
175 | 175 | ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" |
176 | 176 | ts_key_value_ttl: "${TS_KV_TTL:0}" |
177 | 177 | events_ttl: "${TS_EVENTS_TTL:0}" |
... | ... | @@ -214,6 +214,8 @@ sql: |
214 | 214 | stats_print_interval_ms: "${SQL_TS_TIMESCALE_BATCH_STATS_PRINT_MS:10000}" |
215 | 215 | # Specify whether to remove null characters from strValue of attributes and timeseries before insert |
216 | 216 | remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" |
217 | + # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE | |
218 | + ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" | |
217 | 219 | |
218 | 220 | # Actor system parameters |
219 | 221 | actors: | ... | ... |
... | ... | @@ -30,7 +30,7 @@ public class ControllerSqlTestSuite { |
30 | 30 | |
31 | 31 | @ClassRule |
32 | 32 | public static CustomSqlUnit sqlUnit = new CustomSqlUnit( |
33 | - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), | |
33 | + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), | |
34 | 34 | "sql/drop-all-tables.sql", |
35 | 35 | "sql-test.properties"); |
36 | 36 | } | ... | ... |
... | ... | @@ -29,7 +29,7 @@ public class MqttSqlTestSuite { |
29 | 29 | |
30 | 30 | @ClassRule |
31 | 31 | public static CustomSqlUnit sqlUnit = new CustomSqlUnit( |
32 | - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
32 | + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
33 | 33 | "sql/drop-all-tables.sql", |
34 | 34 | "sql-test.properties"); |
35 | 35 | } | ... | ... |
... | ... | @@ -30,7 +30,7 @@ public class RuleEngineSqlTestSuite { |
30 | 30 | |
31 | 31 | @ClassRule |
32 | 32 | public static CustomSqlUnit sqlUnit = new CustomSqlUnit( |
33 | - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
33 | + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
34 | 34 | "sql/drop-all-tables.sql", |
35 | 35 | "sql-test.properties"); |
36 | 36 | } | ... | ... |
... | ... | @@ -31,7 +31,7 @@ public class SystemSqlTestSuite { |
31 | 31 | |
32 | 32 | @ClassRule |
33 | 33 | public static CustomSqlUnit sqlUnit = new CustomSqlUnit( |
34 | - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
34 | + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
35 | 35 | "sql/drop-all-tables.sql", |
36 | 36 | "sql-test.properties"); |
37 | 37 | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.util; | |
17 | + | |
18 | +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; | |
19 | + | |
20 | +@ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.entities.type}'=='timescale') " + | |
21 | + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") | |
22 | +public @interface PsqlTsAnyDao { | |
23 | +} | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java
... | ... | @@ -21,15 +21,17 @@ import org.springframework.context.annotation.ComponentScan; |
21 | 21 | import org.springframework.context.annotation.Configuration; |
22 | 22 | import org.springframework.data.jpa.repository.config.EnableJpaRepositories; |
23 | 23 | import org.springframework.transaction.annotation.EnableTransactionManagement; |
24 | +import org.thingsboard.server.dao.util.HsqlDao; | |
24 | 25 | import org.thingsboard.server.dao.util.SqlTsDao; |
25 | 26 | |
26 | 27 | @Configuration |
27 | 28 | @EnableAutoConfiguration |
28 | -@ComponentScan("org.thingsboard.server.dao.sqlts.ts") | |
29 | -@EnableJpaRepositories("org.thingsboard.server.dao.sqlts.ts") | |
30 | -@EntityScan("org.thingsboard.server.dao.model.sqlts.ts") | |
29 | +@ComponentScan({"org.thingsboard.server.dao.sqlts.hsql", "org.thingsboard.server.dao.sqlts.latest"}) | |
30 | +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.hsql", "org.thingsboard.server.dao.sqlts.latest"}) | |
31 | +@EntityScan({"org.thingsboard.server.dao.model.sqlts.hsql", "org.thingsboard.server.dao.model.sqlts.latest"}) | |
31 | 32 | @EnableTransactionManagement |
32 | 33 | @SqlTsDao |
33 | -public class SqlTsDaoConfig { | |
34 | +@HsqlDao | |
35 | +public class HsqlTsDaoConfig { | |
34 | 36 | |
35 | 37 | } |
\ No newline at end of file | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao; | |
17 | + | |
18 | +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; | |
19 | +import org.springframework.boot.autoconfigure.domain.EntityScan; | |
20 | +import org.springframework.context.annotation.ComponentScan; | |
21 | +import org.springframework.context.annotation.Configuration; | |
22 | +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; | |
23 | +import org.springframework.transaction.annotation.EnableTransactionManagement; | |
24 | +import org.thingsboard.server.dao.util.PsqlDao; | |
25 | +import org.thingsboard.server.dao.util.SqlTsDao; | |
26 | + | |
27 | +@Configuration | |
28 | +@EnableAutoConfiguration | |
29 | +@ComponentScan({"org.thingsboard.server.dao.sqlts.psql", "org.thingsboard.server.dao.sqlts.latest"}) | |
30 | +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.psql", "org.thingsboard.server.dao.sqlts.latest", "org.thingsboard.server.dao.sqlts.dictionary"}) | |
31 | +@EntityScan({"org.thingsboard.server.dao.model.sqlts.psql", "org.thingsboard.server.dao.model.sqlts.latest", "org.thingsboard.server.dao.model.sqlts.dictionary"}) | |
32 | +@EnableTransactionManagement | |
33 | +@SqlTsDao | |
34 | +@PsqlDao | |
35 | +public class PsqlTsDaoConfig { | |
36 | + | |
37 | +} | |
\ No newline at end of file | ... | ... |
... | ... | @@ -25,9 +25,9 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
25 | 25 | |
26 | 26 | @Configuration |
27 | 27 | @EnableAutoConfiguration |
28 | -@ComponentScan("org.thingsboard.server.dao.sqlts.timescale") | |
29 | -@EnableJpaRepositories("org.thingsboard.server.dao.sqlts.timescale") | |
30 | -@EntityScan("org.thingsboard.server.dao.model.sqlts.timescale") | |
28 | +@ComponentScan({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.latest"}) | |
29 | +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.dictionary", "org.thingsboard.server.dao.sqlts.latest"}) | |
30 | +@EntityScan({"org.thingsboard.server.dao.model.sqlts.timescale", "org.thingsboard.server.dao.model.sqlts.dictionary", "org.thingsboard.server.dao.model.sqlts.latest"}) | |
31 | 31 | @EnableTransactionManagement |
32 | 32 | @TimescaleDBTsDao |
33 | 33 | public class TimescaleDaoConfig { | ... | ... |
... | ... | @@ -41,7 +41,7 @@ import org.thingsboard.server.dao.DaoUtil; |
41 | 41 | import org.thingsboard.server.dao.model.ModelConstants; |
42 | 42 | import org.thingsboard.server.dao.model.nosql.AuditLogEntity; |
43 | 43 | import org.thingsboard.server.dao.nosql.CassandraAbstractSearchTimeDao; |
44 | -import org.thingsboard.server.dao.timeseries.TsPartitionDate; | |
44 | +import org.thingsboard.server.dao.timeseries.NoSqlTsPartitionDate; | |
45 | 45 | import org.thingsboard.server.dao.util.NoSqlDao; |
46 | 46 | |
47 | 47 | import javax.annotation.Nullable; |
... | ... | @@ -92,7 +92,7 @@ public class CassandraAuditLogDao extends CassandraAbstractSearchTimeDao<AuditLo |
92 | 92 | |
93 | 93 | @Value("${audit-log.by_tenant_partitioning}") |
94 | 94 | private String partitioning; |
95 | - private TsPartitionDate tsFormat; | |
95 | + private NoSqlTsPartitionDate tsFormat; | |
96 | 96 | |
97 | 97 | @Value("${audit-log.default_query_period}") |
98 | 98 | private Integer defaultQueryPeriodInDays; |
... | ... | @@ -110,7 +110,7 @@ public class CassandraAuditLogDao extends CassandraAbstractSearchTimeDao<AuditLo |
110 | 110 | @PostConstruct |
111 | 111 | public void init() { |
112 | 112 | if (!isInstall()) { |
113 | - Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning); | |
113 | + Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning); | |
114 | 114 | if (partition.isPresent()) { |
115 | 115 | tsFormat = partition.get(); |
116 | 116 | } else { | ... | ... |
... | ... | @@ -359,6 +359,7 @@ public class ModelConstants { |
359 | 359 | |
360 | 360 | public static final String PARTITION_COLUMN = "partition"; |
361 | 361 | public static final String KEY_COLUMN = "key"; |
362 | + public static final String KEY_ID_COLUMN = "key_id"; | |
362 | 363 | public static final String TS_COLUMN = "ts"; |
363 | 364 | |
364 | 365 | /** | ... | ... |
... | ... | @@ -16,11 +16,6 @@ |
16 | 16 | package org.thingsboard.server.dao.model.sql; |
17 | 17 | |
18 | 18 | import lombok.Data; |
19 | -import org.thingsboard.server.common.data.kv.BooleanDataEntry; | |
20 | -import org.thingsboard.server.common.data.kv.DoubleDataEntry; | |
21 | -import org.thingsboard.server.common.data.kv.KvEntry; | |
22 | -import org.thingsboard.server.common.data.kv.LongDataEntry; | |
23 | -import org.thingsboard.server.common.data.kv.StringDataEntry; | |
24 | 19 | |
25 | 20 | import javax.persistence.Column; |
26 | 21 | import javax.persistence.Id; |
... | ... | @@ -28,10 +23,9 @@ import javax.persistence.MappedSuperclass; |
28 | 23 | |
29 | 24 | import static org.thingsboard.server.dao.model.ModelConstants.BOOLEAN_VALUE_COLUMN; |
30 | 25 | import static org.thingsboard.server.dao.model.ModelConstants.DOUBLE_VALUE_COLUMN; |
31 | -import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; | |
32 | -import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; | |
33 | 26 | import static org.thingsboard.server.dao.model.ModelConstants.LONG_VALUE_COLUMN; |
34 | 27 | import static org.thingsboard.server.dao.model.ModelConstants.STRING_VALUE_COLUMN; |
28 | +import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; | |
35 | 29 | |
36 | 30 | @Data |
37 | 31 | @MappedSuperclass |
... | ... | @@ -43,12 +37,8 @@ public abstract class AbstractTsKvEntity { |
43 | 37 | protected static final String MAX = "MAX"; |
44 | 38 | |
45 | 39 | @Id |
46 | - @Column(name = ENTITY_ID_COLUMN) | |
47 | - protected String entityId; | |
48 | - | |
49 | - @Id | |
50 | - @Column(name = KEY_COLUMN) | |
51 | - protected String key; | |
40 | + @Column(name = TS_COLUMN) | |
41 | + protected Long ts; | |
52 | 42 | |
53 | 43 | @Column(name = BOOLEAN_VALUE_COLUMN) |
54 | 44 | protected Boolean booleanValue; |
... | ... | @@ -62,20 +52,6 @@ public abstract class AbstractTsKvEntity { |
62 | 52 | @Column(name = DOUBLE_VALUE_COLUMN) |
63 | 53 | protected Double doubleValue; |
64 | 54 | |
65 | - protected KvEntry getKvEntry() { | |
66 | - KvEntry kvEntry = null; | |
67 | - if (strValue != null) { | |
68 | - kvEntry = new StringDataEntry(key, strValue); | |
69 | - } else if (longValue != null) { | |
70 | - kvEntry = new LongDataEntry(key, longValue); | |
71 | - } else if (doubleValue != null) { | |
72 | - kvEntry = new DoubleDataEntry(key, doubleValue); | |
73 | - } else if (booleanValue != null) { | |
74 | - kvEntry = new BooleanDataEntry(key, booleanValue); | |
75 | - } | |
76 | - return kvEntry; | |
77 | - } | |
78 | - | |
79 | 55 | public abstract boolean isNotEmpty(); |
80 | 56 | |
81 | 57 | protected static boolean isAllNull(Object... args) { | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.model.sqlts.dictionary; | |
17 | + | |
18 | +import lombok.Data; | |
19 | +import org.hibernate.annotations.Generated; | |
20 | +import org.hibernate.annotations.GenerationTime; | |
21 | + | |
22 | +import javax.persistence.Column; | |
23 | +import javax.persistence.Entity; | |
24 | +import javax.persistence.Id; | |
25 | +import javax.persistence.IdClass; | |
26 | +import javax.persistence.Table; | |
27 | + | |
28 | +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; | |
29 | +import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN; | |
30 | + | |
31 | +@Data | |
32 | +@Entity | |
33 | +@Table(name = "ts_kv_dictionary") | |
34 | +@IdClass(TsKvDictionaryCompositeKey.class) | |
35 | +public final class TsKvDictionary { | |
36 | + | |
37 | + @Id | |
38 | + @Column(name = KEY_COLUMN) | |
39 | + private String key; | |
40 | + | |
41 | + @Column(name = KEY_ID_COLUMN, unique = true, columnDefinition="serial") | |
42 | + @Generated(GenerationTime.INSERT) | |
43 | + private int keyId; | |
44 | + | |
45 | +} | |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java
0 → 100644
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.model.sqlts.dictionary; | |
17 | + | |
18 | +import lombok.AllArgsConstructor; | |
19 | +import lombok.Data; | |
20 | +import lombok.NoArgsConstructor; | |
21 | + | |
22 | +import javax.persistence.Transient; | |
23 | +import java.io.Serializable; | |
24 | + | |
25 | +@Data | |
26 | +@NoArgsConstructor | |
27 | +@AllArgsConstructor | |
28 | +public class TsKvDictionaryCompositeKey implements Serializable{ | |
29 | + | |
30 | + @Transient | |
31 | + private static final long serialVersionUID = -4089175869616037523L; | |
32 | + | |
33 | + private String key; | |
34 | +} | |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvCompositeKey.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvCompositeKey.java
... | ... | @@ -13,7 +13,7 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.model.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.model.sqlts.hsql; | |
17 | 17 | |
18 | 18 | import lombok.AllArgsConstructor; |
19 | 19 | import lombok.Data; |
... | ... | @@ -35,4 +35,5 @@ public class TsKvCompositeKey implements Serializable { |
35 | 35 | private String entityId; |
36 | 36 | private String key; |
37 | 37 | private long ts; |
38 | + | |
38 | 39 | } |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvEntity.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java
... | ... | @@ -13,11 +13,16 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.model.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.model.sqlts.hsql; | |
17 | 17 | |
18 | 18 | import lombok.Data; |
19 | 19 | import org.thingsboard.server.common.data.EntityType; |
20 | 20 | import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
21 | +import org.thingsboard.server.common.data.kv.BooleanDataEntry; | |
22 | +import org.thingsboard.server.common.data.kv.DoubleDataEntry; | |
23 | +import org.thingsboard.server.common.data.kv.KvEntry; | |
24 | +import org.thingsboard.server.common.data.kv.LongDataEntry; | |
25 | +import org.thingsboard.server.common.data.kv.StringDataEntry; | |
21 | 26 | import org.thingsboard.server.common.data.kv.TsKvEntry; |
22 | 27 | import org.thingsboard.server.dao.model.ToData; |
23 | 28 | import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; |
... | ... | @@ -30,8 +35,9 @@ import javax.persistence.Id; |
30 | 35 | import javax.persistence.IdClass; |
31 | 36 | import javax.persistence.Table; |
32 | 37 | |
38 | +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; | |
33 | 39 | import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_TYPE_COLUMN; |
34 | -import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; | |
40 | +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; | |
35 | 41 | |
36 | 42 | @Data |
37 | 43 | @Entity |
... | ... | @@ -45,8 +51,12 @@ public final class TsKvEntity extends AbstractTsKvEntity implements ToData<TsKvE |
45 | 51 | private EntityType entityType; |
46 | 52 | |
47 | 53 | @Id |
48 | - @Column(name = TS_COLUMN) | |
49 | - protected Long ts; | |
54 | + @Column(name = ENTITY_ID_COLUMN) | |
55 | + private String entityId; | |
56 | + | |
57 | + @Id | |
58 | + @Column(name = KEY_COLUMN) | |
59 | + private String key; | |
50 | 60 | |
51 | 61 | public TsKvEntity() { |
52 | 62 | } |
... | ... | @@ -113,6 +123,16 @@ public final class TsKvEntity extends AbstractTsKvEntity implements ToData<TsKvE |
113 | 123 | |
114 | 124 | @Override |
115 | 125 | public TsKvEntry toData() { |
116 | - return new BasicTsKvEntry(ts, getKvEntry()); | |
126 | + KvEntry kvEntry = null; | |
127 | + if (strValue != null) { | |
128 | + kvEntry = new StringDataEntry(key, strValue); | |
129 | + } else if (longValue != null) { | |
130 | + kvEntry = new LongDataEntry(key, longValue); | |
131 | + } else if (doubleValue != null) { | |
132 | + kvEntry = new DoubleDataEntry(key, doubleValue); | |
133 | + } else if (booleanValue != null) { | |
134 | + kvEntry = new BooleanDataEntry(key, booleanValue); | |
135 | + } | |
136 | + return new BasicTsKvEntry(ts, kvEntry); | |
117 | 137 | } |
118 | -} | |
138 | +} | |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestCompositeKey.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvLatestCompositeKey.java
... | ... | @@ -13,7 +13,7 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.model.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.model.sqlts.latest; | |
17 | 17 | |
18 | 18 | import lombok.AllArgsConstructor; |
19 | 19 | import lombok.Data; | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvLatestEntity.java
... | ... | @@ -13,11 +13,16 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.model.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.model.sqlts.latest; | |
17 | 17 | |
18 | 18 | import lombok.Data; |
19 | 19 | import org.thingsboard.server.common.data.EntityType; |
20 | 20 | import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
21 | +import org.thingsboard.server.common.data.kv.BooleanDataEntry; | |
22 | +import org.thingsboard.server.common.data.kv.DoubleDataEntry; | |
23 | +import org.thingsboard.server.common.data.kv.KvEntry; | |
24 | +import org.thingsboard.server.common.data.kv.LongDataEntry; | |
25 | +import org.thingsboard.server.common.data.kv.StringDataEntry; | |
21 | 26 | import org.thingsboard.server.common.data.kv.TsKvEntry; |
22 | 27 | import org.thingsboard.server.dao.model.ToData; |
23 | 28 | import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; |
... | ... | @@ -30,8 +35,9 @@ import javax.persistence.Id; |
30 | 35 | import javax.persistence.IdClass; |
31 | 36 | import javax.persistence.Table; |
32 | 37 | |
38 | +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; | |
33 | 39 | import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_TYPE_COLUMN; |
34 | -import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; | |
40 | +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; | |
35 | 41 | |
36 | 42 | @Data |
37 | 43 | @Entity |
... | ... | @@ -44,16 +50,32 @@ public final class TsKvLatestEntity extends AbstractTsKvEntity implements ToData |
44 | 50 | @Column(name = ENTITY_TYPE_COLUMN) |
45 | 51 | private EntityType entityType; |
46 | 52 | |
47 | - @Column(name = TS_COLUMN) | |
48 | - private long ts; | |
53 | + @Id | |
54 | + @Column(name = ENTITY_ID_COLUMN) | |
55 | + private String entityId; | |
49 | 56 | |
50 | - @Override | |
51 | - public TsKvEntry toData() { | |
52 | - return new BasicTsKvEntry(ts, getKvEntry()); | |
53 | - } | |
57 | + @Id | |
58 | + @Column(name = KEY_COLUMN) | |
59 | + private String key; | |
54 | 60 | |
55 | 61 | @Override |
56 | 62 | public boolean isNotEmpty() { |
57 | 63 | return strValue != null || longValue != null || doubleValue != null || booleanValue != null; |
58 | 64 | } |
65 | + | |
66 | + @Override | |
67 | + public TsKvEntry toData() { | |
68 | + KvEntry kvEntry = null; | |
69 | + if (strValue != null) { | |
70 | + kvEntry = new StringDataEntry(key, strValue); | |
71 | + } else if (longValue != null) { | |
72 | + kvEntry = new LongDataEntry(key, longValue); | |
73 | + } else if (doubleValue != null) { | |
74 | + kvEntry = new DoubleDataEntry(key, doubleValue); | |
75 | + } else if (booleanValue != null) { | |
76 | + kvEntry = new BooleanDataEntry(key, booleanValue); | |
77 | + } | |
78 | + return new BasicTsKvEntry(ts, kvEntry); | |
79 | + } | |
80 | + | |
59 | 81 | } | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.model.sqlts.psql; | |
17 | + | |
18 | +import lombok.AllArgsConstructor; | |
19 | +import lombok.Data; | |
20 | +import lombok.NoArgsConstructor; | |
21 | + | |
22 | +import javax.persistence.Transient; | |
23 | +import java.io.Serializable; | |
24 | +import java.util.UUID; | |
25 | + | |
26 | +@Data | |
27 | +@AllArgsConstructor | |
28 | +@NoArgsConstructor | |
29 | +public class TsKvCompositeKey implements Serializable { | |
30 | + | |
31 | + @Transient | |
32 | + private static final long serialVersionUID = -4089175869616037523L; | |
33 | + | |
34 | + private UUID entityId; | |
35 | + private int key; | |
36 | + private long ts; | |
37 | +} | |
\ No newline at end of file | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.model.sqlts.psql; | |
17 | + | |
18 | +import lombok.Data; | |
19 | +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; | |
20 | +import org.thingsboard.server.common.data.kv.BooleanDataEntry; | |
21 | +import org.thingsboard.server.common.data.kv.DoubleDataEntry; | |
22 | +import org.thingsboard.server.common.data.kv.KvEntry; | |
23 | +import org.thingsboard.server.common.data.kv.LongDataEntry; | |
24 | +import org.thingsboard.server.common.data.kv.StringDataEntry; | |
25 | +import org.thingsboard.server.common.data.kv.TsKvEntry; | |
26 | +import org.thingsboard.server.dao.model.ToData; | |
27 | +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; | |
28 | + | |
29 | +import javax.persistence.Column; | |
30 | +import javax.persistence.Entity; | |
31 | +import javax.persistence.Id; | |
32 | +import javax.persistence.IdClass; | |
33 | +import javax.persistence.Table; | |
34 | +import javax.persistence.Transient; | |
35 | +import java.util.UUID; | |
36 | + | |
37 | +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; | |
38 | +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; | |
39 | + | |
40 | +@Data | |
41 | +@Entity | |
42 | +@Table(name = "ts_kv") | |
43 | +@IdClass(TsKvCompositeKey.class) | |
44 | +public final class TsKvEntity extends AbstractTsKvEntity implements ToData<TsKvEntry> { | |
45 | + | |
46 | + @Id | |
47 | + @Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid") | |
48 | + protected UUID entityId; | |
49 | + | |
50 | + @Id | |
51 | + @Column(name = KEY_COLUMN) | |
52 | + protected int key; | |
53 | + | |
54 | + @Transient | |
55 | + protected String strKey; | |
56 | + | |
57 | + public TsKvEntity() { | |
58 | + } | |
59 | + | |
60 | + public TsKvEntity(String strValue) { | |
61 | + this.strValue = strValue; | |
62 | + } | |
63 | + | |
64 | + public TsKvEntity(Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String aggType) { | |
65 | + if (!isAllNull(longValue, doubleValue, longCountValue, doubleCountValue)) { | |
66 | + switch (aggType) { | |
67 | + case AVG: | |
68 | + double sum = 0.0; | |
69 | + if (longValue != null) { | |
70 | + sum += longValue; | |
71 | + } | |
72 | + if (doubleValue != null) { | |
73 | + sum += doubleValue; | |
74 | + } | |
75 | + long totalCount = longCountValue + doubleCountValue; | |
76 | + if (totalCount > 0) { | |
77 | + this.doubleValue = sum / (longCountValue + doubleCountValue); | |
78 | + } else { | |
79 | + this.doubleValue = 0.0; | |
80 | + } | |
81 | + break; | |
82 | + case SUM: | |
83 | + if (doubleCountValue > 0) { | |
84 | + this.doubleValue = doubleValue + (longValue != null ? longValue.doubleValue() : 0.0); | |
85 | + } else { | |
86 | + this.longValue = longValue; | |
87 | + } | |
88 | + break; | |
89 | + case MIN: | |
90 | + case MAX: | |
91 | + if (longCountValue > 0 && doubleCountValue > 0) { | |
92 | + this.doubleValue = MAX.equals(aggType) ? Math.max(doubleValue, longValue.doubleValue()) : Math.min(doubleValue, longValue.doubleValue()); | |
93 | + } else if (doubleCountValue > 0) { | |
94 | + this.doubleValue = doubleValue; | |
95 | + } else if (longCountValue > 0) { | |
96 | + this.longValue = longValue; | |
97 | + } | |
98 | + break; | |
99 | + } | |
100 | + } | |
101 | + } | |
102 | + | |
103 | + public TsKvEntity(Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount) { | |
104 | + if (!isAllNull(booleanValueCount, strValueCount, longValueCount, doubleValueCount)) { | |
105 | + if (booleanValueCount != 0) { | |
106 | + this.longValue = booleanValueCount; | |
107 | + } else if (strValueCount != 0) { | |
108 | + this.longValue = strValueCount; | |
109 | + } else { | |
110 | + this.longValue = longValueCount + doubleValueCount; | |
111 | + } | |
112 | + } | |
113 | + } | |
114 | + | |
115 | + @Override | |
116 | + public boolean isNotEmpty() { | |
117 | + return strValue != null || longValue != null || doubleValue != null || booleanValue != null; | |
118 | + } | |
119 | + | |
120 | + @Override | |
121 | + public TsKvEntry toData() { | |
122 | + KvEntry kvEntry = null; | |
123 | + if (strValue != null) { | |
124 | + kvEntry = new StringDataEntry(strKey, strValue); | |
125 | + } else if (longValue != null) { | |
126 | + kvEntry = new LongDataEntry(strKey, longValue); | |
127 | + } else if (doubleValue != null) { | |
128 | + kvEntry = new DoubleDataEntry(strKey, doubleValue); | |
129 | + } else if (booleanValue != null) { | |
130 | + kvEntry = new BooleanDataEntry(strKey, booleanValue); | |
131 | + } | |
132 | + return new BasicTsKvEntry(ts, kvEntry); | |
133 | + } | |
134 | + | |
135 | +} | ... | ... |
... | ... | @@ -21,6 +21,7 @@ import lombok.NoArgsConstructor; |
21 | 21 | |
22 | 22 | import javax.persistence.Transient; |
23 | 23 | import java.io.Serializable; |
24 | +import java.util.UUID; | |
24 | 25 | |
25 | 26 | @Data |
26 | 27 | @AllArgsConstructor |
... | ... | @@ -30,8 +31,8 @@ public class TimescaleTsKvCompositeKey implements Serializable { |
30 | 31 | @Transient |
31 | 32 | private static final long serialVersionUID = -4089175869616037523L; |
32 | 33 | |
33 | - private String tenantId; | |
34 | - private String entityId; | |
35 | - private String key; | |
34 | + private UUID tenantId; | |
35 | + private UUID entityId; | |
36 | + private int key; | |
36 | 37 | private long ts; |
37 | 38 | } |
\ No newline at end of file | ... | ... |
... | ... | @@ -19,6 +19,11 @@ import lombok.Data; |
19 | 19 | import lombok.EqualsAndHashCode; |
20 | 20 | import org.springframework.util.StringUtils; |
21 | 21 | import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
22 | +import org.thingsboard.server.common.data.kv.BooleanDataEntry; | |
23 | +import org.thingsboard.server.common.data.kv.DoubleDataEntry; | |
24 | +import org.thingsboard.server.common.data.kv.KvEntry; | |
25 | +import org.thingsboard.server.common.data.kv.LongDataEntry; | |
26 | +import org.thingsboard.server.common.data.kv.StringDataEntry; | |
22 | 27 | import org.thingsboard.server.common.data.kv.TsKvEntry; |
23 | 28 | import org.thingsboard.server.dao.model.ToData; |
24 | 29 | import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; |
... | ... | @@ -34,9 +39,12 @@ import javax.persistence.NamedNativeQuery; |
34 | 39 | import javax.persistence.SqlResultSetMapping; |
35 | 40 | import javax.persistence.SqlResultSetMappings; |
36 | 41 | import javax.persistence.Table; |
42 | +import javax.persistence.Transient; | |
43 | +import java.util.UUID; | |
37 | 44 | |
45 | +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; | |
46 | +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; | |
38 | 47 | import static org.thingsboard.server.dao.model.ModelConstants.TENANT_ID_COLUMN; |
39 | -import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; | |
40 | 48 | import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG; |
41 | 49 | import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG_QUERY; |
42 | 50 | import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_COUNT; |
... | ... | @@ -118,21 +126,30 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F |
118 | 126 | public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToData<TsKvEntry> { |
119 | 127 | |
120 | 128 | @Id |
121 | - @Column(name = TENANT_ID_COLUMN) | |
122 | - private String tenantId; | |
129 | + @Column(name = TENANT_ID_COLUMN, columnDefinition = "uuid") | |
130 | + private UUID tenantId; | |
123 | 131 | |
124 | 132 | @Id |
125 | - @Column(name = TS_COLUMN) | |
126 | - protected Long ts; | |
133 | + @Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid") | |
134 | + protected UUID entityId; | |
127 | 135 | |
128 | - public TimescaleTsKvEntity() { } | |
136 | + @Id | |
137 | + @Column(name = KEY_COLUMN) | |
138 | + protected int key; | |
139 | + | |
140 | + @Transient | |
141 | + protected String strKey; | |
142 | + | |
143 | + | |
144 | + public TimescaleTsKvEntity() { | |
145 | + } | |
129 | 146 | |
130 | 147 | public TimescaleTsKvEntity(Long tsBucket, Long interval, Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String strValue, String aggType) { |
131 | 148 | if (!StringUtils.isEmpty(strValue)) { |
132 | 149 | this.strValue = strValue; |
133 | 150 | } |
134 | 151 | if (!isAllNull(tsBucket, interval, longValue, doubleValue, longCountValue, doubleCountValue)) { |
135 | - this.ts = tsBucket + interval/2; | |
152 | + this.ts = tsBucket + interval / 2; | |
136 | 153 | switch (aggType) { |
137 | 154 | case AVG: |
138 | 155 | double sum = 0.0; |
... | ... | @@ -172,7 +189,7 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToD |
172 | 189 | |
173 | 190 | public TimescaleTsKvEntity(Long tsBucket, Long interval, Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount) { |
174 | 191 | if (!isAllNull(tsBucket, interval, booleanValueCount, strValueCount, longValueCount, doubleValueCount)) { |
175 | - this.ts = tsBucket + interval/2; | |
192 | + this.ts = tsBucket + interval / 2; | |
176 | 193 | if (booleanValueCount != 0) { |
177 | 194 | this.longValue = booleanValueCount; |
178 | 195 | } else if (strValueCount != 0) { |
... | ... | @@ -190,6 +207,17 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToD |
190 | 207 | |
191 | 208 | @Override |
192 | 209 | public TsKvEntry toData() { |
193 | - return new BasicTsKvEntry(ts, getKvEntry()); | |
210 | + KvEntry kvEntry = null; | |
211 | + if (strValue != null) { | |
212 | + kvEntry = new StringDataEntry(strKey, strValue); | |
213 | + } else if (longValue != null) { | |
214 | + kvEntry = new LongDataEntry(strKey, longValue); | |
215 | + } else if (doubleValue != null) { | |
216 | + kvEntry = new DoubleDataEntry(strKey, doubleValue); | |
217 | + } else if (booleanValue != null) { | |
218 | + kvEntry = new BooleanDataEntry(strKey, booleanValue); | |
219 | + } | |
220 | + return new BasicTsKvEntry(ts, kvEntry); | |
194 | 221 | } |
222 | + | |
195 | 223 | } |
\ No newline at end of file | ... | ... |
... | ... | @@ -15,13 +15,8 @@ |
15 | 15 | */ |
16 | 16 | package org.thingsboard.server.dao.sql; |
17 | 17 | |
18 | -import com.google.common.util.concurrent.ListeningExecutorService; | |
19 | -import com.google.common.util.concurrent.MoreExecutors; | |
20 | 18 | import org.springframework.beans.factory.annotation.Autowired; |
21 | 19 | |
22 | -import javax.annotation.PreDestroy; | |
23 | -import java.util.concurrent.Executors; | |
24 | - | |
25 | 20 | public abstract class JpaAbstractDaoListeningExecutorService { |
26 | 21 | |
27 | 22 | @Autowired | ... | ... |
... | ... | @@ -21,8 +21,6 @@ import org.springframework.jdbc.core.JdbcTemplate; |
21 | 21 | import org.springframework.stereotype.Repository; |
22 | 22 | import org.springframework.transaction.support.TransactionTemplate; |
23 | 23 | |
24 | -import javax.persistence.EntityManager; | |
25 | -import javax.persistence.PersistenceContext; | |
26 | 24 | import java.util.regex.Pattern; |
27 | 25 | |
28 | 26 | @Repository |
... | ... | @@ -31,64 +29,15 @@ public abstract class AbstractInsertRepository { |
31 | 29 | private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE))); |
32 | 30 | private static final String EMPTY_STR = ""; |
33 | 31 | |
34 | - protected static final String BOOL_V = "bool_v"; | |
35 | - protected static final String STR_V = "str_v"; | |
36 | - protected static final String LONG_V = "long_v"; | |
37 | - protected static final String DBL_V = "dbl_v"; | |
38 | - | |
39 | - protected static final String TS_KV_LATEST_TABLE = "ts_kv_latest"; | |
40 | - protected static final String TS_KV_TABLE = "ts_kv"; | |
41 | - | |
42 | - protected static final String HSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, BOOL_V); | |
43 | - protected static final String HSQL_ON_STR_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, STR_V); | |
44 | - protected static final String HSQL_ON_LONG_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, LONG_V); | |
45 | - protected static final String HSQL_ON_DBL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, DBL_V); | |
46 | - | |
47 | - protected static final String HSQL_LATEST_ON_BOOL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, BOOL_V); | |
48 | - protected static final String HSQL_LATEST_ON_STR_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, STR_V); | |
49 | - protected static final String HSQL_LATEST_ON_LONG_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, LONG_V); | |
50 | - protected static final String HSQL_LATEST_ON_DBL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, DBL_V); | |
51 | - | |
52 | - protected static final String PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS = "str_v = null, long_v = null, dbl_v = null"; | |
53 | - protected static final String PSQL_ON_STR_VALUE_UPDATE_SET_NULLS = "bool_v = null, long_v = null, dbl_v = null"; | |
54 | - protected static final String PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS = "str_v = null, bool_v = null, dbl_v = null"; | |
55 | - protected static final String PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS = "str_v = null, long_v = null, bool_v = null"; | |
56 | - | |
57 | 32 | @Value("${sql.remove_null_chars}") |
58 | 33 | private boolean removeNullChars; |
59 | 34 | |
60 | - @PersistenceContext | |
61 | - protected EntityManager entityManager; | |
62 | - | |
63 | 35 | @Autowired |
64 | 36 | protected JdbcTemplate jdbcTemplate; |
65 | 37 | |
66 | 38 | @Autowired |
67 | 39 | protected TransactionTemplate transactionTemplate; |
68 | 40 | |
69 | - protected static String getInsertOrUpdateStringHsql(String tableName, String constraint, String value, String nullValues) { | |
70 | - return "MERGE INTO " + tableName + " USING(VALUES :entity_type, :entity_id, :key, :ts, :" + value + ") A (entity_type, entity_id, key, ts, " + value + ") ON " + constraint + " WHEN MATCHED THEN UPDATE SET " + tableName + "." + value + " = A." + value + ", " + tableName + ".ts = A.ts," + nullValues + "WHEN NOT MATCHED THEN INSERT (entity_type, entity_id, key, ts, " + value + ") VALUES (A.entity_type, A.entity_id, A.key, A.ts, A." + value + ")"; | |
71 | - } | |
72 | - | |
73 | - protected static String getInsertOrUpdateStringPsql(String tableName, String constraint, String value, String nullValues) { | |
74 | - return "INSERT INTO " + tableName + " (entity_type, entity_id, key, ts, " + value + ") VALUES (:entity_type, :entity_id, :key, :ts, :" + value + ") ON CONFLICT " + constraint + " DO UPDATE SET " + value + " = :" + value + ", ts = :ts," + nullValues; | |
75 | - } | |
76 | - | |
77 | - private static String getHsqlNullValues(String tableName, String notNullValue) { | |
78 | - switch (notNullValue) { | |
79 | - case BOOL_V: | |
80 | - return " " + tableName + ".str_v = null, " + tableName + ".long_v = null, " + tableName + ".dbl_v = null "; | |
81 | - case STR_V: | |
82 | - return " " + tableName + ".bool_v = null, " + tableName + ".long_v = null, " + tableName + ".dbl_v = null "; | |
83 | - case LONG_V: | |
84 | - return " " + tableName + ".str_v = null, " + tableName + ".bool_v = null, " + tableName + ".dbl_v = null "; | |
85 | - case DBL_V: | |
86 | - return " " + tableName + ".str_v = null, " + tableName + ".long_v = null, " + tableName + ".bool_v = null "; | |
87 | - default: | |
88 | - throw new RuntimeException("Unsupported insert value: [" + notNullValue + "]"); | |
89 | - } | |
90 | - } | |
91 | - | |
92 | 41 | protected String replaceNullChars(String strValue) { |
93 | 42 | if (removeNullChars && strValue != null) { |
94 | 43 | return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractLatestInsertRepository.java
deleted
100644 → 0
1 | -/** | |
2 | - * Copyright © 2016-2020 The Thingsboard Authors | |
3 | - * | |
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | - * you may not use this file except in compliance with the License. | |
6 | - * You may obtain a copy of the License at | |
7 | - * | |
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | - * | |
10 | - * Unless required by applicable law or agreed to in writing, software | |
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | - * See the License for the specific language governing permissions and | |
14 | - * limitations under the License. | |
15 | - */ | |
16 | -package org.thingsboard.server.dao.sqlts; | |
17 | - | |
18 | -import org.springframework.data.jpa.repository.Modifying; | |
19 | -import org.springframework.stereotype.Repository; | |
20 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; | |
21 | - | |
22 | -import java.util.List; | |
23 | - | |
24 | -@Repository | |
25 | -public abstract class AbstractLatestInsertRepository extends AbstractInsertRepository { | |
26 | - | |
27 | - public abstract void saveOrUpdate(TsKvLatestEntity entity); | |
28 | - | |
29 | - public abstract void saveOrUpdate(List<TsKvLatestEntity> entities); | |
30 | - | |
31 | - protected void processSaveOrUpdate(TsKvLatestEntity entity, String requestBoolValue, String requestStrValue, String requestLongValue, String requestDblValue) { | |
32 | - if (entity.getBooleanValue() != null) { | |
33 | - saveOrUpdateBoolean(entity, requestBoolValue); | |
34 | - } | |
35 | - if (entity.getStrValue() != null) { | |
36 | - saveOrUpdateString(entity, requestStrValue); | |
37 | - } | |
38 | - if (entity.getLongValue() != null) { | |
39 | - saveOrUpdateLong(entity, requestLongValue); | |
40 | - } | |
41 | - if (entity.getDoubleValue() != null) { | |
42 | - saveOrUpdateDouble(entity, requestDblValue); | |
43 | - } | |
44 | - } | |
45 | - | |
46 | - @Modifying | |
47 | - protected abstract void saveOrUpdateBoolean(TsKvLatestEntity entity, String query); | |
48 | - | |
49 | - @Modifying | |
50 | - protected abstract void saveOrUpdateString(TsKvLatestEntity entity, String query); | |
51 | - | |
52 | - @Modifying | |
53 | - protected abstract void saveOrUpdateLong(TsKvLatestEntity entity, String query); | |
54 | - | |
55 | - @Modifying | |
56 | - protected abstract void saveOrUpdateDouble(TsKvLatestEntity entity, String query); | |
57 | - | |
58 | -} | |
\ No newline at end of file |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts; | |
17 | + | |
18 | +import com.google.common.util.concurrent.Futures; | |
19 | +import com.google.common.util.concurrent.ListenableFuture; | |
20 | +import com.google.common.util.concurrent.SettableFuture; | |
21 | +import lombok.extern.slf4j.Slf4j; | |
22 | +import org.springframework.beans.factory.annotation.Autowired; | |
23 | +import org.springframework.beans.factory.annotation.Value; | |
24 | +import org.thingsboard.server.common.data.id.EntityId; | |
25 | +import org.thingsboard.server.common.data.id.TenantId; | |
26 | +import org.thingsboard.server.common.data.kv.Aggregation; | |
27 | +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; | |
28 | +import org.thingsboard.server.common.data.kv.TsKvEntry; | |
29 | +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; | |
30 | +import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; | |
31 | +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; | |
32 | + | |
33 | +import javax.annotation.PostConstruct; | |
34 | +import javax.annotation.PreDestroy; | |
35 | +import java.util.ArrayList; | |
36 | +import java.util.List; | |
37 | +import java.util.Optional; | |
38 | +import java.util.concurrent.CompletableFuture; | |
39 | +import java.util.stream.Collectors; | |
40 | + | |
41 | +@Slf4j | |
42 | +public abstract class AbstractSimpleSqlTimeseriesDao<T extends AbstractTsKvEntity> extends AbstractSqlTimeseriesDao { | |
43 | + | |
44 | + @Autowired | |
45 | + private InsertTsRepository<T> insertRepository; | |
46 | + | |
47 | + @Value("${sql.ts.batch_size:1000}") | |
48 | + private int tsBatchSize; | |
49 | + | |
50 | + @Value("${sql.ts.batch_max_delay:100}") | |
51 | + private long tsMaxDelay; | |
52 | + | |
53 | + @Value("${sql.ts.stats_print_interval_ms:1000}") | |
54 | + private long tsStatsPrintIntervalMs; | |
55 | + | |
56 | + protected TbSqlBlockingQueue<EntityContainer<T>> tsQueue; | |
57 | + | |
58 | + @PostConstruct | |
59 | + protected void init() { | |
60 | + super.init(); | |
61 | + TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() | |
62 | + .logName("TS") | |
63 | + .batchSize(tsBatchSize) | |
64 | + .maxDelay(tsMaxDelay) | |
65 | + .statsPrintIntervalMs(tsStatsPrintIntervalMs) | |
66 | + .build(); | |
67 | + tsQueue = new TbSqlBlockingQueue<>(tsParams); | |
68 | + tsQueue.init(logExecutor, v -> insertRepository.saveOrUpdate(v)); | |
69 | + } | |
70 | + | |
71 | + @PreDestroy | |
72 | + protected void destroy() { | |
73 | + super.init(); | |
74 | + if (tsQueue != null) { | |
75 | + tsQueue.destroy(); | |
76 | + } | |
77 | + } | |
78 | + | |
79 | + protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { | |
80 | + if (query.getAggregation() == Aggregation.NONE) { | |
81 | + return findAllAsyncWithLimit(entityId, query); | |
82 | + } else { | |
83 | + long stepTs = query.getStartTs(); | |
84 | + List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>(); | |
85 | + while (stepTs < query.getEndTs()) { | |
86 | + long startTs = stepTs; | |
87 | + long endTs = stepTs + query.getInterval(); | |
88 | + long ts = startTs + (endTs - startTs) / 2; | |
89 | + futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); | |
90 | + stepTs = endTs; | |
91 | + } | |
92 | + return getTskvEntriesFuture(Futures.allAsList(futures)); | |
93 | + } | |
94 | + } | |
95 | + | |
96 | + protected abstract ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation); | |
97 | + | |
98 | + protected abstract ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query); | |
99 | + | |
100 | + protected SettableFuture<T> setFutures(List<CompletableFuture<T>> entitiesFutures) { | |
101 | + SettableFuture<T> listenableFuture = SettableFuture.create(); | |
102 | + CompletableFuture<List<T>> entities = | |
103 | + CompletableFuture.allOf(entitiesFutures.toArray(new CompletableFuture[entitiesFutures.size()])) | |
104 | + .thenApply(v -> entitiesFutures.stream() | |
105 | + .map(CompletableFuture::join) | |
106 | + .collect(Collectors.toList())); | |
107 | + | |
108 | + entities.whenComplete((tsKvEntities, throwable) -> { | |
109 | + if (throwable != null) { | |
110 | + listenableFuture.setException(throwable); | |
111 | + } else { | |
112 | + T result = null; | |
113 | + for (T entity : tsKvEntities) { | |
114 | + if (entity.isNotEmpty()) { | |
115 | + result = entity; | |
116 | + break; | |
117 | + } | |
118 | + } | |
119 | + listenableFuture.set(result); | |
120 | + } | |
121 | + }); | |
122 | + return listenableFuture; | |
123 | + } | |
124 | + | |
125 | + protected void switchAgregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List<CompletableFuture<T>> entitiesFutures) { | |
126 | + switch (aggregation) { | |
127 | + case AVG: | |
128 | + findAvg(entityId, key, startTs, endTs, entitiesFutures); | |
129 | + break; | |
130 | + case MAX: | |
131 | + findMax(entityId, key, startTs, endTs, entitiesFutures); | |
132 | + break; | |
133 | + case MIN: | |
134 | + findMin(entityId, key, startTs, endTs, entitiesFutures); | |
135 | + break; | |
136 | + case SUM: | |
137 | + findSum(entityId, key, startTs, endTs, entitiesFutures); | |
138 | + break; | |
139 | + case COUNT: | |
140 | + findCount(entityId, key, startTs, endTs, entitiesFutures); | |
141 | + break; | |
142 | + default: | |
143 | + throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); | |
144 | + } | |
145 | + } | |
146 | + | |
147 | + protected abstract void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures); | |
148 | + | |
149 | + protected abstract void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures); | |
150 | + | |
151 | + protected abstract void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures); | |
152 | + | |
153 | + protected abstract void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures); | |
154 | + | |
155 | + protected abstract void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<T>> entitiesFutures); | |
156 | +} | |
\ No newline at end of file | ... | ... |
... | ... | @@ -16,20 +16,32 @@ |
16 | 16 | package org.thingsboard.server.dao.sqlts; |
17 | 17 | |
18 | 18 | import com.google.common.base.Function; |
19 | +import com.google.common.collect.Lists; | |
20 | +import com.google.common.util.concurrent.FutureCallback; | |
19 | 21 | import com.google.common.util.concurrent.Futures; |
20 | 22 | import com.google.common.util.concurrent.ListenableFuture; |
21 | -import com.google.common.util.concurrent.ListeningExecutorService; | |
22 | -import com.google.common.util.concurrent.MoreExecutors; | |
23 | +import lombok.extern.slf4j.Slf4j; | |
24 | +import org.springframework.beans.factory.annotation.Autowired; | |
23 | 25 | import org.springframework.beans.factory.annotation.Value; |
26 | +import org.thingsboard.server.common.data.UUIDConverter; | |
24 | 27 | import org.thingsboard.server.common.data.id.EntityId; |
25 | 28 | import org.thingsboard.server.common.data.id.TenantId; |
26 | 29 | import org.thingsboard.server.common.data.kv.Aggregation; |
27 | 30 | import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; |
31 | +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; | |
28 | 32 | import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; |
29 | 33 | import org.thingsboard.server.common.data.kv.ReadTsKvQuery; |
34 | +import org.thingsboard.server.common.data.kv.StringDataEntry; | |
30 | 35 | import org.thingsboard.server.common.data.kv.TsKvEntry; |
36 | +import org.thingsboard.server.dao.DaoUtil; | |
37 | +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; | |
38 | +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; | |
31 | 39 | import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; |
32 | -import org.thingsboard.server.dao.timeseries.TsInsertExecutorType; | |
40 | +import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; | |
41 | +import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; | |
42 | +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; | |
43 | +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; | |
44 | +import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; | |
33 | 45 | |
34 | 46 | import javax.annotation.Nullable; |
35 | 47 | import javax.annotation.PostConstruct; |
... | ... | @@ -37,13 +49,55 @@ import javax.annotation.PreDestroy; |
37 | 49 | import java.util.List; |
38 | 50 | import java.util.Objects; |
39 | 51 | import java.util.Optional; |
40 | -import java.util.concurrent.Executors; | |
52 | +import java.util.concurrent.ExecutionException; | |
41 | 53 | import java.util.stream.Collectors; |
42 | 54 | |
55 | +import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; | |
56 | + | |
57 | +@Slf4j | |
43 | 58 | public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { |
44 | 59 | |
45 | 60 | private static final String DESC_ORDER = "DESC"; |
46 | 61 | |
62 | + @Autowired | |
63 | + private TsKvLatestRepository tsKvLatestRepository; | |
64 | + | |
65 | + @Autowired | |
66 | + private InsertLatestRepository insertLatestRepository; | |
67 | + | |
68 | + @Autowired | |
69 | + protected ScheduledLogExecutorComponent logExecutor; | |
70 | + | |
71 | + @Value("${sql.ts_latest.batch_size:1000}") | |
72 | + private int tsLatestBatchSize; | |
73 | + | |
74 | + @Value("${sql.ts_latest.batch_max_delay:100}") | |
75 | + private long tsLatestMaxDelay; | |
76 | + | |
77 | + @Value("${sql.ts_latest.stats_print_interval_ms:1000}") | |
78 | + private long tsLatestStatsPrintIntervalMs; | |
79 | + | |
80 | + private TbSqlBlockingQueue<TsKvLatestEntity> tsLatestQueue; | |
81 | + | |
82 | + @PostConstruct | |
83 | + protected void init() { | |
84 | + TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() | |
85 | + .logName("TS Latest") | |
86 | + .batchSize(tsLatestBatchSize) | |
87 | + .maxDelay(tsLatestMaxDelay) | |
88 | + .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) | |
89 | + .build(); | |
90 | + tsLatestQueue = new TbSqlBlockingQueue<>(tsLatestParams); | |
91 | + tsLatestQueue.init(logExecutor, v -> insertLatestRepository.saveOrUpdate(v)); | |
92 | + } | |
93 | + | |
94 | + @PreDestroy | |
95 | + protected void destroy() { | |
96 | + if (tsLatestQueue != null) { | |
97 | + tsLatestQueue.destroy(); | |
98 | + } | |
99 | + } | |
100 | + | |
47 | 101 | protected ListenableFuture<List<TsKvEntry>> processFindAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) { |
48 | 102 | List<ListenableFuture<List<TsKvEntry>>> futures = queries |
49 | 103 | .stream() |
... | ... | @@ -89,4 +143,105 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx |
89 | 143 | Aggregation.NONE, DESC_ORDER); |
90 | 144 | return findAllAsync(tenantId, entityId, findNewLatestQuery); |
91 | 145 | } |
146 | + | |
147 | + protected ListenableFuture<TsKvEntry> getFindLatestFuture(EntityId entityId, String key) { | |
148 | + TsKvLatestCompositeKey compositeKey = | |
149 | + new TsKvLatestCompositeKey( | |
150 | + entityId.getEntityType(), | |
151 | + fromTimeUUID(entityId.getId()), | |
152 | + key); | |
153 | + Optional<TsKvLatestEntity> entry = tsKvLatestRepository.findById(compositeKey); | |
154 | + TsKvEntry result; | |
155 | + if (entry.isPresent()) { | |
156 | + result = DaoUtil.getData(entry.get()); | |
157 | + } else { | |
158 | + result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); | |
159 | + } | |
160 | + return Futures.immediateFuture(result); | |
161 | + } | |
162 | + | |
163 | + protected ListenableFuture<Void> getRemoveLatestFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
164 | + ListenableFuture<TsKvEntry> latestFuture = getFindLatestFuture(entityId, query.getKey()); | |
165 | + | |
166 | + ListenableFuture<Boolean> booleanFuture = Futures.transform(latestFuture, tsKvEntry -> { | |
167 | + long ts = tsKvEntry.getTs(); | |
168 | + return ts > query.getStartTs() && ts <= query.getEndTs(); | |
169 | + }, service); | |
170 | + | |
171 | + ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { | |
172 | + if (isRemove) { | |
173 | + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); | |
174 | + latestEntity.setEntityType(entityId.getEntityType()); | |
175 | + latestEntity.setEntityId(fromTimeUUID(entityId.getId())); | |
176 | + latestEntity.setKey(query.getKey()); | |
177 | + return service.submit(() -> { | |
178 | + tsKvLatestRepository.delete(latestEntity); | |
179 | + return null; | |
180 | + }); | |
181 | + } | |
182 | + return Futures.immediateFuture(null); | |
183 | + }, service); | |
184 | + | |
185 | + final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>(); | |
186 | + Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() { | |
187 | + @Override | |
188 | + public void onSuccess(@Nullable Void result) { | |
189 | + if (query.getRewriteLatestIfDeleted()) { | |
190 | + ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { | |
191 | + if (isRemove) { | |
192 | + return getNewLatestEntryFuture(tenantId, entityId, query); | |
193 | + } | |
194 | + return Futures.immediateFuture(null); | |
195 | + }, service); | |
196 | + | |
197 | + try { | |
198 | + resultFuture.set(savedLatestFuture.get()); | |
199 | + } catch (InterruptedException | ExecutionException e) { | |
200 | + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); | |
201 | + } | |
202 | + } else { | |
203 | + resultFuture.set(null); | |
204 | + } | |
205 | + } | |
206 | + | |
207 | + @Override | |
208 | + public void onFailure(Throwable t) { | |
209 | + log.warn("[{}] Failed to process remove of the latest value", entityId, t); | |
210 | + } | |
211 | + }); | |
212 | + return resultFuture; | |
213 | + } | |
214 | + | |
215 | + protected ListenableFuture<List<TsKvEntry>> getFindAllLatestFuture(EntityId entityId) { | |
216 | + return Futures.immediateFuture( | |
217 | + DaoUtil.convertDataList(Lists.newArrayList( | |
218 | + tsKvLatestRepository.findAllByEntityTypeAndEntityId( | |
219 | + entityId.getEntityType(), | |
220 | + UUIDConverter.fromTimeUUID(entityId.getId()))))); | |
221 | + } | |
222 | + | |
223 | + protected ListenableFuture<Void> getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) { | |
224 | + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); | |
225 | + latestEntity.setEntityType(entityId.getEntityType()); | |
226 | + latestEntity.setEntityId(fromTimeUUID(entityId.getId())); | |
227 | + latestEntity.setTs(tsKvEntry.getTs()); | |
228 | + latestEntity.setKey(tsKvEntry.getKey()); | |
229 | + latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); | |
230 | + latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); | |
231 | + latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); | |
232 | + latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); | |
233 | + return tsLatestQueue.add(latestEntity); | |
234 | + } | |
235 | + | |
236 | + private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
237 | + ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query); | |
238 | + return Futures.transformAsync(future, entryList -> { | |
239 | + if (entryList.size() == 1) { | |
240 | + return getSaveLatestFuture(entityId, entryList.get(0)); | |
241 | + } else { | |
242 | + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); | |
243 | + } | |
244 | + return Futures.immediateFuture(null); | |
245 | + }, service); | |
246 | + } | |
92 | 247 | } | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts; | |
17 | + | |
18 | +import lombok.AllArgsConstructor; | |
19 | +import lombok.Data; | |
20 | +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; | |
21 | + | |
22 | +@Data | |
23 | +@AllArgsConstructor | |
24 | +public class EntityContainer<T extends AbstractTsKvEntity> { | |
25 | + | |
26 | + private T entity; | |
27 | + private String partitionDate; | |
28 | + | |
29 | +} | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts; | |
17 | + | |
18 | +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; | |
19 | + | |
20 | +import java.util.List; | |
21 | + | |
22 | +public interface InsertLatestRepository { | |
23 | + | |
24 | + void saveOrUpdate(List<TsKvLatestEntity> entities); | |
25 | + | |
26 | +} | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertTsRepository.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractTimeseriesInsertRepository.java
... | ... | @@ -15,44 +15,12 @@ |
15 | 15 | */ |
16 | 16 | package org.thingsboard.server.dao.sqlts; |
17 | 17 | |
18 | -import org.springframework.data.jpa.repository.Modifying; | |
19 | -import org.springframework.stereotype.Repository; | |
20 | 18 | import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; |
21 | 19 | |
22 | 20 | import java.util.List; |
23 | 21 | |
24 | -@Repository | |
25 | -public abstract class AbstractTimeseriesInsertRepository<T extends AbstractTsKvEntity> extends AbstractInsertRepository { | |
22 | +public interface InsertTsRepository<T extends AbstractTsKvEntity> { | |
26 | 23 | |
27 | - public abstract void saveOrUpdate(T entity); | |
24 | + void saveOrUpdate(List<EntityContainer<T>> entities); | |
28 | 25 | |
29 | - public abstract void saveOrUpdate(List<T> entities); | |
30 | - | |
31 | - protected void processSaveOrUpdate(T entity, String requestBoolValue, String requestStrValue, String requestLongValue, String requestDblValue) { | |
32 | - if (entity.getBooleanValue() != null) { | |
33 | - saveOrUpdateBoolean(entity, requestBoolValue); | |
34 | - } | |
35 | - if (entity.getStrValue() != null) { | |
36 | - saveOrUpdateString(entity, requestStrValue); | |
37 | - } | |
38 | - if (entity.getLongValue() != null) { | |
39 | - saveOrUpdateLong(entity, requestLongValue); | |
40 | - } | |
41 | - if (entity.getDoubleValue() != null) { | |
42 | - saveOrUpdateDouble(entity, requestDblValue); | |
43 | - } | |
44 | - } | |
45 | - | |
46 | - @Modifying | |
47 | - protected abstract void saveOrUpdateBoolean(T entity, String query); | |
48 | - | |
49 | - @Modifying | |
50 | - protected abstract void saveOrUpdateString(T entity, String query); | |
51 | - | |
52 | - @Modifying | |
53 | - protected abstract void saveOrUpdateLong(T entity, String query); | |
54 | - | |
55 | - @Modifying | |
56 | - protected abstract void saveOrUpdateDouble(T entity, String query); | |
57 | - | |
58 | -} | |
\ No newline at end of file | ||
26 | +} | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java
0 → 100644
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts.dictionary; | |
17 | + | |
18 | +import org.springframework.data.repository.CrudRepository; | |
19 | +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; | |
20 | +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; | |
21 | +import org.thingsboard.server.dao.util.PsqlDao; | |
22 | + | |
23 | +import java.util.Optional; | |
24 | + | |
25 | +@PsqlDao | |
26 | +public interface TsKvDictionaryRepository extends CrudRepository<TsKvDictionary, TsKvDictionaryCompositeKey> { | |
27 | + | |
28 | + Optional<TsKvDictionary> findByKeyId(int keyId); | |
29 | + | |
30 | +} | |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/HsqlTimeseriesInsertRepository.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlTimeseriesInsertRepository.java
... | ... | @@ -13,13 +13,15 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.sqlts.hsql; | |
17 | 17 | |
18 | 18 | import org.springframework.jdbc.core.BatchPreparedStatementSetter; |
19 | 19 | import org.springframework.stereotype.Repository; |
20 | 20 | import org.springframework.transaction.annotation.Transactional; |
21 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; | |
22 | -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; | |
21 | +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity; | |
22 | +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; | |
23 | +import org.thingsboard.server.dao.sqlts.EntityContainer; | |
24 | +import org.thingsboard.server.dao.sqlts.InsertTsRepository; | |
23 | 25 | import org.thingsboard.server.dao.util.HsqlDao; |
24 | 26 | import org.thingsboard.server.dao.util.SqlTsDao; |
25 | 27 | |
... | ... | @@ -32,14 +34,7 @@ import java.util.List; |
32 | 34 | @HsqlDao |
33 | 35 | @Repository |
34 | 36 | @Transactional |
35 | -public class HsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepository<TsKvEntity> { | |
36 | - | |
37 | - private static final String TS_KV_CONSTRAINT = "(ts_kv.entity_type=A.entity_type AND ts_kv.entity_id=A.entity_id AND ts_kv.key=A.key AND ts_kv.ts=A.ts)"; | |
38 | - | |
39 | - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, BOOL_V, HSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS); | |
40 | - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, STR_V, HSQL_ON_STR_VALUE_UPDATE_SET_NULLS); | |
41 | - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, LONG_V, HSQL_ON_LONG_VALUE_UPDATE_SET_NULLS); | |
42 | - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, DBL_V, HSQL_ON_DBL_VALUE_UPDATE_SET_NULLS); | |
37 | +public class HsqlTimeseriesInsertRepository extends AbstractInsertRepository implements InsertTsRepository<TsKvEntity> { | |
43 | 38 | |
44 | 39 | private static final String INSERT_OR_UPDATE = |
45 | 40 | "MERGE INTO ts_kv USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " + |
... | ... | @@ -53,36 +48,33 @@ public class HsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepo |
53 | 48 | "VALUES (T.entity_type, T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);"; |
54 | 49 | |
55 | 50 | @Override |
56 | - public void saveOrUpdate(TsKvEntity entity) { | |
57 | - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); | |
58 | - } | |
59 | - | |
60 | - @Override | |
61 | - public void saveOrUpdate(List<TsKvEntity> entities) { | |
51 | + public void saveOrUpdate(List<EntityContainer<TsKvEntity>> entities) { | |
62 | 52 | jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { |
63 | 53 | @Override |
64 | 54 | public void setValues(PreparedStatement ps, int i) throws SQLException { |
65 | - ps.setString(1, entities.get(i).getEntityType().name()); | |
66 | - ps.setString(2, entities.get(i).getEntityId()); | |
67 | - ps.setString(3, entities.get(i).getKey()); | |
68 | - ps.setLong(4, entities.get(i).getTs()); | |
69 | - | |
70 | - if (entities.get(i).getBooleanValue() != null) { | |
71 | - ps.setBoolean(5, entities.get(i).getBooleanValue()); | |
55 | + EntityContainer<TsKvEntity> tsKvEntityEntityContainer = entities.get(i); | |
56 | + TsKvEntity tsKvEntity = tsKvEntityEntityContainer.getEntity(); | |
57 | + ps.setString(1, tsKvEntity.getEntityType().name()); | |
58 | + ps.setString(2, tsKvEntity.getEntityId()); | |
59 | + ps.setString(3, tsKvEntity.getKey()); | |
60 | + ps.setLong(4, tsKvEntity.getTs()); | |
61 | + | |
62 | + if (tsKvEntity.getBooleanValue() != null) { | |
63 | + ps.setBoolean(5, tsKvEntity.getBooleanValue()); | |
72 | 64 | } else { |
73 | 65 | ps.setNull(5, Types.BOOLEAN); |
74 | 66 | } |
75 | 67 | |
76 | - ps.setString(6, entities.get(i).getStrValue()); | |
68 | + ps.setString(6, tsKvEntity.getStrValue()); | |
77 | 69 | |
78 | - if (entities.get(i).getLongValue() != null) { | |
79 | - ps.setLong(7, entities.get(i).getLongValue()); | |
70 | + if (tsKvEntity.getLongValue() != null) { | |
71 | + ps.setLong(7, tsKvEntity.getLongValue()); | |
80 | 72 | } else { |
81 | 73 | ps.setNull(7, Types.BIGINT); |
82 | 74 | } |
83 | 75 | |
84 | - if (entities.get(i).getDoubleValue() != null) { | |
85 | - ps.setDouble(8, entities.get(i).getDoubleValue()); | |
76 | + if (tsKvEntity.getDoubleValue() != null) { | |
77 | + ps.setDouble(8, tsKvEntity.getDoubleValue()); | |
86 | 78 | } else { |
87 | 79 | ps.setNull(8, Types.DOUBLE); |
88 | 80 | } |
... | ... | @@ -94,48 +86,4 @@ public class HsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepo |
94 | 86 | } |
95 | 87 | }); |
96 | 88 | } |
97 | - | |
98 | - @Override | |
99 | - protected void saveOrUpdateBoolean(TsKvEntity entity, String query) { | |
100 | - entityManager.createNativeQuery(query) | |
101 | - .setParameter("entity_type", entity.getEntityType().name()) | |
102 | - .setParameter("entity_id", entity.getEntityId()) | |
103 | - .setParameter("key", entity.getKey()) | |
104 | - .setParameter("ts", entity.getTs()) | |
105 | - .setParameter("bool_v", entity.getBooleanValue()) | |
106 | - .executeUpdate(); | |
107 | - } | |
108 | - | |
109 | - @Override | |
110 | - protected void saveOrUpdateString(TsKvEntity entity, String query) { | |
111 | - entityManager.createNativeQuery(query) | |
112 | - .setParameter("entity_type", entity.getEntityType().name()) | |
113 | - .setParameter("entity_id", entity.getEntityId()) | |
114 | - .setParameter("key", entity.getKey()) | |
115 | - .setParameter("ts", entity.getTs()) | |
116 | - .setParameter("str_v", entity.getStrValue()) | |
117 | - .executeUpdate(); | |
118 | - } | |
119 | - | |
120 | - @Override | |
121 | - protected void saveOrUpdateLong(TsKvEntity entity, String query) { | |
122 | - entityManager.createNativeQuery(query) | |
123 | - .setParameter("entity_type", entity.getEntityType().name()) | |
124 | - .setParameter("entity_id", entity.getEntityId()) | |
125 | - .setParameter("key", entity.getKey()) | |
126 | - .setParameter("ts", entity.getTs()) | |
127 | - .setParameter("long_v", entity.getLongValue()) | |
128 | - .executeUpdate(); | |
129 | - } | |
130 | - | |
131 | - @Override | |
132 | - protected void saveOrUpdateDouble(TsKvEntity entity, String query) { | |
133 | - entityManager.createNativeQuery(query) | |
134 | - .setParameter("entity_type", entity.getEntityType().name()) | |
135 | - .setParameter("entity_id", entity.getEntityId()) | |
136 | - .setParameter("key", entity.getKey()) | |
137 | - .setParameter("ts", entity.getTs()) | |
138 | - .setParameter("dbl_v", entity.getDoubleValue()) | |
139 | - .executeUpdate(); | |
140 | - } | |
141 | 89 | } |
\ No newline at end of file | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts.hsql; | |
17 | + | |
18 | +import com.google.common.util.concurrent.Futures; | |
19 | +import com.google.common.util.concurrent.ListenableFuture; | |
20 | +import lombok.extern.slf4j.Slf4j; | |
21 | +import org.springframework.beans.factory.annotation.Autowired; | |
22 | +import org.springframework.data.domain.PageRequest; | |
23 | +import org.springframework.data.domain.Sort; | |
24 | +import org.springframework.stereotype.Component; | |
25 | +import org.thingsboard.server.common.data.id.EntityId; | |
26 | +import org.thingsboard.server.common.data.id.TenantId; | |
27 | +import org.thingsboard.server.common.data.kv.Aggregation; | |
28 | +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; | |
29 | +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; | |
30 | +import org.thingsboard.server.common.data.kv.TsKvEntry; | |
31 | +import org.thingsboard.server.dao.DaoUtil; | |
32 | +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity; | |
33 | +import org.thingsboard.server.dao.sqlts.AbstractSimpleSqlTimeseriesDao; | |
34 | +import org.thingsboard.server.dao.sqlts.EntityContainer; | |
35 | +import org.thingsboard.server.dao.timeseries.TimeseriesDao; | |
36 | +import org.thingsboard.server.dao.util.HsqlDao; | |
37 | +import org.thingsboard.server.dao.util.SqlTsDao; | |
38 | + | |
39 | +import java.util.ArrayList; | |
40 | +import java.util.List; | |
41 | +import java.util.Optional; | |
42 | +import java.util.concurrent.CompletableFuture; | |
43 | + | |
44 | +import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; | |
45 | + | |
46 | + | |
47 | +@Component | |
48 | +@Slf4j | |
49 | +@SqlTsDao | |
50 | +@HsqlDao | |
51 | +public class JpaHsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEntity> implements TimeseriesDao { | |
52 | + | |
53 | + @Autowired | |
54 | + private TsKvHsqlRepository tsKvRepository; | |
55 | + | |
56 | + @Override | |
57 | + public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) { | |
58 | + return processFindAllAsync(tenantId, entityId, queries); | |
59 | + } | |
60 | + | |
61 | + @Override | |
62 | + public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { | |
63 | + TsKvEntity entity = new TsKvEntity(); | |
64 | + entity.setEntityType(entityId.getEntityType()); | |
65 | + entity.setEntityId(fromTimeUUID(entityId.getId())); | |
66 | + entity.setTs(tsKvEntry.getTs()); | |
67 | + entity.setKey(tsKvEntry.getKey()); | |
68 | + entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); | |
69 | + entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); | |
70 | + entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); | |
71 | + entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); | |
72 | + log.trace("Saving entity: {}", entity); | |
73 | + return tsQueue.add(new EntityContainer(entity, null)); | |
74 | + } | |
75 | + | |
76 | + @Override | |
77 | + public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
78 | + return service.submit(() -> { | |
79 | + tsKvRepository.delete( | |
80 | + fromTimeUUID(entityId.getId()), | |
81 | + entityId.getEntityType(), | |
82 | + query.getKey(), | |
83 | + query.getStartTs(), | |
84 | + query.getEndTs()); | |
85 | + return null; | |
86 | + }); | |
87 | + } | |
88 | + | |
89 | + @Override | |
90 | + public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { | |
91 | + return getSaveLatestFuture(entityId, tsKvEntry); | |
92 | + } | |
93 | + | |
94 | + @Override | |
95 | + public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
96 | + return getRemoveLatestFuture(tenantId, entityId, query); | |
97 | + } | |
98 | + | |
99 | + @Override | |
100 | + public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) { | |
101 | + return getFindLatestFuture(entityId, key); | |
102 | + } | |
103 | + | |
104 | + @Override | |
105 | + public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) { | |
106 | + return getFindAllLatestFuture(entityId); | |
107 | + } | |
108 | + | |
109 | + @Override | |
110 | + public ListenableFuture<Void> savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { | |
111 | + return Futures.immediateFuture(null); | |
112 | + } | |
113 | + | |
114 | + @Override | |
115 | + public ListenableFuture<Void> removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
116 | + return Futures.immediateFuture(null); | |
117 | + } | |
118 | + | |
119 | + protected ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { | |
120 | + List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>(); | |
121 | + switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); | |
122 | + return Futures.transform(setFutures(entitiesFutures), entity -> { | |
123 | + if (entity != null && entity.isNotEmpty()) { | |
124 | + entity.setEntityId(fromTimeUUID(entityId.getId())); | |
125 | + entity.setEntityType(entityId.getEntityType()); | |
126 | + entity.setKey(key); | |
127 | + entity.setTs(ts); | |
128 | + return Optional.of(DaoUtil.getData(entity)); | |
129 | + } else { | |
130 | + return Optional.empty(); | |
131 | + } | |
132 | + }); | |
133 | + } | |
134 | + | |
135 | + protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { | |
136 | + return Futures.immediateFuture( | |
137 | + DaoUtil.convertDataList( | |
138 | + tsKvRepository.findAllWithLimit( | |
139 | + fromTimeUUID(entityId.getId()), | |
140 | + entityId.getEntityType(), | |
141 | + query.getKey(), | |
142 | + query.getStartTs(), | |
143 | + query.getEndTs(), | |
144 | + new PageRequest(0, query.getLimit(), | |
145 | + new Sort(Sort.Direction.fromString( | |
146 | + query.getOrderBy()), "ts"))))); | |
147 | + } | |
148 | + | |
149 | + protected void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
150 | + entitiesFutures.add(tsKvRepository.findCount( | |
151 | + fromTimeUUID(entityId.getId()), | |
152 | + entityId.getEntityType(), | |
153 | + key, | |
154 | + startTs, | |
155 | + endTs)); | |
156 | + } | |
157 | + | |
158 | + protected void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
159 | + entitiesFutures.add(tsKvRepository.findSum( | |
160 | + fromTimeUUID(entityId.getId()), | |
161 | + entityId.getEntityType(), | |
162 | + key, | |
163 | + startTs, | |
164 | + endTs)); | |
165 | + } | |
166 | + | |
167 | + protected void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
168 | + entitiesFutures.add(tsKvRepository.findStringMin( | |
169 | + fromTimeUUID(entityId.getId()), | |
170 | + entityId.getEntityType(), | |
171 | + key, | |
172 | + startTs, | |
173 | + endTs)); | |
174 | + entitiesFutures.add(tsKvRepository.findNumericMin( | |
175 | + fromTimeUUID(entityId.getId()), | |
176 | + entityId.getEntityType(), | |
177 | + key, | |
178 | + startTs, | |
179 | + endTs)); | |
180 | + } | |
181 | + | |
182 | + protected void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
183 | + entitiesFutures.add(tsKvRepository.findStringMax( | |
184 | + fromTimeUUID(entityId.getId()), | |
185 | + entityId.getEntityType(), | |
186 | + key, | |
187 | + startTs, | |
188 | + endTs)); | |
189 | + entitiesFutures.add(tsKvRepository.findNumericMax( | |
190 | + fromTimeUUID(entityId.getId()), | |
191 | + entityId.getEntityType(), | |
192 | + key, | |
193 | + startTs, | |
194 | + endTs)); | |
195 | + } | |
196 | + | |
197 | + protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
198 | + entitiesFutures.add(tsKvRepository.findAvg( | |
199 | + fromTimeUUID(entityId.getId()), | |
200 | + entityId.getEntityType(), | |
201 | + key, | |
202 | + startTs, | |
203 | + endTs)); | |
204 | + } | |
205 | + | |
206 | +} | |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/TsKvHsqlRepository.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java
... | ... | @@ -13,7 +13,7 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.sqlts.hsql; | |
17 | 17 | |
18 | 18 | import org.springframework.data.domain.Pageable; |
19 | 19 | import org.springframework.data.jpa.repository.Modifying; |
... | ... | @@ -23,15 +23,15 @@ import org.springframework.data.repository.query.Param; |
23 | 23 | import org.springframework.scheduling.annotation.Async; |
24 | 24 | import org.springframework.transaction.annotation.Transactional; |
25 | 25 | import org.thingsboard.server.common.data.EntityType; |
26 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvCompositeKey; | |
27 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; | |
26 | +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvCompositeKey; | |
27 | +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity; | |
28 | 28 | import org.thingsboard.server.dao.util.SqlDao; |
29 | 29 | |
30 | 30 | import java.util.List; |
31 | 31 | import java.util.concurrent.CompletableFuture; |
32 | 32 | |
33 | 33 | @SqlDao |
34 | -public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvCompositeKey> { | |
34 | +public interface TsKvHsqlRepository extends CrudRepository<TsKvEntity, TsKvCompositeKey> { | |
35 | 35 | |
36 | 36 | @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + |
37 | 37 | "AND tskv.entityType = :entityType AND tskv.key = :entityKey " + |
... | ... | @@ -146,4 +146,4 @@ public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvComposite |
146 | 146 | @Param("startTs") long startTs, |
147 | 147 | @Param("endTs") long endTs); |
148 | 148 | |
149 | -} | |
149 | +} | |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/HsqlLatestInsertRepository.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlLatestInsertRepository.java
... | ... | @@ -13,13 +13,14 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.sqlts.latest; | |
17 | 17 | |
18 | 18 | import org.springframework.jdbc.core.BatchPreparedStatementSetter; |
19 | 19 | import org.springframework.stereotype.Repository; |
20 | 20 | import org.springframework.transaction.annotation.Transactional; |
21 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; | |
22 | -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository; | |
21 | +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; | |
22 | +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; | |
23 | +import org.thingsboard.server.dao.sqlts.InsertLatestRepository; | |
23 | 24 | import org.thingsboard.server.dao.util.HsqlDao; |
24 | 25 | import org.thingsboard.server.dao.util.SqlTsDao; |
25 | 26 | |
... | ... | @@ -32,14 +33,7 @@ import java.util.List; |
32 | 33 | @HsqlDao |
33 | 34 | @Repository |
34 | 35 | @Transactional |
35 | -public class HsqlLatestInsertRepository extends AbstractLatestInsertRepository { | |
36 | - | |
37 | - private static final String TS_KV_LATEST_CONSTRAINT = "(ts_kv_latest.entity_type=A.entity_type AND ts_kv_latest.entity_id=A.entity_id AND ts_kv_latest.key=A.key)"; | |
38 | - | |
39 | - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, BOOL_V, HSQL_LATEST_ON_BOOL_VALUE_UPDATE_SET_NULLS); | |
40 | - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, STR_V, HSQL_LATEST_ON_STR_VALUE_UPDATE_SET_NULLS); | |
41 | - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, LONG_V, HSQL_LATEST_ON_LONG_VALUE_UPDATE_SET_NULLS); | |
42 | - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, DBL_V, HSQL_LATEST_ON_DBL_VALUE_UPDATE_SET_NULLS); | |
36 | +public class HsqlLatestInsertRepository extends AbstractInsertRepository implements InsertLatestRepository { | |
43 | 37 | |
44 | 38 | private static final String INSERT_OR_UPDATE = |
45 | 39 | "MERGE INTO ts_kv_latest USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " + |
... | ... | @@ -52,11 +46,6 @@ public class HsqlLatestInsertRepository extends AbstractLatestInsertRepository { |
52 | 46 | "VALUES (T.entity_type, T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);"; |
53 | 47 | |
54 | 48 | @Override |
55 | - public void saveOrUpdate(TsKvLatestEntity entity) { | |
56 | - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); | |
57 | - } | |
58 | - | |
59 | - @Override | |
60 | 49 | public void saveOrUpdate(List<TsKvLatestEntity> entities) { |
61 | 50 | jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { |
62 | 51 | @Override |
... | ... | @@ -93,48 +82,4 @@ public class HsqlLatestInsertRepository extends AbstractLatestInsertRepository { |
93 | 82 | } |
94 | 83 | }); |
95 | 84 | } |
96 | - | |
97 | - @Override | |
98 | - protected void saveOrUpdateBoolean(TsKvLatestEntity entity, String query) { | |
99 | - entityManager.createNativeQuery(query) | |
100 | - .setParameter("entity_type", entity.getEntityType().name()) | |
101 | - .setParameter("entity_id", entity.getEntityId()) | |
102 | - .setParameter("key", entity.getKey()) | |
103 | - .setParameter("ts", entity.getTs()) | |
104 | - .setParameter("bool_v", entity.getBooleanValue()) | |
105 | - .executeUpdate(); | |
106 | - } | |
107 | - | |
108 | - @Override | |
109 | - protected void saveOrUpdateString(TsKvLatestEntity entity, String query) { | |
110 | - entityManager.createNativeQuery(query) | |
111 | - .setParameter("entity_type", entity.getEntityType().name()) | |
112 | - .setParameter("entity_id", entity.getEntityId()) | |
113 | - .setParameter("key", entity.getKey()) | |
114 | - .setParameter("ts", entity.getTs()) | |
115 | - .setParameter("str_v", entity.getStrValue()) | |
116 | - .executeUpdate(); | |
117 | - } | |
118 | - | |
119 | - @Override | |
120 | - protected void saveOrUpdateLong(TsKvLatestEntity entity, String query) { | |
121 | - entityManager.createNativeQuery(query) | |
122 | - .setParameter("entity_type", entity.getEntityType().name()) | |
123 | - .setParameter("entity_id", entity.getEntityId()) | |
124 | - .setParameter("key", entity.getKey()) | |
125 | - .setParameter("ts", entity.getTs()) | |
126 | - .setParameter("long_v", entity.getLongValue()) | |
127 | - .executeUpdate(); | |
128 | - } | |
129 | - | |
130 | - @Override | |
131 | - protected void saveOrUpdateDouble(TsKvLatestEntity entity, String query) { | |
132 | - entityManager.createNativeQuery(query) | |
133 | - .setParameter("entity_type", entity.getEntityType().name()) | |
134 | - .setParameter("entity_id", entity.getEntityId()) | |
135 | - .setParameter("key", entity.getKey()) | |
136 | - .setParameter("ts", entity.getTs()) | |
137 | - .setParameter("dbl_v", entity.getDoubleValue()) | |
138 | - .executeUpdate(); | |
139 | - } | |
140 | 85 | } |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/PsqlLatestInsertRepository.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java
... | ... | @@ -13,17 +13,17 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.sqlts.latest; | |
17 | 17 | |
18 | 18 | import org.springframework.jdbc.core.BatchPreparedStatementSetter; |
19 | 19 | import org.springframework.stereotype.Repository; |
20 | 20 | import org.springframework.transaction.TransactionStatus; |
21 | 21 | import org.springframework.transaction.annotation.Transactional; |
22 | 22 | import org.springframework.transaction.support.TransactionCallbackWithoutResult; |
23 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; | |
24 | -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository; | |
25 | -import org.thingsboard.server.dao.util.PsqlDao; | |
26 | -import org.thingsboard.server.dao.util.SqlTsDao; | |
23 | +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; | |
24 | +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; | |
25 | +import org.thingsboard.server.dao.sqlts.InsertLatestRepository; | |
26 | +import org.thingsboard.server.dao.util.PsqlTsAnyDao; | |
27 | 27 | |
28 | 28 | import java.sql.PreparedStatement; |
29 | 29 | import java.sql.SQLException; |
... | ... | @@ -31,18 +31,11 @@ import java.sql.Types; |
31 | 31 | import java.util.ArrayList; |
32 | 32 | import java.util.List; |
33 | 33 | |
34 | -@SqlTsDao | |
35 | -@PsqlDao | |
34 | + | |
35 | +@PsqlTsAnyDao | |
36 | 36 | @Repository |
37 | 37 | @Transactional |
38 | -public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository { | |
39 | - | |
40 | - private static final String TS_KV_LATEST_CONSTRAINT = "(entity_type, entity_id, key)"; | |
41 | - | |
42 | - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS); | |
43 | - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS); | |
44 | - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS); | |
45 | - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS); | |
38 | +public class PsqlLatestInsertRepository extends AbstractInsertRepository implements InsertLatestRepository { | |
46 | 39 | |
47 | 40 | private static final String BATCH_UPDATE = |
48 | 41 | "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ? WHERE entity_type = ? AND entity_id = ? and key = ?"; |
... | ... | @@ -53,11 +46,6 @@ public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository { |
53 | 46 | "ON CONFLICT (entity_type, entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;"; |
54 | 47 | |
55 | 48 | @Override |
56 | - public void saveOrUpdate(TsKvLatestEntity entity) { | |
57 | - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); | |
58 | - } | |
59 | - | |
60 | - @Override | |
61 | 49 | public void saveOrUpdate(List<TsKvLatestEntity> entities) { |
62 | 50 | transactionTemplate.execute(new TransactionCallbackWithoutResult() { |
63 | 51 | @Override |
... | ... | @@ -160,48 +148,4 @@ public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository { |
160 | 148 | } |
161 | 149 | }); |
162 | 150 | } |
163 | - | |
164 | - @Override | |
165 | - protected void saveOrUpdateBoolean(TsKvLatestEntity entity, String query) { | |
166 | - entityManager.createNativeQuery(query) | |
167 | - .setParameter("entity_type", entity.getEntityType().name()) | |
168 | - .setParameter("entity_id", entity.getEntityId()) | |
169 | - .setParameter("key", entity.getKey()) | |
170 | - .setParameter("ts", entity.getTs()) | |
171 | - .setParameter("bool_v", entity.getBooleanValue()) | |
172 | - .executeUpdate(); | |
173 | - } | |
174 | - | |
175 | - @Override | |
176 | - protected void saveOrUpdateString(TsKvLatestEntity entity, String query) { | |
177 | - entityManager.createNativeQuery(query) | |
178 | - .setParameter("entity_type", entity.getEntityType().name()) | |
179 | - .setParameter("entity_id", entity.getEntityId()) | |
180 | - .setParameter("key", entity.getKey()) | |
181 | - .setParameter("ts", entity.getTs()) | |
182 | - .setParameter("str_v", replaceNullChars(entity.getStrValue())) | |
183 | - .executeUpdate(); | |
184 | - } | |
185 | - | |
186 | - @Override | |
187 | - protected void saveOrUpdateLong(TsKvLatestEntity entity, String query) { | |
188 | - entityManager.createNativeQuery(query) | |
189 | - .setParameter("entity_type", entity.getEntityType().name()) | |
190 | - .setParameter("entity_id", entity.getEntityId()) | |
191 | - .setParameter("key", entity.getKey()) | |
192 | - .setParameter("ts", entity.getTs()) | |
193 | - .setParameter("long_v", entity.getLongValue()) | |
194 | - .executeUpdate(); | |
195 | - } | |
196 | - | |
197 | - @Override | |
198 | - protected void saveOrUpdateDouble(TsKvLatestEntity entity, String query) { | |
199 | - entityManager.createNativeQuery(query) | |
200 | - .setParameter("entity_type", entity.getEntityType().name()) | |
201 | - .setParameter("entity_id", entity.getEntityId()) | |
202 | - .setParameter("key", entity.getKey()) | |
203 | - .setParameter("ts", entity.getTs()) | |
204 | - .setParameter("dbl_v", entity.getDoubleValue()) | |
205 | - .executeUpdate(); | |
206 | - } | |
207 | 151 | } |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvLatestRepository.java
... | ... | @@ -13,12 +13,12 @@ |
13 | 13 | * See the License for the specific language governing permissions and |
14 | 14 | * limitations under the License. |
15 | 15 | */ |
16 | -package org.thingsboard.server.dao.sqlts.ts; | |
16 | +package org.thingsboard.server.dao.sqlts.latest; | |
17 | 17 | |
18 | 18 | import org.springframework.data.repository.CrudRepository; |
19 | 19 | import org.thingsboard.server.common.data.EntityType; |
20 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestCompositeKey; | |
21 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; | |
20 | +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; | |
21 | +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; | |
22 | 22 | import org.thingsboard.server.dao.util.SqlDao; |
23 | 23 | |
24 | 24 | import java.util.List; | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts.psql; | |
17 | + | |
18 | +import com.google.common.util.concurrent.Futures; | |
19 | +import com.google.common.util.concurrent.ListenableFuture; | |
20 | +import lombok.extern.slf4j.Slf4j; | |
21 | +import org.hibernate.exception.ConstraintViolationException; | |
22 | +import org.springframework.beans.factory.annotation.Autowired; | |
23 | +import org.springframework.beans.factory.annotation.Value; | |
24 | +import org.springframework.data.domain.PageRequest; | |
25 | +import org.springframework.data.domain.Sort; | |
26 | +import org.springframework.stereotype.Component; | |
27 | +import org.thingsboard.server.common.data.id.EntityId; | |
28 | +import org.thingsboard.server.common.data.id.TenantId; | |
29 | +import org.thingsboard.server.common.data.kv.Aggregation; | |
30 | +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; | |
31 | +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; | |
32 | +import org.thingsboard.server.common.data.kv.TsKvEntry; | |
33 | +import org.thingsboard.server.dao.DaoUtil; | |
34 | +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; | |
35 | +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; | |
36 | +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity; | |
37 | +import org.thingsboard.server.dao.sqlts.AbstractSimpleSqlTimeseriesDao; | |
38 | +import org.thingsboard.server.dao.sqlts.EntityContainer; | |
39 | +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; | |
40 | +import org.thingsboard.server.dao.timeseries.PsqlPartition; | |
41 | +import org.thingsboard.server.dao.timeseries.SqlTsPartitionDate; | |
42 | +import org.thingsboard.server.dao.timeseries.TimeseriesDao; | |
43 | +import org.thingsboard.server.dao.util.PsqlDao; | |
44 | +import org.thingsboard.server.dao.util.SqlTsDao; | |
45 | + | |
46 | +import java.time.Instant; | |
47 | +import java.time.LocalDateTime; | |
48 | +import java.time.ZoneOffset; | |
49 | +import java.util.ArrayList; | |
50 | +import java.util.List; | |
51 | +import java.util.Optional; | |
52 | +import java.util.Set; | |
53 | +import java.util.concurrent.CompletableFuture; | |
54 | +import java.util.concurrent.ConcurrentHashMap; | |
55 | +import java.util.concurrent.ConcurrentMap; | |
56 | +import java.util.concurrent.locks.ReentrantLock; | |
57 | + | |
58 | +import static org.thingsboard.server.dao.timeseries.SqlTsPartitionDate.EPOCH_START; | |
59 | + | |
60 | + | |
61 | +@Component | |
62 | +@Slf4j | |
63 | +@SqlTsDao | |
64 | +@PsqlDao | |
65 | +public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEntity> implements TimeseriesDao { | |
66 | + | |
67 | + private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>(); | |
68 | + private final Set<PsqlPartition> partitions = ConcurrentHashMap.newKeySet(); | |
69 | + | |
70 | + private static final ReentrantLock tsCreationLock = new ReentrantLock(); | |
71 | + private static final ReentrantLock partitionCreationLock = new ReentrantLock(); | |
72 | + | |
73 | + @Autowired | |
74 | + private TsKvDictionaryRepository dictionaryRepository; | |
75 | + | |
76 | + @Autowired | |
77 | + private TsKvPsqlRepository tsKvRepository; | |
78 | + | |
79 | + @Autowired | |
80 | + private PsqlPartitioningRepository partitioningRepository; | |
81 | + | |
82 | + private SqlTsPartitionDate tsFormat; | |
83 | + | |
84 | + @Value("${sql.ts_key_value_partitioning}") | |
85 | + private String partitioning; | |
86 | + | |
87 | + @Override | |
88 | + protected void init() { | |
89 | + super.init(); | |
90 | + Optional<SqlTsPartitionDate> partition = SqlTsPartitionDate.parse(partitioning); | |
91 | + if (partition.isPresent()) { | |
92 | + tsFormat = partition.get(); | |
93 | + } else { | |
94 | + log.warn("Incorrect configuration of partitioning {}", partitioning); | |
95 | + throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!"); | |
96 | + } | |
97 | + } | |
98 | + | |
99 | + @Override | |
100 | + public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) { | |
101 | + return processFindAllAsync(tenantId, entityId, queries); | |
102 | + } | |
103 | + | |
104 | + @Override | |
105 | + public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { | |
106 | + String strKey = tsKvEntry.getKey(); | |
107 | + Integer keyId = getOrSaveKeyId(strKey); | |
108 | + TsKvEntity entity = new TsKvEntity(); | |
109 | + entity.setEntityId(entityId.getId()); | |
110 | + entity.setTs(tsKvEntry.getTs()); | |
111 | + entity.setKey(keyId); | |
112 | + entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); | |
113 | + entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); | |
114 | + entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); | |
115 | + entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); | |
116 | + PsqlPartition psqlPartition = toPartition(tsKvEntry.getTs()); | |
117 | + savePartition(psqlPartition); | |
118 | + log.trace("Saving entity: {}", entity); | |
119 | + return tsQueue.add(new EntityContainer(entity, psqlPartition.getPartitionDate())); | |
120 | + } | |
121 | + | |
122 | + @Override | |
123 | + public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
124 | + return service.submit(() -> { | |
125 | + String strKey = query.getKey(); | |
126 | + Integer keyId = getOrSaveKeyId(strKey); | |
127 | + tsKvRepository.delete( | |
128 | + entityId.getId(), | |
129 | + keyId, | |
130 | + query.getStartTs(), | |
131 | + query.getEndTs()); | |
132 | + return null; | |
133 | + }); | |
134 | + } | |
135 | + | |
136 | + @Override | |
137 | + public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
138 | + return getRemoveLatestFuture(tenantId, entityId, query); | |
139 | + } | |
140 | + | |
141 | + @Override | |
142 | + public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { | |
143 | + return getSaveLatestFuture(entityId, tsKvEntry); | |
144 | + } | |
145 | + | |
146 | + @Override | |
147 | + public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) { | |
148 | + return getFindLatestFuture(entityId, key); | |
149 | + } | |
150 | + | |
151 | + @Override | |
152 | + public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) { | |
153 | + return getFindAllLatestFuture(entityId); | |
154 | + } | |
155 | + | |
156 | + @Override | |
157 | + public ListenableFuture<Void> savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { | |
158 | + return Futures.immediateFuture(null); | |
159 | + } | |
160 | + | |
161 | + @Override | |
162 | + public ListenableFuture<Void> removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
163 | + return Futures.immediateFuture(null); | |
164 | + } | |
165 | + | |
166 | + protected ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { | |
167 | + List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>(); | |
168 | + switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); | |
169 | + return Futures.transform(setFutures(entitiesFutures), entity -> { | |
170 | + if (entity != null && entity.isNotEmpty()) { | |
171 | + entity.setEntityId(entityId.getId()); | |
172 | + entity.setStrKey(key); | |
173 | + entity.setTs(ts); | |
174 | + return Optional.of(DaoUtil.getData(entity)); | |
175 | + } else { | |
176 | + return Optional.empty(); | |
177 | + } | |
178 | + }); | |
179 | + } | |
180 | + | |
181 | + protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { | |
182 | + Integer keyId = getOrSaveKeyId(query.getKey()); | |
183 | + List<TsKvEntity> tsKvEntities = tsKvRepository.findAllWithLimit( | |
184 | + entityId.getId(), | |
185 | + keyId, | |
186 | + query.getStartTs(), | |
187 | + query.getEndTs(), | |
188 | + new PageRequest(0, query.getLimit(), | |
189 | + new Sort(Sort.Direction.fromString( | |
190 | + query.getOrderBy()), "ts"))); | |
191 | + tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey())); | |
192 | + return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); | |
193 | + } | |
194 | + | |
195 | + protected void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
196 | + Integer keyId = getOrSaveKeyId(key); | |
197 | + entitiesFutures.add(tsKvRepository.findCount( | |
198 | + entityId.getId(), | |
199 | + keyId, | |
200 | + startTs, | |
201 | + endTs)); | |
202 | + } | |
203 | + | |
204 | + protected void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
205 | + Integer keyId = getOrSaveKeyId(key); | |
206 | + entitiesFutures.add(tsKvRepository.findSum( | |
207 | + entityId.getId(), | |
208 | + keyId, | |
209 | + startTs, | |
210 | + endTs)); | |
211 | + } | |
212 | + | |
213 | + protected void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
214 | + Integer keyId = getOrSaveKeyId(key); | |
215 | + entitiesFutures.add(tsKvRepository.findStringMin( | |
216 | + entityId.getId(), | |
217 | + keyId, | |
218 | + startTs, | |
219 | + endTs)); | |
220 | + entitiesFutures.add(tsKvRepository.findNumericMin( | |
221 | + entityId.getId(), | |
222 | + keyId, | |
223 | + startTs, | |
224 | + endTs)); | |
225 | + } | |
226 | + | |
227 | + protected void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
228 | + Integer keyId = getOrSaveKeyId(key); | |
229 | + entitiesFutures.add(tsKvRepository.findStringMax( | |
230 | + entityId.getId(), | |
231 | + keyId, | |
232 | + startTs, | |
233 | + endTs)); | |
234 | + entitiesFutures.add(tsKvRepository.findNumericMax( | |
235 | + entityId.getId(), | |
236 | + keyId, | |
237 | + startTs, | |
238 | + endTs)); | |
239 | + } | |
240 | + | |
241 | + protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) { | |
242 | + Integer keyId = getOrSaveKeyId(key); | |
243 | + entitiesFutures.add(tsKvRepository.findAvg( | |
244 | + entityId.getId(), | |
245 | + keyId, | |
246 | + startTs, | |
247 | + endTs)); | |
248 | + } | |
249 | + | |
250 | + private Integer getOrSaveKeyId(String strKey) { | |
251 | + Integer keyId = tsKvDictionaryMap.get(strKey); | |
252 | + if (keyId == null) { | |
253 | + Optional<TsKvDictionary> tsKvDictionaryOptional; | |
254 | + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); | |
255 | + if (!tsKvDictionaryOptional.isPresent()) { | |
256 | + tsCreationLock.lock(); | |
257 | + try { | |
258 | + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); | |
259 | + if (!tsKvDictionaryOptional.isPresent()) { | |
260 | + TsKvDictionary tsKvDictionary = new TsKvDictionary(); | |
261 | + tsKvDictionary.setKey(strKey); | |
262 | + try { | |
263 | + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); | |
264 | + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); | |
265 | + keyId = saved.getKeyId(); | |
266 | + } catch (ConstraintViolationException e) { | |
267 | + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); | |
268 | + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); | |
269 | + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); | |
270 | + keyId = dictionary.getKeyId(); | |
271 | + } | |
272 | + } else { | |
273 | + keyId = tsKvDictionaryOptional.get().getKeyId(); | |
274 | + } | |
275 | + } finally { | |
276 | + tsCreationLock.unlock(); | |
277 | + } | |
278 | + } else { | |
279 | + keyId = tsKvDictionaryOptional.get().getKeyId(); | |
280 | + tsKvDictionaryMap.put(strKey, keyId); | |
281 | + } | |
282 | + } | |
283 | + return keyId; | |
284 | + } | |
285 | + | |
286 | + private void savePartition(PsqlPartition psqlPartition) { | |
287 | + if (!partitions.contains(psqlPartition)) { | |
288 | + partitionCreationLock.lock(); | |
289 | + try { | |
290 | + log.trace("Saving partition: {}", psqlPartition); | |
291 | + partitioningRepository.save(psqlPartition); | |
292 | + log.trace("Adding partition to Set: {}", psqlPartition); | |
293 | + partitions.add(psqlPartition); | |
294 | + } finally { | |
295 | + partitionCreationLock.unlock(); | |
296 | + } | |
297 | + } | |
298 | + } | |
299 | + | |
300 | + private PsqlPartition toPartition(long ts) { | |
301 | + LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); | |
302 | + LocalDateTime localDateTimeStart = tsFormat.trancateTo(time); | |
303 | + if (localDateTimeStart == SqlTsPartitionDate.EPOCH_START) { | |
304 | + return new PsqlPartition(toMills(EPOCH_START), Long.MAX_VALUE, tsFormat.getPattern()); | |
305 | + } else { | |
306 | + LocalDateTime localDateTimeEnd = tsFormat.plusTo(localDateTimeStart); | |
307 | + return new PsqlPartition(toMills(localDateTimeStart), toMills(localDateTimeEnd), tsFormat.getPattern()); | |
308 | + } | |
309 | + } | |
310 | + | |
311 | + private long toMills(LocalDateTime time) { return time.toInstant(ZoneOffset.UTC).toEpochMilli(); } | |
312 | +} | |
\ No newline at end of file | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts.psql; | |
17 | + | |
18 | +import org.springframework.stereotype.Repository; | |
19 | +import org.springframework.transaction.annotation.Transactional; | |
20 | +import org.thingsboard.server.dao.timeseries.PsqlPartition; | |
21 | +import org.thingsboard.server.dao.util.PsqlDao; | |
22 | +import org.thingsboard.server.dao.util.SqlTsDao; | |
23 | + | |
24 | +import javax.persistence.EntityManager; | |
25 | +import javax.persistence.PersistenceContext; | |
26 | + | |
27 | +@SqlTsDao | |
28 | +@PsqlDao | |
29 | +@Repository | |
30 | +@Transactional | |
31 | +public class PsqlPartitioningRepository { | |
32 | + | |
33 | + @PersistenceContext | |
34 | + private EntityManager entityManager; | |
35 | + | |
36 | + public void save(PsqlPartition partition) { | |
37 | + entityManager.createNativeQuery(partition.getQuery()) | |
38 | + .executeUpdate(); | |
39 | + } | |
40 | + | |
41 | +} | |
\ No newline at end of file | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlTimeseriesInsertRepository.java
0 → 100644
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts.psql; | |
17 | + | |
18 | +import org.springframework.jdbc.core.BatchPreparedStatementSetter; | |
19 | +import org.springframework.stereotype.Repository; | |
20 | +import org.springframework.transaction.annotation.Transactional; | |
21 | +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity; | |
22 | +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; | |
23 | +import org.thingsboard.server.dao.sqlts.EntityContainer; | |
24 | +import org.thingsboard.server.dao.sqlts.InsertTsRepository; | |
25 | +import org.thingsboard.server.dao.util.PsqlDao; | |
26 | +import org.thingsboard.server.dao.util.SqlTsDao; | |
27 | + | |
28 | +import java.sql.PreparedStatement; | |
29 | +import java.sql.SQLException; | |
30 | +import java.sql.Types; | |
31 | +import java.util.ArrayList; | |
32 | +import java.util.HashMap; | |
33 | +import java.util.List; | |
34 | +import java.util.Map; | |
35 | + | |
36 | +@SqlTsDao | |
37 | +@PsqlDao | |
38 | +@Repository | |
39 | +@Transactional | |
40 | +public class PsqlTimeseriesInsertRepository extends AbstractInsertRepository implements InsertTsRepository<TsKvEntity> { | |
41 | + | |
42 | + private static final String INSERT_INTO_TS_KV = "INSERT INTO ts_kv_"; | |
43 | + | |
44 | + private static final String VALUES_ON_CONFLICT_DO_UPDATE = " (entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES (?, ?, ?, ?, ?, ?, ?) " + | |
45 | + "ON CONFLICT (entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;"; | |
46 | + | |
47 | + @Override | |
48 | + public void saveOrUpdate(List<EntityContainer<TsKvEntity>> entities) { | |
49 | + Map<String, List<TsKvEntity>> partitionMap = new HashMap<>(); | |
50 | + for (EntityContainer<TsKvEntity> entityContainer : entities) { | |
51 | + List<TsKvEntity> tsKvEntities = partitionMap.computeIfAbsent(entityContainer.getPartitionDate(), k -> new ArrayList<>()); | |
52 | + tsKvEntities.add(entityContainer.getEntity()); | |
53 | + } | |
54 | + partitionMap.forEach((partition, entries) -> jdbcTemplate.batchUpdate(getInsertOrUpdateQuery(partition), new BatchPreparedStatementSetter() { | |
55 | + @Override | |
56 | + public void setValues(PreparedStatement ps, int i) throws SQLException { | |
57 | + TsKvEntity tsKvEntity = entries.get(i); | |
58 | + ps.setObject(1, tsKvEntity.getEntityId()); | |
59 | + ps.setInt(2, tsKvEntity.getKey()); | |
60 | + ps.setLong(3, tsKvEntity.getTs()); | |
61 | + | |
62 | + if (tsKvEntity.getBooleanValue() != null) { | |
63 | + ps.setBoolean(4, tsKvEntity.getBooleanValue()); | |
64 | + ps.setBoolean(8, tsKvEntity.getBooleanValue()); | |
65 | + } else { | |
66 | + ps.setNull(4, Types.BOOLEAN); | |
67 | + ps.setNull(8, Types.BOOLEAN); | |
68 | + } | |
69 | + | |
70 | + ps.setString(5, replaceNullChars(tsKvEntity.getStrValue())); | |
71 | + ps.setString(9, replaceNullChars(tsKvEntity.getStrValue())); | |
72 | + | |
73 | + | |
74 | + if (tsKvEntity.getLongValue() != null) { | |
75 | + ps.setLong(6, tsKvEntity.getLongValue()); | |
76 | + ps.setLong(10, tsKvEntity.getLongValue()); | |
77 | + } else { | |
78 | + ps.setNull(6, Types.BIGINT); | |
79 | + ps.setNull(10, Types.BIGINT); | |
80 | + } | |
81 | + | |
82 | + if (tsKvEntity.getDoubleValue() != null) { | |
83 | + ps.setDouble(7, tsKvEntity.getDoubleValue()); | |
84 | + ps.setDouble(11, tsKvEntity.getDoubleValue()); | |
85 | + } else { | |
86 | + ps.setNull(7, Types.DOUBLE); | |
87 | + ps.setNull(11, Types.DOUBLE); | |
88 | + } | |
89 | + } | |
90 | + | |
91 | + @Override | |
92 | + public int getBatchSize() { | |
93 | + return entries.size(); | |
94 | + } | |
95 | + })); | |
96 | + } | |
97 | + | |
98 | + private String getInsertOrUpdateQuery(String partitionDate) { | |
99 | + return INSERT_INTO_TS_KV + partitionDate + VALUES_ON_CONFLICT_DO_UPDATE; | |
100 | + } | |
101 | +} | |
\ No newline at end of file | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.sqlts.psql; | |
17 | + | |
18 | +import org.springframework.data.domain.Pageable; | |
19 | +import org.springframework.data.jpa.repository.Modifying; | |
20 | +import org.springframework.data.jpa.repository.Query; | |
21 | +import org.springframework.data.repository.CrudRepository; | |
22 | +import org.springframework.data.repository.query.Param; | |
23 | +import org.springframework.scheduling.annotation.Async; | |
24 | +import org.springframework.transaction.annotation.Transactional; | |
25 | +import org.thingsboard.server.dao.model.sqlts.psql.TsKvCompositeKey; | |
26 | +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity; | |
27 | +import org.thingsboard.server.dao.util.SqlDao; | |
28 | + | |
29 | +import java.util.List; | |
30 | +import java.util.UUID; | |
31 | +import java.util.concurrent.CompletableFuture; | |
32 | + | |
33 | +@SqlDao | |
34 | +public interface TsKvPsqlRepository extends CrudRepository<TsKvEntity, TsKvCompositeKey> { | |
35 | + | |
36 | + @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + | |
37 | + "AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
38 | + List<TsKvEntity> findAllWithLimit(@Param("entityId") UUID entityId, | |
39 | + @Param("entityKey") int key, | |
40 | + @Param("startTs") long startTs, | |
41 | + @Param("endTs") long endTs, | |
42 | + Pageable pageable); | |
43 | + | |
44 | + @Transactional | |
45 | + @Modifying | |
46 | + @Query("DELETE FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + | |
47 | + "AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
48 | + void delete(@Param("entityId") UUID entityId, | |
49 | + @Param("entityKey") int key, | |
50 | + @Param("startTs") long startTs, | |
51 | + @Param("endTs") long endTs); | |
52 | + | |
53 | + @Async | |
54 | + @Query("SELECT new TsKvEntity(MAX(tskv.strValue)) FROM TsKvEntity tskv " + | |
55 | + "WHERE tskv.strValue IS NOT NULL " + | |
56 | + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
57 | + CompletableFuture<TsKvEntity> findStringMax(@Param("entityId") UUID entityId, | |
58 | + @Param("entityKey") int entityKey, | |
59 | + @Param("startTs") long startTs, | |
60 | + @Param("endTs") long endTs); | |
61 | + | |
62 | + @Async | |
63 | + @Query("SELECT new TsKvEntity(MAX(COALESCE(tskv.longValue, -9223372036854775807)), " + | |
64 | + "MAX(COALESCE(tskv.doubleValue, -1.79769E+308)), " + | |
65 | + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + | |
66 | + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + | |
67 | + "'MAX') FROM TsKvEntity tskv " + | |
68 | + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
69 | + CompletableFuture<TsKvEntity> findNumericMax(@Param("entityId") UUID entityId, | |
70 | + @Param("entityKey") int entityKey, | |
71 | + @Param("startTs") long startTs, | |
72 | + @Param("endTs") long endTs); | |
73 | + | |
74 | + | |
75 | + @Async | |
76 | + @Query("SELECT new TsKvEntity(MIN(tskv.strValue)) FROM TsKvEntity tskv " + | |
77 | + "WHERE tskv.strValue IS NOT NULL " + | |
78 | + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
79 | + CompletableFuture<TsKvEntity> findStringMin(@Param("entityId") UUID entityId, | |
80 | + @Param("entityKey") int entityKey, | |
81 | + @Param("startTs") long startTs, | |
82 | + @Param("endTs") long endTs); | |
83 | + | |
84 | + @Async | |
85 | + @Query("SELECT new TsKvEntity(MIN(COALESCE(tskv.longValue, 9223372036854775807)), " + | |
86 | + "MIN(COALESCE(tskv.doubleValue, 1.79769E+308)), " + | |
87 | + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + | |
88 | + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + | |
89 | + "'MIN') FROM TsKvEntity tskv " + | |
90 | + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
91 | + CompletableFuture<TsKvEntity> findNumericMin( | |
92 | + @Param("entityId") UUID entityId, | |
93 | + @Param("entityKey") int entityKey, | |
94 | + @Param("startTs") long startTs, | |
95 | + @Param("endTs") long endTs); | |
96 | + | |
97 | + @Async | |
98 | + @Query("SELECT new TsKvEntity(SUM(CASE WHEN tskv.booleanValue IS NULL THEN 0 ELSE 1 END), " + | |
99 | + "SUM(CASE WHEN tskv.strValue IS NULL THEN 0 ELSE 1 END), " + | |
100 | + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + | |
101 | + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END)) FROM TsKvEntity tskv " + | |
102 | + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
103 | + CompletableFuture<TsKvEntity> findCount(@Param("entityId") UUID entityId, | |
104 | + @Param("entityKey") int entityKey, | |
105 | + @Param("startTs") long startTs, | |
106 | + @Param("endTs") long endTs); | |
107 | + | |
108 | + @Async | |
109 | + @Query("SELECT new TsKvEntity(SUM(COALESCE(tskv.longValue, 0)), " + | |
110 | + "SUM(COALESCE(tskv.doubleValue, 0.0)), " + | |
111 | + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + | |
112 | + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + | |
113 | + "'AVG') FROM TsKvEntity tskv " + | |
114 | + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
115 | + CompletableFuture<TsKvEntity> findAvg(@Param("entityId") UUID entityId, | |
116 | + @Param("entityKey") int entityKey, | |
117 | + @Param("startTs") long startTs, | |
118 | + @Param("endTs") long endTs); | |
119 | + | |
120 | + @Async | |
121 | + @Query("SELECT new TsKvEntity(SUM(COALESCE(tskv.longValue, 0)), " + | |
122 | + "SUM(COALESCE(tskv.doubleValue, 0.0)), " + | |
123 | + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + | |
124 | + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + | |
125 | + "'SUM') FROM TsKvEntity tskv " + | |
126 | + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") | |
127 | + CompletableFuture<TsKvEntity> findSum(@Param("entityId") UUID entityId, | |
128 | + @Param("entityKey") int entityKey, | |
129 | + @Param("startTs") long startTs, | |
130 | + @Param("endTs") long endTs); | |
131 | + | |
132 | +} | ... | ... |
... | ... | @@ -23,6 +23,7 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
23 | 23 | import javax.persistence.EntityManager; |
24 | 24 | import javax.persistence.PersistenceContext; |
25 | 25 | import java.util.List; |
26 | +import java.util.UUID; | |
26 | 27 | import java.util.concurrent.CompletableFuture; |
27 | 28 | |
28 | 29 | @Repository |
... | ... | @@ -36,7 +37,7 @@ public class AggregationRepository { |
36 | 37 | public static final String FIND_COUNT = "findCount"; |
37 | 38 | |
38 | 39 | |
39 | - public static final String FROM_WHERE_CLAUSE = "FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS varchar) AND tskv.entity_id = cast(:entityId AS varchar) AND tskv.key= cast(:entityKey AS varchar) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket"; | |
40 | + public static final String FROM_WHERE_CLAUSE = "FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS uuid) AND tskv.entity_id = cast(:entityId AS uuid) AND tskv.key= cast(:entityKey AS int) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket"; | |
40 | 41 | |
41 | 42 | public static final String FIND_AVG_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts) AS tsBucket, :timeBucket AS interval, SUM(COALESCE(tskv.long_v, 0)) AS longValue, SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, null AS strValue, 'AVG' AS aggType "; |
42 | 43 | |
... | ... | @@ -52,41 +53,41 @@ public class AggregationRepository { |
52 | 53 | private EntityManager entityManager; |
53 | 54 | |
54 | 55 | @Async |
55 | - public CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { | |
56 | + public CompletableFuture<List<TimescaleTsKvEntity>> findAvg(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { | |
56 | 57 | @SuppressWarnings("unchecked") |
57 | 58 | List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG); |
58 | 59 | return CompletableFuture.supplyAsync(() -> resultList); |
59 | 60 | } |
60 | 61 | |
61 | 62 | @Async |
62 | - public CompletableFuture<List<TimescaleTsKvEntity>> findMax(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { | |
63 | + public CompletableFuture<List<TimescaleTsKvEntity>> findMax(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { | |
63 | 64 | @SuppressWarnings("unchecked") |
64 | 65 | List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX); |
65 | 66 | return CompletableFuture.supplyAsync(() -> resultList); |
66 | 67 | } |
67 | 68 | |
68 | 69 | @Async |
69 | - public CompletableFuture<List<TimescaleTsKvEntity>> findMin(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { | |
70 | + public CompletableFuture<List<TimescaleTsKvEntity>> findMin(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { | |
70 | 71 | @SuppressWarnings("unchecked") |
71 | 72 | List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN); |
72 | 73 | return CompletableFuture.supplyAsync(() -> resultList); |
73 | 74 | } |
74 | 75 | |
75 | 76 | @Async |
76 | - public CompletableFuture<List<TimescaleTsKvEntity>> findSum(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { | |
77 | + public CompletableFuture<List<TimescaleTsKvEntity>> findSum(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { | |
77 | 78 | @SuppressWarnings("unchecked") |
78 | 79 | List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM); |
79 | 80 | return CompletableFuture.supplyAsync(() -> resultList); |
80 | 81 | } |
81 | 82 | |
82 | 83 | @Async |
83 | - public CompletableFuture<List<TimescaleTsKvEntity>> findCount(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { | |
84 | + public CompletableFuture<List<TimescaleTsKvEntity>> findCount(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { | |
84 | 85 | @SuppressWarnings("unchecked") |
85 | 86 | List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT); |
86 | 87 | return CompletableFuture.supplyAsync(() -> resultList); |
87 | 88 | } |
88 | 89 | |
89 | - private List getResultList(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs, String query) { | |
90 | + private List getResultList(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs, String query) { | |
90 | 91 | return entityManager.createNamedQuery(query) |
91 | 92 | .setParameter("tenantId", tenantId) |
92 | 93 | .setParameter("entityId", entityId) | ... | ... |
... | ... | @@ -19,7 +19,9 @@ import org.springframework.jdbc.core.BatchPreparedStatementSetter; |
19 | 19 | import org.springframework.stereotype.Repository; |
20 | 20 | import org.springframework.transaction.annotation.Transactional; |
21 | 21 | import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity; |
22 | -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; | |
22 | +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; | |
23 | +import org.thingsboard.server.dao.sqlts.EntityContainer; | |
24 | +import org.thingsboard.server.dao.sqlts.InsertTsRepository; | |
23 | 25 | import org.thingsboard.server.dao.util.PsqlDao; |
24 | 26 | import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
25 | 27 | |
... | ... | @@ -32,35 +34,21 @@ import java.util.List; |
32 | 34 | @PsqlDao |
33 | 35 | @Repository |
34 | 36 | @Transactional |
35 | -public class TimescaleInsertRepository extends AbstractTimeseriesInsertRepository<TimescaleTsKvEntity> { | |
36 | - | |
37 | - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateString(BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS); | |
38 | - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateString(STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS); | |
39 | - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateString(LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS); | |
40 | - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateString(DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS); | |
41 | - | |
42 | - private static final String BATCH_UPDATE = | |
43 | - "UPDATE tenant_ts_kv SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ? WHERE entity_type = ? AND entity_id = ? and key = ? and ts = ?"; | |
44 | - | |
37 | +public class TimescaleInsertRepository extends AbstractInsertRepository implements InsertTsRepository<TimescaleTsKvEntity> { | |
45 | 38 | |
46 | 39 | private static final String INSERT_OR_UPDATE = |
47 | 40 | "INSERT INTO tenant_ts_kv (tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?) " + |
48 | 41 | "ON CONFLICT (tenant_id, entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;"; |
49 | 42 | |
50 | 43 | @Override |
51 | - public void saveOrUpdate(TimescaleTsKvEntity entity) { | |
52 | - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); | |
53 | - } | |
54 | - | |
55 | - @Override | |
56 | - public void saveOrUpdate(List<TimescaleTsKvEntity> entities) { | |
44 | + public void saveOrUpdate(List<EntityContainer<TimescaleTsKvEntity>> entities) { | |
57 | 45 | jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { |
58 | 46 | @Override |
59 | 47 | public void setValues(PreparedStatement ps, int i) throws SQLException { |
60 | - TimescaleTsKvEntity tsKvEntity = entities.get(i); | |
61 | - ps.setString(1, tsKvEntity.getTenantId()); | |
62 | - ps.setString(2, tsKvEntity.getEntityId()); | |
63 | - ps.setString(3, tsKvEntity.getKey()); | |
48 | + TimescaleTsKvEntity tsKvEntity = entities.get(i).getEntity(); | |
49 | + ps.setObject(1, tsKvEntity.getTenantId()); | |
50 | + ps.setObject(2, tsKvEntity.getEntityId()); | |
51 | + ps.setInt(3, tsKvEntity.getKey()); | |
64 | 52 | ps.setLong(4, tsKvEntity.getTs()); |
65 | 53 | |
66 | 54 | if (tsKvEntity.getBooleanValue() != null) { |
... | ... | @@ -98,52 +86,4 @@ public class TimescaleInsertRepository extends AbstractTimeseriesInsertRepositor |
98 | 86 | } |
99 | 87 | }); |
100 | 88 | } |
101 | - | |
102 | - @Override | |
103 | - protected void saveOrUpdateBoolean(TimescaleTsKvEntity entity, String query) { | |
104 | - entityManager.createNativeQuery(query) | |
105 | - .setParameter("tenant_id", entity.getTenantId()) | |
106 | - .setParameter("entity_id", entity.getEntityId()) | |
107 | - .setParameter("key", entity.getKey()) | |
108 | - .setParameter("ts", entity.getTs()) | |
109 | - .setParameter("bool_v", entity.getBooleanValue()) | |
110 | - .executeUpdate(); | |
111 | - } | |
112 | - | |
113 | - @Override | |
114 | - protected void saveOrUpdateString(TimescaleTsKvEntity entity, String query) { | |
115 | - entityManager.createNativeQuery(query) | |
116 | - .setParameter("tenant_id", entity.getTenantId()) | |
117 | - .setParameter("entity_id", entity.getEntityId()) | |
118 | - .setParameter("key", entity.getKey()) | |
119 | - .setParameter("ts", entity.getTs()) | |
120 | - .setParameter("str_v", replaceNullChars(entity.getStrValue())) | |
121 | - .executeUpdate(); | |
122 | - } | |
123 | - | |
124 | - @Override | |
125 | - protected void saveOrUpdateLong(TimescaleTsKvEntity entity, String query) { | |
126 | - entityManager.createNativeQuery(query) | |
127 | - .setParameter("tenant_id", entity.getTenantId()) | |
128 | - .setParameter("entity_id", entity.getEntityId()) | |
129 | - .setParameter("key", entity.getKey()) | |
130 | - .setParameter("ts", entity.getTs()) | |
131 | - .setParameter("long_v", entity.getLongValue()) | |
132 | - .executeUpdate(); | |
133 | - } | |
134 | - | |
135 | - @Override | |
136 | - protected void saveOrUpdateDouble(TimescaleTsKvEntity entity, String query) { | |
137 | - entityManager.createNativeQuery(query) | |
138 | - .setParameter("tenant_id", entity.getTenantId()) | |
139 | - .setParameter("entity_id", entity.getEntityId()) | |
140 | - .setParameter("key", entity.getKey()) | |
141 | - .setParameter("ts", entity.getTs()) | |
142 | - .setParameter("dbl_v", entity.getDoubleValue()) | |
143 | - .executeUpdate(); | |
144 | - } | |
145 | - | |
146 | - private static String getInsertOrUpdateString(String value, String nullValues) { | |
147 | - return "INSERT INTO tenant_ts_kv(tenant_id, entity_id, key, ts, " + value + ") VALUES (:tenant_id, :entity_id, :key, :ts, :" + value + ") ON CONFLICT (tenant_id, entity_id, key, ts) DO UPDATE SET " + value + " = :" + value + ", ts = :ts," + nullValues; | |
148 | - } | |
149 | 89 | } |
\ No newline at end of file | ... | ... |
... | ... | @@ -15,11 +15,11 @@ |
15 | 15 | */ |
16 | 16 | package org.thingsboard.server.dao.sqlts.timescale; |
17 | 17 | |
18 | -import com.google.common.collect.Lists; | |
19 | 18 | import com.google.common.util.concurrent.Futures; |
20 | 19 | import com.google.common.util.concurrent.ListenableFuture; |
21 | 20 | import com.google.common.util.concurrent.SettableFuture; |
22 | 21 | import lombok.extern.slf4j.Slf4j; |
22 | +import org.hibernate.exception.ConstraintViolationException; | |
23 | 23 | import org.springframework.beans.factory.annotation.Autowired; |
24 | 24 | import org.springframework.beans.factory.annotation.Value; |
25 | 25 | import org.springframework.data.domain.PageRequest; |
... | ... | @@ -29,19 +29,19 @@ import org.springframework.util.CollectionUtils; |
29 | 29 | import org.thingsboard.server.common.data.id.EntityId; |
30 | 30 | import org.thingsboard.server.common.data.id.TenantId; |
31 | 31 | import org.thingsboard.server.common.data.kv.Aggregation; |
32 | -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; | |
33 | 32 | import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; |
34 | 33 | import org.thingsboard.server.common.data.kv.ReadTsKvQuery; |
35 | -import org.thingsboard.server.common.data.kv.StringDataEntry; | |
36 | 34 | import org.thingsboard.server.common.data.kv.TsKvEntry; |
37 | -import org.thingsboard.server.common.data.kv.TsKvQuery; | |
38 | 35 | import org.thingsboard.server.dao.DaoUtil; |
36 | +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; | |
37 | +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; | |
39 | 38 | import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity; |
40 | -import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; | |
41 | 39 | import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; |
42 | 40 | import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; |
43 | 41 | import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; |
44 | -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; | |
42 | +import org.thingsboard.server.dao.sqlts.EntityContainer; | |
43 | +import org.thingsboard.server.dao.sqlts.InsertTsRepository; | |
44 | +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; | |
45 | 45 | import org.thingsboard.server.dao.timeseries.TimeseriesDao; |
46 | 46 | import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
47 | 47 | |
... | ... | @@ -51,9 +51,11 @@ import java.util.ArrayList; |
51 | 51 | import java.util.Collections; |
52 | 52 | import java.util.List; |
53 | 53 | import java.util.Optional; |
54 | +import java.util.UUID; | |
54 | 55 | import java.util.concurrent.CompletableFuture; |
55 | - | |
56 | -import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; | |
56 | +import java.util.concurrent.ConcurrentHashMap; | |
57 | +import java.util.concurrent.ConcurrentMap; | |
58 | +import java.util.concurrent.locks.ReentrantLock; | |
57 | 59 | |
58 | 60 | |
59 | 61 | @Component |
... | ... | @@ -63,17 +65,21 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
63 | 65 | |
64 | 66 | private static final String TS = "ts"; |
65 | 67 | |
68 | + private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>(); | |
69 | + | |
70 | + private static final ReentrantLock tsCreationLock = new ReentrantLock(); | |
71 | + | |
66 | 72 | @Autowired |
67 | - private TsKvTimescaleRepository tsKvRepository; | |
73 | + private TsKvDictionaryRepository dictionaryRepository; | |
68 | 74 | |
69 | 75 | @Autowired |
70 | - private AggregationRepository aggregationRepository; | |
76 | + private TsKvTimescaleRepository tsKvRepository; | |
71 | 77 | |
72 | 78 | @Autowired |
73 | - private AbstractTimeseriesInsertRepository insertRepository; | |
79 | + private AggregationRepository aggregationRepository; | |
74 | 80 | |
75 | 81 | @Autowired |
76 | - ScheduledLogExecutorComponent logExecutor; | |
82 | + private InsertTsRepository<TimescaleTsKvEntity> insertRepository; | |
77 | 83 | |
78 | 84 | @Value("${sql.ts_timescale.batch_size:1000}") |
79 | 85 | private int batchSize; |
... | ... | @@ -84,10 +90,11 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
84 | 90 | @Value("${sql.ts_timescale.stats_print_interval_ms:1000}") |
85 | 91 | private long statsPrintIntervalMs; |
86 | 92 | |
87 | - private TbSqlBlockingQueue<TimescaleTsKvEntity> queue; | |
93 | + private TbSqlBlockingQueue<EntityContainer<TimescaleTsKvEntity>> queue; | |
88 | 94 | |
89 | 95 | @PostConstruct |
90 | - private void init() { | |
96 | + protected void init() { | |
97 | + super.init(); | |
91 | 98 | TbSqlBlockingQueueParams params = TbSqlBlockingQueueParams.builder() |
92 | 99 | .logName("TS Timescale") |
93 | 100 | .batchSize(batchSize) |
... | ... | @@ -99,17 +106,13 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
99 | 106 | } |
100 | 107 | |
101 | 108 | @PreDestroy |
102 | - private void destroy() { | |
109 | + protected void destroy() { | |
110 | + super.init(); | |
103 | 111 | if (queue != null) { |
104 | 112 | queue.destroy(); |
105 | 113 | } |
106 | 114 | } |
107 | 115 | |
108 | - @Override | |
109 | - public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) { | |
110 | - return processFindAllAsync(tenantId, entityId, queries); | |
111 | - } | |
112 | - | |
113 | 116 | protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { |
114 | 117 | if (query.getAggregation() == Aggregation.NONE) { |
115 | 118 | return findAllAsyncWithLimit(tenantId, entityId, query); |
... | ... | @@ -122,50 +125,36 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
122 | 125 | } |
123 | 126 | } |
124 | 127 | |
125 | - private ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { | |
126 | - return Futures.immediateFuture( | |
127 | - DaoUtil.convertDataList( | |
128 | - tsKvRepository.findAllWithLimit( | |
129 | - fromTimeUUID(tenantId.getId()), | |
130 | - fromTimeUUID(entityId.getId()), | |
131 | - query.getKey(), | |
132 | - query.getStartTs(), | |
133 | - query.getEndTs(), | |
134 | - new PageRequest(0, query.getLimit(), | |
135 | - new Sort(Sort.Direction.fromString( | |
136 | - query.getOrderBy()), "ts"))))); | |
128 | + @Override | |
129 | + public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) { | |
130 | + return processFindAllAsync(tenantId, entityId, queries); | |
137 | 131 | } |
138 | 132 | |
139 | - | |
140 | 133 | @Override |
141 | 134 | public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) { |
142 | - ListenableFuture<List<TimescaleTsKvEntity>> future = getLatest(tenantId, entityId, key, 0L, System.currentTimeMillis()); | |
143 | - return Futures.transform(future, latest -> { | |
144 | - if (!CollectionUtils.isEmpty(latest)) { | |
145 | - return DaoUtil.getData(latest.get(0)); | |
146 | - } else { | |
147 | - return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); | |
148 | - } | |
149 | - }, service); | |
135 | + return getFindLatestFuture(entityId, key); | |
150 | 136 | } |
151 | 137 | |
152 | 138 | @Override |
153 | 139 | public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) { |
154 | - return Futures.immediateFuture(DaoUtil.convertDataList(Lists.newArrayList(tsKvRepository.findAllLatestValues(fromTimeUUID(tenantId.getId()), fromTimeUUID(entityId.getId()))))); | |
140 | + return getFindAllLatestFuture(entityId); | |
155 | 141 | } |
156 | 142 | |
157 | 143 | @Override |
158 | 144 | public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { |
145 | + String strKey = tsKvEntry.getKey(); | |
146 | + Integer keyId = getOrSaveKeyId(strKey); | |
159 | 147 | TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); |
160 | - entity.setTenantId(fromTimeUUID(tenantId.getId())); | |
161 | - entity.setEntityId(fromTimeUUID(entityId.getId())); | |
148 | + entity.setTenantId(tenantId.getId()); | |
149 | + entity.setEntityId(entityId.getId()); | |
162 | 150 | entity.setTs(tsKvEntry.getTs()); |
163 | - entity.setKey(tsKvEntry.getKey()); | |
151 | + entity.setKey(keyId); | |
164 | 152 | entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); |
165 | 153 | entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); |
166 | 154 | entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); |
167 | 155 | entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); |
168 | - return queue.add(entity); | |
156 | + log.trace("Saving entity to timescale db: {}", entity); | |
157 | + return queue.add(new EntityContainer(entity, null)); | |
169 | 158 | } |
170 | 159 | |
171 | 160 | @Override |
... | ... | @@ -175,16 +164,18 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
175 | 164 | |
176 | 165 | @Override |
177 | 166 | public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { |
178 | - return Futures.immediateFuture(null); | |
167 | + return getSaveLatestFuture(entityId, tsKvEntry); | |
179 | 168 | } |
180 | 169 | |
181 | 170 | @Override |
182 | 171 | public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { |
172 | + String strKey = query.getKey(); | |
173 | + Integer keyId = getOrSaveKeyId(strKey); | |
183 | 174 | return service.submit(() -> { |
184 | 175 | tsKvRepository.delete( |
185 | - fromTimeUUID(tenantId.getId()), | |
186 | - fromTimeUUID(entityId.getId()), | |
187 | - query.getKey(), | |
176 | + tenantId.getId(), | |
177 | + entityId.getId(), | |
178 | + keyId, | |
188 | 179 | query.getStartTs(), |
189 | 180 | query.getEndTs()); |
190 | 181 | return null; |
... | ... | @@ -193,7 +184,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
193 | 184 | |
194 | 185 | @Override |
195 | 186 | public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { |
196 | - return service.submit(() -> null); | |
187 | + return getRemoveLatestFuture(tenantId, entityId, query); | |
197 | 188 | } |
198 | 189 | |
199 | 190 | @Override |
... | ... | @@ -201,37 +192,60 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
201 | 192 | return service.submit(() -> null); |
202 | 193 | } |
203 | 194 | |
204 | - private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
205 | - ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query); | |
206 | - return Futures.transformAsync(future, entryList -> { | |
207 | - if (entryList.size() == 1) { | |
208 | - return save(tenantId, entityId, entryList.get(0), 0L); | |
209 | - } else { | |
210 | - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); | |
211 | - } | |
212 | - return Futures.immediateFuture(null); | |
213 | - }, service); | |
214 | - } | |
215 | - | |
216 | - private ListenableFuture<List<TimescaleTsKvEntity>> findLatestByQuery(TenantId tenantId, EntityId entityId, TsKvQuery query) { | |
217 | - return getLatest(tenantId, entityId, query.getKey(), query.getStartTs(), query.getEndTs()); | |
195 | + private ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { | |
196 | + String strKey = query.getKey(); | |
197 | + Integer keyId = getOrSaveKeyId(strKey); | |
198 | + List<TimescaleTsKvEntity> timescaleTsKvEntities = tsKvRepository.findAllWithLimit( | |
199 | + tenantId.getId(), | |
200 | + entityId.getId(), | |
201 | + keyId, | |
202 | + query.getStartTs(), | |
203 | + query.getEndTs(), | |
204 | + new PageRequest(0, query.getLimit(), | |
205 | + new Sort(Sort.Direction.fromString( | |
206 | + query.getOrderBy()), TS))); | |
207 | + timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); | |
208 | + return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); | |
218 | 209 | } |
219 | 210 | |
220 | - private ListenableFuture<List<TimescaleTsKvEntity>> getLatest(TenantId tenantId, EntityId entityId, String key, long start, long end) { | |
221 | - return Futures.immediateFuture(tsKvRepository.findAllWithLimit( | |
222 | - fromTimeUUID(tenantId.getId()), | |
223 | - fromTimeUUID(entityId.getId()), | |
224 | - key, | |
225 | - start, | |
226 | - end, | |
227 | - new PageRequest(0, 1, | |
228 | - new Sort(Sort.Direction.DESC, TS)))); | |
211 | + private Integer getOrSaveKeyId(String strKey) { | |
212 | + Integer keyId = tsKvDictionaryMap.get(strKey); | |
213 | + if (keyId == null) { | |
214 | + Optional<TsKvDictionary> tsKvDictionaryOptional; | |
215 | + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); | |
216 | + if (!tsKvDictionaryOptional.isPresent()) { | |
217 | + tsCreationLock.lock(); | |
218 | + try { | |
219 | + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); | |
220 | + if (!tsKvDictionaryOptional.isPresent()) { | |
221 | + TsKvDictionary tsKvDictionary = new TsKvDictionary(); | |
222 | + tsKvDictionary.setKey(strKey); | |
223 | + try { | |
224 | + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); | |
225 | + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); | |
226 | + keyId = saved.getKeyId(); | |
227 | + } catch (ConstraintViolationException e) { | |
228 | + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); | |
229 | + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); | |
230 | + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); | |
231 | + keyId = dictionary.getKeyId(); | |
232 | + } | |
233 | + } else { | |
234 | + keyId = tsKvDictionaryOptional.get().getKeyId(); | |
235 | + } | |
236 | + } finally { | |
237 | + tsCreationLock.unlock(); | |
238 | + } | |
239 | + } else { | |
240 | + keyId = tsKvDictionaryOptional.get().getKeyId(); | |
241 | + tsKvDictionaryMap.put(strKey, keyId); | |
242 | + } | |
243 | + } | |
244 | + return keyId; | |
229 | 245 | } |
230 | 246 | |
231 | 247 | private ListenableFuture<List<Optional<TsKvEntry>>> findAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) { |
232 | - String entityIdStr = fromTimeUUID(entityId.getId()); | |
233 | - String tenantIdStr = fromTimeUUID(tenantId.getId()); | |
234 | - CompletableFuture<List<TimescaleTsKvEntity>> listCompletableFuture = switchAgregation(key, startTs, endTs, timeBucket, aggregation, entityIdStr, tenantIdStr); | |
248 | + CompletableFuture<List<TimescaleTsKvEntity>> listCompletableFuture = switchAgregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId(), tenantId.getId()); | |
235 | 249 | SettableFuture<List<TimescaleTsKvEntity>> listenableFuture = SettableFuture.create(); |
236 | 250 | listCompletableFuture.whenComplete((timescaleTsKvEntities, throwable) -> { |
237 | 251 | if (throwable != null) { |
... | ... | @@ -245,9 +259,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
245 | 259 | List<Optional<TsKvEntry>> result = new ArrayList<>(); |
246 | 260 | timescaleTsKvEntities.forEach(entity -> { |
247 | 261 | if (entity != null && entity.isNotEmpty()) { |
248 | - entity.setEntityId(entityIdStr); | |
249 | - entity.setTenantId(tenantIdStr); | |
250 | - entity.setKey(key); | |
262 | + entity.setEntityId(entityId.getId()); | |
263 | + entity.setTenantId(tenantId.getId()); | |
264 | + entity.setStrKey(key); | |
251 | 265 | result.add(Optional.of(DaoUtil.getData(entity))); |
252 | 266 | } else { |
253 | 267 | result.add(Optional.empty()); |
... | ... | @@ -260,69 +274,74 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements |
260 | 274 | }); |
261 | 275 | } |
262 | 276 | |
263 | - private CompletableFuture<List<TimescaleTsKvEntity>> switchAgregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, String entityIdStr, String tenantIdStr) { | |
277 | + private CompletableFuture<List<TimescaleTsKvEntity>> switchAgregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId, UUID tenantId) { | |
264 | 278 | switch (aggregation) { |
265 | 279 | case AVG: |
266 | - return findAvg(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); | |
280 | + return findAvg(key, startTs, endTs, timeBucket, entityId, tenantId); | |
267 | 281 | case MAX: |
268 | - return findMax(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); | |
282 | + return findMax(key, startTs, endTs, timeBucket, entityId, tenantId); | |
269 | 283 | case MIN: |
270 | - return findMin(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); | |
284 | + return findMin(key, startTs, endTs, timeBucket, entityId, tenantId); | |
271 | 285 | case SUM: |
272 | - return findSum(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); | |
286 | + return findSum(key, startTs, endTs, timeBucket, entityId, tenantId); | |
273 | 287 | case COUNT: |
274 | - return findCount(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); | |
288 | + return findCount(key, startTs, endTs, timeBucket, entityId, tenantId); | |
275 | 289 | default: |
276 | 290 | throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); |
277 | 291 | } |
278 | 292 | } |
279 | 293 | |
280 | - private CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { | |
294 | + private CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { | |
295 | + Integer keyId = getOrSaveKeyId(key); | |
281 | 296 | return aggregationRepository.findAvg( |
282 | - tenantIdStr, | |
283 | - entityIdStr, | |
284 | - key, | |
297 | + tenantId, | |
298 | + entityId, | |
299 | + keyId, | |
285 | 300 | timeBucket, |
286 | 301 | startTs, |
287 | 302 | endTs); |
288 | 303 | } |
289 | 304 | |
290 | - private CompletableFuture<List<TimescaleTsKvEntity>> findMax(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { | |
305 | + private CompletableFuture<List<TimescaleTsKvEntity>> findMax(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { | |
306 | + Integer keyId = getOrSaveKeyId(key); | |
291 | 307 | return aggregationRepository.findMax( |
292 | - tenantIdStr, | |
293 | - entityIdStr, | |
294 | - key, | |
308 | + tenantId, | |
309 | + entityId, | |
310 | + keyId, | |
295 | 311 | timeBucket, |
296 | 312 | startTs, |
297 | 313 | endTs); |
298 | 314 | } |
299 | 315 | |
300 | - private CompletableFuture<List<TimescaleTsKvEntity>> findMin(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { | |
316 | + private CompletableFuture<List<TimescaleTsKvEntity>> findMin(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { | |
317 | + Integer keyId = getOrSaveKeyId(key); | |
301 | 318 | return aggregationRepository.findMin( |
302 | - tenantIdStr, | |
303 | - entityIdStr, | |
304 | - key, | |
319 | + tenantId, | |
320 | + entityId, | |
321 | + keyId, | |
305 | 322 | timeBucket, |
306 | 323 | startTs, |
307 | 324 | endTs); |
308 | 325 | |
309 | 326 | } |
310 | 327 | |
311 | - private CompletableFuture<List<TimescaleTsKvEntity>> findSum(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { | |
328 | + private CompletableFuture<List<TimescaleTsKvEntity>> findSum(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { | |
329 | + Integer keyId = getOrSaveKeyId(key); | |
312 | 330 | return aggregationRepository.findSum( |
313 | - tenantIdStr, | |
314 | - entityIdStr, | |
315 | - key, | |
331 | + tenantId, | |
332 | + entityId, | |
333 | + keyId, | |
316 | 334 | timeBucket, |
317 | 335 | startTs, |
318 | 336 | endTs); |
319 | 337 | } |
320 | 338 | |
321 | - private CompletableFuture<List<TimescaleTsKvEntity>> findCount(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { | |
339 | + private CompletableFuture<List<TimescaleTsKvEntity>> findCount(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { | |
340 | + Integer keyId = getOrSaveKeyId(key); | |
322 | 341 | return aggregationRepository.findCount( |
323 | - tenantIdStr, | |
324 | - entityIdStr, | |
325 | - key, | |
342 | + tenantId, | |
343 | + entityId, | |
344 | + keyId, | |
326 | 345 | timeBucket, |
327 | 346 | startTs, |
328 | 347 | endTs); | ... | ... |
... | ... | @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity; |
26 | 26 | import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
27 | 27 | |
28 | 28 | import java.util.List; |
29 | +import java.util.UUID; | |
29 | 30 | |
30 | 31 | @TimescaleDBTsDao |
31 | 32 | public interface TsKvTimescaleRepository extends CrudRepository<TimescaleTsKvEntity, TimescaleTsKvCompositeKey> { |
... | ... | @@ -35,31 +36,21 @@ public interface TsKvTimescaleRepository extends CrudRepository<TimescaleTsKvEnt |
35 | 36 | "AND tskv.key = :entityKey " + |
36 | 37 | "AND tskv.ts > :startTs AND tskv.ts <= :endTs") |
37 | 38 | List<TimescaleTsKvEntity> findAllWithLimit( |
38 | - @Param("tenantId") String tenantId, | |
39 | - @Param("entityId") String entityId, | |
40 | - @Param("entityKey") String key, | |
39 | + @Param("tenantId") UUID tenantId, | |
40 | + @Param("entityId") UUID entityId, | |
41 | + @Param("entityKey") int key, | |
41 | 42 | @Param("startTs") long startTs, |
42 | 43 | @Param("endTs") long endTs, Pageable pageable); |
43 | 44 | |
44 | - @Query(value = "SELECT tskv.tenant_id as tenant_id, tskv.entity_id as entity_id, tskv.key as key, last(tskv.ts,tskv.ts) as ts," + | |
45 | - " last(tskv.bool_v, tskv.ts) as bool_v, last(tskv.str_v, tskv.ts) as str_v," + | |
46 | - " last(tskv.long_v, tskv.ts) as long_v, last(tskv.dbl_v, tskv.ts) as dbl_v" + | |
47 | - " FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS varchar) " + | |
48 | - "AND tskv.entity_id = cast(:entityId AS varchar) " + | |
49 | - "GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key", nativeQuery = true) | |
50 | - List<TimescaleTsKvEntity> findAllLatestValues( | |
51 | - @Param("tenantId") String tenantId, | |
52 | - @Param("entityId") String entityId); | |
53 | - | |
54 | 45 | @Transactional |
55 | 46 | @Modifying |
56 | 47 | @Query("DELETE FROM TimescaleTsKvEntity tskv WHERE tskv.tenantId = :tenantId " + |
57 | 48 | "AND tskv.entityId = :entityId " + |
58 | 49 | "AND tskv.key = :entityKey " + |
59 | 50 | "AND tskv.ts > :startTs AND tskv.ts <= :endTs") |
60 | - void delete(@Param("tenantId") String tenantId, | |
61 | - @Param("entityId") String entityId, | |
62 | - @Param("entityKey") String key, | |
51 | + void delete(@Param("tenantId") UUID tenantId, | |
52 | + @Param("entityId") UUID entityId, | |
53 | + @Param("entityKey") int key, | |
63 | 54 | @Param("startTs") long startTs, |
64 | 55 | @Param("endTs") long endTs); |
65 | 56 | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/JpaTimeseriesDao.java
deleted
100644 → 0
1 | -/** | |
2 | - * Copyright © 2016-2020 The Thingsboard Authors | |
3 | - * | |
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | - * you may not use this file except in compliance with the License. | |
6 | - * You may obtain a copy of the License at | |
7 | - * | |
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | - * | |
10 | - * Unless required by applicable law or agreed to in writing, software | |
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | - * See the License for the specific language governing permissions and | |
14 | - * limitations under the License. | |
15 | - */ | |
16 | -package org.thingsboard.server.dao.sqlts.ts; | |
17 | - | |
18 | -import com.google.common.collect.Lists; | |
19 | -import com.google.common.util.concurrent.FutureCallback; | |
20 | -import com.google.common.util.concurrent.Futures; | |
21 | -import com.google.common.util.concurrent.ListenableFuture; | |
22 | -import com.google.common.util.concurrent.SettableFuture; | |
23 | -import lombok.extern.slf4j.Slf4j; | |
24 | -import org.springframework.beans.factory.annotation.Autowired; | |
25 | -import org.springframework.beans.factory.annotation.Value; | |
26 | -import org.springframework.data.domain.PageRequest; | |
27 | -import org.springframework.data.domain.Sort; | |
28 | -import org.springframework.stereotype.Component; | |
29 | -import org.thingsboard.server.common.data.UUIDConverter; | |
30 | -import org.thingsboard.server.common.data.id.EntityId; | |
31 | -import org.thingsboard.server.common.data.id.TenantId; | |
32 | -import org.thingsboard.server.common.data.kv.Aggregation; | |
33 | -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; | |
34 | -import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; | |
35 | -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; | |
36 | -import org.thingsboard.server.common.data.kv.StringDataEntry; | |
37 | -import org.thingsboard.server.common.data.kv.TsKvEntry; | |
38 | -import org.thingsboard.server.dao.DaoUtil; | |
39 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; | |
40 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestCompositeKey; | |
41 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; | |
42 | -import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; | |
43 | -import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; | |
44 | -import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; | |
45 | -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository; | |
46 | -import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; | |
47 | -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; | |
48 | -import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; | |
49 | -import org.thingsboard.server.dao.timeseries.TimeseriesDao; | |
50 | -import org.thingsboard.server.dao.util.SqlTsDao; | |
51 | - | |
52 | -import javax.annotation.Nullable; | |
53 | -import javax.annotation.PostConstruct; | |
54 | -import javax.annotation.PreDestroy; | |
55 | -import java.util.ArrayList; | |
56 | -import java.util.List; | |
57 | -import java.util.Optional; | |
58 | -import java.util.concurrent.CompletableFuture; | |
59 | -import java.util.concurrent.ExecutionException; | |
60 | -import java.util.stream.Collectors; | |
61 | - | |
62 | -import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; | |
63 | - | |
64 | - | |
65 | -@Component | |
66 | -@Slf4j | |
67 | -@SqlTsDao | |
68 | -public class JpaTimeseriesDao extends AbstractSqlTimeseriesDao implements TimeseriesDao { | |
69 | - | |
70 | - @Autowired | |
71 | - private TsKvRepository tsKvRepository; | |
72 | - | |
73 | - @Autowired | |
74 | - private TsKvLatestRepository tsKvLatestRepository; | |
75 | - | |
76 | - @Autowired | |
77 | - private AbstractTimeseriesInsertRepository insertRepository; | |
78 | - | |
79 | - @Autowired | |
80 | - private AbstractLatestInsertRepository insertLatestRepository; | |
81 | - | |
82 | - @Autowired | |
83 | - ScheduledLogExecutorComponent logExecutor; | |
84 | - | |
85 | - @Value("${sql.ts.batch_size:1000}") | |
86 | - private int tsBatchSize; | |
87 | - | |
88 | - @Value("${sql.ts.batch_max_delay:100}") | |
89 | - private long tsMaxDelay; | |
90 | - | |
91 | - @Value("${sql.ts.stats_print_interval_ms:1000}") | |
92 | - private long tsStatsPrintIntervalMs; | |
93 | - | |
94 | - @Value("${sql.ts_latest.batch_size:1000}") | |
95 | - private int tsLatestBatchSize; | |
96 | - | |
97 | - @Value("${sql.ts_latest.batch_max_delay:100}") | |
98 | - private long tsLatestMaxDelay; | |
99 | - | |
100 | - @Value("${sql.ts_latest.stats_print_interval_ms:1000}") | |
101 | - private long tsLatestStatsPrintIntervalMs; | |
102 | - | |
103 | - private TbSqlBlockingQueue<TsKvEntity> tsQueue; | |
104 | - private TbSqlBlockingQueue<TsKvLatestEntity> tsLatestQueue; | |
105 | - | |
106 | - | |
107 | - @PostConstruct | |
108 | - private void init() { | |
109 | - TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() | |
110 | - .logName("TS") | |
111 | - .batchSize(tsBatchSize) | |
112 | - .maxDelay(tsMaxDelay) | |
113 | - .statsPrintIntervalMs(tsStatsPrintIntervalMs) | |
114 | - .build(); | |
115 | - tsQueue = new TbSqlBlockingQueue<>(tsParams); | |
116 | - tsQueue.init(logExecutor, v -> insertRepository.saveOrUpdate(v)); | |
117 | - | |
118 | - TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() | |
119 | - .logName("TS Latest") | |
120 | - .batchSize(tsLatestBatchSize) | |
121 | - .maxDelay(tsLatestMaxDelay) | |
122 | - .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) | |
123 | - .build(); | |
124 | - tsLatestQueue = new TbSqlBlockingQueue<>(tsLatestParams); | |
125 | - tsLatestQueue.init(logExecutor, v -> insertLatestRepository.saveOrUpdate(v)); | |
126 | - } | |
127 | - | |
128 | - @PreDestroy | |
129 | - private void destroy() { | |
130 | - if (tsQueue != null) { | |
131 | - tsQueue.destroy(); | |
132 | - } | |
133 | - | |
134 | - if (tsLatestQueue != null) { | |
135 | - tsLatestQueue.destroy(); | |
136 | - } | |
137 | - } | |
138 | - | |
139 | - @Override | |
140 | - public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) { | |
141 | - return processFindAllAsync(tenantId, entityId, queries); | |
142 | - } | |
143 | - | |
144 | - protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { | |
145 | - if (query.getAggregation() == Aggregation.NONE) { | |
146 | - return findAllAsyncWithLimit(entityId, query); | |
147 | - } else { | |
148 | - long stepTs = query.getStartTs(); | |
149 | - List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>(); | |
150 | - while (stepTs < query.getEndTs()) { | |
151 | - long startTs = stepTs; | |
152 | - long endTs = stepTs + query.getInterval(); | |
153 | - long ts = startTs + (endTs - startTs) / 2; | |
154 | - futures.add(findAndAggregateAsync(tenantId, entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); | |
155 | - stepTs = endTs; | |
156 | - } | |
157 | - return getTskvEntriesFuture(Futures.allAsList(futures)); | |
158 | - } | |
159 | - } | |
160 | - | |
161 | - private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { | |
162 | - List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>(); | |
163 | - String entityIdStr = fromTimeUUID(entityId.getId()); | |
164 | - switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures, entityIdStr); | |
165 | - | |
166 | - SettableFuture<TsKvEntity> listenableFuture = SettableFuture.create(); | |
167 | - | |
168 | - CompletableFuture<List<TsKvEntity>> entities = | |
169 | - CompletableFuture.allOf(entitiesFutures.toArray(new CompletableFuture[entitiesFutures.size()])) | |
170 | - .thenApply(v -> entitiesFutures.stream() | |
171 | - .map(CompletableFuture::join) | |
172 | - .collect(Collectors.toList())); | |
173 | - | |
174 | - entities.whenComplete((tsKvEntities, throwable) -> { | |
175 | - if (throwable != null) { | |
176 | - listenableFuture.setException(throwable); | |
177 | - } else { | |
178 | - TsKvEntity result = null; | |
179 | - for (TsKvEntity entity : tsKvEntities) { | |
180 | - if (entity.isNotEmpty()) { | |
181 | - result = entity; | |
182 | - break; | |
183 | - } | |
184 | - } | |
185 | - listenableFuture.set(result); | |
186 | - } | |
187 | - }); | |
188 | - return Futures.transform(listenableFuture, entity -> { | |
189 | - if (entity != null && entity.isNotEmpty()) { | |
190 | - entity.setEntityId(entityIdStr); | |
191 | - entity.setEntityType(entityId.getEntityType()); | |
192 | - entity.setKey(key); | |
193 | - entity.setTs(ts); | |
194 | - return Optional.of(DaoUtil.getData(entity)); | |
195 | - } else { | |
196 | - return Optional.empty(); | |
197 | - } | |
198 | - }); | |
199 | - } | |
200 | - | |
201 | - private void switchAgregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) { | |
202 | - switch (aggregation) { | |
203 | - case AVG: | |
204 | - findAvg(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); | |
205 | - break; | |
206 | - case MAX: | |
207 | - findMax(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); | |
208 | - break; | |
209 | - case MIN: | |
210 | - findMin(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); | |
211 | - break; | |
212 | - case SUM: | |
213 | - findSum(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); | |
214 | - break; | |
215 | - case COUNT: | |
216 | - findCount(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); | |
217 | - break; | |
218 | - default: | |
219 | - throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); | |
220 | - } | |
221 | - } | |
222 | - | |
223 | - private void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) { | |
224 | - entitiesFutures.add(tsKvRepository.findCount( | |
225 | - entityIdStr, | |
226 | - entityId.getEntityType(), | |
227 | - key, | |
228 | - startTs, | |
229 | - endTs)); | |
230 | - } | |
231 | - | |
232 | - private void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) { | |
233 | - entitiesFutures.add(tsKvRepository.findSum( | |
234 | - entityIdStr, | |
235 | - entityId.getEntityType(), | |
236 | - key, | |
237 | - startTs, | |
238 | - endTs)); | |
239 | - } | |
240 | - | |
241 | - private void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) { | |
242 | - entitiesFutures.add(tsKvRepository.findStringMin( | |
243 | - entityIdStr, | |
244 | - entityId.getEntityType(), | |
245 | - key, | |
246 | - startTs, | |
247 | - endTs)); | |
248 | - entitiesFutures.add(tsKvRepository.findNumericMin( | |
249 | - entityIdStr, | |
250 | - entityId.getEntityType(), | |
251 | - key, | |
252 | - startTs, | |
253 | - endTs)); | |
254 | - } | |
255 | - | |
256 | - private void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) { | |
257 | - entitiesFutures.add(tsKvRepository.findStringMax( | |
258 | - entityIdStr, | |
259 | - entityId.getEntityType(), | |
260 | - key, | |
261 | - startTs, | |
262 | - endTs)); | |
263 | - entitiesFutures.add(tsKvRepository.findNumericMax( | |
264 | - entityIdStr, | |
265 | - entityId.getEntityType(), | |
266 | - key, | |
267 | - startTs, | |
268 | - endTs)); | |
269 | - } | |
270 | - | |
271 | - private void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures, String entityIdStr) { | |
272 | - entitiesFutures.add(tsKvRepository.findAvg( | |
273 | - entityIdStr, | |
274 | - entityId.getEntityType(), | |
275 | - key, | |
276 | - startTs, | |
277 | - endTs)); | |
278 | - } | |
279 | - | |
280 | - private ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { | |
281 | - return Futures.immediateFuture( | |
282 | - DaoUtil.convertDataList( | |
283 | - tsKvRepository.findAllWithLimit( | |
284 | - fromTimeUUID(entityId.getId()), | |
285 | - entityId.getEntityType(), | |
286 | - query.getKey(), | |
287 | - query.getStartTs(), | |
288 | - query.getEndTs(), | |
289 | - new PageRequest(0, query.getLimit(), | |
290 | - new Sort(Sort.Direction.fromString( | |
291 | - query.getOrderBy()), "ts"))))); | |
292 | - } | |
293 | - | |
294 | - @Override | |
295 | - public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) { | |
296 | - TsKvLatestCompositeKey compositeKey = | |
297 | - new TsKvLatestCompositeKey( | |
298 | - entityId.getEntityType(), | |
299 | - fromTimeUUID(entityId.getId()), | |
300 | - key); | |
301 | - Optional<TsKvLatestEntity> entry = tsKvLatestRepository.findById(compositeKey); | |
302 | - TsKvEntry result; | |
303 | - if (entry.isPresent()) { | |
304 | - result = DaoUtil.getData(entry.get()); | |
305 | - } else { | |
306 | - result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); | |
307 | - } | |
308 | - return Futures.immediateFuture(result); | |
309 | - } | |
310 | - | |
311 | - @Override | |
312 | - public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) { | |
313 | - return Futures.immediateFuture( | |
314 | - DaoUtil.convertDataList(Lists.newArrayList( | |
315 | - tsKvLatestRepository.findAllByEntityTypeAndEntityId( | |
316 | - entityId.getEntityType(), | |
317 | - UUIDConverter.fromTimeUUID(entityId.getId()))))); | |
318 | - } | |
319 | - | |
320 | - @Override | |
321 | - public ListenableFuture<Void> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { | |
322 | - TsKvEntity entity = new TsKvEntity(); | |
323 | - entity.setEntityType(entityId.getEntityType()); | |
324 | - entity.setEntityId(fromTimeUUID(entityId.getId())); | |
325 | - entity.setTs(tsKvEntry.getTs()); | |
326 | - entity.setKey(tsKvEntry.getKey()); | |
327 | - entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); | |
328 | - entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); | |
329 | - entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); | |
330 | - entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); | |
331 | - log.trace("Saving entity: {}", entity); | |
332 | - return tsQueue.add(entity); | |
333 | - } | |
334 | - | |
335 | - @Override | |
336 | - public ListenableFuture<Void> savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { | |
337 | - return Futures.immediateFuture(null); | |
338 | - } | |
339 | - | |
340 | - @Override | |
341 | - public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { | |
342 | - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); | |
343 | - latestEntity.setEntityType(entityId.getEntityType()); | |
344 | - latestEntity.setEntityId(fromTimeUUID(entityId.getId())); | |
345 | - latestEntity.setTs(tsKvEntry.getTs()); | |
346 | - latestEntity.setKey(tsKvEntry.getKey()); | |
347 | - latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); | |
348 | - latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); | |
349 | - latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); | |
350 | - latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); | |
351 | - return tsLatestQueue.add(latestEntity); | |
352 | - } | |
353 | - | |
354 | - @Override | |
355 | - public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
356 | - return service.submit(() -> { | |
357 | - tsKvRepository.delete( | |
358 | - fromTimeUUID(entityId.getId()), | |
359 | - entityId.getEntityType(), | |
360 | - query.getKey(), | |
361 | - query.getStartTs(), | |
362 | - query.getEndTs()); | |
363 | - return null; | |
364 | - }); | |
365 | - } | |
366 | - | |
367 | - @Override | |
368 | - public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
369 | - ListenableFuture<TsKvEntry> latestFuture = findLatest(tenantId, entityId, query.getKey()); | |
370 | - | |
371 | - ListenableFuture<Boolean> booleanFuture = Futures.transform(latestFuture, tsKvEntry -> { | |
372 | - long ts = tsKvEntry.getTs(); | |
373 | - return ts > query.getStartTs() && ts <= query.getEndTs(); | |
374 | - }, service); | |
375 | - | |
376 | - ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { | |
377 | - if (isRemove) { | |
378 | - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); | |
379 | - latestEntity.setEntityType(entityId.getEntityType()); | |
380 | - latestEntity.setEntityId(fromTimeUUID(entityId.getId())); | |
381 | - latestEntity.setKey(query.getKey()); | |
382 | - return service.submit(() -> { | |
383 | - tsKvLatestRepository.delete(latestEntity); | |
384 | - return null; | |
385 | - }); | |
386 | - } | |
387 | - return Futures.immediateFuture(null); | |
388 | - }, service); | |
389 | - | |
390 | - final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>(); | |
391 | - Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() { | |
392 | - @Override | |
393 | - public void onSuccess(@Nullable Void result) { | |
394 | - if (query.getRewriteLatestIfDeleted()) { | |
395 | - ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { | |
396 | - if (isRemove) { | |
397 | - return getNewLatestEntryFuture(tenantId, entityId, query); | |
398 | - } | |
399 | - return Futures.immediateFuture(null); | |
400 | - }, service); | |
401 | - | |
402 | - try { | |
403 | - resultFuture.set(savedLatestFuture.get()); | |
404 | - } catch (InterruptedException | ExecutionException e) { | |
405 | - log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); | |
406 | - } | |
407 | - } else { | |
408 | - resultFuture.set(null); | |
409 | - } | |
410 | - } | |
411 | - | |
412 | - @Override | |
413 | - public void onFailure(Throwable t) { | |
414 | - log.warn("[{}] Failed to process remove of the latest value", entityId, t); | |
415 | - } | |
416 | - }); | |
417 | - return resultFuture; | |
418 | - } | |
419 | - | |
420 | - private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
421 | - ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query); | |
422 | - return Futures.transformAsync(future, entryList -> { | |
423 | - if (entryList.size() == 1) { | |
424 | - return saveLatest(tenantId, entityId, entryList.get(0)); | |
425 | - } else { | |
426 | - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); | |
427 | - } | |
428 | - return Futures.immediateFuture(null); | |
429 | - }, service); | |
430 | - } | |
431 | - | |
432 | - @Override | |
433 | - public ListenableFuture<Void> removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { | |
434 | - return service.submit(() -> null); | |
435 | - } | |
436 | -} | |
\ No newline at end of file |
dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java
deleted
100644 → 0
1 | -/** | |
2 | - * Copyright © 2016-2020 The Thingsboard Authors | |
3 | - * | |
4 | - * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | - * you may not use this file except in compliance with the License. | |
6 | - * You may obtain a copy of the License at | |
7 | - * | |
8 | - * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | - * | |
10 | - * Unless required by applicable law or agreed to in writing, software | |
11 | - * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | - * See the License for the specific language governing permissions and | |
14 | - * limitations under the License. | |
15 | - */ | |
16 | -package org.thingsboard.server.dao.sqlts.ts; | |
17 | - | |
18 | -import org.springframework.jdbc.core.BatchPreparedStatementSetter; | |
19 | -import org.springframework.stereotype.Repository; | |
20 | -import org.springframework.transaction.annotation.Transactional; | |
21 | -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; | |
22 | -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; | |
23 | -import org.thingsboard.server.dao.util.PsqlDao; | |
24 | -import org.thingsboard.server.dao.util.SqlTsDao; | |
25 | - | |
26 | -import java.sql.PreparedStatement; | |
27 | -import java.sql.SQLException; | |
28 | -import java.sql.Types; | |
29 | -import java.util.List; | |
30 | - | |
31 | -@SqlTsDao | |
32 | -@PsqlDao | |
33 | -@Repository | |
34 | -@Transactional | |
35 | -public class PsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepository<TsKvEntity> { | |
36 | - | |
37 | - private static final String TS_KV_CONSTRAINT = "(entity_type, entity_id, key, ts)"; | |
38 | - | |
39 | - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS); | |
40 | - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS); | |
41 | - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS); | |
42 | - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS); | |
43 | - | |
44 | - private static final String INSERT_OR_UPDATE = | |
45 | - "INSERT INTO ts_kv (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?) " + | |
46 | - "ON CONFLICT (entity_type, entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;"; | |
47 | - | |
48 | - @Override | |
49 | - public void saveOrUpdate(TsKvEntity entity) { | |
50 | - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); | |
51 | - } | |
52 | - | |
53 | - @Override | |
54 | - protected void saveOrUpdateBoolean(TsKvEntity entity, String query) { | |
55 | - entityManager.createNativeQuery(query) | |
56 | - .setParameter("entity_type", entity.getEntityType().name()) | |
57 | - .setParameter("entity_id", entity.getEntityId()) | |
58 | - .setParameter("key", entity.getKey()) | |
59 | - .setParameter("ts", entity.getTs()) | |
60 | - .setParameter("bool_v", entity.getBooleanValue()) | |
61 | - .executeUpdate(); | |
62 | - } | |
63 | - | |
64 | - @Override | |
65 | - protected void saveOrUpdateString(TsKvEntity entity, String query) { | |
66 | - entityManager.createNativeQuery(query) | |
67 | - .setParameter("entity_type", entity.getEntityType().name()) | |
68 | - .setParameter("entity_id", entity.getEntityId()) | |
69 | - .setParameter("key", entity.getKey()) | |
70 | - .setParameter("ts", entity.getTs()) | |
71 | - .setParameter("str_v", replaceNullChars(entity.getStrValue())) | |
72 | - .executeUpdate(); | |
73 | - } | |
74 | - | |
75 | - @Override | |
76 | - protected void saveOrUpdateLong(TsKvEntity entity, String query) { | |
77 | - entityManager.createNativeQuery(query) | |
78 | - .setParameter("entity_type", entity.getEntityType().name()) | |
79 | - .setParameter("entity_id", entity.getEntityId()) | |
80 | - .setParameter("key", entity.getKey()) | |
81 | - .setParameter("ts", entity.getTs()) | |
82 | - .setParameter("long_v", entity.getLongValue()) | |
83 | - .executeUpdate(); | |
84 | - } | |
85 | - | |
86 | - @Override | |
87 | - protected void saveOrUpdateDouble(TsKvEntity entity, String query) { | |
88 | - entityManager.createNativeQuery(query) | |
89 | - .setParameter("entity_type", entity.getEntityType().name()) | |
90 | - .setParameter("entity_id", entity.getEntityId()) | |
91 | - .setParameter("key", entity.getKey()) | |
92 | - .setParameter("ts", entity.getTs()) | |
93 | - .setParameter("dbl_v", entity.getDoubleValue()) | |
94 | - .executeUpdate(); | |
95 | - } | |
96 | - | |
97 | - @Override | |
98 | - public void saveOrUpdate(List<TsKvEntity> entities) { | |
99 | - jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { | |
100 | - @Override | |
101 | - public void setValues(PreparedStatement ps, int i) throws SQLException { | |
102 | - TsKvEntity tsKvEntity = entities.get(i); | |
103 | - ps.setString(1, tsKvEntity.getEntityType().name()); | |
104 | - ps.setString(2, tsKvEntity.getEntityId()); | |
105 | - ps.setString(3, tsKvEntity.getKey()); | |
106 | - ps.setLong(4, tsKvEntity.getTs()); | |
107 | - | |
108 | - if (tsKvEntity.getBooleanValue() != null) { | |
109 | - ps.setBoolean(5, tsKvEntity.getBooleanValue()); | |
110 | - ps.setBoolean(9, tsKvEntity.getBooleanValue()); | |
111 | - } else { | |
112 | - ps.setNull(5, Types.BOOLEAN); | |
113 | - ps.setNull(9, Types.BOOLEAN); | |
114 | - } | |
115 | - | |
116 | - ps.setString(6, replaceNullChars(tsKvEntity.getStrValue())); | |
117 | - ps.setString(10, replaceNullChars(tsKvEntity.getStrValue())); | |
118 | - | |
119 | - | |
120 | - if (tsKvEntity.getLongValue() != null) { | |
121 | - ps.setLong(7, tsKvEntity.getLongValue()); | |
122 | - ps.setLong(11, tsKvEntity.getLongValue()); | |
123 | - } else { | |
124 | - ps.setNull(7, Types.BIGINT); | |
125 | - ps.setNull(11, Types.BIGINT); | |
126 | - } | |
127 | - | |
128 | - if (tsKvEntity.getDoubleValue() != null) { | |
129 | - ps.setDouble(8, tsKvEntity.getDoubleValue()); | |
130 | - ps.setDouble(12, tsKvEntity.getDoubleValue()); | |
131 | - } else { | |
132 | - ps.setNull(8, Types.DOUBLE); | |
133 | - ps.setNull(12, Types.DOUBLE); | |
134 | - } | |
135 | - } | |
136 | - | |
137 | - @Override | |
138 | - public int getBatchSize() { | |
139 | - return entities.size(); | |
140 | - } | |
141 | - }); | |
142 | - } | |
143 | -} | |
\ No newline at end of file |
... | ... | @@ -97,7 +97,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem |
97 | 97 | @Value("${cassandra.query.set_null_values_enabled}") |
98 | 98 | private boolean setNullValuesEnabled; |
99 | 99 | |
100 | - private TsPartitionDate tsFormat; | |
100 | + private NoSqlTsPartitionDate tsFormat; | |
101 | 101 | |
102 | 102 | private PreparedStatement partitionInsertStmt; |
103 | 103 | private PreparedStatement partitionInsertTtlStmt; |
... | ... | @@ -120,7 +120,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem |
120 | 120 | super.startExecutor(); |
121 | 121 | if (!isInstall()) { |
122 | 122 | getFetchStmt(Aggregation.NONE, DESC_ORDER); |
123 | - Optional<TsPartitionDate> partition = TsPartitionDate.parse(partitioning); | |
123 | + Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning); | |
124 | 124 | if (partition.isPresent()) { |
125 | 125 | tsFormat = partition.get(); |
126 | 126 | } else { | ... | ... |
dao/src/main/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDate.java
renamed from
dao/src/main/java/org/thingsboard/server/dao/timeseries/TsPartitionDate.java
... | ... | @@ -21,7 +21,7 @@ import java.time.temporal.ChronoUnit; |
21 | 21 | import java.time.temporal.TemporalUnit; |
22 | 22 | import java.util.Optional; |
23 | 23 | |
24 | -public enum TsPartitionDate { | |
24 | +public enum NoSqlTsPartitionDate { | |
25 | 25 | |
26 | 26 | MINUTES("yyyy-MM-dd-HH-mm", ChronoUnit.MINUTES), HOURS("yyyy-MM-dd-HH", ChronoUnit.HOURS), DAYS("yyyy-MM-dd", ChronoUnit.DAYS), MONTHS("yyyy-MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS),INDEFINITE("",ChronoUnit.FOREVER); |
27 | 27 | |
... | ... | @@ -29,7 +29,7 @@ public enum TsPartitionDate { |
29 | 29 | private final transient TemporalUnit truncateUnit; |
30 | 30 | public final static LocalDateTime EPOCH_START = LocalDateTime.ofEpochSecond(0,0, ZoneOffset.UTC); |
31 | 31 | |
32 | - TsPartitionDate(String pattern, TemporalUnit truncateUnit) { | |
32 | + NoSqlTsPartitionDate(String pattern, TemporalUnit truncateUnit) { | |
33 | 33 | this.pattern = pattern; |
34 | 34 | this.truncateUnit = truncateUnit; |
35 | 35 | } |
... | ... | @@ -56,10 +56,10 @@ public enum TsPartitionDate { |
56 | 56 | } |
57 | 57 | } |
58 | 58 | |
59 | - public static Optional<TsPartitionDate> parse(String name) { | |
60 | - TsPartitionDate partition = null; | |
59 | + public static Optional<NoSqlTsPartitionDate> parse(String name) { | |
60 | + NoSqlTsPartitionDate partition = null; | |
61 | 61 | if (name != null) { |
62 | - for (TsPartitionDate partitionDate : TsPartitionDate.values()) { | |
62 | + for (NoSqlTsPartitionDate partitionDate : NoSqlTsPartitionDate.values()) { | |
63 | 63 | if (partitionDate.name().equalsIgnoreCase(name)) { |
64 | 64 | partition = partitionDate; |
65 | 65 | break; | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.timeseries; | |
17 | + | |
18 | +import lombok.Data; | |
19 | + | |
20 | +import java.text.SimpleDateFormat; | |
21 | +import java.util.Date; | |
22 | + | |
23 | +@Data | |
24 | +public class PsqlPartition { | |
25 | + | |
26 | + private static final String TABLE_REGEX = "ts_kv_"; | |
27 | + | |
28 | + private long start; | |
29 | + private long end; | |
30 | + private String partitionDate; | |
31 | + private String query; | |
32 | + | |
33 | + public PsqlPartition(long start, long end, String pattern) { | |
34 | + this.start = start; | |
35 | + this.end = end; | |
36 | + this.partitionDate = new SimpleDateFormat(pattern).format(new Date(start)); | |
37 | + this.query = createStatement(start, end, partitionDate); | |
38 | + } | |
39 | + | |
40 | + private String createStatement(long start, long end, String partitionDate) { | |
41 | + return "CREATE TABLE IF NOT EXISTS " + TABLE_REGEX + partitionDate + " PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (" + start + ") TO (" + end + ")"; | |
42 | + } | |
43 | +} | |
\ No newline at end of file | ... | ... |
1 | +/** | |
2 | + * Copyright © 2016-2020 The Thingsboard Authors | |
3 | + * | |
4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | + * you may not use this file except in compliance with the License. | |
6 | + * You may obtain a copy of the License at | |
7 | + * | |
8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
9 | + * | |
10 | + * Unless required by applicable law or agreed to in writing, software | |
11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | + * See the License for the specific language governing permissions and | |
14 | + * limitations under the License. | |
15 | + */ | |
16 | +package org.thingsboard.server.dao.timeseries; | |
17 | + | |
18 | +import java.time.LocalDateTime; | |
19 | +import java.time.ZoneOffset; | |
20 | +import java.time.temporal.ChronoUnit; | |
21 | +import java.time.temporal.TemporalUnit; | |
22 | +import java.util.Optional; | |
23 | + | |
24 | +public enum SqlTsPartitionDate { | |
25 | + | |
26 | + MINUTES("yyyy_MM_dd_HH_mm", ChronoUnit.MINUTES), HOURS("yyyy_MM_dd_HH", ChronoUnit.HOURS), DAYS("yyyy_MM_dd", ChronoUnit.DAYS), MONTHS("yyyy_MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS), INDEFINITE("indefinite", ChronoUnit.FOREVER); | |
27 | + | |
28 | + private final String pattern; | |
29 | + private final transient TemporalUnit truncateUnit; | |
30 | + public final static LocalDateTime EPOCH_START = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC); | |
31 | + | |
32 | + SqlTsPartitionDate(String pattern, TemporalUnit truncateUnit) { | |
33 | + this.pattern = pattern; | |
34 | + this.truncateUnit = truncateUnit; | |
35 | + } | |
36 | + | |
37 | + public String getPattern() { | |
38 | + return pattern; | |
39 | + } | |
40 | + | |
41 | + public TemporalUnit getTruncateUnit() { | |
42 | + return truncateUnit; | |
43 | + } | |
44 | + | |
45 | + public LocalDateTime trancateTo(LocalDateTime time) { | |
46 | + switch (this) { | |
47 | + case MINUTES: | |
48 | + return time.truncatedTo(ChronoUnit.MINUTES); | |
49 | + case HOURS: | |
50 | + return time.truncatedTo(ChronoUnit.HOURS); | |
51 | + case DAYS: | |
52 | + return time.truncatedTo(ChronoUnit.DAYS); | |
53 | + case MONTHS: | |
54 | + return time.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1); | |
55 | + case YEARS: | |
56 | + return time.truncatedTo(ChronoUnit.DAYS).withDayOfYear(1); | |
57 | + case INDEFINITE: | |
58 | + return EPOCH_START; | |
59 | + default: | |
60 | + throw new RuntimeException("Failed to parse partitioning property!"); | |
61 | + } | |
62 | + } | |
63 | + | |
64 | + public LocalDateTime plusTo(LocalDateTime time) { | |
65 | + switch (this) { | |
66 | + case MINUTES: | |
67 | + return time.plusMinutes(1); | |
68 | + case HOURS: | |
69 | + return time.plusHours(1); | |
70 | + case DAYS: | |
71 | + return time.plusDays(1); | |
72 | + case MONTHS: | |
73 | + return time.plusMonths(1); | |
74 | + case YEARS: | |
75 | + return time.plusYears(1); | |
76 | + default: | |
77 | + throw new RuntimeException("Failed to parse partitioning property!"); | |
78 | + } | |
79 | + } | |
80 | + | |
81 | + public static Optional<SqlTsPartitionDate> parse(String name) { | |
82 | + SqlTsPartitionDate partition = null; | |
83 | + if (name != null) { | |
84 | + for (SqlTsPartitionDate partitionDate : SqlTsPartitionDate.values()) { | |
85 | + if (partitionDate.name().equalsIgnoreCase(name)) { | |
86 | + partition = partitionDate; | |
87 | + break; | |
88 | + } | |
89 | + } | |
90 | + } | |
91 | + return Optional.of(partition); | |
92 | + } | |
93 | +} | |
\ No newline at end of file | ... | ... |
... | ... | @@ -17,7 +17,25 @@ |
17 | 17 | CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; |
18 | 18 | |
19 | 19 | CREATE TABLE IF NOT EXISTS tenant_ts_kv ( |
20 | - tenant_id varchar(31) NOT NULL, | |
20 | + tenant_id uuid NOT NULL, | |
21 | + entity_id uuid NOT NULL, | |
22 | + key int NOT NULL, | |
23 | + ts bigint NOT NULL, | |
24 | + bool_v boolean, | |
25 | + str_v varchar(10000000), | |
26 | + long_v bigint, | |
27 | + dbl_v double precision, | |
28 | + CONSTRAINT ts_kv_pkey PRIMARY KEY (tenant_id, entity_id, key, ts) | |
29 | +); | |
30 | + | |
31 | +CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( | |
32 | + key varchar(255) NOT NULL, | |
33 | + key_id serial UNIQUE, | |
34 | + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) | |
35 | +); | |
36 | + | |
37 | +CREATE TABLE IF NOT EXISTS ts_kv_latest ( | |
38 | + entity_type varchar(255) NOT NULL, | |
21 | 39 | entity_id varchar(31) NOT NULL, |
22 | 40 | key varchar(255) NOT NULL, |
23 | 41 | ts bigint NOT NULL, |
... | ... | @@ -25,7 +43,7 @@ CREATE TABLE IF NOT EXISTS tenant_ts_kv ( |
25 | 43 | str_v varchar(10000000), |
26 | 44 | long_v bigint, |
27 | 45 | dbl_v double precision, |
28 | - CONSTRAINT ts_kv_pkey PRIMARY KEY (tenant_id, entity_id, key, ts) | |
46 | + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_type, entity_id, key) | |
29 | 47 | ); |
30 | 48 | |
31 | 49 | SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => 86400000, if_not_exists => true); |
\ No newline at end of file | ... | ... |
dao/src/main/resources/sql/schema-ts-hsql.sql
renamed from
dao/src/main/resources/sql/schema-ts.sql
1 | +-- | |
2 | +-- Copyright © 2016-2020 The Thingsboard Authors | |
3 | +-- | |
4 | +-- Licensed under the Apache License, Version 2.0 (the "License"); | |
5 | +-- you may not use this file except in compliance with the License. | |
6 | +-- You may obtain a copy of the License at | |
7 | +-- | |
8 | +-- http://www.apache.org/licenses/LICENSE-2.0 | |
9 | +-- | |
10 | +-- Unless required by applicable law or agreed to in writing, software | |
11 | +-- distributed under the License is distributed on an "AS IS" BASIS, | |
12 | +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | +-- See the License for the specific language governing permissions and | |
14 | +-- limitations under the License. | |
15 | +-- | |
16 | + | |
17 | +CREATE TABLE IF NOT EXISTS ts_kv ( | |
18 | + entity_id uuid NOT NULL, | |
19 | + key int NOT NULL, | |
20 | + ts bigint NOT NULL, | |
21 | + bool_v boolean, | |
22 | + str_v varchar(10000000), | |
23 | + long_v bigint, | |
24 | + dbl_v double precision | |
25 | +) PARTITION BY RANGE (ts); | |
26 | + | |
27 | +CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( | |
28 | + key varchar(255) NOT NULL, | |
29 | + key_id serial UNIQUE, | |
30 | + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) | |
31 | +); | |
32 | + | |
33 | +CREATE TABLE IF NOT EXISTS ts_kv_latest ( | |
34 | + entity_type varchar(255) NOT NULL, | |
35 | + entity_id varchar(31) NOT NULL, | |
36 | + key varchar(255) NOT NULL, | |
37 | + ts bigint NOT NULL, | |
38 | + bool_v boolean, | |
39 | + str_v varchar(10000000), | |
40 | + long_v bigint, | |
41 | + dbl_v double precision, | |
42 | + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_type, entity_id, key) | |
43 | +); | |
\ No newline at end of file | ... | ... |
... | ... | @@ -30,7 +30,7 @@ import org.springframework.test.context.support.DirtiesContextTestExecutionListe |
30 | 30 | * Created by Valerii Sosliuk on 4/22/2017. |
31 | 31 | */ |
32 | 32 | @RunWith(SpringRunner.class) |
33 | -@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, JpaDbunitTestConfig.class}) | |
33 | +@ContextConfiguration(classes = {JpaDaoConfig.class, HsqlTsDaoConfig.class, JpaDbunitTestConfig.class}) | |
34 | 34 | @TestPropertySource("classpath:sql-test.properties") |
35 | 35 | @TestExecutionListeners({ |
36 | 36 | DependencyInjectionTestExecutionListener.class, | ... | ... |
... | ... | @@ -30,9 +30,23 @@ public class JpaDaoTestSuite { |
30 | 30 | |
31 | 31 | @ClassRule |
32 | 32 | public static CustomSqlUnit sqlUnit = new CustomSqlUnit( |
33 | - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
33 | + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
34 | 34 | "sql/drop-all-tables.sql", |
35 | 35 | "sql-test.properties" |
36 | 36 | ); |
37 | 37 | |
38 | +// @ClassRule | |
39 | +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( | |
40 | +// Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
41 | +// "sql/drop-all-tables.sql", | |
42 | +// "sql-test.properties" | |
43 | +// ); | |
44 | + | |
45 | +// @ClassRule | |
46 | +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( | |
47 | +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-timescale-idx.sql", "sql/schema-entities.sql", "sql/system-data.sql"), | |
48 | +// "sql/timescale/drop-all-tables.sql", | |
49 | +// "sql-test.properties" | |
50 | +// ); | |
51 | + | |
38 | 52 | } | ... | ... |
... | ... | @@ -30,9 +30,23 @@ public class SqlDaoServiceTestSuite { |
30 | 30 | |
31 | 31 | @ClassRule |
32 | 32 | public static CustomSqlUnit sqlUnit = new CustomSqlUnit( |
33 | - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), | |
33 | + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), | |
34 | 34 | "sql/drop-all-tables.sql", |
35 | 35 | "sql-test.properties" |
36 | 36 | ); |
37 | 37 | |
38 | +// @ClassRule | |
39 | +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( | |
40 | +// Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), | |
41 | +// "sql/drop-all-tables.sql", | |
42 | +// "sql-test.properties" | |
43 | +// ); | |
44 | + | |
45 | +// @ClassRule | |
46 | +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( | |
47 | +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-timescale-idx.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), | |
48 | +// "sql/timescale/drop-all-tables.sql", | |
49 | +// "sql-test.properties" | |
50 | +// ); | |
51 | + | |
38 | 52 | } | ... | ... |
... | ... | @@ -2,7 +2,8 @@ database.ts.type=sql |
2 | 2 | database.entities.type=sql |
3 | 3 | |
4 | 4 | sql.ts_inserts_executor_type=fixed |
5 | -sql.ts_inserts_fixed_thread_pool_size=10 | |
5 | +sql.ts_inserts_fixed_thread_pool_size=200 | |
6 | +sql.ts_key_value_partitioning=MONTHS | |
6 | 7 | |
7 | 8 | spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true |
8 | 9 | spring.jpa.show-sql=false |
... | ... | @@ -13,4 +14,23 @@ spring.datasource.username=sa |
13 | 14 | spring.datasource.password= |
14 | 15 | spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false |
15 | 16 | spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver |
16 | -spring.datasource.hikari.maximumPoolSize = 50 | |
\ No newline at end of file | ||
17 | +spring.datasource.hikari.maximumPoolSize = 50 | |
18 | + | |
19 | +#database.ts.type=timescale | |
20 | +#database.ts.type=sql | |
21 | +#database.entities.type=sql | |
22 | +# | |
23 | +#sql.ts_inserts_executor_type=fixed | |
24 | +#sql.ts_inserts_fixed_thread_pool_size=200 | |
25 | +#sql.ts_key_value_partitioning=MONTHS | |
26 | +# | |
27 | +#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true | |
28 | +#spring.jpa.show-sql=false | |
29 | +#spring.jpa.hibernate.ddl-auto=none | |
30 | +#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect | |
31 | +# | |
32 | +#spring.datasource.username=postgres | |
33 | +#spring.datasource.password=postgres | |
34 | +#spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest | |
35 | +#spring.datasource.driverClassName=org.postgresql.Driver | |
36 | +#spring.datasource.hikari.maximumPoolSize = 50 | |
\ No newline at end of file | ... | ... |
... | ... | @@ -13,6 +13,7 @@ DROP TABLE IF EXISTS relation; |
13 | 13 | DROP TABLE IF EXISTS tb_user; |
14 | 14 | DROP TABLE IF EXISTS tenant; |
15 | 15 | DROP TABLE IF EXISTS tenant_ts_kv; |
16 | +DROP TABLE IF EXISTS ts_kv_latest; | |
16 | 17 | DROP TABLE IF EXISTS user_credentials; |
17 | 18 | DROP TABLE IF EXISTS widget_type; |
18 | 19 | DROP TABLE IF EXISTS widgets_bundle; | ... | ... |
... | ... | @@ -20,10 +20,10 @@ firstlaunch=${DATA_FOLDER}/.firstlaunch |
20 | 20 | if [ ! -d ${PGDATA} ]; then |
21 | 21 | mkdir -p ${PGDATA} |
22 | 22 | chown -R postgres:postgres ${PGDATA} |
23 | - su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl initdb -U postgres' | |
23 | + su postgres -c '/usr/lib/postgresql/10/bin/pg_ctl initdb -U postgres' | |
24 | 24 | fi |
25 | 25 | |
26 | -su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -l /var/log/postgres/postgres.log -w start' | |
26 | +su postgres -c '/usr/lib/postgresql/10/bin/pg_ctl -l /var/log/postgres/postgres.log -w start' | |
27 | 27 | |
28 | 28 | if [ ! -f ${firstlaunch} ]; then |
29 | 29 | su postgres -c 'psql -U postgres -d postgres -c "CREATE DATABASE thingsboard"' | ... | ... |