events: performance improvements: UI fast response with latest events, ttl clean…

…up speedup (with schema update)

+--
+-- Copyright © 2016-2021 The Thingsboard Authors
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- PROCEDURE: public.cleanup_events_by_ttl(bigint, bigint, bigint)
+
+DROP PROCEDURE IF EXISTS public.cleanup_events_by_ttl(bigint, bigint, bigint);
+
+CREATE OR REPLACE PROCEDURE public.cleanup_events_by_ttl(
+	ttl bigint,
+	debug_ttl bigint,
+	INOUT deleted bigint)
+LANGUAGE 'plpgsql'
+AS $BODY$
+DECLARE
+    ttl_ts bigint;
+    debug_ttl_ts bigint;
+    ttl_deleted_count bigint DEFAULT 0;
+    debug_ttl_deleted_count bigint DEFAULT 0;
+BEGIN
+    IF ttl > 0 THEN
+        ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint;
+
+        DELETE FROM event
+		  WHERE ts < ttl_ts
+		    AND NOT event_type IN ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN', 'DEBUG_CONVERTER', 'DEBUG_INTEGRATION');
+
+		GET DIAGNOSTICS ttl_deleted_count = ROW_COUNT;
+    END IF;
+
+    IF debug_ttl > 0 THEN
+        debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint;
+
+		DELETE FROM event
+		  WHERE ts < debug_ttl_ts
+		    AND     event_type IN ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN', 'DEBUG_CONVERTER', 'DEBUG_INTEGRATION');
+
+		GET DIAGNOSTICS debug_ttl_deleted_count = ROW_COUNT;
+    END IF;
+
+    RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count;
+    RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count;
+    deleted := ttl_deleted_count + debug_ttl_deleted_count;
+END
+$BODY$;
+
+
+-- Index: idx_event_ts
+
+DROP INDEX IF EXISTS public.idx_event_ts;
+
+-- Hint: add CONCURRENTLY to CREATE INDEX query in case of more then 1 million records or during live update
+-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_ts
+CREATE INDEX IF NOT EXISTS idx_event_ts
+    ON public.event USING btree
+    (ts DESC NULLS LAST)
+    WITH (FILLFACTOR=95);
+
+COMMENT ON INDEX public.idx_event_ts
+    IS 'This index helps to delete events by TTL using timestamp';
+
+
+-- Index: idx_event_tenant_entity_type_entity_event_type_created_time_des
+
+DROP INDEX IF EXISTS public.idx_event_tenant_entity_type_entity_event_type_created_time_des;
+
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des
+CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des
+    ON public.event USING btree
+    (tenant_id ASC NULLS LAST, entity_type ASC NULLS LAST, entity_id ASC NULLS LAST, event_type ASC NULLS LAST, created_time DESC NULLS LAST)
+    WITH (FILLFACTOR=95);
+
+COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des
+    IS 'This index helps to open latest events on UI fast';
+
+-- Index: idx_event_type_entity_id
+-- Description: replaced with more suitable idx_event_tenant_entity_type_entity_event_type_created_time_des
+DROP INDEX IF EXISTS public.idx_event_type_entity_id;
\ No newline at end of file

@@ -465,6 +465,17 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
                     log.error("Failed updating schema!!!", e);
                 }
                 break;
+            case "3.3.0":
+                try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
+                    log.info("Updating schema ...");
+                    log.info("Updating indexes and procedure for event table...");
+                    schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.0", "schema_update_event.sql");
+                    loadSql(schemaUpdateFile, conn);
+                    log.info("Schema updated.");
+                } catch (Exception e) {
+                    log.error("Failed updating schema!!!", e);
+                }
+                break;
             default:
                 throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
         }

@@ -45,7 +45,7 @@ public class EventsCleanUpService extends AbstractCleanUpService {
         this.eventService = eventService;
     }
 
-    @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}")
+    @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}")
     public void cleanUp() {
         if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) {
             eventService.cleanupEvents(ttl, debugTtl);

@@ -266,7 +266,7 @@ sql:
         ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds
       events:
         enabled: "${SQL_TTL_EVENTS_ENABLED:true}"
-        execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day
+        execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:2220000}" # Number of milliseconds (max random initial delay and fixed period). # 37minutes to avoid common interval spikes
         events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds
         debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week
       edge_events:

+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.ttl;
+
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+@Slf4j
+public class EventsCleanUpServiceTest {
+
+    @Test
+    public void givenInterval_whenRandomDelay_ThenDelayInInterval() {
+        final long executionIntervalMs = 2220000; //37min
+        final long randomDelay = org.apache.commons.lang3.RandomUtils.nextLong(0, executionIntervalMs); //same as @Scheduled(initialDelayString = ...
+        log.info("randomDelay {}", randomDelay);
+        assertThat(randomDelay, greaterThanOrEqualTo(0L));
+        assertThat(randomDelay, lessThanOrEqualTo(executionIntervalMs));
+    }
+}