Revert "Feature Purging System"

Reverting selective purge feature until we add tests

collector/collector/config.py

@@ -25,20 +25,9 @@ class Config(object):
     SQLALCHEMY_TRACK_MODIFICATIONS = True
     LOG_FILE = 'handler.log'
 
-    # When PURGE_OLD_RECORDS == True then a purging system of old records will
-    # be triggered daily. If this variable is not present, then no purging will be done.
-    # If the purging system is enabled, then the following two variables must be set
-    PURGE_OLD_RECORDS = True
-    # The maximum retention time in days for records stored in the database
-    # which do not match the filters in PURGE_FILTERED_RECORDS.
-    # Use 0 to avoid deletion of all unfiltered records.
-    MAX_DAYS_KEEP_UNFILTERED_RECORDS = 35
-    # See config_example.py for details about PURGE_FILTERED_RECORDS
-    PURGE_FILTERED_RECORDS = {
-        "classification": {
-            "org.clearlinux/hello/world": 1,
-        }
-    }
+    # The maximum retention time for records stored in the database, measured
+    # from the time received by the `collector` app.
+    MAX_WEEK_KEEP_RECORDS = 5
 
     # The Telemetry ID (TID) accepted by this `collector` app. The ID should be a
     # random UUID, generated with (for example) `uuidgen`. The default value

collector/collector/config_example.py

@@ -1,77 +0,0 @@
-#
-# Copyright 2015-2017 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import logging
-
-
-class Config(object):
-    DEBUG = False
-    TESTING = False
-    LOG_LEVEL = logging.ERROR
-    SQLALCHEMY_DATABASE_URI = 'postgres://postgres:@@db_password@@@localhost/telemdb'
-    SQLALCHEMY_TRACK_MODIFICATIONS = True
-    LOG_FILE = 'handler.log'
-
-    # When PURGE_OLD_RECORDS == True then a purging system of old records will
-    # be triggered daily. If this variable is not present, then no purging will be done.
-    # If the purging system is enabled, then the following two variables must be set
-    PURGE_OLD_RECORDS = True
-    # The maximum retention time in days for records stored in the database
-    # which do not match the filters in PURGE_FILTERED_RECORDS.
-    # Use 0 to avoid deletion of all unfiltered records.
-    MAX_DAYS_KEEP_UNFILTERED_RECORDS = 35
-    # A dictionary in the following format:
-    # {
-    #     "<field_name>": {
-    #         "<field_value>": <max_days_to_keep>,
-    #         ...
-    #         },
-    #     ...
-    # }
-    # Currently supported fields to filter:
-    # [
-    #     'severity',
-    #     'classification',
-    #     'machine_id'
-    # ]
-    # Use 0 to avoid deletion of records that matches the filter.
-    # If you do not want to filter records to delete, just set an empty dict '{}'
-    PURGE_FILTERED_RECORDS = {
-        "severity": {
-            1: 5,
-            4: 0
-        },
-        "classification": {
-            "org.clearlinux/mce/*": 0,
-            "org.clearlinux/hello/world": 1,
-            "org.clearlinux/heartbeat/ping": 1,
-        }
-    }
-
-    # The Telemetry ID (TID) accepted by this `collector` app. The ID should be a
-    # random UUID, generated with (for example) `uuidgen`. The default value
-    # set here is used for records from the Clear Linux OS for Intel
-    # Architecture.
-    TELEMETRY_ID = "6907c830-eed9-4ce9-81ae-76daf8d88f0f"
-
-
-class Testing(Config):
-    TESTING = True
-    SQLALCHEMY_DATABASE_URI = 'postgres://postgres:@@db_password@@@localhost/testdb'
-    SQLALCHEMY_TRACK_MODIFICATIONS = True
-
-
-# vi: ts=4 et sw=4 sts=4

collector/collector/config_local.py

@@ -31,6 +31,10 @@ class Config(object):
     SQLALCHEMY_TRACK_MODIFICATIONS = True
     LOG_FILE = 'handler.log'
 
+    # The maximum retention time for records stored in the database, measured
+    # from the time received by the `collector` app.
+    MAX_WEEK_KEEP_RECORDS = 5
+
 
 class Testing(Config):
     TESTING = True

collector/collector/purge.py

@@ -21,15 +21,12 @@ try:
     import uwsgi
     from uwsgidecorators import cron
 
-    PURGE_OLD_RECORDS = app.config.get("PURGE_OLD_RECORDS", True)
-
     # Runs cron job at 4:30 every day
     @cron(30, 4, -1, -1, -1, target='spooler')
     def purge_task(signum):
-        if PURGE_OLD_RECORDS:
-            app.logger.info("Running cron job for purging records")
-            with app.app_context():
-                Record.delete_records()
+        app.logger.info("Running cron job for purging records")
+        with app.app_context():
+            Record.delete_records()
 
 except ImportError:
         app.logger.info("Import error for uwsgi")

shared/model.py

@@ -26,8 +26,7 @@ from . import app
 
 db = SQLAlchemy(app)
 
-MAX_DAYS_KEEP_UNFILTERED_RECORDS = app.config.get("MAX_DAYS_KEEP_UNFILTERED_RECORDS", 35)
-PURGE_FILTERED_RECORDS = app.config.get("PURGE_FILTERED_RECORDS", {})
+MAX_WEEK_KEEP_RECORDS = app.config.get("MAX_WEEK_KEEP_RECORDS", 5)
 
 
 class Classification(db.Model):
@@ -292,39 +291,14 @@ class Record(db.Model):
     @staticmethod
     def delete_records():
         try:
-            def purge_field(field):
-                for name in PURGE_FILTERED_RECORDS[field].keys():
-                    if PURGE_FILTERED_RECORDS[field][name]:
-                        age = time() - PURGE_FILTERED_RECORDS[field][name] * 24 * 60 * 60
-                        q = db.session.query(Record)
-                        if field == 'classification':
-                            q = q.join(Record.classification)
-                            q = q.filter(Classification.classification.like(name))
-                        else:
-                            q = q.filter(getattr(Record, field) == name)
-                        q = q.filter(Record.tsp_server < age)
-                        count = db.session.query(Record).filter(Record.id.in_(q)).delete(synchronize_session=False)
-                        print("Deleted {} {} records".format(count, name))
-            for field in PURGE_FILTERED_RECORDS.keys():
-                purge_field(field)
-            if MAX_DAYS_KEEP_UNFILTERED_RECORDS:
-                unfiltered_age = time() - MAX_DAYS_KEEP_UNFILTERED_RECORDS * 24 * 60 * 60
-                q = db.session.query(Record.id)
-                for field in PURGE_FILTERED_RECORDS.keys():
-                    if field == 'classification':
-                        q = q.join(Record.classification)
-                        for classification in PURGE_FILTERED_RECORDS[field].keys():
-                            q = q.filter(~Classification.classification.like(classification))
-                    else:
-                        for name in PURGE_FILTERED_RECORDS[field].keys():
-                            q = q.filter(getattr(Record, field) != name)
-                q = q.filter(Record.tsp_server < unfiltered_age)
-                count = db.session.query(Record).filter(Record.id.in_(q)).delete(synchronize_session=False)
-                print("Deleted {} old records".format(count))
+            sec_weeks = MAX_WEEK_KEEP_RECORDS * 7 * 24 * 60 * 60
+            current_time = time()
+            time_weeks_ago = current_time - sec_weeks
+            q = db.session.query(Record).filter(Record.tsp_server < time_weeks_ago)
+            count = q.delete(synchronize_session=False)
             db.session.commit()
-        except Exception as e:
-            app.logger.error("Record purging failed")
-            app.logger.error(e)
+            print("Deleted {} old records".format(count))
+        except:
             db.session.rollback()
 
     @staticmethod

telemetryui/telemetryui/config.py

@@ -28,6 +28,7 @@ class Config(object):
     SECRET_KEY = '@@flask_key@@'
     RECORDS_PER_PAGE = 50
     MAX_RECORDS_PER_PAGE = 1000
+    MAX_WEEK_KEEP_RECORDS = 5
 
 
 class Testing(Config):