diff --git a/collector/collector/config.py b/collector/collector/config.py index 921d72d..81cb285 100644 --- a/collector/collector/config.py +++ b/collector/collector/config.py @@ -25,20 +25,9 @@ class Config(object): SQLALCHEMY_TRACK_MODIFICATIONS = True LOG_FILE = 'handler.log' - # When PURGE_OLD_RECORDS == True then a purging system of old records will - # be triggered daily. If this variable is not present, then no purging will be done. - # If the purging system is enabled, then the following two variables must be set - PURGE_OLD_RECORDS = True - # The maximum retention time in days for records stored in the database - # which do not match the filters in PURGE_FILTERED_RECORDS. - # Use 0 to avoid deletion of all unfiltered records. - MAX_DAYS_KEEP_UNFILTERED_RECORDS = 35 - # See config_example.py for details about PURGE_FILTERED_RECORDS - PURGE_FILTERED_RECORDS = { - "classification": { - "org.clearlinux/hello/world": 1, - } - } + # The maximum retention time for records stored in the database, measured + # from the time received by the `collector` app. + MAX_WEEK_KEEP_RECORDS = 5 # The Telemetry ID (TID) accepted by this `collector` app. The ID should be a # random UUID, generated with (for example) `uuidgen`. The default value diff --git a/collector/collector/config_example.py b/collector/collector/config_example.py deleted file mode 100644 index 8a59637..0000000 --- a/collector/collector/config_example.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright 2015-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - - -class Config(object): - DEBUG = False - TESTING = False - LOG_LEVEL = logging.ERROR - SQLALCHEMY_DATABASE_URI = 'postgres://postgres:@@db_password@@@localhost/telemdb' - SQLALCHEMY_TRACK_MODIFICATIONS = True - LOG_FILE = 'handler.log' - - # When PURGE_OLD_RECORDS == True then a purging system of old records will - # be triggered daily. If this variable is not present, then no purging will be done. - # If the purging system is enabled, then the following two variables must be set - PURGE_OLD_RECORDS = True - # The maximum retention time in days for records stored in the database - # which do not match the filters in PURGE_FILTERED_RECORDS. - # Use 0 to avoid deletion of all unfiltered records. - MAX_DAYS_KEEP_UNFILTERED_RECORDS = 35 - # A dictionary in the following format: - # { - # "": { - # "": , - # ... - # }, - # ... - # } - # Currently supported fields to filter: - # [ - # 'severity', - # 'classification', - # 'machine_id' - # ] - # Use 0 to avoid deletion of records that matches the filter. - # If you do not want to filter records to delete, just set an empty dict '{}' - PURGE_FILTERED_RECORDS = { - "severity": { - 1: 5, - 4: 0 - }, - "classification": { - "org.clearlinux/mce/*": 0, - "org.clearlinux/hello/world": 1, - "org.clearlinux/heartbeat/ping": 1, - } - } - - # The Telemetry ID (TID) accepted by this `collector` app. The ID should be a - # random UUID, generated with (for example) `uuidgen`. The default value - # set here is used for records from the Clear Linux OS for Intel - # Architecture. - TELEMETRY_ID = "6907c830-eed9-4ce9-81ae-76daf8d88f0f" - - -class Testing(Config): - TESTING = True - SQLALCHEMY_DATABASE_URI = 'postgres://postgres:@@db_password@@@localhost/testdb' - SQLALCHEMY_TRACK_MODIFICATIONS = True - - -# vi: ts=4 et sw=4 sts=4 diff --git a/collector/collector/config_local.py b/collector/collector/config_local.py index 27b3a81..2d11cb3 100644 --- a/collector/collector/config_local.py +++ b/collector/collector/config_local.py @@ -31,6 +31,10 @@ class Config(object): SQLALCHEMY_TRACK_MODIFICATIONS = True LOG_FILE = 'handler.log' + # The maximum retention time for records stored in the database, measured + # from the time received by the `collector` app. + MAX_WEEK_KEEP_RECORDS = 5 + class Testing(Config): TESTING = True diff --git a/collector/collector/purge.py b/collector/collector/purge.py index cb6808a..e14fd1b 100644 --- a/collector/collector/purge.py +++ b/collector/collector/purge.py @@ -21,15 +21,12 @@ try: import uwsgi from uwsgidecorators import cron - PURGE_OLD_RECORDS = app.config.get("PURGE_OLD_RECORDS", True) - # Runs cron job at 4:30 every day @cron(30, 4, -1, -1, -1, target='spooler') def purge_task(signum): - if PURGE_OLD_RECORDS: - app.logger.info("Running cron job for purging records") - with app.app_context(): - Record.delete_records() + app.logger.info("Running cron job for purging records") + with app.app_context(): + Record.delete_records() except ImportError: app.logger.info("Import error for uwsgi") diff --git a/shared/model.py b/shared/model.py index abfa56d..ff155ad 100644 --- a/shared/model.py +++ b/shared/model.py @@ -26,8 +26,7 @@ from . import app db = SQLAlchemy(app) -MAX_DAYS_KEEP_UNFILTERED_RECORDS = app.config.get("MAX_DAYS_KEEP_UNFILTERED_RECORDS", 35) -PURGE_FILTERED_RECORDS = app.config.get("PURGE_FILTERED_RECORDS", {}) +MAX_WEEK_KEEP_RECORDS = app.config.get("MAX_WEEK_KEEP_RECORDS", 5) class Classification(db.Model): @@ -292,39 +291,14 @@ class Record(db.Model): @staticmethod def delete_records(): try: - def purge_field(field): - for name in PURGE_FILTERED_RECORDS[field].keys(): - if PURGE_FILTERED_RECORDS[field][name]: - age = time() - PURGE_FILTERED_RECORDS[field][name] * 24 * 60 * 60 - q = db.session.query(Record) - if field == 'classification': - q = q.join(Record.classification) - q = q.filter(Classification.classification.like(name)) - else: - q = q.filter(getattr(Record, field) == name) - q = q.filter(Record.tsp_server < age) - count = db.session.query(Record).filter(Record.id.in_(q)).delete(synchronize_session=False) - print("Deleted {} {} records".format(count, name)) - for field in PURGE_FILTERED_RECORDS.keys(): - purge_field(field) - if MAX_DAYS_KEEP_UNFILTERED_RECORDS: - unfiltered_age = time() - MAX_DAYS_KEEP_UNFILTERED_RECORDS * 24 * 60 * 60 - q = db.session.query(Record.id) - for field in PURGE_FILTERED_RECORDS.keys(): - if field == 'classification': - q = q.join(Record.classification) - for classification in PURGE_FILTERED_RECORDS[field].keys(): - q = q.filter(~Classification.classification.like(classification)) - else: - for name in PURGE_FILTERED_RECORDS[field].keys(): - q = q.filter(getattr(Record, field) != name) - q = q.filter(Record.tsp_server < unfiltered_age) - count = db.session.query(Record).filter(Record.id.in_(q)).delete(synchronize_session=False) - print("Deleted {} old records".format(count)) + sec_weeks = MAX_WEEK_KEEP_RECORDS * 7 * 24 * 60 * 60 + current_time = time() + time_weeks_ago = current_time - sec_weeks + q = db.session.query(Record).filter(Record.tsp_server < time_weeks_ago) + count = q.delete(synchronize_session=False) db.session.commit() - except Exception as e: - app.logger.error("Record purging failed") - app.logger.error(e) + print("Deleted {} old records".format(count)) + except: db.session.rollback() @staticmethod diff --git a/telemetryui/telemetryui/config.py b/telemetryui/telemetryui/config.py index 2f8c472..89b6f13 100644 --- a/telemetryui/telemetryui/config.py +++ b/telemetryui/telemetryui/config.py @@ -28,6 +28,7 @@ class Config(object): SECRET_KEY = '@@flask_key@@' RECORDS_PER_PAGE = 50 MAX_RECORDS_PER_PAGE = 1000 + MAX_WEEK_KEEP_RECORDS = 5 class Testing(Config):