From edd02159df591a0d7ffca4697c427bd4b861c8ce Mon Sep 17 00:00:00 2001 From: Alex Jaramillo Date: Wed, 19 Feb 2020 18:52:42 +0000 Subject: [PATCH] docker-compose prod and test clean up Using docker-compose.yaml for testing and making it more consistent with docker-compose.prod.yaml Signed-off-by: Alex Jaramillo --- docker-compose.prod.yaml | 5 +- docker-compose.yaml | 77 ++++ docker/Dockerfile | 7 - docker/docker-compose.yml | 21 - docker/entrypoint.sh | 8 - services/production.env | 1 + services/testing.env | 10 + services/webapp/entrypoint.py | 18 + services/webapp/migrate_entrypoint.sh | 2 +- shared/crash.py | 436 ------------------ shared/model.py | 617 -------------------------- 11 files changed, 111 insertions(+), 1091 deletions(-) create mode 100644 docker-compose.yaml delete mode 100644 docker/Dockerfile delete mode 100644 docker/docker-compose.yml delete mode 100755 docker/entrypoint.sh create mode 100644 services/testing.env create mode 100644 services/webapp/entrypoint.py mode change 100644 => 100755 services/webapp/migrate_entrypoint.sh delete mode 100644 shared/crash.py delete mode 100644 shared/model.py diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index 40ee9f2..07ab07b 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -19,6 +19,7 @@ services: image: telemetry/proxy:1.0 networks: - frontend + restart: always build: context: ./services/nginx volumes: @@ -43,6 +44,7 @@ services: volumes: - "telemetry-socket-volume:/var/www/collector/socket" - "telemetry-uwsgi-logs:/var/www/collector/log" + restart: always build: context: . dockerfile: ./services/collector/Dockerfile @@ -58,6 +60,7 @@ services: volumes: - "telemetry-socket-volume:/var/www/webapp/socket" - "telemetry-uwsgi-logs:/var/www/webapp/log" + restart: always build: context: . dockerfile: ./services/webapp/Dockerfile @@ -76,7 +79,7 @@ services: FLASK_APP: /var/www/webapp/run.py working_dir: /var/www/webapp depends_on: ["db"] - command: flask db upgrade + command: /var/www/webapp/migrate_entrypoint.sh volumes: telemetry-socket-volume: diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..babde50 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,77 @@ +version: '3.7' +services: + db: + image: postgres:latest + networks: + - backend + env_file: + - ./services/testing.env + + nginx: + image: telemetry/proxy:1.0 + networks: + - frontend + volumes: + - "telemetry-socket-volume:/var/run/uwsgi/" + build: + context: ./services/nginx + ports: + - 80:80 + - 443:443 + + redis: + image: redis:latest + networks: + - backend + env_file: + - ./services/testing.env + + collector: + image: telemetry/collector:1.0 + networks: + - backend + volumes: + - "telemetry-socket-volume:/var/www/collector/socket" + build: + context: . + dockerfile: ./services/collector/Dockerfile + env_file: + - services/testing.env + depends_on: ["db", "redis"] + command: uwsgi --ini /var/www/collector/collector.ini + + webapp: + image: telemetry/webapp:1.0 + networks: + - backend + volumes: + - "telemetry-socket-volume:/var/www/webapp/socket" + build: + context: . + dockerfile: ./services/webapp/Dockerfile + env_file: + - ./services/testing.env + depends_on: ["db", "redis"] + command: uwsgi --ini /var/www/webapp/webapp.ini + + migrate: + image: telemetry/webapp:1.0 + networks: + - backend + env_file: + - ./services/testing.env + environment: + FLASK_APP: /var/www/webapp/run.py + working_dir: /var/www/webapp + depends_on: ["db"] + command: /var/www/webapp/migrate_entrypoint.sh + +volumes: + telemetry-socket-volume: + +networks: + frontend: + driver: bridge + backend: + driver: overlay + internal: true diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index a698a85..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) 2018 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -FROM clearlinux:latest - -ADD ./requirements.txt /tmp/requirements.txt -RUN swupd bundle-add sudo uwsgi nginx python-basic redis-native -RUN pip3 install -r /tmp/requirements.txt diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 52a2ef3..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: '3.1' -services: - db: - image: postgres:latest - environment: - POSTGRES_DB: telemetry - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - app: - build: - context: ${PWD} - dockerfile: ./docker/Dockerfile - environment: - FLASK_APP: /var/www/telemetry/telemetryui/run.py - depends_on: - - db - ports: - - 5000:5000 - volumes: - - ${PWD}:/var/www/telemetry - command: /usr/bin/bash /var/www/telemetry/docker/entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index 8e51a9a..0000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -export FLASK_APP=/var/www/telemetry/telemetryui/run.py -pushd /var/www/telemetry/telemetryui/ - -flask db upgrade -uwsgi --http 0.0.0.0:5000 --chdir /var/www/telemetry/telemetryui/ -w telemetryui:app \ - --spooler /var/www/telemetry/telemetryui/uwsgi-spool --py-autoreload 1 diff --git a/services/production.env b/services/production.env index 0c495b2..f445e22 100644 --- a/services/production.env +++ b/services/production.env @@ -1,4 +1,5 @@ # Database access +POSTGRES_HOSTNAME=db POSTGRES_PASSWORD= POSTGRES_USER=telemetry # Redis configuration diff --git a/services/testing.env b/services/testing.env new file mode 100644 index 0000000..8d1ee0d --- /dev/null +++ b/services/testing.env @@ -0,0 +1,10 @@ +# Database access +POSTGRES_HOSTNAME=db +POSTGRES_PASSWORD=postgres +POSTGRES_USER=telemetry +POSTGRES_DB=telemetry +PGDATA=/var/lib/postgresql/data/pgdata +# Redis configuration +REDIS_HOSTNAME=redis +REDIS_PORT=6379 +REDIS_PASSWD= diff --git a/services/webapp/entrypoint.py b/services/webapp/entrypoint.py new file mode 100644 index 0000000..2bbbfde --- /dev/null +++ b/services/webapp/entrypoint.py @@ -0,0 +1,18 @@ +import os +import psycopg2 + +CONN = "dbname='{}' user='{}' host='{}' password='{}'" + +def main(): + db_host = os.environ['POSTGRES_HOSTNAME'] + db_user = os.environ['POSTGRES_USER'] + db_passwd = os.environ['POSTGRES_PASSWORD'] + try: + conn = psycopg2.connect(CONN.format('telemetry', db_user, db_host, db_passwd)) + conn.close() + except psycopg2.OperationalError as op_error: + print("could not connect to server: Connection refused") + exit(1) + +if __name__ == '__main__': + main() diff --git a/services/webapp/migrate_entrypoint.sh b/services/webapp/migrate_entrypoint.sh old mode 100644 new mode 100755 index 714b7bf..93e096c --- a/services/webapp/migrate_entrypoint.sh +++ b/services/webapp/migrate_entrypoint.sh @@ -2,7 +2,7 @@ set -e -until PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOSTNAME -U "postgres" -c '\q'; do +until python3 /var/www/webapp/entrypoint.py; do >&2 echo "Postgres is unavailable - sleeping" sleep 1 done diff --git a/shared/crash.py b/shared/crash.py deleted file mode 100644 index cfe77a4..0000000 --- a/shared/crash.py +++ /dev/null @@ -1,436 +0,0 @@ -# -# Copyright 2015-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import math -from operator import itemgetter -from collections import namedtuple -import subprocess -import re -from .model import Record, GuiltyBlacklist -from . import app - -try: - from uwsgidecorators import spool -except ImportError: - def spool(f): - f.spool = f - return f - -filters = [] - -# Groups for the frame_pattern below -# 1 - frame number + one space -# 2 - function name + optional arguments -# 3 - rest of the line -# 4 - module name (inside the []) -# 5 - optional frame source file and line number info - -# TODO: The current c++filt logic depends on properly subsituting c++filt -# output for the function name. Thus, it is very, very important to keep a -# capture group that extends from the function name to the end of the frame as -# long as this logic remains the same. Probably better to rework the code to -# *not* destructively overwrite the backtrace field. Maybe store the filtered -# output in a different field. - -frame_pattern = "^(#\d+ )(.+)( - \[(.*)\](.*))$" - -backtrace_classes = [ - 'org.clearlinux/crash/clr', - 'org.clearlinux/kernel/bug', - 'org.clearlinux/kernel/stackoverflow', - 'org.clearlinux/kernel/warning' -] - -other_classes = [ - 'org.clearlinux/crash/unknown', - 'org.clearlinux/crash/clr-build', - 'org.clearlinux/crash/error' -] - -CRASH_PAGE_SIZE = 10 - -def get_all_classes(): - return backtrace_classes + other_classes - - -def get_backtrace_classes(): - return backtrace_classes - - -def get_other_classes(): - return other_classes - - -def is_crash_classification(klass): - return (klass in backtrace_classes) and True or False - - -def is_blacklisted(function, module): - funcmod = (function, module) - if funcmod in filters: - return True - - return False - - -def demangle_backtrace(bt): - new_bt = [] - # FIXME: c++filt should be replaced before demangle_backtrace is used again - prog = '/usr/bin/c++filt' - - frame_regex = re.compile(frame_pattern) - lines = bt.splitlines() - - for line in lines: - m = frame_regex.match(line) - if m: - func = m.group(2) - - # A frame with missing symbols is a special case, so skip it - if func == '???': - new_bt.append(line) - continue - - # FIXME: this logic will break once the crash probe starts sending - # function argument values; make this more generic! - if func[-2:] == '()': - # The crash probe adds the () to the function name, but c++filt - # cannot demangle a symbol with the () suffix - func_name = func[:-2] - else: - # Assume already demangled, or this is from a kernel crash record - new_bt.append(line) - continue - - try: - new_func = subprocess.check_output([prog, func_name], universal_newlines=True) - except: - new_bt.append(line) - continue - - # c++filt adds a trailing newline to the output - new_func = new_func.rstrip() - - # Restore () if this was not a mangled symbol - if new_func == func_name: - new_func = func_name + '()' - - repl_str = r'\1{}\3'.format(new_func) - new_line = frame_regex.sub(repl_str, line) - new_bt.append(new_line) - else: - new_bt.append(line) - - return '\n'.join(new_bt) - - -def find_guilty(backtrace): - frame_regex = re.compile(frame_pattern) - - guilty = {} - - lines = backtrace.splitlines() - - first_unknown = None - in_backtrace = False - prev_frame = None - found_match = False - found_unknown = False - - # Begin guilty detection process - for line in lines[1:]: - m = frame_regex.match(line) - - if m: - # Either this is the first frame of the backtrace, or we are still - # iterating through the backtrace. - in_backtrace = True - - func = m.group(2) - mod = m.group(4) - - # Only consider blacklisted function/module pairs as a last resort. - # It's likely that the blacklisted pairs will never be chosen as - # worthy candidates... if they are, the guilty blacklist may be - # filtering too much. - if is_blacklisted(func, mod): - prev_frame = (func, mod) - continue - - # Consider the first frame without function symbols ('???') only if - # there are no function symbols for any frames lower in the stack. - if (func == '???' or func[:2] == '? ') and not found_unknown: - found_unknown = True - first_unknown = (func, mod) - prev_frame = (func, mod) - continue - elif func == '???': - # In this case, we've already encountered a frame with missing - # function symbols, so skip it, but save the info for backup. - prev_frame = (func, mod) - continue - - # If the previous three conditional checks fail, then we have found - # the best guilty candidate: it is not in the blacklist, and it has - # function symbols. - guilty['function'] = func - guilty['module'] = mod - guilty['count'] = 1 - - found_match = True - return (guilty, found_match) - - elif in_backtrace: - # We have processed the entire backtrace for the crashing thread of - # the process, but no solid guilty has been found. Since we only - # consider the crashing thread for guilty detection, stop iterating - # through the remainder of the threads at this point. - break - - # Implement a backup plan to ensure that a guilty is chosen. - if found_unknown: - # Take preference for '???' - guilty['function'] = first_unknown[0] - guilty['module'] = first_unknown[1] - guilty['count'] = 1 - found_match = True - elif prev_frame: - # Choose the previous frame as a last resort - guilty['function'] = prev_frame[0] - guilty['module'] = prev_frame[1] - guilty['count'] = 1 - found_match = True - - return (guilty, found_match) - - -def _process_guilties(args): - if isinstance(args['klass'], bytes): - klass = args['klass'].decode() - else: - klass = args['klass'] - # In case the caller does not check for proper classification, bail early - if not is_crash_classification(klass): - return - if 'id' in args: - record_id = int(args['id']) - else: - record_id = None - global filters - with app.app_context(): - crashes = Record.get_new_crash_records(classes=get_backtrace_classes(), id=record_id) - filters = GuiltyBlacklist.get_guilties() - for rec in crashes: - if rec.payload: - # TODO: re-add demangling capabilities when a better solution is found - # TODO: update the rec.payload field as well - Record.commit_guilty_changes() - g, match = find_guilty(rec.payload) - if match: - function = g['function'] - module = g['module'] - db_guilty = Record.get_guilty_for_funcmod(function, module) - if db_guilty is None: - db_guilty = Record.init_guilty(function, module) - Record.create_guilty_for_record(rec, db_guilty) - Record.set_processed_flag(rec) - - Record.commit_guilty_changes() - - -@spool -def process_guilties(args): - _process_guilties(args) - - -def process_guilties_sync(**args): - _process_guilties(args) - - -def guilty_list_per_build(guilties, page=1): - # TODO: should compute max values per build with a subquery instead - build_maxcount = {} - - buildset = set() - buildlist = [] - newlist = [] - - for g in guilties: - found_entry = False - guilty_str = g[0] + ' - [' + g[1] + ']' - build, count, guilty_id, comment = (g[2], g[3], g[4], g[5]) - for i, n in enumerate(newlist): - if guilty_str == n['guilty']: - newlist[i]['total'] += count - newlist[i]['builds'].append((build, count)) - if build in build_maxcount: - build_maxcount[build] = max(build_maxcount[build], count) - else: - build_maxcount[build] = count - found_entry = True - break - - if found_entry: - continue - - entry = {} - entry['guilty'] = guilty_str - entry['total'] = count - entry['guilty_id'] = guilty_id - entry['comment'] = comment - entry['builds'] = [] - entry['builds'].append((build, count)) - if build in build_maxcount: - build_maxcount[build] = max(build_maxcount[build], count) - else: - build_maxcount[build] = count - newlist.append(entry) - # We only care about the top 10 guilties - start = CRASH_PAGE_SIZE * (page - 1) - end = (CRASH_PAGE_SIZE * page) - newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[start:end] - for guilty in newlist: - for build in guilty['builds']: - buildset.add(build[0]) - - buildlist = list(buildset) - buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True) - - # For crashes not occuring in a particular build, provide a "0" value for - # the count. This simplifies table generation in the jinja template. - for i, g in enumerate(newlist): - builds, counts = list(zip(*g['builds'])) - counter = 0 - for b in buildlist: - if b not in builds: - newlist[i]['builds'].insert(counter, (b, "0")) - counter += 1 - - for i, b in enumerate(buildlist): - buildlist[i] = (b, build_maxcount[b]) - - buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True) - - for i, b in enumerate(newlist): - newlist[i]['builds'] = sorted(newlist[i]['builds'], key=lambda b: int(b[0]), reverse=True) - - return math.ceil(len(newlist)/CRASH_PAGE_SIZE), buildlist, newlist - - -def guilty_list_for_build(guilties, filter='overall'): - newlist = [] - - for g in guilties: - found_entry = False - guilty_str = g[0] + ' - [' + g[1] + ']' - build, count, guilty_id, comment = (g[2], g[3], g[4], g[5]) - for i, n in enumerate(newlist): - if guilty_str == n['guilty'] and filter in ['overall', build]: - newlist[i]['total'] += count - found_entry = True - break - - if found_entry: - continue - - if filter in ['overall', build]: - entry = {} - entry['guilty'] = guilty_str - entry['total'] = count - entry['guilty_id'] = guilty_id - entry['comment'] = comment - newlist.append(entry) - - # We only care about the top 10 guilties - newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:CRASH_PAGE_SIZE] - - return newlist - - -def get_all_funcmods(): - frame_regex = re.compile(frame_pattern) - funcmodset = set() - funcmodlist = [] - backtraces = Record.get_crash_backtraces(classes=get_backtrace_classes()) - - for b in backtraces: - lines = b[0].splitlines() - for line in lines: - match = frame_regex.match(line) - if match: - funcmodset.add((match.group(2), match.group(4))) - - return sorted(funcmodset) - - -def parse_backtrace(backtrace): - program_regex = re.compile('^Process: (.*)$') - pid_regex = re.compile('^PID: ([0-9]+)$') - signal_regex = re.compile('^Signal: ([0-9]+)$') - bt_header_regex = re.compile('^Backtrace \(TID ([0-9]+)\):$') - frame_regex = re.compile(frame_pattern) - - Crash = namedtuple('Crash', ['record_id', 'program', 'pid', 'signal', 'backtrace']) - program = '' - pid = '' - signal = '' - frames = [] - parsed_header = False - - lines = backtrace[0].splitlines() - for line in lines: - # Header info - match = program_regex.match(line) - if match: - program = match.group(1) - continue - match = pid_regex.match(line) - if match: - pid = match.group(1) - continue - match = signal_regex.match(line) - if match: - signal = match.group(1) - continue - - # We only care about the backtrace from the crashing thread, which - # is listed first in the payload. - match = bt_header_regex.match(line) - if match and not parsed_header: - parsed_header = True - elif match: - break - - match = frame_regex.match(line) - if match: - frames.append((match.group(2), match.group(4), match.group(5))) - - # Populate a namedtuple for convenience - record_id = backtrace[1] - c = Crash(record_id, program, pid, signal, frames) - return c - - -def explode_backtraces(classes=None, guilty_id=None, machine_id=None, build=None): - crashes = [] - backtraces = Record.get_crash_backtraces(classes, guilty_id, machine_id, build) - for b in backtraces: - crashes.append(parse_backtrace(b)) - return crashes - - -# vi: ts=4 et sw=4 sts=4 diff --git a/shared/model.py b/shared/model.py deleted file mode 100644 index b7ddb06..0000000 --- a/shared/model.py +++ /dev/null @@ -1,617 +0,0 @@ -# -# Copyright 2015-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import itertools -from flask_sqlalchemy import SQLAlchemy -from sqlalchemy.sql.expression import cast -from sqlalchemy.sql.expression import desc -from sqlalchemy.sql.expression import case -from time import time, localtime, strftime, mktime, strptime, gmtime -from distutils.version import LooseVersion - -from . import app - -db = SQLAlchemy(app) - - -class Guilty(db.Model): - __tablename__ = 'guilty' - id = db.Column(db.Integer, primary_key=True) - function = db.Column(db.String) - module = db.Column(db.String) - comment = db.Column(db.String) - hide = db.Column(db.Boolean, default=False) - - def __init__(self, func, mod): - self.function = func - self.module = mod - - @staticmethod - def update_comment(guilty_id, comment): - guilty = Guilty.query.filter_by(id=guilty_id).first() - guilty.comment = comment - db.session.commit() - - @staticmethod - def get_function(guilty_id): - guilty = Guilty.query.filter_by(id=guilty_id).first() - return guilty and guilty.function or "" - - @staticmethod - def get_module(guilty_id): - guilty = Guilty.query.filter_by(id=guilty_id).first() - return guilty and guilty.module or "" - - @staticmethod - def update_hidden(guilty_id, status): - guilty = Guilty.query.filter_by(id=guilty_id).first() - guilty.hide = status - db.session.commit() - - @staticmethod - def get_hidden_value(guilty_id): - guilty = Guilty.query.filter_by(id=guilty_id).first() - return guilty and guilty.hide or False - - @staticmethod - def get_hidden_guilties(): - q = db.session.query(Guilty.id, Guilty.function, Guilty.module) - q = q.filter(Guilty.hide == True) - q = q.order_by(Guilty.function) - return q.all() - - -class Record(db.Model): - __tablename__ = 'records' - id = db.Column(db.Integer, primary_key=True) - architecture = db.Column(db.Text) - bios_version = db.Column(db.Text, default='') - board_name = db.Column(db.Text, default='') - build = db.Column(db.Text, nullable=False) - classification = db.Column(db.Text, nullable=False) - cpu_model = db.Column(db.Text, default='') - event_id = db.Column(db.Text, default='') - external = db.Column(db.Boolean, default=False) - host_type = db.Column(db.Text, default='') - kernel_version = db.Column(db.Text, default=0) - machine_id = db.Column(db.Text, default='') - payload_version = db.Column(db.Integer) - record_version = db.Column(db.Integer, default=0) - severity = db.Column(db.Integer) - system_name = db.Column(db.Text) - timestamp_client = db.Column(db.Numeric) - timestamp_server = db.Column(db.Numeric, nullable=False) - payload = db.Column(db.Text, nullable=False) - - processed = db.Column(db.Boolean, default=False) - guilty_id = db.Column(db.Integer, db.ForeignKey('guilty.id')) - - guilty = db.Column(db.Text, default='') - guilty = db.relationship('Guilty', backref=db.backref('records', lazy='dynamic'), lazy='joined') - - def __init__(self, machine_id, host_type, severity, classification, build, architecture, kernel_version, - record_version, ts_capture, ts_reception, payload_version, system_name, - board_name, bios_version, cpu_model, event_id, external, payload): - self.machine_id = machine_id - self.host_type = host_type - self.architecture = architecture - self.classification = classification - self.build = build - self.kernel_version = kernel_version - self.record_version = record_version - self.severity = severity - self.timestamp_client = ts_capture - self.timestamp_server = ts_reception - self.payload_version = payload_version - self.system_name = system_name - self.external = external - self.board_name = board_name - self.bios_version = bios_version - self.cpu_model = cpu_model - self.event_id = event_id - self.payload = payload - - - def __repr__(self): - return "".format(self.id, self.classification, self.build, strftime("%a, %d %b %Y %H:%M:%S", localtime(self.timestamp_client))) - - def __str__(self): - return str(self.to_dict()) - - def to_dict(self): - record = { - 'id': self.id, - 'machine_id': self.machine_id, - 'machine_type': self.host_type, - 'arch': self.architecture, - 'build': self.build, - 'kernel_version': self.kernel_version, - 'ts_capture': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_client)), - 'ts_reception': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_server)), - 'severity': self.severity, - 'classification': self.classification, - 'record_version': self.record_version, - 'payload': self.payload, - 'board_name': self.board_name, - 'bios_version': self.bios_version, - 'cpu_model': self.cpu_model, - 'event_id': self.event_id, - 'external': self.external, - } - return record - - # for the exported CSV rows - def to_list(self): - record = [ - self.id, - self.external, - self.timestamp_server, - self.severity, - self.classification, - self.build, - self.machine_id, - self.payload - ] - return record - - @staticmethod - def list(): - return Record.query.all() - - @staticmethod - def create(machine_id, host_type, severity, classification, build, architecture, kernel_version, - record_version, ts_capture, ts_reception, payload_version, system_name, - board_name, bios_version, cpu_model, event_id, external, payload): - try: - record = Record(machine_id, host_type, severity, classification, build, architecture, kernel_version, - record_version, ts_capture, ts_reception, payload_version, system_name, - board_name, bios_version, cpu_model, event_id, external, payload) - db.session.add(record) - db.session.commit() - return record - except: - db.session.rollback() - raise - - @staticmethod - def query_records(build, classification, severity, machine_id, limit, - interval_sec=None, ts_capture=None, from_id=None): - records = Record.query - if build is not None: - records = records.filter_by(build=build) - if classification is not None: - records = records.filter_by(classification=classification) - if severity is not None: - records = records.filter(Record.severity == severity) - if machine_id is not None: - records = records.filter(Record.machine_id == machine_id) - if from_id is not None: - records = records.filter(Record.id >= from_id) - if ts_capture is not None: - records = records.filter(Record.timestamp_client > ts_capture) - - if interval_sec is not None: - current_time = time() - secs_in_past = current_time - interval_sec - # Due to time skew on client systems, delayed sends due to - # spooling, etc, timestamp_server works better as the reference - # timestamp. - records = records.filter(Record.timestamp_server > secs_in_past) - - records = records.order_by(Record.id.desc()) - - if limit is not None: - records = records.limit(limit) - - return records.all() - - @staticmethod - def get_record(record_id): - record = Record.query.filter_by(id=record_id).first() - return record - - @staticmethod - def filter_records(build, classification, severity, machine_id=None, system_name=None, limit=None, from_date=None, - to_date=None, payload=None, not_payload=None, data_source=None): - records = Record.query - if build is not None: - records = records.filter_by(build=build) - if classification is not None: - if isinstance(classification, list): - records = records.filter(Record.classification.in_(classification)) - else: - records = records.filter(Record.classification.like(classification)) - if severity is not None: - records = records.filter(Record.severity == severity) - if system_name is not None: - records = records.filter(Record.system_name == system_name) - if machine_id is not None: - records = records.filter(Record.machine_id == machine_id) - if from_date is not None: - from_date = mktime(strptime(from_date, "%Y-%m-%d")) - records = records.filter(Record.timestamp_client >= from_date) - if to_date is not None: - to_date = mktime(strptime(to_date, "%Y-%m-%d")) - records = records.filter(Record.timestamp_client < to_date) - if payload is not None: - records = records.filter(Record.payload.op('~')(payload)) - if not_payload is not None: - records = records.filter(~Record.payload.op('~')(not_payload)) - if data_source is not None: - if data_source == "external": - records = records.filter(Record.external == True) - elif data_source == "internal": - records = records.filter(Record.external == False) - - records = records.order_by(Record.id.desc()) - - if limit is not None: - records = records.limit(limit) - - return records - - @staticmethod - def delete_records(): - MAX_DAYS_KEEP_UNFILTERED_RECORDS = app.config.get("MAX_DAYS_KEEP_UNFILTERED_RECORDS", 35) - PURGE_FILTERED_RECORDS = app.config.get("PURGE_FILTERED_RECORDS", {}) - try: - def purge_field(field): - for name in PURGE_FILTERED_RECORDS[field].keys(): - if PURGE_FILTERED_RECORDS[field][name]: - age = time() - PURGE_FILTERED_RECORDS[field][name] * 24 * 60 * 60 - q = db.session.query(Record) - if field == 'classification': - q = q.filter(Record.classification.like(name.replace("*", "%"))) - else: - q = q.filter(getattr(Record, field) == name) - q = q.filter(Record.timestamp_server < age) - if q.all(): - count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False) - print("Deleted {} {} records".format(count, name)) - for field in PURGE_FILTERED_RECORDS.keys(): - purge_field(field) - if MAX_DAYS_KEEP_UNFILTERED_RECORDS: - unfiltered_age = time() - MAX_DAYS_KEEP_UNFILTERED_RECORDS * 24 * 60 * 60 - q = db.session.query(Record.id) - for field in PURGE_FILTERED_RECORDS.keys(): - if field == 'classification': - for classification in PURGE_FILTERED_RECORDS[field].keys(): - q = q.filter(~Record.classification.like(classification.replace("*", "%"))) - else: - for name in PURGE_FILTERED_RECORDS[field].keys(): - q = q.filter(getattr(Record, field) != name) - q = q.filter(Record.timestamp_server < unfiltered_age) - if q.all(): - count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False) - print("Deleted {} old records".format(count)) - db.session.commit() - except Exception as e: - app.logger.error("Record purging failed") - app.logger.error(e) - db.session.rollback() - - @staticmethod - def get_recordcnts_by_build(): - q = db.session.query(Record.build, db.func.count(Record.id)) - q = q.filter(Record.build.op('~')('^[0-9]+$')) - q = q.group_by(Record.build).order_by(cast(Record.build, db.Integer)).all() - return q - - @staticmethod - def get_builds(): - q = db.session.query(Record.build).distinct() - q = q.filter(Record.build.op('~')('^[0-9]+$')) - q = q.order_by(Record.build) - return sorted(q.all(), key=lambda x: LooseVersion(x[0]), reverse=True) - - @staticmethod - def get_recordcnts_by_classification(): - q = db.session.query(Record.classification, db.func.count(Record.id).label('total')) - q = q.group_by(Record.classification) - q = q.order_by(desc('total')) - return q.all() - - @staticmethod - def expand_class(D): - A, B, C = D - return ["{}/*".format(A), "{}/{}/*".format(A, B), "{}/{}/{}".format(A, B, C)] - - @staticmethod - def get_classifications(with_regex=False): - q = db.session.query(Record.classification).distinct() - if with_regex: - classes = [Record.expand_class(c[0].split('/')) for c in q.all()] - return sorted(set(itertools.chain(*classes))) - else: - return q.all() - - @staticmethod - def get_os_map(): - q = db.session.query(Record.system_name, Record.build).order_by(Record.system_name).group_by(Record.system_name, Record.build).all() - result = {} - for x in q: - result.setdefault(x[0], []).append(x[1]) - return result - - @staticmethod - def get_recordcnts_by_machine_type(): - q = db.session.query(Record.host_type, db.func.count(Record.id).label('total')) - q = q.group_by(Record.host_type) - q = q.order_by(desc('total')) - return q.all() - - @staticmethod - def get_recordcnts_by_severity(): - q = db.session.query(Record.severity, db.func.count(Record.id)).group_by(Record.severity).all() - return q - - @staticmethod - def get_crashcnts_by_class(classes=None): - q = db.session.query(Record.classification, db.func.count(Record.id)) - if classes: - q = q.filter(Record.classification.in_(classes)) - else: - q = q.filter(Record.classification.like('org.clearlinux/crash/%')) - q = q.group_by(Record.classification) - return q.all() - - @staticmethod - def get_crashcnts_by_build(classes=None): - q = db.session.query(Record.build, db.func.count(Record.id)) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.build.op('~')('^[0-9]+$')) - q = q.group_by(Record.build) - q = q.order_by(desc(cast(Record.build, db.Integer))) - q = q.limit(10) - return q.all() - - @staticmethod - def get_top_crash_guilties(classes=None): - q = db.session.query(Guilty.function, Guilty.module, Record.build, db.func.count(Record.id).label('total'), Guilty.id, Guilty.comment) - q = q.join(Record) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.build.op('~')('^[0-9][0-9]+$')) - q = q.filter(cast(Record.build, db.Integer) <= 100000) - q = q.filter(Guilty.hide == False) - q = q.group_by(Guilty.function, Guilty.module, Guilty.comment, Guilty.id, Record.build) - q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total')) - # query for records created in the last week (~ 10 Clear builds) - q = q.filter(Record.build.in_(sorted(tuple(set([x[2] for x in q.all()])), key=lambda x: int(x))[-8:])) - interval_sec = 24 * 60 * 60 * 7 - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - return q.all() - - @staticmethod - def get_new_crash_records(classes=None, id=None): - q = db.session.query(Record) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.system_name == 'clear-linux-os') - q = q.filter(Record.processed == False) - if id: - q = q.filter(Record.id == id) - return q.all() - - @staticmethod - def set_processed_flag(record): - record.processed = True - - @staticmethod - def get_guilty_for_funcmod(func, mod): - q = db.session.query(Guilty).filter_by(function=func, module=mod).first() - return q - - @staticmethod - def get_guilty_id_for_record(record_id): - q = db.session.query(Guilty.id).join(Record) - q = q.filter(Record.id == record_id) - return q.first() - - @staticmethod - def init_guilty(func, mod): - return Guilty(func, mod) - - @staticmethod - def create_guilty_for_record(record, guilty): - record.guilty = guilty - - @staticmethod - def commit_guilty_changes(): - # just commit for now - db.session.commit() - - @staticmethod - def get_crash_backtraces(classes=None, guilty_id=None, machine_id=None, build=None, most_recent=None, record_id=None): - q = db.session.query(Record.payload, Record.id) - # Short circuit if we know the record ID - if record_id: - q = q.filter(Record.id == record_id) - return q.first() - if build: - q = q.filter(Record.build == build) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.system_name == 'clear-linux-os') - if guilty_id: - q = q.filter(Record.guilty_id == guilty_id) - if machine_id: - q = q.filter(Record.machine_id == machine_id) - if most_recent: - interval_sec = 24 * 60 * 60 * int(most_recent) - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - return q.all() - - @staticmethod - def reset_processed_records(classes=None, id=None): - q = db.session.query(Record) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.system_name == 'clear-linux-os') - if id: - q = q.filter(Record.id == id) - records = q.all() - for r in records: - r.processed = False - db.session.commit() - - @staticmethod - def get_machine_ids_for_guilty(id, most_recent=None): - q = db.session.query(Record.build, Record.machine_id, db.func.count(Record.id).label('total'), Record.guilty_id) - q = q.filter(Record.guilty_id == id) - q = q.filter(Record.system_name == 'clear-linux-os') - q = q.filter(Record.build.op('~')('^[0-9][0-9]+$')) - q = q.group_by(Record.build, Record.machine_id, Record.guilty_id) - q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total')) - if most_recent: - interval_sec = 24 * 60 * 60 * int(most_recent) - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - return q.all() - - @staticmethod - def get_update_msgs(): - q = db.session.query(Record.payload) - q = q.filter(Record.classification == "org.clearlinux/swupd-client/update") - - sec_2_weeks = 24 * 60 * 60 * 7 - current_time = time() - time_2_weeks_ago = current_time - sec_2_weeks - - # query for records created in that last 2 weeks - q = q.filter(Record.timestamp_client > time_2_weeks_ago) - return q.all() - - @staticmethod - def get_swupd_msgs(most_recent=None): - q = db.session.query(Record.timestamp_client, Record.machine_id, Record.payload) - q = q.filter(Record.classification.like('org.clearlinux/swupd-client/%')) - - if most_recent: - interval_sec = 24 * 60 * 60 * int(most_recent) - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - - q = q.order_by(desc(Record.timestamp_client)) - return q - - @staticmethod - def get_heartbeat_msgs(most_recent=None): - # These two expressions are SQL CASE conditional expressions, later - # used within count(distinct ...) aggregates for the query. - internal_expr = case([(Record.external == False, Record.machine_id), ]).label('internal_count') - external_expr = case([(Record.external == True, Record.machine_id), ]).label('external_count') - - q = db.session.query(Record.build, db.func.count(db.distinct(internal_expr)), db.func.count(db.distinct(external_expr))) - q = q.filter(Record.classification == "org.clearlinux/heartbeat/ping") - q = q.filter(Record.system_name == 'clear-linux-os') - q = q.group_by(Record.build) - - if most_recent: - interval_sec = 24 * 60 * 60 * int(most_recent) - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - - q = q.order_by(cast(Record.build, db.Integer)) - return q.all() - - -class GuiltyBlacklist(db.Model): - __tablename__ = 'guilty_blacklisted' - id = db.Column(db.Integer, primary_key=True) - function = db.Column(db.String) - module = db.Column(db.String) - - def __init__(self, func, mod): - self.function = func - self.module = mod - - def __repr__(self): - return "".format(self.id, self.function, self.module) - - def __str__(self): - return str(self.to_dict()) - - def to_dict(self): - guilty = { - 'function': self.function, - 'module': self.module - } - return guilty - - @staticmethod - def add(func, mod): - try: - g = GuiltyBlacklist(func, mod) - db.session.add(g) - db.session.commit() - return g - except: - db.session.rollback() - raise - - @staticmethod - def remove(func, mod): - q = db.session.query(GuiltyBlacklist) - q = q.filter_by(function=func, module=mod) - entry = q.first() - db.session.delete(entry) - db.session.commit() - - @staticmethod - def get_guilties(): - q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module) - q = q.order_by(GuiltyBlacklist.function) - return q.all() - - @staticmethod - def exists(func, mod): - q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module) - q = q.filter_by(function=func, module=mod) - return len(q.all()) != 0 and True or False - - @staticmethod - def update(to_add, to_remove): - try: - for i in to_add: - if not GuiltyBlacklist.exists(i[0], i[1]): - g = GuiltyBlacklist(i[0], i[1]) - db.session.add(g) - for i in to_remove: - if GuiltyBlacklist.exists(i[0], i[1]): - q = db.session.query(GuiltyBlacklist) - q = q.filter_by(function=i[0], module=i[1]) - entry = q.first() - db.session.delete(entry) - db.session.commit() - except: - db.session.rollback() - raise - -# vi: ts=4 et sw=4 sts=4