diff --git a/shared/crash.py b/shared/crash.py deleted file mode 100644 index f828a09..0000000 --- a/shared/crash.py +++ /dev/null @@ -1,433 +0,0 @@ -# -# Copyright 2015-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -from operator import itemgetter -from collections import namedtuple -import subprocess -import re -from .model import Record, GuiltyBlacklist -from . import app - -try: - from uwsgidecorators import spool -except ImportError: - def spool(f): - f.spool = f - return f - -filters = [] - -# Groups for the frame_pattern below -# 1 - frame number + one space -# 2 - function name + optional arguments -# 3 - rest of the line -# 4 - module name (inside the []) -# 5 - optional frame source file and line number info - -# TODO: The current c++filt logic depends on properly subsituting c++filt -# output for the function name. Thus, it is very, very important to keep a -# capture group that extends from the function name to the end of the frame as -# long as this logic remains the same. Probably better to rework the code to -# *not* destructively overwrite the backtrace field. Maybe store the filtered -# output in a different field. - -frame_pattern = "^(#\d+ )(.+)( - \[(.*)\](.*))$" - -backtrace_classes = [ - 'org.clearlinux/crash/clr', - 'org.clearlinux/kernel/bug', - 'org.clearlinux/kernel/stackoverflow', - 'org.clearlinux/kernel/warning' -] - -other_classes = [ - 'org.clearlinux/crash/unknown', - 'org.clearlinux/crash/clr-build', - 'org.clearlinux/crash/error' -] - - -def get_all_classes(): - return backtrace_classes + other_classes - - -def get_backtrace_classes(): - return backtrace_classes - - -def get_other_classes(): - return other_classes - - -def is_crash_classification(klass): - return (klass in backtrace_classes) and True or False - - -def is_blacklisted(function, module): - funcmod = (function, module) - if funcmod in filters: - return True - - return False - - -def demangle_backtrace(bt): - new_bt = [] - prog = '/usr/bin/c++filt' - - frame_regex = re.compile(frame_pattern) - lines = bt.splitlines() - - for line in lines: - m = frame_regex.match(line) - if m: - func = m.group(2) - - # A frame with missing symbols is a special case, so skip it - if func == '???': - new_bt.append(line) - continue - - # FIXME: this logic will break once the crash probe starts sending - # function argument values; make this more generic! - if func[-2:] == '()': - # The crash probe adds the () to the function name, but c++filt - # cannot demangle a symbol with the () suffix - func_name = func[:-2] - else: - # Assume already demangled, or this is from a kernel crash record - new_bt.append(line) - continue - - try: - new_func = subprocess.check_output([prog, func_name], universal_newlines=True) - except: - new_bt.append(line) - continue - - # c++filt adds a trailing newline to the output - new_func = new_func.rstrip() - - # Restore () if this was not a mangled symbol - if new_func == func_name: - new_func = func_name + '()' - - repl_str = r'\1{}\3'.format(new_func) - new_line = frame_regex.sub(repl_str, line) - new_bt.append(new_line) - else: - new_bt.append(line) - - return '\n'.join(new_bt) - - -def find_guilty(backtrace): - frame_regex = re.compile(frame_pattern) - - guilty = {} - - lines = backtrace.splitlines() - - first_unknown = None - in_backtrace = False - prev_frame = None - found_match = False - found_unknown = False - - # Begin guilty detection process - for line in lines[1:]: - m = frame_regex.match(line) - - if m: - # Either this is the first frame of the backtrace, or we are still - # iterating through the backtrace. - in_backtrace = True - - func = m.group(2) - mod = m.group(4) - - # Only consider blacklisted function/module pairs as a last resort. - # It's likely that the blacklisted pairs will never be chosen as - # worthy candidates... if they are, the guilty blacklist may be - # filtering too much. - if is_blacklisted(func, mod): - prev_frame = (func, mod) - continue - - # Consider the first frame without function symbols ('???') only if - # there are no function symbols for any frames lower in the stack. - if (func == '???' or func[:2] == '? ') and not found_unknown: - found_unknown = True - first_unknown = (func, mod) - prev_frame = (func, mod) - continue - elif func == '???': - # In this case, we've already encountered a frame with missing - # function symbols, so skip it, but save the info for backup. - prev_frame = (func, mod) - continue - - # If the previous three conditional checks fail, then we have found - # the best guilty candidate: it is not in the blacklist, and it has - # function symbols. - guilty['function'] = func - guilty['module'] = mod - guilty['count'] = 1 - - found_match = True - return (guilty, found_match) - - elif in_backtrace: - # We have processed the entire backtrace for the crashing thread of - # the process, but no solid guilty has been found. Since we only - # consider the crashing thread for guilty detection, stop iterating - # through the remainder of the threads at this point. - break - - # Implement a backup plan to ensure that a guilty is chosen. - if found_unknown: - # Take preference for '???' - guilty['function'] = first_unknown[0] - guilty['module'] = first_unknown[1] - guilty['count'] = 1 - found_match = True - elif prev_frame: - # Choose the previous frame as a last resort - guilty['function'] = prev_frame[0] - guilty['module'] = prev_frame[1] - guilty['count'] = 1 - found_match = True - - return (guilty, found_match) - - -def _process_guilties(args): - if isinstance(args['klass'], bytes): - klass = args['klass'].decode() - else: - klass = args['klass'] - # In case the caller does not check for proper classification, bail early - if not is_crash_classification(klass): - return - if 'id' in args: - record_id = int(args['id']) - else: - record_id = None - global filters - with app.app_context(): - crashes = Record.get_new_crash_records(classes=get_backtrace_classes(), id=record_id) - filters = GuiltyBlacklist.get_guilties() - for rec in crashes: - if rec.payload: - new_bt = demangle_backtrace(rec.payload) - rec.payload = new_bt - # TODO: update the rec.payload field as well - Record.commit_guilty_changes() - g, match = find_guilty(rec.payload) - if match: - function = g['function'] - module = g['module'] - db_guilty = Record.get_guilty_for_funcmod(function, module) - if db_guilty is None: - db_guilty = Record.init_guilty(function, module) - Record.create_guilty_for_record(rec, db_guilty) - Record.set_processed_flag(rec) - - Record.commit_guilty_changes() - - -@spool -def process_guilties(args): - _process_guilties(args) - - -def process_guilties_sync(**args): - _process_guilties(args) - - -def guilty_list_per_build(guilties): - # TODO: should compute max values per build with a subquery instead - build_maxcount = {} - - buildset = set() - buildlist = [] - newlist = [] - - for g in guilties: - found_entry = False - guilty_str = g[0] + ' - [' + g[1] + ']' - build, count, guilty_id, comment = (g[2], g[3], g[4], g[5]) - for i, n in enumerate(newlist): - if guilty_str == n['guilty']: - newlist[i]['total'] += count - newlist[i]['builds'].append((build, count)) - if build in build_maxcount: - build_maxcount[build] = max(build_maxcount[build], count) - else: - build_maxcount[build] = count - found_entry = True - break - - if found_entry: - continue - - entry = {} - entry['guilty'] = guilty_str - entry['total'] = count - entry['guilty_id'] = guilty_id - entry['comment'] = comment - entry['builds'] = [] - entry['builds'].append((build, count)) - if build in build_maxcount: - build_maxcount[build] = max(build_maxcount[build], count) - else: - build_maxcount[build] = count - newlist.append(entry) - - # We only care about the top 10 guilties - newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:10] - for guilty in newlist: - for build in guilty['builds']: - buildset.add(build[0]) - - buildlist = list(buildset) - buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True) - - # For crashes not occuring in a particular build, provide a "0" value for - # the count. This simplifies table generation in the jinja template. - for i, g in enumerate(newlist): - builds, counts = list(zip(*g['builds'])) - counter = 0 - for b in buildlist: - if b not in builds: - newlist[i]['builds'].insert(counter, (b, "0")) - counter += 1 - - for i, b in enumerate(buildlist): - buildlist[i] = (b, build_maxcount[b]) - - buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True) - - for i, b in enumerate(newlist): - newlist[i]['builds'] = sorted(newlist[i]['builds'], key=lambda b: int(b[0]), reverse=True) - - return (buildlist, newlist) - - -def guilty_list_for_build(guilties, filter='overall'): - newlist = [] - - for g in guilties: - found_entry = False - guilty_str = g[0] + ' - [' + g[1] + ']' - build, count, guilty_id, comment = (g[2], g[3], g[4], g[5]) - for i, n in enumerate(newlist): - if guilty_str == n['guilty'] and filter in ['overall', build]: - newlist[i]['total'] += count - found_entry = True - break - - if found_entry: - continue - - if filter in ['overall', build]: - entry = {} - entry['guilty'] = guilty_str - entry['total'] = count - entry['guilty_id'] = guilty_id - entry['comment'] = comment - newlist.append(entry) - - # We only care about the top 10 guilties - newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:10] - - return newlist - - -def get_all_funcmods(): - frame_regex = re.compile(frame_pattern) - funcmodset = set() - funcmodlist = [] - backtraces = Record.get_crash_backtraces(classes=get_backtrace_classes()) - - for b in backtraces: - lines = b[0].splitlines() - for line in lines: - match = frame_regex.match(line) - if match: - funcmodset.add((match.group(2), match.group(4))) - - return sorted(funcmodset) - - -def parse_backtrace(backtrace): - program_regex = re.compile('^Process: (.*)$') - pid_regex = re.compile('^PID: ([0-9]+)$') - signal_regex = re.compile('^Signal: ([0-9]+)$') - bt_header_regex = re.compile('^Backtrace \(TID ([0-9]+)\):$') - frame_regex = re.compile(frame_pattern) - - Crash = namedtuple('Crash', ['record_id', 'program', 'pid', 'signal', 'backtrace']) - program = '' - pid = '' - signal = '' - frames = [] - parsed_header = False - - lines = backtrace[0].splitlines() - for line in lines: - # Header info - match = program_regex.match(line) - if match: - program = match.group(1) - continue - match = pid_regex.match(line) - if match: - pid = match.group(1) - continue - match = signal_regex.match(line) - if match: - signal = match.group(1) - continue - - # We only care about the backtrace from the crashing thread, which - # is listed first in the payload. - match = bt_header_regex.match(line) - if match and not parsed_header: - parsed_header = True - elif match: - break - - match = frame_regex.match(line) - if match: - frames.append((match.group(2), match.group(4), match.group(5))) - - # Populate a namedtuple for convenience - record_id = backtrace[1] - c = Crash(record_id, program, pid, signal, frames) - return c - - -def explode_backtraces(classes=None, guilty_id=None, machine_id=None, build=None): - crashes = [] - backtraces = Record.get_crash_backtraces(classes, guilty_id, machine_id, build) - for b in backtraces: - crashes.append(parse_backtrace(b)) - return crashes - - -# vi: ts=4 et sw=4 sts=4 diff --git a/shared/model.py b/shared/model.py deleted file mode 100644 index 549b0e8..0000000 --- a/shared/model.py +++ /dev/null @@ -1,619 +0,0 @@ -# -# Copyright 2015-2017 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import itertools -from flask_sqlalchemy import SQLAlchemy -from sqlalchemy.sql.expression import cast -from sqlalchemy.sql.expression import desc -from sqlalchemy.sql.expression import case -from time import time, localtime, strftime, mktime, strptime, gmtime -from distutils.version import LooseVersion - -from . import app - -db = SQLAlchemy(app) - - -class Guilty(db.Model): - __tablename__ = 'guilty' - id = db.Column(db.Integer, primary_key=True) - function = db.Column(db.String) - module = db.Column(db.String) - comment = db.Column(db.String) - hide = db.Column(db.Boolean, default=False) - - def __init__(self, func, mod): - self.function = func - self.module = mod - - @staticmethod - def update_comment(guilty_id, comment): - guilty = Guilty.query.filter_by(id=guilty_id).first() - guilty.comment = comment - db.session.commit() - - @staticmethod - def get_function(guilty_id): - guilty = Guilty.query.filter_by(id=guilty_id).first() - return guilty and guilty.function or "" - - @staticmethod - def get_module(guilty_id): - guilty = Guilty.query.filter_by(id=guilty_id).first() - return guilty and guilty.module or "" - - @staticmethod - def update_hidden(guilty_id, status): - guilty = Guilty.query.filter_by(id=guilty_id).first() - guilty.hide = status - db.session.commit() - - @staticmethod - def get_hidden_value(guilty_id): - guilty = Guilty.query.filter_by(id=guilty_id).first() - return guilty and guilty.hide or False - - @staticmethod - def get_hidden_guilties(): - q = db.session.query(Guilty.id, Guilty.function, Guilty.module) - q = q.filter(Guilty.hide == True) - q = q.order_by(Guilty.function) - return q.all() - - -class Record(db.Model): - __tablename__ = 'records' - id = db.Column(db.Integer, primary_key=True) - architecture = db.Column(db.Text) - bios_version = db.Column(db.Text, default='') - board_name = db.Column(db.Text, default='') - build = db.Column(db.Text, nullable=False) - classification = db.Column(db.Text, nullable=False) - cpu_model = db.Column(db.Text, default='') - event_id = db.Column(db.Text, default='') - external = db.Column(db.Boolean, default=False) - host_type = db.Column(db.Text, default='') - kernel_version = db.Column(db.Text, default=0) - machine_id = db.Column(db.Text, default='') - payload_version = db.Column(db.Integer) - record_version = db.Column(db.Integer, default=0) - severity = db.Column(db.Integer) - system_name = db.Column(db.Text) - timestamp_client = db.Column(db.Numeric) - timestamp_server = db.Column(db.Numeric, nullable=False) - payload = db.Column(db.Text, nullable=False) - - processed = db.Column(db.Boolean, default=False) - guilty_id = db.Column(db.Integer, db.ForeignKey('guilty.id')) - - guilty = db.Column(db.Text, default='') - guilty = db.relationship('Guilty', backref=db.backref('records', lazy='dynamic'), lazy='joined') - - def __init__(self, machine_id, host_type, severity, classification, build, architecture, kernel_version, - record_version, ts_capture, ts_reception, payload_version, system_name, - board_name, bios_version, cpu_model, event_id, external, payload): - self.machine_id = machine_id - self.host_type = host_type - self.architecture = architecture - self.classification = classification - self.build = build - self.kernel_version = kernel_version - self.record_version = record_version - self.severity = severity - self.timestamp_client = ts_capture - self.timestamp_server = ts_reception - self.payload_version = payload_version - self.system_name = system_name - self.external = external - self.board_name = board_name - self.bios_version = bios_version - self.cpu_model = cpu_model - self.event_id = event_id - - try: - self.payload = payload.encode('utf-8') - except UnicodeError: - self.payload = payload.encode('latin-1') - - def __repr__(self): - return "".format(self.id, self.classification, self.build, strftime("%a, %d %b %Y %H:%M:%S", localtime(self.timestamp_client))) - - def __str__(self): - return str(self.to_dict()) - - def to_dict(self): - record = { - 'id': self.id, - 'machine_id': self.machine_id, - 'machine_type': self.host_type, - 'arch': self.architecture, - 'build': self.build, - 'kernel_version': self.kernel_version, - 'ts_capture': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_client)), - 'ts_reception': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_server)), - 'severity': self.severity, - 'classification': self.classification, - 'record_version': self.record_version, - 'payload': self.payload, - 'board_name': self.board_name, - 'bios_version': self.bios_version, - 'cpu_model': self.cpu_model, - 'event_id': self.event_id, - 'external': self.external, - } - return record - - # for the exported CSV rows - def to_list(self): - record = [ - self.id, - self.external, - self.timestamp_server, - self.severity, - self.classification, - self.build, - self.machine_id, - self.payload - ] - return record - - @staticmethod - def list(): - return Record.query.all() - - @staticmethod - def create(machine_id, host_type, severity, classification, build, architecture, kernel_version, - record_version, ts_capture, ts_reception, payload_version, system_name, - board_name, bios_version, cpu_model, event_id, external, payload): - try: - record = Record(machine_id, host_type, severity, classification, build, architecture, kernel_version, - record_version, ts_capture, ts_reception, payload_version, system_name, - board_name, bios_version, cpu_model, event_id, external, payload) - db.session.add(record) - db.session.commit() - return record - except: - db.session.rollback() - raise - - @staticmethod - def query_records(build, classification, severity, machine_id, limit, - interval_sec=None, ts_capture=None, from_id=None): - records = Record.query - if build is not None: - records = records.filter_by(build=build) - if classification is not None: - records = records.filter_by(classification=classification) - if severity is not None: - records = records.filter(Record.severity == severity) - if machine_id is not None: - records = records.filter(Record.machine_id == machine_id) - if from_id is not None: - records = records.filter(Record.id >= from_id) - if ts_capture is not None: - records = records.filter(Record.timestamp_client > ts_capture) - - if interval_sec is not None: - current_time = time() - secs_in_past = current_time - interval_sec - # Due to time skew on client systems, delayed sends due to - # spooling, etc, timestamp_server works better as the reference - # timestamp. - records = records.filter(Record.timestamp_server > secs_in_past) - - records = records.order_by(Record.id.desc()) - - if limit is not None: - records = records.limit(limit) - - return records.all() - - @staticmethod - def get_record(record_id): - record = Record.query.filter_by(id=record_id).first() - return record - - @staticmethod - def filter_records(build, classification, severity, machine_id=None, system_name=None, limit=None, from_date=None, - to_date=None, payload=None, not_payload=None, data_source=None): - records = Record.query - if build is not None: - records = records.filter_by(build=build) - if classification is not None: - if isinstance(classification, list): - records = records.filter(Record.classification.in_(classification)) - else: - records = records.filter(Record.classification.like(classification)) - if severity is not None: - records = records.filter(Record.severity == severity) - if system_name is not None: - records = records.filter(Record.system_name == system_name) - if machine_id is not None: - records = records.filter(Record.machine_id == machine_id) - if from_date is not None: - from_date = mktime(strptime(from_date, "%Y-%m-%d")) - records = records.filter(Record.timestamp_client >= from_date) - if to_date is not None: - to_date = mktime(strptime(to_date, "%Y-%m-%d")) - records = records.filter(Record.timestamp_client < to_date) - if payload is not None: - records = records.filter(Record.payload.op('~')(payload)) - if not_payload is not None: - records = records.filter(~Record.payload.op('~')(not_payload)) - if data_source is not None: - if data_source == "external": - records = records.filter(Record.external == True) - elif data_source == "internal": - records = records.filter(Record.external == False) - - records = records.order_by(Record.id.desc()) - - if limit is not None: - records = records.limit(limit) - - return records - - @staticmethod - def delete_records(): - MAX_DAYS_KEEP_UNFILTERED_RECORDS = app.config.get("MAX_DAYS_KEEP_UNFILTERED_RECORDS", 35) - PURGE_FILTERED_RECORDS = app.config.get("PURGE_FILTERED_RECORDS", {}) - try: - def purge_field(field): - for name in PURGE_FILTERED_RECORDS[field].keys(): - if PURGE_FILTERED_RECORDS[field][name]: - age = time() - PURGE_FILTERED_RECORDS[field][name] * 24 * 60 * 60 - q = db.session.query(Record) - if field == 'classification': - q = q.filter(Record.classification.like(name.replace("*", "%"))) - else: - q = q.filter(getattr(Record, field) == name) - q = q.filter(Record.timestamp_server < age) - if q.all(): - count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False) - print("Deleted {} {} records".format(count, name)) - for field in PURGE_FILTERED_RECORDS.keys(): - purge_field(field) - if MAX_DAYS_KEEP_UNFILTERED_RECORDS: - unfiltered_age = time() - MAX_DAYS_KEEP_UNFILTERED_RECORDS * 24 * 60 * 60 - q = db.session.query(Record.id) - for field in PURGE_FILTERED_RECORDS.keys(): - if field == 'classification': - for classification in PURGE_FILTERED_RECORDS[field].keys(): - q = q.filter(~Record.classification.like(classification.replace("*", "%"))) - else: - for name in PURGE_FILTERED_RECORDS[field].keys(): - q = q.filter(getattr(Record, field) != name) - q = q.filter(Record.timestamp_server < unfiltered_age) - if q.all(): - count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False) - print("Deleted {} old records".format(count)) - db.session.commit() - except Exception as e: - app.logger.error("Record purging failed") - app.logger.error(e) - db.session.rollback() - - @staticmethod - def get_recordcnts_by_build(): - q = db.session.query(Record.build, db.func.count(Record.id)) - q = q.filter(Record.build.op('~')('^[0-9]+$')) - q = q.group_by(Record.build).order_by(cast(Record.build, db.Integer)).all() - return q - - @staticmethod - def get_builds(): - q = db.session.query(Record.build).distinct() - q = q.order_by(Record.build) - return sorted(q.all(), key=lambda x: LooseVersion(x[0]), reverse=True) - - @staticmethod - def get_recordcnts_by_classification(): - q = db.session.query(Record.classification, db.func.count(Record.id).label('total')) - q = q.group_by(Record.classification) - q = q.order_by(desc('total')) - return q.all() - - @staticmethod - def expand_class(D): - A, B, C = D - return ["{}/*".format(A), "{}/{}/*".format(A, B), "{}/{}/{}".format(A, B, C)] - - @staticmethod - def get_classifications(with_regex=False): - q = db.session.query(Record.classification).distinct() - if with_regex: - classes = [Record.expand_class(c[0].split('/')) for c in q.all()] - return sorted(set(itertools.chain(*classes))) - else: - return q.all() - - @staticmethod - def get_os_map(): - q = db.session.query(Record.system_name, Record.build).order_by(Record.system_name).group_by(Record.system_name, Record.build).all() - result = {} - for x in q: - result.setdefault(x[0], []).append(x[1]) - return result - - @staticmethod - def get_recordcnts_by_machine_type(): - q = db.session.query(Record.host_type, db.func.count(Record.id).label('total')) - q = q.group_by(Record.host_type) - q = q.order_by(desc('total')) - return q.all() - - @staticmethod - def get_recordcnts_by_severity(): - q = db.session.query(Record.severity, db.func.count(Record.id)).group_by(Record.severity).all() - return q - - @staticmethod - def get_crashcnts_by_class(classes=None): - q = db.session.query(Record.classification, db.func.count(Record.id)) - if classes: - q = q.filter(Record.classification.in_(classes)) - else: - q = q.filter(Record.classification.like('org.clearlinux/crash/%')) - q = q.group_by(Record.classification) - return q.all() - - @staticmethod - def get_crashcnts_by_build(classes=None): - q = db.session.query(Record.build, db.func.count(Record.id)) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.build.op('~')('^[0-9]+$')) - q = q.group_by(Record.build) - q = q.order_by(desc(cast(Record.build, db.Integer))) - q = q.limit(10) - return q.all() - - @staticmethod - def get_top_crash_guilties(classes=None): - q = db.session.query(Guilty.function, Guilty.module, Record.build, db.func.count(Record.id).label('total'), Guilty.id, Guilty.comment) - q = q.join(Record) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.build.op('~')('^[0-9][0-9]+$')) - q = q.filter(cast(Record.build, db.Integer) <= 100000) - q = q.filter(Guilty.hide == False) - q = q.group_by(Guilty.function, Guilty.module, Guilty.comment, Guilty.id, Record.build) - q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total')) - # query for records created in the last week (~ 10 Clear builds) - q = q.filter(Record.build.in_(sorted(tuple(set([x[2] for x in q.all()])), key=lambda x: int(x))[-8:])) - interval_sec = 24 * 60 * 60 * 7 - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - return q.all() - - @staticmethod - def get_new_crash_records(classes=None, id=None): - q = db.session.query(Record) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.system_name == 'clear-linux-os') - q = q.filter(Record.processed == False) - if id: - q = q.filter(Record.id == id) - return q.all() - - @staticmethod - def set_processed_flag(record): - record.processed = True - - @staticmethod - def get_guilty_for_funcmod(func, mod): - q = db.session.query(Guilty).filter_by(function=func, module=mod).first() - return q - - @staticmethod - def get_guilty_id_for_record(record_id): - q = db.session.query(Guilty.id).join(Record) - q = q.filter(Record.id == record_id) - return q.first() - - @staticmethod - def init_guilty(func, mod): - return Guilty(func, mod) - - @staticmethod - def create_guilty_for_record(record, guilty): - record.guilty = guilty - - @staticmethod - def commit_guilty_changes(): - # just commit for now - db.session.commit() - - @staticmethod - def get_crash_backtraces(classes=None, guilty_id=None, machine_id=None, build=None, most_recent=None, record_id=None): - q = db.session.query(Record.payload, Record.id) - # Short circuit if we know the record ID - if record_id: - q = q.filter(Record.id == record_id) - return q.first() - if build: - q = q.filter(Record.build == build) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.system_name == 'clear-linux-os') - if guilty_id: - q = q.filter(Record.guilty_id == guilty_id) - if machine_id: - q = q.filter(Record.machine_id == machine_id) - if most_recent: - interval_sec = 24 * 60 * 60 * int(most_recent) - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - return q.all() - - @staticmethod - def reset_processed_records(classes=None, id=None): - q = db.session.query(Record) - if not classes: - classes = ['org.clearlinux/crash/clr'] - q = q.filter(Record.classification.in_(classes)) - q = q.filter(Record.system_name == 'clear-linux-os') - if id: - q = q.filter(Record.id == id) - records = q.all() - for r in records: - r.processed = False - db.session.commit() - - @staticmethod - def get_machine_ids_for_guilty(id, most_recent=None): - q = db.session.query(Record.build, Record.machine_id, db.func.count(Record.id).label('total'), Record.guilty_id) - q = q.filter(Record.guilty_id == id) - q = q.filter(Record.system_name == 'clear-linux-os') - q = q.filter(Record.build.op('~')('^[0-9][0-9]+$')) - q = q.group_by(Record.build, Record.machine_id, Record.guilty_id) - q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total')) - if most_recent: - interval_sec = 24 * 60 * 60 * int(most_recent) - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - return q.all() - - @staticmethod - def get_update_msgs(): - q = db.session.query(Record.payload) - q = q.filter(Record.classification == "org.clearlinux/swupd-client/update") - - sec_2_weeks = 24 * 60 * 60 * 7 - current_time = time() - time_2_weeks_ago = current_time - sec_2_weeks - - # query for records created in that last 2 weeks - q = q.filter(Record.timestamp_client > time_2_weeks_ago) - return q - - @staticmethod - def get_swupd_msgs(most_recent=None): - q = db.session.query(Record.timestamp_client, Record.machine_id, Record.payload) - q = q.filter(Record.classification.like('org.clearlinux/swupd-client/%')) - - if most_recent: - interval_sec = 24 * 60 * 60 * int(most_recent) - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - - q = q.order_by(desc(Record.timestamp_client)) - return q - - @staticmethod - def get_heartbeat_msgs(most_recent=None): - # These two expressions are SQL CASE conditional expressions, later - # used within count(distinct ...) aggregates for the query. - internal_expr = case([(Record.external == False, Record.machine_id), ]).label('internal_count') - external_expr = case([(Record.external == True, Record.machine_id), ]).label('external_count') - - q = db.session.query(Record.build, db.func.count(db.distinct(internal_expr)), db.func.count(db.distinct(external_expr))) - q = q.filter(Record.classification == "org.clearlinux/heartbeat/ping") - q = q.filter(Record.system_name == 'clear-linux-os') - q = q.group_by(Record.build) - - if most_recent: - interval_sec = 24 * 60 * 60 * int(most_recent) - current_time = time() - sec_in_past = current_time - interval_sec - q = q.filter(Record.timestamp_client > sec_in_past) - - q = q.order_by(cast(Record.build, db.Integer)) - return q.all() - - -class GuiltyBlacklist(db.Model): - __tablename__ = 'guilty_blacklisted' - id = db.Column(db.Integer, primary_key=True) - function = db.Column(db.String) - module = db.Column(db.String) - - def __init__(self, func, mod): - self.function = func - self.module = mod - - def __repr__(self): - return "".format(self.id, self.function, self.module) - - def __str__(self): - return str(self.to_dict()) - - def to_dict(self): - guilty = { - 'function': self.function, - 'module': self.module - } - return guilty - - @staticmethod - def add(func, mod): - try: - g = GuiltyBlacklist(func, mod) - db.session.add(g) - db.session.commit() - return g - except: - db.session.rollback() - raise - - @staticmethod - def remove(func, mod): - q = db.session.query(GuiltyBlacklist) - q = q.filter_by(function=func, module=mod) - entry = q.first() - db.session.delete(entry) - db.session.commit() - - @staticmethod - def get_guilties(): - q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module) - q = q.order_by(GuiltyBlacklist.function) - return q.all() - - @staticmethod - def exists(func, mod): - q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module) - q = q.filter_by(function=func, module=mod) - return len(q.all()) != 0 and True or False - - @staticmethod - def update(to_add, to_remove): - try: - for i in to_add: - if not GuiltyBlacklist.exists(i[0], i[1]): - g = GuiltyBlacklist(i[0], i[1]) - db.session.add(g) - for i in to_remove: - if GuiltyBlacklist.exists(i[0], i[1]): - q = db.session.query(GuiltyBlacklist) - q = q.filter_by(function=i[0], module=i[1]) - entry = q.first() - db.session.delete(entry) - db.session.commit() - except: - db.session.rollback() - raise - -# vi: ts=4 et sw=4 sts=4 diff --git a/telemetryui/telemetryui/crash.py b/telemetryui/telemetryui/crash.py deleted file mode 120000 index 68dab23..0000000 --- a/telemetryui/telemetryui/crash.py +++ /dev/null @@ -1 +0,0 @@ -../../shared/crash.py \ No newline at end of file diff --git a/telemetryui/telemetryui/crash.py b/telemetryui/telemetryui/crash.py new file mode 100644 index 0000000..f828a09 --- /dev/null +++ b/telemetryui/telemetryui/crash.py @@ -0,0 +1,433 @@ +# +# Copyright 2015-2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +from operator import itemgetter +from collections import namedtuple +import subprocess +import re +from .model import Record, GuiltyBlacklist +from . import app + +try: + from uwsgidecorators import spool +except ImportError: + def spool(f): + f.spool = f + return f + +filters = [] + +# Groups for the frame_pattern below +# 1 - frame number + one space +# 2 - function name + optional arguments +# 3 - rest of the line +# 4 - module name (inside the []) +# 5 - optional frame source file and line number info + +# TODO: The current c++filt logic depends on properly subsituting c++filt +# output for the function name. Thus, it is very, very important to keep a +# capture group that extends from the function name to the end of the frame as +# long as this logic remains the same. Probably better to rework the code to +# *not* destructively overwrite the backtrace field. Maybe store the filtered +# output in a different field. + +frame_pattern = "^(#\d+ )(.+)( - \[(.*)\](.*))$" + +backtrace_classes = [ + 'org.clearlinux/crash/clr', + 'org.clearlinux/kernel/bug', + 'org.clearlinux/kernel/stackoverflow', + 'org.clearlinux/kernel/warning' +] + +other_classes = [ + 'org.clearlinux/crash/unknown', + 'org.clearlinux/crash/clr-build', + 'org.clearlinux/crash/error' +] + + +def get_all_classes(): + return backtrace_classes + other_classes + + +def get_backtrace_classes(): + return backtrace_classes + + +def get_other_classes(): + return other_classes + + +def is_crash_classification(klass): + return (klass in backtrace_classes) and True or False + + +def is_blacklisted(function, module): + funcmod = (function, module) + if funcmod in filters: + return True + + return False + + +def demangle_backtrace(bt): + new_bt = [] + prog = '/usr/bin/c++filt' + + frame_regex = re.compile(frame_pattern) + lines = bt.splitlines() + + for line in lines: + m = frame_regex.match(line) + if m: + func = m.group(2) + + # A frame with missing symbols is a special case, so skip it + if func == '???': + new_bt.append(line) + continue + + # FIXME: this logic will break once the crash probe starts sending + # function argument values; make this more generic! + if func[-2:] == '()': + # The crash probe adds the () to the function name, but c++filt + # cannot demangle a symbol with the () suffix + func_name = func[:-2] + else: + # Assume already demangled, or this is from a kernel crash record + new_bt.append(line) + continue + + try: + new_func = subprocess.check_output([prog, func_name], universal_newlines=True) + except: + new_bt.append(line) + continue + + # c++filt adds a trailing newline to the output + new_func = new_func.rstrip() + + # Restore () if this was not a mangled symbol + if new_func == func_name: + new_func = func_name + '()' + + repl_str = r'\1{}\3'.format(new_func) + new_line = frame_regex.sub(repl_str, line) + new_bt.append(new_line) + else: + new_bt.append(line) + + return '\n'.join(new_bt) + + +def find_guilty(backtrace): + frame_regex = re.compile(frame_pattern) + + guilty = {} + + lines = backtrace.splitlines() + + first_unknown = None + in_backtrace = False + prev_frame = None + found_match = False + found_unknown = False + + # Begin guilty detection process + for line in lines[1:]: + m = frame_regex.match(line) + + if m: + # Either this is the first frame of the backtrace, or we are still + # iterating through the backtrace. + in_backtrace = True + + func = m.group(2) + mod = m.group(4) + + # Only consider blacklisted function/module pairs as a last resort. + # It's likely that the blacklisted pairs will never be chosen as + # worthy candidates... if they are, the guilty blacklist may be + # filtering too much. + if is_blacklisted(func, mod): + prev_frame = (func, mod) + continue + + # Consider the first frame without function symbols ('???') only if + # there are no function symbols for any frames lower in the stack. + if (func == '???' or func[:2] == '? ') and not found_unknown: + found_unknown = True + first_unknown = (func, mod) + prev_frame = (func, mod) + continue + elif func == '???': + # In this case, we've already encountered a frame with missing + # function symbols, so skip it, but save the info for backup. + prev_frame = (func, mod) + continue + + # If the previous three conditional checks fail, then we have found + # the best guilty candidate: it is not in the blacklist, and it has + # function symbols. + guilty['function'] = func + guilty['module'] = mod + guilty['count'] = 1 + + found_match = True + return (guilty, found_match) + + elif in_backtrace: + # We have processed the entire backtrace for the crashing thread of + # the process, but no solid guilty has been found. Since we only + # consider the crashing thread for guilty detection, stop iterating + # through the remainder of the threads at this point. + break + + # Implement a backup plan to ensure that a guilty is chosen. + if found_unknown: + # Take preference for '???' + guilty['function'] = first_unknown[0] + guilty['module'] = first_unknown[1] + guilty['count'] = 1 + found_match = True + elif prev_frame: + # Choose the previous frame as a last resort + guilty['function'] = prev_frame[0] + guilty['module'] = prev_frame[1] + guilty['count'] = 1 + found_match = True + + return (guilty, found_match) + + +def _process_guilties(args): + if isinstance(args['klass'], bytes): + klass = args['klass'].decode() + else: + klass = args['klass'] + # In case the caller does not check for proper classification, bail early + if not is_crash_classification(klass): + return + if 'id' in args: + record_id = int(args['id']) + else: + record_id = None + global filters + with app.app_context(): + crashes = Record.get_new_crash_records(classes=get_backtrace_classes(), id=record_id) + filters = GuiltyBlacklist.get_guilties() + for rec in crashes: + if rec.payload: + new_bt = demangle_backtrace(rec.payload) + rec.payload = new_bt + # TODO: update the rec.payload field as well + Record.commit_guilty_changes() + g, match = find_guilty(rec.payload) + if match: + function = g['function'] + module = g['module'] + db_guilty = Record.get_guilty_for_funcmod(function, module) + if db_guilty is None: + db_guilty = Record.init_guilty(function, module) + Record.create_guilty_for_record(rec, db_guilty) + Record.set_processed_flag(rec) + + Record.commit_guilty_changes() + + +@spool +def process_guilties(args): + _process_guilties(args) + + +def process_guilties_sync(**args): + _process_guilties(args) + + +def guilty_list_per_build(guilties): + # TODO: should compute max values per build with a subquery instead + build_maxcount = {} + + buildset = set() + buildlist = [] + newlist = [] + + for g in guilties: + found_entry = False + guilty_str = g[0] + ' - [' + g[1] + ']' + build, count, guilty_id, comment = (g[2], g[3], g[4], g[5]) + for i, n in enumerate(newlist): + if guilty_str == n['guilty']: + newlist[i]['total'] += count + newlist[i]['builds'].append((build, count)) + if build in build_maxcount: + build_maxcount[build] = max(build_maxcount[build], count) + else: + build_maxcount[build] = count + found_entry = True + break + + if found_entry: + continue + + entry = {} + entry['guilty'] = guilty_str + entry['total'] = count + entry['guilty_id'] = guilty_id + entry['comment'] = comment + entry['builds'] = [] + entry['builds'].append((build, count)) + if build in build_maxcount: + build_maxcount[build] = max(build_maxcount[build], count) + else: + build_maxcount[build] = count + newlist.append(entry) + + # We only care about the top 10 guilties + newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:10] + for guilty in newlist: + for build in guilty['builds']: + buildset.add(build[0]) + + buildlist = list(buildset) + buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True) + + # For crashes not occuring in a particular build, provide a "0" value for + # the count. This simplifies table generation in the jinja template. + for i, g in enumerate(newlist): + builds, counts = list(zip(*g['builds'])) + counter = 0 + for b in buildlist: + if b not in builds: + newlist[i]['builds'].insert(counter, (b, "0")) + counter += 1 + + for i, b in enumerate(buildlist): + buildlist[i] = (b, build_maxcount[b]) + + buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True) + + for i, b in enumerate(newlist): + newlist[i]['builds'] = sorted(newlist[i]['builds'], key=lambda b: int(b[0]), reverse=True) + + return (buildlist, newlist) + + +def guilty_list_for_build(guilties, filter='overall'): + newlist = [] + + for g in guilties: + found_entry = False + guilty_str = g[0] + ' - [' + g[1] + ']' + build, count, guilty_id, comment = (g[2], g[3], g[4], g[5]) + for i, n in enumerate(newlist): + if guilty_str == n['guilty'] and filter in ['overall', build]: + newlist[i]['total'] += count + found_entry = True + break + + if found_entry: + continue + + if filter in ['overall', build]: + entry = {} + entry['guilty'] = guilty_str + entry['total'] = count + entry['guilty_id'] = guilty_id + entry['comment'] = comment + newlist.append(entry) + + # We only care about the top 10 guilties + newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:10] + + return newlist + + +def get_all_funcmods(): + frame_regex = re.compile(frame_pattern) + funcmodset = set() + funcmodlist = [] + backtraces = Record.get_crash_backtraces(classes=get_backtrace_classes()) + + for b in backtraces: + lines = b[0].splitlines() + for line in lines: + match = frame_regex.match(line) + if match: + funcmodset.add((match.group(2), match.group(4))) + + return sorted(funcmodset) + + +def parse_backtrace(backtrace): + program_regex = re.compile('^Process: (.*)$') + pid_regex = re.compile('^PID: ([0-9]+)$') + signal_regex = re.compile('^Signal: ([0-9]+)$') + bt_header_regex = re.compile('^Backtrace \(TID ([0-9]+)\):$') + frame_regex = re.compile(frame_pattern) + + Crash = namedtuple('Crash', ['record_id', 'program', 'pid', 'signal', 'backtrace']) + program = '' + pid = '' + signal = '' + frames = [] + parsed_header = False + + lines = backtrace[0].splitlines() + for line in lines: + # Header info + match = program_regex.match(line) + if match: + program = match.group(1) + continue + match = pid_regex.match(line) + if match: + pid = match.group(1) + continue + match = signal_regex.match(line) + if match: + signal = match.group(1) + continue + + # We only care about the backtrace from the crashing thread, which + # is listed first in the payload. + match = bt_header_regex.match(line) + if match and not parsed_header: + parsed_header = True + elif match: + break + + match = frame_regex.match(line) + if match: + frames.append((match.group(2), match.group(4), match.group(5))) + + # Populate a namedtuple for convenience + record_id = backtrace[1] + c = Crash(record_id, program, pid, signal, frames) + return c + + +def explode_backtraces(classes=None, guilty_id=None, machine_id=None, build=None): + crashes = [] + backtraces = Record.get_crash_backtraces(classes, guilty_id, machine_id, build) + for b in backtraces: + crashes.append(parse_backtrace(b)) + return crashes + + +# vi: ts=4 et sw=4 sts=4 diff --git a/telemetryui/telemetryui/model.py b/telemetryui/telemetryui/model.py deleted file mode 120000 index ac6394a..0000000 --- a/telemetryui/telemetryui/model.py +++ /dev/null @@ -1 +0,0 @@ -../../shared/model.py \ No newline at end of file diff --git a/telemetryui/telemetryui/model.py b/telemetryui/telemetryui/model.py new file mode 100644 index 0000000..549b0e8 --- /dev/null +++ b/telemetryui/telemetryui/model.py @@ -0,0 +1,619 @@ +# +# Copyright 2015-2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import itertools +from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.sql.expression import cast +from sqlalchemy.sql.expression import desc +from sqlalchemy.sql.expression import case +from time import time, localtime, strftime, mktime, strptime, gmtime +from distutils.version import LooseVersion + +from . import app + +db = SQLAlchemy(app) + + +class Guilty(db.Model): + __tablename__ = 'guilty' + id = db.Column(db.Integer, primary_key=True) + function = db.Column(db.String) + module = db.Column(db.String) + comment = db.Column(db.String) + hide = db.Column(db.Boolean, default=False) + + def __init__(self, func, mod): + self.function = func + self.module = mod + + @staticmethod + def update_comment(guilty_id, comment): + guilty = Guilty.query.filter_by(id=guilty_id).first() + guilty.comment = comment + db.session.commit() + + @staticmethod + def get_function(guilty_id): + guilty = Guilty.query.filter_by(id=guilty_id).first() + return guilty and guilty.function or "" + + @staticmethod + def get_module(guilty_id): + guilty = Guilty.query.filter_by(id=guilty_id).first() + return guilty and guilty.module or "" + + @staticmethod + def update_hidden(guilty_id, status): + guilty = Guilty.query.filter_by(id=guilty_id).first() + guilty.hide = status + db.session.commit() + + @staticmethod + def get_hidden_value(guilty_id): + guilty = Guilty.query.filter_by(id=guilty_id).first() + return guilty and guilty.hide or False + + @staticmethod + def get_hidden_guilties(): + q = db.session.query(Guilty.id, Guilty.function, Guilty.module) + q = q.filter(Guilty.hide == True) + q = q.order_by(Guilty.function) + return q.all() + + +class Record(db.Model): + __tablename__ = 'records' + id = db.Column(db.Integer, primary_key=True) + architecture = db.Column(db.Text) + bios_version = db.Column(db.Text, default='') + board_name = db.Column(db.Text, default='') + build = db.Column(db.Text, nullable=False) + classification = db.Column(db.Text, nullable=False) + cpu_model = db.Column(db.Text, default='') + event_id = db.Column(db.Text, default='') + external = db.Column(db.Boolean, default=False) + host_type = db.Column(db.Text, default='') + kernel_version = db.Column(db.Text, default=0) + machine_id = db.Column(db.Text, default='') + payload_version = db.Column(db.Integer) + record_version = db.Column(db.Integer, default=0) + severity = db.Column(db.Integer) + system_name = db.Column(db.Text) + timestamp_client = db.Column(db.Numeric) + timestamp_server = db.Column(db.Numeric, nullable=False) + payload = db.Column(db.Text, nullable=False) + + processed = db.Column(db.Boolean, default=False) + guilty_id = db.Column(db.Integer, db.ForeignKey('guilty.id')) + + guilty = db.Column(db.Text, default='') + guilty = db.relationship('Guilty', backref=db.backref('records', lazy='dynamic'), lazy='joined') + + def __init__(self, machine_id, host_type, severity, classification, build, architecture, kernel_version, + record_version, ts_capture, ts_reception, payload_version, system_name, + board_name, bios_version, cpu_model, event_id, external, payload): + self.machine_id = machine_id + self.host_type = host_type + self.architecture = architecture + self.classification = classification + self.build = build + self.kernel_version = kernel_version + self.record_version = record_version + self.severity = severity + self.timestamp_client = ts_capture + self.timestamp_server = ts_reception + self.payload_version = payload_version + self.system_name = system_name + self.external = external + self.board_name = board_name + self.bios_version = bios_version + self.cpu_model = cpu_model + self.event_id = event_id + + try: + self.payload = payload.encode('utf-8') + except UnicodeError: + self.payload = payload.encode('latin-1') + + def __repr__(self): + return "".format(self.id, self.classification, self.build, strftime("%a, %d %b %Y %H:%M:%S", localtime(self.timestamp_client))) + + def __str__(self): + return str(self.to_dict()) + + def to_dict(self): + record = { + 'id': self.id, + 'machine_id': self.machine_id, + 'machine_type': self.host_type, + 'arch': self.architecture, + 'build': self.build, + 'kernel_version': self.kernel_version, + 'ts_capture': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_client)), + 'ts_reception': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_server)), + 'severity': self.severity, + 'classification': self.classification, + 'record_version': self.record_version, + 'payload': self.payload, + 'board_name': self.board_name, + 'bios_version': self.bios_version, + 'cpu_model': self.cpu_model, + 'event_id': self.event_id, + 'external': self.external, + } + return record + + # for the exported CSV rows + def to_list(self): + record = [ + self.id, + self.external, + self.timestamp_server, + self.severity, + self.classification, + self.build, + self.machine_id, + self.payload + ] + return record + + @staticmethod + def list(): + return Record.query.all() + + @staticmethod + def create(machine_id, host_type, severity, classification, build, architecture, kernel_version, + record_version, ts_capture, ts_reception, payload_version, system_name, + board_name, bios_version, cpu_model, event_id, external, payload): + try: + record = Record(machine_id, host_type, severity, classification, build, architecture, kernel_version, + record_version, ts_capture, ts_reception, payload_version, system_name, + board_name, bios_version, cpu_model, event_id, external, payload) + db.session.add(record) + db.session.commit() + return record + except: + db.session.rollback() + raise + + @staticmethod + def query_records(build, classification, severity, machine_id, limit, + interval_sec=None, ts_capture=None, from_id=None): + records = Record.query + if build is not None: + records = records.filter_by(build=build) + if classification is not None: + records = records.filter_by(classification=classification) + if severity is not None: + records = records.filter(Record.severity == severity) + if machine_id is not None: + records = records.filter(Record.machine_id == machine_id) + if from_id is not None: + records = records.filter(Record.id >= from_id) + if ts_capture is not None: + records = records.filter(Record.timestamp_client > ts_capture) + + if interval_sec is not None: + current_time = time() + secs_in_past = current_time - interval_sec + # Due to time skew on client systems, delayed sends due to + # spooling, etc, timestamp_server works better as the reference + # timestamp. + records = records.filter(Record.timestamp_server > secs_in_past) + + records = records.order_by(Record.id.desc()) + + if limit is not None: + records = records.limit(limit) + + return records.all() + + @staticmethod + def get_record(record_id): + record = Record.query.filter_by(id=record_id).first() + return record + + @staticmethod + def filter_records(build, classification, severity, machine_id=None, system_name=None, limit=None, from_date=None, + to_date=None, payload=None, not_payload=None, data_source=None): + records = Record.query + if build is not None: + records = records.filter_by(build=build) + if classification is not None: + if isinstance(classification, list): + records = records.filter(Record.classification.in_(classification)) + else: + records = records.filter(Record.classification.like(classification)) + if severity is not None: + records = records.filter(Record.severity == severity) + if system_name is not None: + records = records.filter(Record.system_name == system_name) + if machine_id is not None: + records = records.filter(Record.machine_id == machine_id) + if from_date is not None: + from_date = mktime(strptime(from_date, "%Y-%m-%d")) + records = records.filter(Record.timestamp_client >= from_date) + if to_date is not None: + to_date = mktime(strptime(to_date, "%Y-%m-%d")) + records = records.filter(Record.timestamp_client < to_date) + if payload is not None: + records = records.filter(Record.payload.op('~')(payload)) + if not_payload is not None: + records = records.filter(~Record.payload.op('~')(not_payload)) + if data_source is not None: + if data_source == "external": + records = records.filter(Record.external == True) + elif data_source == "internal": + records = records.filter(Record.external == False) + + records = records.order_by(Record.id.desc()) + + if limit is not None: + records = records.limit(limit) + + return records + + @staticmethod + def delete_records(): + MAX_DAYS_KEEP_UNFILTERED_RECORDS = app.config.get("MAX_DAYS_KEEP_UNFILTERED_RECORDS", 35) + PURGE_FILTERED_RECORDS = app.config.get("PURGE_FILTERED_RECORDS", {}) + try: + def purge_field(field): + for name in PURGE_FILTERED_RECORDS[field].keys(): + if PURGE_FILTERED_RECORDS[field][name]: + age = time() - PURGE_FILTERED_RECORDS[field][name] * 24 * 60 * 60 + q = db.session.query(Record) + if field == 'classification': + q = q.filter(Record.classification.like(name.replace("*", "%"))) + else: + q = q.filter(getattr(Record, field) == name) + q = q.filter(Record.timestamp_server < age) + if q.all(): + count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False) + print("Deleted {} {} records".format(count, name)) + for field in PURGE_FILTERED_RECORDS.keys(): + purge_field(field) + if MAX_DAYS_KEEP_UNFILTERED_RECORDS: + unfiltered_age = time() - MAX_DAYS_KEEP_UNFILTERED_RECORDS * 24 * 60 * 60 + q = db.session.query(Record.id) + for field in PURGE_FILTERED_RECORDS.keys(): + if field == 'classification': + for classification in PURGE_FILTERED_RECORDS[field].keys(): + q = q.filter(~Record.classification.like(classification.replace("*", "%"))) + else: + for name in PURGE_FILTERED_RECORDS[field].keys(): + q = q.filter(getattr(Record, field) != name) + q = q.filter(Record.timestamp_server < unfiltered_age) + if q.all(): + count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False) + print("Deleted {} old records".format(count)) + db.session.commit() + except Exception as e: + app.logger.error("Record purging failed") + app.logger.error(e) + db.session.rollback() + + @staticmethod + def get_recordcnts_by_build(): + q = db.session.query(Record.build, db.func.count(Record.id)) + q = q.filter(Record.build.op('~')('^[0-9]+$')) + q = q.group_by(Record.build).order_by(cast(Record.build, db.Integer)).all() + return q + + @staticmethod + def get_builds(): + q = db.session.query(Record.build).distinct() + q = q.order_by(Record.build) + return sorted(q.all(), key=lambda x: LooseVersion(x[0]), reverse=True) + + @staticmethod + def get_recordcnts_by_classification(): + q = db.session.query(Record.classification, db.func.count(Record.id).label('total')) + q = q.group_by(Record.classification) + q = q.order_by(desc('total')) + return q.all() + + @staticmethod + def expand_class(D): + A, B, C = D + return ["{}/*".format(A), "{}/{}/*".format(A, B), "{}/{}/{}".format(A, B, C)] + + @staticmethod + def get_classifications(with_regex=False): + q = db.session.query(Record.classification).distinct() + if with_regex: + classes = [Record.expand_class(c[0].split('/')) for c in q.all()] + return sorted(set(itertools.chain(*classes))) + else: + return q.all() + + @staticmethod + def get_os_map(): + q = db.session.query(Record.system_name, Record.build).order_by(Record.system_name).group_by(Record.system_name, Record.build).all() + result = {} + for x in q: + result.setdefault(x[0], []).append(x[1]) + return result + + @staticmethod + def get_recordcnts_by_machine_type(): + q = db.session.query(Record.host_type, db.func.count(Record.id).label('total')) + q = q.group_by(Record.host_type) + q = q.order_by(desc('total')) + return q.all() + + @staticmethod + def get_recordcnts_by_severity(): + q = db.session.query(Record.severity, db.func.count(Record.id)).group_by(Record.severity).all() + return q + + @staticmethod + def get_crashcnts_by_class(classes=None): + q = db.session.query(Record.classification, db.func.count(Record.id)) + if classes: + q = q.filter(Record.classification.in_(classes)) + else: + q = q.filter(Record.classification.like('org.clearlinux/crash/%')) + q = q.group_by(Record.classification) + return q.all() + + @staticmethod + def get_crashcnts_by_build(classes=None): + q = db.session.query(Record.build, db.func.count(Record.id)) + if not classes: + classes = ['org.clearlinux/crash/clr'] + q = q.filter(Record.classification.in_(classes)) + q = q.filter(Record.build.op('~')('^[0-9]+$')) + q = q.group_by(Record.build) + q = q.order_by(desc(cast(Record.build, db.Integer))) + q = q.limit(10) + return q.all() + + @staticmethod + def get_top_crash_guilties(classes=None): + q = db.session.query(Guilty.function, Guilty.module, Record.build, db.func.count(Record.id).label('total'), Guilty.id, Guilty.comment) + q = q.join(Record) + if not classes: + classes = ['org.clearlinux/crash/clr'] + q = q.filter(Record.classification.in_(classes)) + q = q.filter(Record.build.op('~')('^[0-9][0-9]+$')) + q = q.filter(cast(Record.build, db.Integer) <= 100000) + q = q.filter(Guilty.hide == False) + q = q.group_by(Guilty.function, Guilty.module, Guilty.comment, Guilty.id, Record.build) + q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total')) + # query for records created in the last week (~ 10 Clear builds) + q = q.filter(Record.build.in_(sorted(tuple(set([x[2] for x in q.all()])), key=lambda x: int(x))[-8:])) + interval_sec = 24 * 60 * 60 * 7 + current_time = time() + sec_in_past = current_time - interval_sec + q = q.filter(Record.timestamp_client > sec_in_past) + return q.all() + + @staticmethod + def get_new_crash_records(classes=None, id=None): + q = db.session.query(Record) + if not classes: + classes = ['org.clearlinux/crash/clr'] + q = q.filter(Record.classification.in_(classes)) + q = q.filter(Record.system_name == 'clear-linux-os') + q = q.filter(Record.processed == False) + if id: + q = q.filter(Record.id == id) + return q.all() + + @staticmethod + def set_processed_flag(record): + record.processed = True + + @staticmethod + def get_guilty_for_funcmod(func, mod): + q = db.session.query(Guilty).filter_by(function=func, module=mod).first() + return q + + @staticmethod + def get_guilty_id_for_record(record_id): + q = db.session.query(Guilty.id).join(Record) + q = q.filter(Record.id == record_id) + return q.first() + + @staticmethod + def init_guilty(func, mod): + return Guilty(func, mod) + + @staticmethod + def create_guilty_for_record(record, guilty): + record.guilty = guilty + + @staticmethod + def commit_guilty_changes(): + # just commit for now + db.session.commit() + + @staticmethod + def get_crash_backtraces(classes=None, guilty_id=None, machine_id=None, build=None, most_recent=None, record_id=None): + q = db.session.query(Record.payload, Record.id) + # Short circuit if we know the record ID + if record_id: + q = q.filter(Record.id == record_id) + return q.first() + if build: + q = q.filter(Record.build == build) + if not classes: + classes = ['org.clearlinux/crash/clr'] + q = q.filter(Record.classification.in_(classes)) + q = q.filter(Record.system_name == 'clear-linux-os') + if guilty_id: + q = q.filter(Record.guilty_id == guilty_id) + if machine_id: + q = q.filter(Record.machine_id == machine_id) + if most_recent: + interval_sec = 24 * 60 * 60 * int(most_recent) + current_time = time() + sec_in_past = current_time - interval_sec + q = q.filter(Record.timestamp_client > sec_in_past) + return q.all() + + @staticmethod + def reset_processed_records(classes=None, id=None): + q = db.session.query(Record) + if not classes: + classes = ['org.clearlinux/crash/clr'] + q = q.filter(Record.classification.in_(classes)) + q = q.filter(Record.system_name == 'clear-linux-os') + if id: + q = q.filter(Record.id == id) + records = q.all() + for r in records: + r.processed = False + db.session.commit() + + @staticmethod + def get_machine_ids_for_guilty(id, most_recent=None): + q = db.session.query(Record.build, Record.machine_id, db.func.count(Record.id).label('total'), Record.guilty_id) + q = q.filter(Record.guilty_id == id) + q = q.filter(Record.system_name == 'clear-linux-os') + q = q.filter(Record.build.op('~')('^[0-9][0-9]+$')) + q = q.group_by(Record.build, Record.machine_id, Record.guilty_id) + q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total')) + if most_recent: + interval_sec = 24 * 60 * 60 * int(most_recent) + current_time = time() + sec_in_past = current_time - interval_sec + q = q.filter(Record.timestamp_client > sec_in_past) + return q.all() + + @staticmethod + def get_update_msgs(): + q = db.session.query(Record.payload) + q = q.filter(Record.classification == "org.clearlinux/swupd-client/update") + + sec_2_weeks = 24 * 60 * 60 * 7 + current_time = time() + time_2_weeks_ago = current_time - sec_2_weeks + + # query for records created in that last 2 weeks + q = q.filter(Record.timestamp_client > time_2_weeks_ago) + return q + + @staticmethod + def get_swupd_msgs(most_recent=None): + q = db.session.query(Record.timestamp_client, Record.machine_id, Record.payload) + q = q.filter(Record.classification.like('org.clearlinux/swupd-client/%')) + + if most_recent: + interval_sec = 24 * 60 * 60 * int(most_recent) + current_time = time() + sec_in_past = current_time - interval_sec + q = q.filter(Record.timestamp_client > sec_in_past) + + q = q.order_by(desc(Record.timestamp_client)) + return q + + @staticmethod + def get_heartbeat_msgs(most_recent=None): + # These two expressions are SQL CASE conditional expressions, later + # used within count(distinct ...) aggregates for the query. + internal_expr = case([(Record.external == False, Record.machine_id), ]).label('internal_count') + external_expr = case([(Record.external == True, Record.machine_id), ]).label('external_count') + + q = db.session.query(Record.build, db.func.count(db.distinct(internal_expr)), db.func.count(db.distinct(external_expr))) + q = q.filter(Record.classification == "org.clearlinux/heartbeat/ping") + q = q.filter(Record.system_name == 'clear-linux-os') + q = q.group_by(Record.build) + + if most_recent: + interval_sec = 24 * 60 * 60 * int(most_recent) + current_time = time() + sec_in_past = current_time - interval_sec + q = q.filter(Record.timestamp_client > sec_in_past) + + q = q.order_by(cast(Record.build, db.Integer)) + return q.all() + + +class GuiltyBlacklist(db.Model): + __tablename__ = 'guilty_blacklisted' + id = db.Column(db.Integer, primary_key=True) + function = db.Column(db.String) + module = db.Column(db.String) + + def __init__(self, func, mod): + self.function = func + self.module = mod + + def __repr__(self): + return "".format(self.id, self.function, self.module) + + def __str__(self): + return str(self.to_dict()) + + def to_dict(self): + guilty = { + 'function': self.function, + 'module': self.module + } + return guilty + + @staticmethod + def add(func, mod): + try: + g = GuiltyBlacklist(func, mod) + db.session.add(g) + db.session.commit() + return g + except: + db.session.rollback() + raise + + @staticmethod + def remove(func, mod): + q = db.session.query(GuiltyBlacklist) + q = q.filter_by(function=func, module=mod) + entry = q.first() + db.session.delete(entry) + db.session.commit() + + @staticmethod + def get_guilties(): + q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module) + q = q.order_by(GuiltyBlacklist.function) + return q.all() + + @staticmethod + def exists(func, mod): + q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module) + q = q.filter_by(function=func, module=mod) + return len(q.all()) != 0 and True or False + + @staticmethod + def update(to_add, to_remove): + try: + for i in to_add: + if not GuiltyBlacklist.exists(i[0], i[1]): + g = GuiltyBlacklist(i[0], i[1]) + db.session.add(g) + for i in to_remove: + if GuiltyBlacklist.exists(i[0], i[1]): + q = db.session.query(GuiltyBlacklist) + q = q.filter_by(function=i[0], module=i[1]) + entry = q.first() + db.session.delete(entry) + db.session.commit() + except: + db.session.rollback() + raise + +# vi: ts=4 et sw=4 sts=4