mirror of
https://github.com/clearlinux/telemetrics-backend.git
synced 2026-09-05 13:21:28 +00:00
Removed the "shared" folder
Moved the contents of "shared" folder to "telemetryui" and deleted it. Signed-off-by: Reagan Lopez <reagan.lopez@intel.com>
This commit is contained in:
Reagan Lopez
-433
@@ -1,433 +0,0 @@
|
||||
#
|
||||
# Copyright 2015-2017 Intel Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import json
|
||||
from operator import itemgetter
|
||||
from collections import namedtuple
|
||||
import subprocess
|
||||
import re
|
||||
from .model import Record, GuiltyBlacklist
|
||||
from . import app
|
||||
|
||||
try:
|
||||
from uwsgidecorators import spool
|
||||
except ImportError:
|
||||
def spool(f):
|
||||
f.spool = f
|
||||
return f
|
||||
|
||||
filters = []
|
||||
|
||||
# Groups for the frame_pattern below
|
||||
# 1 - frame number + one space
|
||||
# 2 - function name + optional arguments
|
||||
# 3 - rest of the line
|
||||
# 4 - module name (inside the [])
|
||||
# 5 - optional frame source file and line number info
|
||||
|
||||
# TODO: The current c++filt logic depends on properly subsituting c++filt
|
||||
# output for the function name. Thus, it is very, very important to keep a
|
||||
# capture group that extends from the function name to the end of the frame as
|
||||
# long as this logic remains the same. Probably better to rework the code to
|
||||
# *not* destructively overwrite the backtrace field. Maybe store the filtered
|
||||
# output in a different field.
|
||||
|
||||
frame_pattern = "^(#\d+ )(.+)( - \[(.*)\](.*))$"
|
||||
|
||||
backtrace_classes = [
|
||||
'org.clearlinux/crash/clr',
|
||||
'org.clearlinux/kernel/bug',
|
||||
'org.clearlinux/kernel/stackoverflow',
|
||||
'org.clearlinux/kernel/warning'
|
||||
]
|
||||
|
||||
other_classes = [
|
||||
'org.clearlinux/crash/unknown',
|
||||
'org.clearlinux/crash/clr-build',
|
||||
'org.clearlinux/crash/error'
|
||||
]
|
||||
|
||||
|
||||
def get_all_classes():
|
||||
return backtrace_classes + other_classes
|
||||
|
||||
|
||||
def get_backtrace_classes():
|
||||
return backtrace_classes
|
||||
|
||||
|
||||
def get_other_classes():
|
||||
return other_classes
|
||||
|
||||
|
||||
def is_crash_classification(klass):
|
||||
return (klass in backtrace_classes) and True or False
|
||||
|
||||
|
||||
def is_blacklisted(function, module):
|
||||
funcmod = (function, module)
|
||||
if funcmod in filters:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def demangle_backtrace(bt):
|
||||
new_bt = []
|
||||
prog = '/usr/bin/c++filt'
|
||||
|
||||
frame_regex = re.compile(frame_pattern)
|
||||
lines = bt.splitlines()
|
||||
|
||||
for line in lines:
|
||||
m = frame_regex.match(line)
|
||||
if m:
|
||||
func = m.group(2)
|
||||
|
||||
# A frame with missing symbols is a special case, so skip it
|
||||
if func == '???':
|
||||
new_bt.append(line)
|
||||
continue
|
||||
|
||||
# FIXME: this logic will break once the crash probe starts sending
|
||||
# function argument values; make this more generic!
|
||||
if func[-2:] == '()':
|
||||
# The crash probe adds the () to the function name, but c++filt
|
||||
# cannot demangle a symbol with the () suffix
|
||||
func_name = func[:-2]
|
||||
else:
|
||||
# Assume already demangled, or this is from a kernel crash record
|
||||
new_bt.append(line)
|
||||
continue
|
||||
|
||||
try:
|
||||
new_func = subprocess.check_output([prog, func_name], universal_newlines=True)
|
||||
except:
|
||||
new_bt.append(line)
|
||||
continue
|
||||
|
||||
# c++filt adds a trailing newline to the output
|
||||
new_func = new_func.rstrip()
|
||||
|
||||
# Restore () if this was not a mangled symbol
|
||||
if new_func == func_name:
|
||||
new_func = func_name + '()'
|
||||
|
||||
repl_str = r'\1{}\3'.format(new_func)
|
||||
new_line = frame_regex.sub(repl_str, line)
|
||||
new_bt.append(new_line)
|
||||
else:
|
||||
new_bt.append(line)
|
||||
|
||||
return '\n'.join(new_bt)
|
||||
|
||||
|
||||
def find_guilty(backtrace):
|
||||
frame_regex = re.compile(frame_pattern)
|
||||
|
||||
guilty = {}
|
||||
|
||||
lines = backtrace.splitlines()
|
||||
|
||||
first_unknown = None
|
||||
in_backtrace = False
|
||||
prev_frame = None
|
||||
found_match = False
|
||||
found_unknown = False
|
||||
|
||||
# Begin guilty detection process
|
||||
for line in lines[1:]:
|
||||
m = frame_regex.match(line)
|
||||
|
||||
if m:
|
||||
# Either this is the first frame of the backtrace, or we are still
|
||||
# iterating through the backtrace.
|
||||
in_backtrace = True
|
||||
|
||||
func = m.group(2)
|
||||
mod = m.group(4)
|
||||
|
||||
# Only consider blacklisted function/module pairs as a last resort.
|
||||
# It's likely that the blacklisted pairs will never be chosen as
|
||||
# worthy candidates... if they are, the guilty blacklist may be
|
||||
# filtering too much.
|
||||
if is_blacklisted(func, mod):
|
||||
prev_frame = (func, mod)
|
||||
continue
|
||||
|
||||
# Consider the first frame without function symbols ('???') only if
|
||||
# there are no function symbols for any frames lower in the stack.
|
||||
if (func == '???' or func[:2] == '? ') and not found_unknown:
|
||||
found_unknown = True
|
||||
first_unknown = (func, mod)
|
||||
prev_frame = (func, mod)
|
||||
continue
|
||||
elif func == '???':
|
||||
# In this case, we've already encountered a frame with missing
|
||||
# function symbols, so skip it, but save the info for backup.
|
||||
prev_frame = (func, mod)
|
||||
continue
|
||||
|
||||
# If the previous three conditional checks fail, then we have found
|
||||
# the best guilty candidate: it is not in the blacklist, and it has
|
||||
# function symbols.
|
||||
guilty['function'] = func
|
||||
guilty['module'] = mod
|
||||
guilty['count'] = 1
|
||||
|
||||
found_match = True
|
||||
return (guilty, found_match)
|
||||
|
||||
elif in_backtrace:
|
||||
# We have processed the entire backtrace for the crashing thread of
|
||||
# the process, but no solid guilty has been found. Since we only
|
||||
# consider the crashing thread for guilty detection, stop iterating
|
||||
# through the remainder of the threads at this point.
|
||||
break
|
||||
|
||||
# Implement a backup plan to ensure that a guilty is chosen.
|
||||
if found_unknown:
|
||||
# Take preference for '???'
|
||||
guilty['function'] = first_unknown[0]
|
||||
guilty['module'] = first_unknown[1]
|
||||
guilty['count'] = 1
|
||||
found_match = True
|
||||
elif prev_frame:
|
||||
# Choose the previous frame as a last resort
|
||||
guilty['function'] = prev_frame[0]
|
||||
guilty['module'] = prev_frame[1]
|
||||
guilty['count'] = 1
|
||||
found_match = True
|
||||
|
||||
return (guilty, found_match)
|
||||
|
||||
|
||||
def _process_guilties(args):
|
||||
if isinstance(args['klass'], bytes):
|
||||
klass = args['klass'].decode()
|
||||
else:
|
||||
klass = args['klass']
|
||||
# In case the caller does not check for proper classification, bail early
|
||||
if not is_crash_classification(klass):
|
||||
return
|
||||
if 'id' in args:
|
||||
record_id = int(args['id'])
|
||||
else:
|
||||
record_id = None
|
||||
global filters
|
||||
with app.app_context():
|
||||
crashes = Record.get_new_crash_records(classes=get_backtrace_classes(), id=record_id)
|
||||
filters = GuiltyBlacklist.get_guilties()
|
||||
for rec in crashes:
|
||||
if rec.payload:
|
||||
new_bt = demangle_backtrace(rec.payload)
|
||||
rec.payload = new_bt
|
||||
# TODO: update the rec.payload field as well
|
||||
Record.commit_guilty_changes()
|
||||
g, match = find_guilty(rec.payload)
|
||||
if match:
|
||||
function = g['function']
|
||||
module = g['module']
|
||||
db_guilty = Record.get_guilty_for_funcmod(function, module)
|
||||
if db_guilty is None:
|
||||
db_guilty = Record.init_guilty(function, module)
|
||||
Record.create_guilty_for_record(rec, db_guilty)
|
||||
Record.set_processed_flag(rec)
|
||||
|
||||
Record.commit_guilty_changes()
|
||||
|
||||
|
||||
@spool
|
||||
def process_guilties(args):
|
||||
_process_guilties(args)
|
||||
|
||||
|
||||
def process_guilties_sync(**args):
|
||||
_process_guilties(args)
|
||||
|
||||
|
||||
def guilty_list_per_build(guilties):
|
||||
# TODO: should compute max values per build with a subquery instead
|
||||
build_maxcount = {}
|
||||
|
||||
buildset = set()
|
||||
buildlist = []
|
||||
newlist = []
|
||||
|
||||
for g in guilties:
|
||||
found_entry = False
|
||||
guilty_str = g[0] + ' - [' + g[1] + ']'
|
||||
build, count, guilty_id, comment = (g[2], g[3], g[4], g[5])
|
||||
for i, n in enumerate(newlist):
|
||||
if guilty_str == n['guilty']:
|
||||
newlist[i]['total'] += count
|
||||
newlist[i]['builds'].append((build, count))
|
||||
if build in build_maxcount:
|
||||
build_maxcount[build] = max(build_maxcount[build], count)
|
||||
else:
|
||||
build_maxcount[build] = count
|
||||
found_entry = True
|
||||
break
|
||||
|
||||
if found_entry:
|
||||
continue
|
||||
|
||||
entry = {}
|
||||
entry['guilty'] = guilty_str
|
||||
entry['total'] = count
|
||||
entry['guilty_id'] = guilty_id
|
||||
entry['comment'] = comment
|
||||
entry['builds'] = []
|
||||
entry['builds'].append((build, count))
|
||||
if build in build_maxcount:
|
||||
build_maxcount[build] = max(build_maxcount[build], count)
|
||||
else:
|
||||
build_maxcount[build] = count
|
||||
newlist.append(entry)
|
||||
|
||||
# We only care about the top 10 guilties
|
||||
newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:10]
|
||||
for guilty in newlist:
|
||||
for build in guilty['builds']:
|
||||
buildset.add(build[0])
|
||||
|
||||
buildlist = list(buildset)
|
||||
buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True)
|
||||
|
||||
# For crashes not occuring in a particular build, provide a "0" value for
|
||||
# the count. This simplifies table generation in the jinja template.
|
||||
for i, g in enumerate(newlist):
|
||||
builds, counts = list(zip(*g['builds']))
|
||||
counter = 0
|
||||
for b in buildlist:
|
||||
if b not in builds:
|
||||
newlist[i]['builds'].insert(counter, (b, "0"))
|
||||
counter += 1
|
||||
|
||||
for i, b in enumerate(buildlist):
|
||||
buildlist[i] = (b, build_maxcount[b])
|
||||
|
||||
buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True)
|
||||
|
||||
for i, b in enumerate(newlist):
|
||||
newlist[i]['builds'] = sorted(newlist[i]['builds'], key=lambda b: int(b[0]), reverse=True)
|
||||
|
||||
return (buildlist, newlist)
|
||||
|
||||
|
||||
def guilty_list_for_build(guilties, filter='overall'):
|
||||
newlist = []
|
||||
|
||||
for g in guilties:
|
||||
found_entry = False
|
||||
guilty_str = g[0] + ' - [' + g[1] + ']'
|
||||
build, count, guilty_id, comment = (g[2], g[3], g[4], g[5])
|
||||
for i, n in enumerate(newlist):
|
||||
if guilty_str == n['guilty'] and filter in ['overall', build]:
|
||||
newlist[i]['total'] += count
|
||||
found_entry = True
|
||||
break
|
||||
|
||||
if found_entry:
|
||||
continue
|
||||
|
||||
if filter in ['overall', build]:
|
||||
entry = {}
|
||||
entry['guilty'] = guilty_str
|
||||
entry['total'] = count
|
||||
entry['guilty_id'] = guilty_id
|
||||
entry['comment'] = comment
|
||||
newlist.append(entry)
|
||||
|
||||
# We only care about the top 10 guilties
|
||||
newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:10]
|
||||
|
||||
return newlist
|
||||
|
||||
|
||||
def get_all_funcmods():
|
||||
frame_regex = re.compile(frame_pattern)
|
||||
funcmodset = set()
|
||||
funcmodlist = []
|
||||
backtraces = Record.get_crash_backtraces(classes=get_backtrace_classes())
|
||||
|
||||
for b in backtraces:
|
||||
lines = b[0].splitlines()
|
||||
for line in lines:
|
||||
match = frame_regex.match(line)
|
||||
if match:
|
||||
funcmodset.add((match.group(2), match.group(4)))
|
||||
|
||||
return sorted(funcmodset)
|
||||
|
||||
|
||||
def parse_backtrace(backtrace):
|
||||
program_regex = re.compile('^Process: (.*)$')
|
||||
pid_regex = re.compile('^PID: ([0-9]+)$')
|
||||
signal_regex = re.compile('^Signal: ([0-9]+)$')
|
||||
bt_header_regex = re.compile('^Backtrace \(TID ([0-9]+)\):$')
|
||||
frame_regex = re.compile(frame_pattern)
|
||||
|
||||
Crash = namedtuple('Crash', ['record_id', 'program', 'pid', 'signal', 'backtrace'])
|
||||
program = ''
|
||||
pid = ''
|
||||
signal = ''
|
||||
frames = []
|
||||
parsed_header = False
|
||||
|
||||
lines = backtrace[0].splitlines()
|
||||
for line in lines:
|
||||
# Header info
|
||||
match = program_regex.match(line)
|
||||
if match:
|
||||
program = match.group(1)
|
||||
continue
|
||||
match = pid_regex.match(line)
|
||||
if match:
|
||||
pid = match.group(1)
|
||||
continue
|
||||
match = signal_regex.match(line)
|
||||
if match:
|
||||
signal = match.group(1)
|
||||
continue
|
||||
|
||||
# We only care about the backtrace from the crashing thread, which
|
||||
# is listed first in the payload.
|
||||
match = bt_header_regex.match(line)
|
||||
if match and not parsed_header:
|
||||
parsed_header = True
|
||||
elif match:
|
||||
break
|
||||
|
||||
match = frame_regex.match(line)
|
||||
if match:
|
||||
frames.append((match.group(2), match.group(4), match.group(5)))
|
||||
|
||||
# Populate a namedtuple for convenience
|
||||
record_id = backtrace[1]
|
||||
c = Crash(record_id, program, pid, signal, frames)
|
||||
return c
|
||||
|
||||
|
||||
def explode_backtraces(classes=None, guilty_id=None, machine_id=None, build=None):
|
||||
crashes = []
|
||||
backtraces = Record.get_crash_backtraces(classes, guilty_id, machine_id, build)
|
||||
for b in backtraces:
|
||||
crashes.append(parse_backtrace(b))
|
||||
return crashes
|
||||
|
||||
|
||||
# vi: ts=4 et sw=4 sts=4
|
||||
-619
@@ -1,619 +0,0 @@
|
||||
#
|
||||
# Copyright 2015-2017 Intel Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import itertools
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from sqlalchemy.sql.expression import cast
|
||||
from sqlalchemy.sql.expression import desc
|
||||
from sqlalchemy.sql.expression import case
|
||||
from time import time, localtime, strftime, mktime, strptime, gmtime
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
from . import app
|
||||
|
||||
db = SQLAlchemy(app)
|
||||
|
||||
|
||||
class Guilty(db.Model):
|
||||
__tablename__ = 'guilty'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
function = db.Column(db.String)
|
||||
module = db.Column(db.String)
|
||||
comment = db.Column(db.String)
|
||||
hide = db.Column(db.Boolean, default=False)
|
||||
|
||||
def __init__(self, func, mod):
|
||||
self.function = func
|
||||
self.module = mod
|
||||
|
||||
@staticmethod
|
||||
def update_comment(guilty_id, comment):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
guilty.comment = comment
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_function(guilty_id):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
return guilty and guilty.function or ""
|
||||
|
||||
@staticmethod
|
||||
def get_module(guilty_id):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
return guilty and guilty.module or ""
|
||||
|
||||
@staticmethod
|
||||
def update_hidden(guilty_id, status):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
guilty.hide = status
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_hidden_value(guilty_id):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
return guilty and guilty.hide or False
|
||||
|
||||
@staticmethod
|
||||
def get_hidden_guilties():
|
||||
q = db.session.query(Guilty.id, Guilty.function, Guilty.module)
|
||||
q = q.filter(Guilty.hide == True)
|
||||
q = q.order_by(Guilty.function)
|
||||
return q.all()
|
||||
|
||||
|
||||
class Record(db.Model):
|
||||
__tablename__ = 'records'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
architecture = db.Column(db.Text)
|
||||
bios_version = db.Column(db.Text, default='')
|
||||
board_name = db.Column(db.Text, default='')
|
||||
build = db.Column(db.Text, nullable=False)
|
||||
classification = db.Column(db.Text, nullable=False)
|
||||
cpu_model = db.Column(db.Text, default='')
|
||||
event_id = db.Column(db.Text, default='')
|
||||
external = db.Column(db.Boolean, default=False)
|
||||
host_type = db.Column(db.Text, default='')
|
||||
kernel_version = db.Column(db.Text, default=0)
|
||||
machine_id = db.Column(db.Text, default='')
|
||||
payload_version = db.Column(db.Integer)
|
||||
record_version = db.Column(db.Integer, default=0)
|
||||
severity = db.Column(db.Integer)
|
||||
system_name = db.Column(db.Text)
|
||||
timestamp_client = db.Column(db.Numeric)
|
||||
timestamp_server = db.Column(db.Numeric, nullable=False)
|
||||
payload = db.Column(db.Text, nullable=False)
|
||||
|
||||
processed = db.Column(db.Boolean, default=False)
|
||||
guilty_id = db.Column(db.Integer, db.ForeignKey('guilty.id'))
|
||||
|
||||
guilty = db.Column(db.Text, default='')
|
||||
guilty = db.relationship('Guilty', backref=db.backref('records', lazy='dynamic'), lazy='joined')
|
||||
|
||||
def __init__(self, machine_id, host_type, severity, classification, build, architecture, kernel_version,
|
||||
record_version, ts_capture, ts_reception, payload_version, system_name,
|
||||
board_name, bios_version, cpu_model, event_id, external, payload):
|
||||
self.machine_id = machine_id
|
||||
self.host_type = host_type
|
||||
self.architecture = architecture
|
||||
self.classification = classification
|
||||
self.build = build
|
||||
self.kernel_version = kernel_version
|
||||
self.record_version = record_version
|
||||
self.severity = severity
|
||||
self.timestamp_client = ts_capture
|
||||
self.timestamp_server = ts_reception
|
||||
self.payload_version = payload_version
|
||||
self.system_name = system_name
|
||||
self.external = external
|
||||
self.board_name = board_name
|
||||
self.bios_version = bios_version
|
||||
self.cpu_model = cpu_model
|
||||
self.event_id = event_id
|
||||
|
||||
try:
|
||||
self.payload = payload.encode('utf-8')
|
||||
except UnicodeError:
|
||||
self.payload = payload.encode('latin-1')
|
||||
|
||||
def __repr__(self):
|
||||
return "<Record(id='{}', class='{}', build='{}', created='{}')>".format(self.id, self.classification, self.build, strftime("%a, %d %b %Y %H:%M:%S", localtime(self.timestamp_client)))
|
||||
|
||||
def __str__(self):
|
||||
return str(self.to_dict())
|
||||
|
||||
def to_dict(self):
|
||||
record = {
|
||||
'id': self.id,
|
||||
'machine_id': self.machine_id,
|
||||
'machine_type': self.host_type,
|
||||
'arch': self.architecture,
|
||||
'build': self.build,
|
||||
'kernel_version': self.kernel_version,
|
||||
'ts_capture': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_client)),
|
||||
'ts_reception': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_server)),
|
||||
'severity': self.severity,
|
||||
'classification': self.classification,
|
||||
'record_version': self.record_version,
|
||||
'payload': self.payload,
|
||||
'board_name': self.board_name,
|
||||
'bios_version': self.bios_version,
|
||||
'cpu_model': self.cpu_model,
|
||||
'event_id': self.event_id,
|
||||
'external': self.external,
|
||||
}
|
||||
return record
|
||||
|
||||
# for the exported CSV rows
|
||||
def to_list(self):
|
||||
record = [
|
||||
self.id,
|
||||
self.external,
|
||||
self.timestamp_server,
|
||||
self.severity,
|
||||
self.classification,
|
||||
self.build,
|
||||
self.machine_id,
|
||||
self.payload
|
||||
]
|
||||
return record
|
||||
|
||||
@staticmethod
|
||||
def list():
|
||||
return Record.query.all()
|
||||
|
||||
@staticmethod
|
||||
def create(machine_id, host_type, severity, classification, build, architecture, kernel_version,
|
||||
record_version, ts_capture, ts_reception, payload_version, system_name,
|
||||
board_name, bios_version, cpu_model, event_id, external, payload):
|
||||
try:
|
||||
record = Record(machine_id, host_type, severity, classification, build, architecture, kernel_version,
|
||||
record_version, ts_capture, ts_reception, payload_version, system_name,
|
||||
board_name, bios_version, cpu_model, event_id, external, payload)
|
||||
db.session.add(record)
|
||||
db.session.commit()
|
||||
return record
|
||||
except:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def query_records(build, classification, severity, machine_id, limit,
|
||||
interval_sec=None, ts_capture=None, from_id=None):
|
||||
records = Record.query
|
||||
if build is not None:
|
||||
records = records.filter_by(build=build)
|
||||
if classification is not None:
|
||||
records = records.filter_by(classification=classification)
|
||||
if severity is not None:
|
||||
records = records.filter(Record.severity == severity)
|
||||
if machine_id is not None:
|
||||
records = records.filter(Record.machine_id == machine_id)
|
||||
if from_id is not None:
|
||||
records = records.filter(Record.id >= from_id)
|
||||
if ts_capture is not None:
|
||||
records = records.filter(Record.timestamp_client > ts_capture)
|
||||
|
||||
if interval_sec is not None:
|
||||
current_time = time()
|
||||
secs_in_past = current_time - interval_sec
|
||||
# Due to time skew on client systems, delayed sends due to
|
||||
# spooling, etc, timestamp_server works better as the reference
|
||||
# timestamp.
|
||||
records = records.filter(Record.timestamp_server > secs_in_past)
|
||||
|
||||
records = records.order_by(Record.id.desc())
|
||||
|
||||
if limit is not None:
|
||||
records = records.limit(limit)
|
||||
|
||||
return records.all()
|
||||
|
||||
@staticmethod
|
||||
def get_record(record_id):
|
||||
record = Record.query.filter_by(id=record_id).first()
|
||||
return record
|
||||
|
||||
@staticmethod
|
||||
def filter_records(build, classification, severity, machine_id=None, system_name=None, limit=None, from_date=None,
|
||||
to_date=None, payload=None, not_payload=None, data_source=None):
|
||||
records = Record.query
|
||||
if build is not None:
|
||||
records = records.filter_by(build=build)
|
||||
if classification is not None:
|
||||
if isinstance(classification, list):
|
||||
records = records.filter(Record.classification.in_(classification))
|
||||
else:
|
||||
records = records.filter(Record.classification.like(classification))
|
||||
if severity is not None:
|
||||
records = records.filter(Record.severity == severity)
|
||||
if system_name is not None:
|
||||
records = records.filter(Record.system_name == system_name)
|
||||
if machine_id is not None:
|
||||
records = records.filter(Record.machine_id == machine_id)
|
||||
if from_date is not None:
|
||||
from_date = mktime(strptime(from_date, "%Y-%m-%d"))
|
||||
records = records.filter(Record.timestamp_client >= from_date)
|
||||
if to_date is not None:
|
||||
to_date = mktime(strptime(to_date, "%Y-%m-%d"))
|
||||
records = records.filter(Record.timestamp_client < to_date)
|
||||
if payload is not None:
|
||||
records = records.filter(Record.payload.op('~')(payload))
|
||||
if not_payload is not None:
|
||||
records = records.filter(~Record.payload.op('~')(not_payload))
|
||||
if data_source is not None:
|
||||
if data_source == "external":
|
||||
records = records.filter(Record.external == True)
|
||||
elif data_source == "internal":
|
||||
records = records.filter(Record.external == False)
|
||||
|
||||
records = records.order_by(Record.id.desc())
|
||||
|
||||
if limit is not None:
|
||||
records = records.limit(limit)
|
||||
|
||||
return records
|
||||
|
||||
@staticmethod
|
||||
def delete_records():
|
||||
MAX_DAYS_KEEP_UNFILTERED_RECORDS = app.config.get("MAX_DAYS_KEEP_UNFILTERED_RECORDS", 35)
|
||||
PURGE_FILTERED_RECORDS = app.config.get("PURGE_FILTERED_RECORDS", {})
|
||||
try:
|
||||
def purge_field(field):
|
||||
for name in PURGE_FILTERED_RECORDS[field].keys():
|
||||
if PURGE_FILTERED_RECORDS[field][name]:
|
||||
age = time() - PURGE_FILTERED_RECORDS[field][name] * 24 * 60 * 60
|
||||
q = db.session.query(Record)
|
||||
if field == 'classification':
|
||||
q = q.filter(Record.classification.like(name.replace("*", "%")))
|
||||
else:
|
||||
q = q.filter(getattr(Record, field) == name)
|
||||
q = q.filter(Record.timestamp_server < age)
|
||||
if q.all():
|
||||
count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False)
|
||||
print("Deleted {} {} records".format(count, name))
|
||||
for field in PURGE_FILTERED_RECORDS.keys():
|
||||
purge_field(field)
|
||||
if MAX_DAYS_KEEP_UNFILTERED_RECORDS:
|
||||
unfiltered_age = time() - MAX_DAYS_KEEP_UNFILTERED_RECORDS * 24 * 60 * 60
|
||||
q = db.session.query(Record.id)
|
||||
for field in PURGE_FILTERED_RECORDS.keys():
|
||||
if field == 'classification':
|
||||
for classification in PURGE_FILTERED_RECORDS[field].keys():
|
||||
q = q.filter(~Record.classification.like(classification.replace("*", "%")))
|
||||
else:
|
||||
for name in PURGE_FILTERED_RECORDS[field].keys():
|
||||
q = q.filter(getattr(Record, field) != name)
|
||||
q = q.filter(Record.timestamp_server < unfiltered_age)
|
||||
if q.all():
|
||||
count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False)
|
||||
print("Deleted {} old records".format(count))
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
app.logger.error("Record purging failed")
|
||||
app.logger.error(e)
|
||||
db.session.rollback()
|
||||
|
||||
@staticmethod
|
||||
def get_recordcnts_by_build():
|
||||
q = db.session.query(Record.build, db.func.count(Record.id))
|
||||
q = q.filter(Record.build.op('~')('^[0-9]+$'))
|
||||
q = q.group_by(Record.build).order_by(cast(Record.build, db.Integer)).all()
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_builds():
|
||||
q = db.session.query(Record.build).distinct()
|
||||
q = q.order_by(Record.build)
|
||||
return sorted(q.all(), key=lambda x: LooseVersion(x[0]), reverse=True)
|
||||
|
||||
@staticmethod
|
||||
def get_recordcnts_by_classification():
|
||||
q = db.session.query(Record.classification, db.func.count(Record.id).label('total'))
|
||||
q = q.group_by(Record.classification)
|
||||
q = q.order_by(desc('total'))
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def expand_class(D):
|
||||
A, B, C = D
|
||||
return ["{}/*".format(A), "{}/{}/*".format(A, B), "{}/{}/{}".format(A, B, C)]
|
||||
|
||||
@staticmethod
|
||||
def get_classifications(with_regex=False):
|
||||
q = db.session.query(Record.classification).distinct()
|
||||
if with_regex:
|
||||
classes = [Record.expand_class(c[0].split('/')) for c in q.all()]
|
||||
return sorted(set(itertools.chain(*classes)))
|
||||
else:
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_os_map():
|
||||
q = db.session.query(Record.system_name, Record.build).order_by(Record.system_name).group_by(Record.system_name, Record.build).all()
|
||||
result = {}
|
||||
for x in q:
|
||||
result.setdefault(x[0], []).append(x[1])
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def get_recordcnts_by_machine_type():
|
||||
q = db.session.query(Record.host_type, db.func.count(Record.id).label('total'))
|
||||
q = q.group_by(Record.host_type)
|
||||
q = q.order_by(desc('total'))
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_recordcnts_by_severity():
|
||||
q = db.session.query(Record.severity, db.func.count(Record.id)).group_by(Record.severity).all()
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_crashcnts_by_class(classes=None):
|
||||
q = db.session.query(Record.classification, db.func.count(Record.id))
|
||||
if classes:
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
else:
|
||||
q = q.filter(Record.classification.like('org.clearlinux/crash/%'))
|
||||
q = q.group_by(Record.classification)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_crashcnts_by_build(classes=None):
|
||||
q = db.session.query(Record.build, db.func.count(Record.id))
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.build.op('~')('^[0-9]+$'))
|
||||
q = q.group_by(Record.build)
|
||||
q = q.order_by(desc(cast(Record.build, db.Integer)))
|
||||
q = q.limit(10)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_top_crash_guilties(classes=None):
|
||||
q = db.session.query(Guilty.function, Guilty.module, Record.build, db.func.count(Record.id).label('total'), Guilty.id, Guilty.comment)
|
||||
q = q.join(Record)
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.build.op('~')('^[0-9][0-9]+$'))
|
||||
q = q.filter(cast(Record.build, db.Integer) <= 100000)
|
||||
q = q.filter(Guilty.hide == False)
|
||||
q = q.group_by(Guilty.function, Guilty.module, Guilty.comment, Guilty.id, Record.build)
|
||||
q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total'))
|
||||
# query for records created in the last week (~ 10 Clear builds)
|
||||
q = q.filter(Record.build.in_(sorted(tuple(set([x[2] for x in q.all()])), key=lambda x: int(x))[-8:]))
|
||||
interval_sec = 24 * 60 * 60 * 7
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_new_crash_records(classes=None, id=None):
|
||||
q = db.session.query(Record)
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
q = q.filter(Record.processed == False)
|
||||
if id:
|
||||
q = q.filter(Record.id == id)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def set_processed_flag(record):
|
||||
record.processed = True
|
||||
|
||||
@staticmethod
|
||||
def get_guilty_for_funcmod(func, mod):
|
||||
q = db.session.query(Guilty).filter_by(function=func, module=mod).first()
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_guilty_id_for_record(record_id):
|
||||
q = db.session.query(Guilty.id).join(Record)
|
||||
q = q.filter(Record.id == record_id)
|
||||
return q.first()
|
||||
|
||||
@staticmethod
|
||||
def init_guilty(func, mod):
|
||||
return Guilty(func, mod)
|
||||
|
||||
@staticmethod
|
||||
def create_guilty_for_record(record, guilty):
|
||||
record.guilty = guilty
|
||||
|
||||
@staticmethod
|
||||
def commit_guilty_changes():
|
||||
# just commit for now
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_crash_backtraces(classes=None, guilty_id=None, machine_id=None, build=None, most_recent=None, record_id=None):
|
||||
q = db.session.query(Record.payload, Record.id)
|
||||
# Short circuit if we know the record ID
|
||||
if record_id:
|
||||
q = q.filter(Record.id == record_id)
|
||||
return q.first()
|
||||
if build:
|
||||
q = q.filter(Record.build == build)
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
if guilty_id:
|
||||
q = q.filter(Record.guilty_id == guilty_id)
|
||||
if machine_id:
|
||||
q = q.filter(Record.machine_id == machine_id)
|
||||
if most_recent:
|
||||
interval_sec = 24 * 60 * 60 * int(most_recent)
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def reset_processed_records(classes=None, id=None):
|
||||
q = db.session.query(Record)
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
if id:
|
||||
q = q.filter(Record.id == id)
|
||||
records = q.all()
|
||||
for r in records:
|
||||
r.processed = False
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_machine_ids_for_guilty(id, most_recent=None):
|
||||
q = db.session.query(Record.build, Record.machine_id, db.func.count(Record.id).label('total'), Record.guilty_id)
|
||||
q = q.filter(Record.guilty_id == id)
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
q = q.filter(Record.build.op('~')('^[0-9][0-9]+$'))
|
||||
q = q.group_by(Record.build, Record.machine_id, Record.guilty_id)
|
||||
q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total'))
|
||||
if most_recent:
|
||||
interval_sec = 24 * 60 * 60 * int(most_recent)
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_update_msgs():
|
||||
q = db.session.query(Record.payload)
|
||||
q = q.filter(Record.classification == "org.clearlinux/swupd-client/update")
|
||||
|
||||
sec_2_weeks = 24 * 60 * 60 * 7
|
||||
current_time = time()
|
||||
time_2_weeks_ago = current_time - sec_2_weeks
|
||||
|
||||
# query for records created in that last 2 weeks
|
||||
q = q.filter(Record.timestamp_client > time_2_weeks_ago)
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_swupd_msgs(most_recent=None):
|
||||
q = db.session.query(Record.timestamp_client, Record.machine_id, Record.payload)
|
||||
q = q.filter(Record.classification.like('org.clearlinux/swupd-client/%'))
|
||||
|
||||
if most_recent:
|
||||
interval_sec = 24 * 60 * 60 * int(most_recent)
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
|
||||
q = q.order_by(desc(Record.timestamp_client))
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_heartbeat_msgs(most_recent=None):
|
||||
# These two expressions are SQL CASE conditional expressions, later
|
||||
# used within count(distinct ...) aggregates for the query.
|
||||
internal_expr = case([(Record.external == False, Record.machine_id), ]).label('internal_count')
|
||||
external_expr = case([(Record.external == True, Record.machine_id), ]).label('external_count')
|
||||
|
||||
q = db.session.query(Record.build, db.func.count(db.distinct(internal_expr)), db.func.count(db.distinct(external_expr)))
|
||||
q = q.filter(Record.classification == "org.clearlinux/heartbeat/ping")
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
q = q.group_by(Record.build)
|
||||
|
||||
if most_recent:
|
||||
interval_sec = 24 * 60 * 60 * int(most_recent)
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
|
||||
q = q.order_by(cast(Record.build, db.Integer))
|
||||
return q.all()
|
||||
|
||||
|
||||
class GuiltyBlacklist(db.Model):
|
||||
__tablename__ = 'guilty_blacklisted'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
function = db.Column(db.String)
|
||||
module = db.Column(db.String)
|
||||
|
||||
def __init__(self, func, mod):
|
||||
self.function = func
|
||||
self.module = mod
|
||||
|
||||
def __repr__(self):
|
||||
return "<GuiltyBlacklist(id='{}', guilty='{}:{}')>".format(self.id, self.function, self.module)
|
||||
|
||||
def __str__(self):
|
||||
return str(self.to_dict())
|
||||
|
||||
def to_dict(self):
|
||||
guilty = {
|
||||
'function': self.function,
|
||||
'module': self.module
|
||||
}
|
||||
return guilty
|
||||
|
||||
@staticmethod
|
||||
def add(func, mod):
|
||||
try:
|
||||
g = GuiltyBlacklist(func, mod)
|
||||
db.session.add(g)
|
||||
db.session.commit()
|
||||
return g
|
||||
except:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def remove(func, mod):
|
||||
q = db.session.query(GuiltyBlacklist)
|
||||
q = q.filter_by(function=func, module=mod)
|
||||
entry = q.first()
|
||||
db.session.delete(entry)
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_guilties():
|
||||
q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module)
|
||||
q = q.order_by(GuiltyBlacklist.function)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def exists(func, mod):
|
||||
q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module)
|
||||
q = q.filter_by(function=func, module=mod)
|
||||
return len(q.all()) != 0 and True or False
|
||||
|
||||
@staticmethod
|
||||
def update(to_add, to_remove):
|
||||
try:
|
||||
for i in to_add:
|
||||
if not GuiltyBlacklist.exists(i[0], i[1]):
|
||||
g = GuiltyBlacklist(i[0], i[1])
|
||||
db.session.add(g)
|
||||
for i in to_remove:
|
||||
if GuiltyBlacklist.exists(i[0], i[1]):
|
||||
q = db.session.query(GuiltyBlacklist)
|
||||
q = q.filter_by(function=i[0], module=i[1])
|
||||
entry = q.first()
|
||||
db.session.delete(entry)
|
||||
db.session.commit()
|
||||
except:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
# vi: ts=4 et sw=4 sts=4
|
||||
@@ -1 +0,0 @@
|
||||
../../shared/crash.py
|
||||
@@ -0,0 +1,433 @@
|
||||
#
|
||||
# Copyright 2015-2017 Intel Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import json
|
||||
from operator import itemgetter
|
||||
from collections import namedtuple
|
||||
import subprocess
|
||||
import re
|
||||
from .model import Record, GuiltyBlacklist
|
||||
from . import app
|
||||
|
||||
try:
|
||||
from uwsgidecorators import spool
|
||||
except ImportError:
|
||||
def spool(f):
|
||||
f.spool = f
|
||||
return f
|
||||
|
||||
filters = []
|
||||
|
||||
# Groups for the frame_pattern below
|
||||
# 1 - frame number + one space
|
||||
# 2 - function name + optional arguments
|
||||
# 3 - rest of the line
|
||||
# 4 - module name (inside the [])
|
||||
# 5 - optional frame source file and line number info
|
||||
|
||||
# TODO: The current c++filt logic depends on properly subsituting c++filt
|
||||
# output for the function name. Thus, it is very, very important to keep a
|
||||
# capture group that extends from the function name to the end of the frame as
|
||||
# long as this logic remains the same. Probably better to rework the code to
|
||||
# *not* destructively overwrite the backtrace field. Maybe store the filtered
|
||||
# output in a different field.
|
||||
|
||||
frame_pattern = "^(#\d+ )(.+)( - \[(.*)\](.*))$"
|
||||
|
||||
backtrace_classes = [
|
||||
'org.clearlinux/crash/clr',
|
||||
'org.clearlinux/kernel/bug',
|
||||
'org.clearlinux/kernel/stackoverflow',
|
||||
'org.clearlinux/kernel/warning'
|
||||
]
|
||||
|
||||
other_classes = [
|
||||
'org.clearlinux/crash/unknown',
|
||||
'org.clearlinux/crash/clr-build',
|
||||
'org.clearlinux/crash/error'
|
||||
]
|
||||
|
||||
|
||||
def get_all_classes():
|
||||
return backtrace_classes + other_classes
|
||||
|
||||
|
||||
def get_backtrace_classes():
|
||||
return backtrace_classes
|
||||
|
||||
|
||||
def get_other_classes():
|
||||
return other_classes
|
||||
|
||||
|
||||
def is_crash_classification(klass):
|
||||
return (klass in backtrace_classes) and True or False
|
||||
|
||||
|
||||
def is_blacklisted(function, module):
|
||||
funcmod = (function, module)
|
||||
if funcmod in filters:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def demangle_backtrace(bt):
|
||||
new_bt = []
|
||||
prog = '/usr/bin/c++filt'
|
||||
|
||||
frame_regex = re.compile(frame_pattern)
|
||||
lines = bt.splitlines()
|
||||
|
||||
for line in lines:
|
||||
m = frame_regex.match(line)
|
||||
if m:
|
||||
func = m.group(2)
|
||||
|
||||
# A frame with missing symbols is a special case, so skip it
|
||||
if func == '???':
|
||||
new_bt.append(line)
|
||||
continue
|
||||
|
||||
# FIXME: this logic will break once the crash probe starts sending
|
||||
# function argument values; make this more generic!
|
||||
if func[-2:] == '()':
|
||||
# The crash probe adds the () to the function name, but c++filt
|
||||
# cannot demangle a symbol with the () suffix
|
||||
func_name = func[:-2]
|
||||
else:
|
||||
# Assume already demangled, or this is from a kernel crash record
|
||||
new_bt.append(line)
|
||||
continue
|
||||
|
||||
try:
|
||||
new_func = subprocess.check_output([prog, func_name], universal_newlines=True)
|
||||
except:
|
||||
new_bt.append(line)
|
||||
continue
|
||||
|
||||
# c++filt adds a trailing newline to the output
|
||||
new_func = new_func.rstrip()
|
||||
|
||||
# Restore () if this was not a mangled symbol
|
||||
if new_func == func_name:
|
||||
new_func = func_name + '()'
|
||||
|
||||
repl_str = r'\1{}\3'.format(new_func)
|
||||
new_line = frame_regex.sub(repl_str, line)
|
||||
new_bt.append(new_line)
|
||||
else:
|
||||
new_bt.append(line)
|
||||
|
||||
return '\n'.join(new_bt)
|
||||
|
||||
|
||||
def find_guilty(backtrace):
|
||||
frame_regex = re.compile(frame_pattern)
|
||||
|
||||
guilty = {}
|
||||
|
||||
lines = backtrace.splitlines()
|
||||
|
||||
first_unknown = None
|
||||
in_backtrace = False
|
||||
prev_frame = None
|
||||
found_match = False
|
||||
found_unknown = False
|
||||
|
||||
# Begin guilty detection process
|
||||
for line in lines[1:]:
|
||||
m = frame_regex.match(line)
|
||||
|
||||
if m:
|
||||
# Either this is the first frame of the backtrace, or we are still
|
||||
# iterating through the backtrace.
|
||||
in_backtrace = True
|
||||
|
||||
func = m.group(2)
|
||||
mod = m.group(4)
|
||||
|
||||
# Only consider blacklisted function/module pairs as a last resort.
|
||||
# It's likely that the blacklisted pairs will never be chosen as
|
||||
# worthy candidates... if they are, the guilty blacklist may be
|
||||
# filtering too much.
|
||||
if is_blacklisted(func, mod):
|
||||
prev_frame = (func, mod)
|
||||
continue
|
||||
|
||||
# Consider the first frame without function symbols ('???') only if
|
||||
# there are no function symbols for any frames lower in the stack.
|
||||
if (func == '???' or func[:2] == '? ') and not found_unknown:
|
||||
found_unknown = True
|
||||
first_unknown = (func, mod)
|
||||
prev_frame = (func, mod)
|
||||
continue
|
||||
elif func == '???':
|
||||
# In this case, we've already encountered a frame with missing
|
||||
# function symbols, so skip it, but save the info for backup.
|
||||
prev_frame = (func, mod)
|
||||
continue
|
||||
|
||||
# If the previous three conditional checks fail, then we have found
|
||||
# the best guilty candidate: it is not in the blacklist, and it has
|
||||
# function symbols.
|
||||
guilty['function'] = func
|
||||
guilty['module'] = mod
|
||||
guilty['count'] = 1
|
||||
|
||||
found_match = True
|
||||
return (guilty, found_match)
|
||||
|
||||
elif in_backtrace:
|
||||
# We have processed the entire backtrace for the crashing thread of
|
||||
# the process, but no solid guilty has been found. Since we only
|
||||
# consider the crashing thread for guilty detection, stop iterating
|
||||
# through the remainder of the threads at this point.
|
||||
break
|
||||
|
||||
# Implement a backup plan to ensure that a guilty is chosen.
|
||||
if found_unknown:
|
||||
# Take preference for '???'
|
||||
guilty['function'] = first_unknown[0]
|
||||
guilty['module'] = first_unknown[1]
|
||||
guilty['count'] = 1
|
||||
found_match = True
|
||||
elif prev_frame:
|
||||
# Choose the previous frame as a last resort
|
||||
guilty['function'] = prev_frame[0]
|
||||
guilty['module'] = prev_frame[1]
|
||||
guilty['count'] = 1
|
||||
found_match = True
|
||||
|
||||
return (guilty, found_match)
|
||||
|
||||
|
||||
def _process_guilties(args):
|
||||
if isinstance(args['klass'], bytes):
|
||||
klass = args['klass'].decode()
|
||||
else:
|
||||
klass = args['klass']
|
||||
# In case the caller does not check for proper classification, bail early
|
||||
if not is_crash_classification(klass):
|
||||
return
|
||||
if 'id' in args:
|
||||
record_id = int(args['id'])
|
||||
else:
|
||||
record_id = None
|
||||
global filters
|
||||
with app.app_context():
|
||||
crashes = Record.get_new_crash_records(classes=get_backtrace_classes(), id=record_id)
|
||||
filters = GuiltyBlacklist.get_guilties()
|
||||
for rec in crashes:
|
||||
if rec.payload:
|
||||
new_bt = demangle_backtrace(rec.payload)
|
||||
rec.payload = new_bt
|
||||
# TODO: update the rec.payload field as well
|
||||
Record.commit_guilty_changes()
|
||||
g, match = find_guilty(rec.payload)
|
||||
if match:
|
||||
function = g['function']
|
||||
module = g['module']
|
||||
db_guilty = Record.get_guilty_for_funcmod(function, module)
|
||||
if db_guilty is None:
|
||||
db_guilty = Record.init_guilty(function, module)
|
||||
Record.create_guilty_for_record(rec, db_guilty)
|
||||
Record.set_processed_flag(rec)
|
||||
|
||||
Record.commit_guilty_changes()
|
||||
|
||||
|
||||
@spool
|
||||
def process_guilties(args):
|
||||
_process_guilties(args)
|
||||
|
||||
|
||||
def process_guilties_sync(**args):
|
||||
_process_guilties(args)
|
||||
|
||||
|
||||
def guilty_list_per_build(guilties):
|
||||
# TODO: should compute max values per build with a subquery instead
|
||||
build_maxcount = {}
|
||||
|
||||
buildset = set()
|
||||
buildlist = []
|
||||
newlist = []
|
||||
|
||||
for g in guilties:
|
||||
found_entry = False
|
||||
guilty_str = g[0] + ' - [' + g[1] + ']'
|
||||
build, count, guilty_id, comment = (g[2], g[3], g[4], g[5])
|
||||
for i, n in enumerate(newlist):
|
||||
if guilty_str == n['guilty']:
|
||||
newlist[i]['total'] += count
|
||||
newlist[i]['builds'].append((build, count))
|
||||
if build in build_maxcount:
|
||||
build_maxcount[build] = max(build_maxcount[build], count)
|
||||
else:
|
||||
build_maxcount[build] = count
|
||||
found_entry = True
|
||||
break
|
||||
|
||||
if found_entry:
|
||||
continue
|
||||
|
||||
entry = {}
|
||||
entry['guilty'] = guilty_str
|
||||
entry['total'] = count
|
||||
entry['guilty_id'] = guilty_id
|
||||
entry['comment'] = comment
|
||||
entry['builds'] = []
|
||||
entry['builds'].append((build, count))
|
||||
if build in build_maxcount:
|
||||
build_maxcount[build] = max(build_maxcount[build], count)
|
||||
else:
|
||||
build_maxcount[build] = count
|
||||
newlist.append(entry)
|
||||
|
||||
# We only care about the top 10 guilties
|
||||
newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:10]
|
||||
for guilty in newlist:
|
||||
for build in guilty['builds']:
|
||||
buildset.add(build[0])
|
||||
|
||||
buildlist = list(buildset)
|
||||
buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True)
|
||||
|
||||
# For crashes not occuring in a particular build, provide a "0" value for
|
||||
# the count. This simplifies table generation in the jinja template.
|
||||
for i, g in enumerate(newlist):
|
||||
builds, counts = list(zip(*g['builds']))
|
||||
counter = 0
|
||||
for b in buildlist:
|
||||
if b not in builds:
|
||||
newlist[i]['builds'].insert(counter, (b, "0"))
|
||||
counter += 1
|
||||
|
||||
for i, b in enumerate(buildlist):
|
||||
buildlist[i] = (b, build_maxcount[b])
|
||||
|
||||
buildlist = sorted(buildlist, key=lambda b: int(b[0]), reverse=True)
|
||||
|
||||
for i, b in enumerate(newlist):
|
||||
newlist[i]['builds'] = sorted(newlist[i]['builds'], key=lambda b: int(b[0]), reverse=True)
|
||||
|
||||
return (buildlist, newlist)
|
||||
|
||||
|
||||
def guilty_list_for_build(guilties, filter='overall'):
|
||||
newlist = []
|
||||
|
||||
for g in guilties:
|
||||
found_entry = False
|
||||
guilty_str = g[0] + ' - [' + g[1] + ']'
|
||||
build, count, guilty_id, comment = (g[2], g[3], g[4], g[5])
|
||||
for i, n in enumerate(newlist):
|
||||
if guilty_str == n['guilty'] and filter in ['overall', build]:
|
||||
newlist[i]['total'] += count
|
||||
found_entry = True
|
||||
break
|
||||
|
||||
if found_entry:
|
||||
continue
|
||||
|
||||
if filter in ['overall', build]:
|
||||
entry = {}
|
||||
entry['guilty'] = guilty_str
|
||||
entry['total'] = count
|
||||
entry['guilty_id'] = guilty_id
|
||||
entry['comment'] = comment
|
||||
newlist.append(entry)
|
||||
|
||||
# We only care about the top 10 guilties
|
||||
newlist = sorted(newlist, key=itemgetter('total'), reverse=True)[:10]
|
||||
|
||||
return newlist
|
||||
|
||||
|
||||
def get_all_funcmods():
|
||||
frame_regex = re.compile(frame_pattern)
|
||||
funcmodset = set()
|
||||
funcmodlist = []
|
||||
backtraces = Record.get_crash_backtraces(classes=get_backtrace_classes())
|
||||
|
||||
for b in backtraces:
|
||||
lines = b[0].splitlines()
|
||||
for line in lines:
|
||||
match = frame_regex.match(line)
|
||||
if match:
|
||||
funcmodset.add((match.group(2), match.group(4)))
|
||||
|
||||
return sorted(funcmodset)
|
||||
|
||||
|
||||
def parse_backtrace(backtrace):
|
||||
program_regex = re.compile('^Process: (.*)$')
|
||||
pid_regex = re.compile('^PID: ([0-9]+)$')
|
||||
signal_regex = re.compile('^Signal: ([0-9]+)$')
|
||||
bt_header_regex = re.compile('^Backtrace \(TID ([0-9]+)\):$')
|
||||
frame_regex = re.compile(frame_pattern)
|
||||
|
||||
Crash = namedtuple('Crash', ['record_id', 'program', 'pid', 'signal', 'backtrace'])
|
||||
program = ''
|
||||
pid = ''
|
||||
signal = ''
|
||||
frames = []
|
||||
parsed_header = False
|
||||
|
||||
lines = backtrace[0].splitlines()
|
||||
for line in lines:
|
||||
# Header info
|
||||
match = program_regex.match(line)
|
||||
if match:
|
||||
program = match.group(1)
|
||||
continue
|
||||
match = pid_regex.match(line)
|
||||
if match:
|
||||
pid = match.group(1)
|
||||
continue
|
||||
match = signal_regex.match(line)
|
||||
if match:
|
||||
signal = match.group(1)
|
||||
continue
|
||||
|
||||
# We only care about the backtrace from the crashing thread, which
|
||||
# is listed first in the payload.
|
||||
match = bt_header_regex.match(line)
|
||||
if match and not parsed_header:
|
||||
parsed_header = True
|
||||
elif match:
|
||||
break
|
||||
|
||||
match = frame_regex.match(line)
|
||||
if match:
|
||||
frames.append((match.group(2), match.group(4), match.group(5)))
|
||||
|
||||
# Populate a namedtuple for convenience
|
||||
record_id = backtrace[1]
|
||||
c = Crash(record_id, program, pid, signal, frames)
|
||||
return c
|
||||
|
||||
|
||||
def explode_backtraces(classes=None, guilty_id=None, machine_id=None, build=None):
|
||||
crashes = []
|
||||
backtraces = Record.get_crash_backtraces(classes, guilty_id, machine_id, build)
|
||||
for b in backtraces:
|
||||
crashes.append(parse_backtrace(b))
|
||||
return crashes
|
||||
|
||||
|
||||
# vi: ts=4 et sw=4 sts=4
|
||||
@@ -1 +0,0 @@
|
||||
../../shared/model.py
|
||||
@@ -0,0 +1,619 @@
|
||||
#
|
||||
# Copyright 2015-2017 Intel Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import itertools
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from sqlalchemy.sql.expression import cast
|
||||
from sqlalchemy.sql.expression import desc
|
||||
from sqlalchemy.sql.expression import case
|
||||
from time import time, localtime, strftime, mktime, strptime, gmtime
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
from . import app
|
||||
|
||||
db = SQLAlchemy(app)
|
||||
|
||||
|
||||
class Guilty(db.Model):
|
||||
__tablename__ = 'guilty'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
function = db.Column(db.String)
|
||||
module = db.Column(db.String)
|
||||
comment = db.Column(db.String)
|
||||
hide = db.Column(db.Boolean, default=False)
|
||||
|
||||
def __init__(self, func, mod):
|
||||
self.function = func
|
||||
self.module = mod
|
||||
|
||||
@staticmethod
|
||||
def update_comment(guilty_id, comment):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
guilty.comment = comment
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_function(guilty_id):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
return guilty and guilty.function or ""
|
||||
|
||||
@staticmethod
|
||||
def get_module(guilty_id):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
return guilty and guilty.module or ""
|
||||
|
||||
@staticmethod
|
||||
def update_hidden(guilty_id, status):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
guilty.hide = status
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_hidden_value(guilty_id):
|
||||
guilty = Guilty.query.filter_by(id=guilty_id).first()
|
||||
return guilty and guilty.hide or False
|
||||
|
||||
@staticmethod
|
||||
def get_hidden_guilties():
|
||||
q = db.session.query(Guilty.id, Guilty.function, Guilty.module)
|
||||
q = q.filter(Guilty.hide == True)
|
||||
q = q.order_by(Guilty.function)
|
||||
return q.all()
|
||||
|
||||
|
||||
class Record(db.Model):
|
||||
__tablename__ = 'records'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
architecture = db.Column(db.Text)
|
||||
bios_version = db.Column(db.Text, default='')
|
||||
board_name = db.Column(db.Text, default='')
|
||||
build = db.Column(db.Text, nullable=False)
|
||||
classification = db.Column(db.Text, nullable=False)
|
||||
cpu_model = db.Column(db.Text, default='')
|
||||
event_id = db.Column(db.Text, default='')
|
||||
external = db.Column(db.Boolean, default=False)
|
||||
host_type = db.Column(db.Text, default='')
|
||||
kernel_version = db.Column(db.Text, default=0)
|
||||
machine_id = db.Column(db.Text, default='')
|
||||
payload_version = db.Column(db.Integer)
|
||||
record_version = db.Column(db.Integer, default=0)
|
||||
severity = db.Column(db.Integer)
|
||||
system_name = db.Column(db.Text)
|
||||
timestamp_client = db.Column(db.Numeric)
|
||||
timestamp_server = db.Column(db.Numeric, nullable=False)
|
||||
payload = db.Column(db.Text, nullable=False)
|
||||
|
||||
processed = db.Column(db.Boolean, default=False)
|
||||
guilty_id = db.Column(db.Integer, db.ForeignKey('guilty.id'))
|
||||
|
||||
guilty = db.Column(db.Text, default='')
|
||||
guilty = db.relationship('Guilty', backref=db.backref('records', lazy='dynamic'), lazy='joined')
|
||||
|
||||
def __init__(self, machine_id, host_type, severity, classification, build, architecture, kernel_version,
|
||||
record_version, ts_capture, ts_reception, payload_version, system_name,
|
||||
board_name, bios_version, cpu_model, event_id, external, payload):
|
||||
self.machine_id = machine_id
|
||||
self.host_type = host_type
|
||||
self.architecture = architecture
|
||||
self.classification = classification
|
||||
self.build = build
|
||||
self.kernel_version = kernel_version
|
||||
self.record_version = record_version
|
||||
self.severity = severity
|
||||
self.timestamp_client = ts_capture
|
||||
self.timestamp_server = ts_reception
|
||||
self.payload_version = payload_version
|
||||
self.system_name = system_name
|
||||
self.external = external
|
||||
self.board_name = board_name
|
||||
self.bios_version = bios_version
|
||||
self.cpu_model = cpu_model
|
||||
self.event_id = event_id
|
||||
|
||||
try:
|
||||
self.payload = payload.encode('utf-8')
|
||||
except UnicodeError:
|
||||
self.payload = payload.encode('latin-1')
|
||||
|
||||
def __repr__(self):
|
||||
return "<Record(id='{}', class='{}', build='{}', created='{}')>".format(self.id, self.classification, self.build, strftime("%a, %d %b %Y %H:%M:%S", localtime(self.timestamp_client)))
|
||||
|
||||
def __str__(self):
|
||||
return str(self.to_dict())
|
||||
|
||||
def to_dict(self):
|
||||
record = {
|
||||
'id': self.id,
|
||||
'machine_id': self.machine_id,
|
||||
'machine_type': self.host_type,
|
||||
'arch': self.architecture,
|
||||
'build': self.build,
|
||||
'kernel_version': self.kernel_version,
|
||||
'ts_capture': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_client)),
|
||||
'ts_reception': strftime('%Y-%m-%d %H:%M:%S UTC', gmtime(self.timestamp_server)),
|
||||
'severity': self.severity,
|
||||
'classification': self.classification,
|
||||
'record_version': self.record_version,
|
||||
'payload': self.payload,
|
||||
'board_name': self.board_name,
|
||||
'bios_version': self.bios_version,
|
||||
'cpu_model': self.cpu_model,
|
||||
'event_id': self.event_id,
|
||||
'external': self.external,
|
||||
}
|
||||
return record
|
||||
|
||||
# for the exported CSV rows
|
||||
def to_list(self):
|
||||
record = [
|
||||
self.id,
|
||||
self.external,
|
||||
self.timestamp_server,
|
||||
self.severity,
|
||||
self.classification,
|
||||
self.build,
|
||||
self.machine_id,
|
||||
self.payload
|
||||
]
|
||||
return record
|
||||
|
||||
@staticmethod
|
||||
def list():
|
||||
return Record.query.all()
|
||||
|
||||
@staticmethod
|
||||
def create(machine_id, host_type, severity, classification, build, architecture, kernel_version,
|
||||
record_version, ts_capture, ts_reception, payload_version, system_name,
|
||||
board_name, bios_version, cpu_model, event_id, external, payload):
|
||||
try:
|
||||
record = Record(machine_id, host_type, severity, classification, build, architecture, kernel_version,
|
||||
record_version, ts_capture, ts_reception, payload_version, system_name,
|
||||
board_name, bios_version, cpu_model, event_id, external, payload)
|
||||
db.session.add(record)
|
||||
db.session.commit()
|
||||
return record
|
||||
except:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def query_records(build, classification, severity, machine_id, limit,
|
||||
interval_sec=None, ts_capture=None, from_id=None):
|
||||
records = Record.query
|
||||
if build is not None:
|
||||
records = records.filter_by(build=build)
|
||||
if classification is not None:
|
||||
records = records.filter_by(classification=classification)
|
||||
if severity is not None:
|
||||
records = records.filter(Record.severity == severity)
|
||||
if machine_id is not None:
|
||||
records = records.filter(Record.machine_id == machine_id)
|
||||
if from_id is not None:
|
||||
records = records.filter(Record.id >= from_id)
|
||||
if ts_capture is not None:
|
||||
records = records.filter(Record.timestamp_client > ts_capture)
|
||||
|
||||
if interval_sec is not None:
|
||||
current_time = time()
|
||||
secs_in_past = current_time - interval_sec
|
||||
# Due to time skew on client systems, delayed sends due to
|
||||
# spooling, etc, timestamp_server works better as the reference
|
||||
# timestamp.
|
||||
records = records.filter(Record.timestamp_server > secs_in_past)
|
||||
|
||||
records = records.order_by(Record.id.desc())
|
||||
|
||||
if limit is not None:
|
||||
records = records.limit(limit)
|
||||
|
||||
return records.all()
|
||||
|
||||
@staticmethod
|
||||
def get_record(record_id):
|
||||
record = Record.query.filter_by(id=record_id).first()
|
||||
return record
|
||||
|
||||
@staticmethod
|
||||
def filter_records(build, classification, severity, machine_id=None, system_name=None, limit=None, from_date=None,
|
||||
to_date=None, payload=None, not_payload=None, data_source=None):
|
||||
records = Record.query
|
||||
if build is not None:
|
||||
records = records.filter_by(build=build)
|
||||
if classification is not None:
|
||||
if isinstance(classification, list):
|
||||
records = records.filter(Record.classification.in_(classification))
|
||||
else:
|
||||
records = records.filter(Record.classification.like(classification))
|
||||
if severity is not None:
|
||||
records = records.filter(Record.severity == severity)
|
||||
if system_name is not None:
|
||||
records = records.filter(Record.system_name == system_name)
|
||||
if machine_id is not None:
|
||||
records = records.filter(Record.machine_id == machine_id)
|
||||
if from_date is not None:
|
||||
from_date = mktime(strptime(from_date, "%Y-%m-%d"))
|
||||
records = records.filter(Record.timestamp_client >= from_date)
|
||||
if to_date is not None:
|
||||
to_date = mktime(strptime(to_date, "%Y-%m-%d"))
|
||||
records = records.filter(Record.timestamp_client < to_date)
|
||||
if payload is not None:
|
||||
records = records.filter(Record.payload.op('~')(payload))
|
||||
if not_payload is not None:
|
||||
records = records.filter(~Record.payload.op('~')(not_payload))
|
||||
if data_source is not None:
|
||||
if data_source == "external":
|
||||
records = records.filter(Record.external == True)
|
||||
elif data_source == "internal":
|
||||
records = records.filter(Record.external == False)
|
||||
|
||||
records = records.order_by(Record.id.desc())
|
||||
|
||||
if limit is not None:
|
||||
records = records.limit(limit)
|
||||
|
||||
return records
|
||||
|
||||
@staticmethod
|
||||
def delete_records():
|
||||
MAX_DAYS_KEEP_UNFILTERED_RECORDS = app.config.get("MAX_DAYS_KEEP_UNFILTERED_RECORDS", 35)
|
||||
PURGE_FILTERED_RECORDS = app.config.get("PURGE_FILTERED_RECORDS", {})
|
||||
try:
|
||||
def purge_field(field):
|
||||
for name in PURGE_FILTERED_RECORDS[field].keys():
|
||||
if PURGE_FILTERED_RECORDS[field][name]:
|
||||
age = time() - PURGE_FILTERED_RECORDS[field][name] * 24 * 60 * 60
|
||||
q = db.session.query(Record)
|
||||
if field == 'classification':
|
||||
q = q.filter(Record.classification.like(name.replace("*", "%")))
|
||||
else:
|
||||
q = q.filter(getattr(Record, field) == name)
|
||||
q = q.filter(Record.timestamp_server < age)
|
||||
if q.all():
|
||||
count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False)
|
||||
print("Deleted {} {} records".format(count, name))
|
||||
for field in PURGE_FILTERED_RECORDS.keys():
|
||||
purge_field(field)
|
||||
if MAX_DAYS_KEEP_UNFILTERED_RECORDS:
|
||||
unfiltered_age = time() - MAX_DAYS_KEEP_UNFILTERED_RECORDS * 24 * 60 * 60
|
||||
q = db.session.query(Record.id)
|
||||
for field in PURGE_FILTERED_RECORDS.keys():
|
||||
if field == 'classification':
|
||||
for classification in PURGE_FILTERED_RECORDS[field].keys():
|
||||
q = q.filter(~Record.classification.like(classification.replace("*", "%")))
|
||||
else:
|
||||
for name in PURGE_FILTERED_RECORDS[field].keys():
|
||||
q = q.filter(getattr(Record, field) != name)
|
||||
q = q.filter(Record.timestamp_server < unfiltered_age)
|
||||
if q.all():
|
||||
count = db.session.query(Record).filter(Record.id.in_([x.id for x in q.all()])).delete(synchronize_session=False)
|
||||
print("Deleted {} old records".format(count))
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
app.logger.error("Record purging failed")
|
||||
app.logger.error(e)
|
||||
db.session.rollback()
|
||||
|
||||
@staticmethod
|
||||
def get_recordcnts_by_build():
|
||||
q = db.session.query(Record.build, db.func.count(Record.id))
|
||||
q = q.filter(Record.build.op('~')('^[0-9]+$'))
|
||||
q = q.group_by(Record.build).order_by(cast(Record.build, db.Integer)).all()
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_builds():
|
||||
q = db.session.query(Record.build).distinct()
|
||||
q = q.order_by(Record.build)
|
||||
return sorted(q.all(), key=lambda x: LooseVersion(x[0]), reverse=True)
|
||||
|
||||
@staticmethod
|
||||
def get_recordcnts_by_classification():
|
||||
q = db.session.query(Record.classification, db.func.count(Record.id).label('total'))
|
||||
q = q.group_by(Record.classification)
|
||||
q = q.order_by(desc('total'))
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def expand_class(D):
|
||||
A, B, C = D
|
||||
return ["{}/*".format(A), "{}/{}/*".format(A, B), "{}/{}/{}".format(A, B, C)]
|
||||
|
||||
@staticmethod
|
||||
def get_classifications(with_regex=False):
|
||||
q = db.session.query(Record.classification).distinct()
|
||||
if with_regex:
|
||||
classes = [Record.expand_class(c[0].split('/')) for c in q.all()]
|
||||
return sorted(set(itertools.chain(*classes)))
|
||||
else:
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_os_map():
|
||||
q = db.session.query(Record.system_name, Record.build).order_by(Record.system_name).group_by(Record.system_name, Record.build).all()
|
||||
result = {}
|
||||
for x in q:
|
||||
result.setdefault(x[0], []).append(x[1])
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def get_recordcnts_by_machine_type():
|
||||
q = db.session.query(Record.host_type, db.func.count(Record.id).label('total'))
|
||||
q = q.group_by(Record.host_type)
|
||||
q = q.order_by(desc('total'))
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_recordcnts_by_severity():
|
||||
q = db.session.query(Record.severity, db.func.count(Record.id)).group_by(Record.severity).all()
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_crashcnts_by_class(classes=None):
|
||||
q = db.session.query(Record.classification, db.func.count(Record.id))
|
||||
if classes:
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
else:
|
||||
q = q.filter(Record.classification.like('org.clearlinux/crash/%'))
|
||||
q = q.group_by(Record.classification)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_crashcnts_by_build(classes=None):
|
||||
q = db.session.query(Record.build, db.func.count(Record.id))
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.build.op('~')('^[0-9]+$'))
|
||||
q = q.group_by(Record.build)
|
||||
q = q.order_by(desc(cast(Record.build, db.Integer)))
|
||||
q = q.limit(10)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_top_crash_guilties(classes=None):
|
||||
q = db.session.query(Guilty.function, Guilty.module, Record.build, db.func.count(Record.id).label('total'), Guilty.id, Guilty.comment)
|
||||
q = q.join(Record)
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.build.op('~')('^[0-9][0-9]+$'))
|
||||
q = q.filter(cast(Record.build, db.Integer) <= 100000)
|
||||
q = q.filter(Guilty.hide == False)
|
||||
q = q.group_by(Guilty.function, Guilty.module, Guilty.comment, Guilty.id, Record.build)
|
||||
q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total'))
|
||||
# query for records created in the last week (~ 10 Clear builds)
|
||||
q = q.filter(Record.build.in_(sorted(tuple(set([x[2] for x in q.all()])), key=lambda x: int(x))[-8:]))
|
||||
interval_sec = 24 * 60 * 60 * 7
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_new_crash_records(classes=None, id=None):
|
||||
q = db.session.query(Record)
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
q = q.filter(Record.processed == False)
|
||||
if id:
|
||||
q = q.filter(Record.id == id)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def set_processed_flag(record):
|
||||
record.processed = True
|
||||
|
||||
@staticmethod
|
||||
def get_guilty_for_funcmod(func, mod):
|
||||
q = db.session.query(Guilty).filter_by(function=func, module=mod).first()
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_guilty_id_for_record(record_id):
|
||||
q = db.session.query(Guilty.id).join(Record)
|
||||
q = q.filter(Record.id == record_id)
|
||||
return q.first()
|
||||
|
||||
@staticmethod
|
||||
def init_guilty(func, mod):
|
||||
return Guilty(func, mod)
|
||||
|
||||
@staticmethod
|
||||
def create_guilty_for_record(record, guilty):
|
||||
record.guilty = guilty
|
||||
|
||||
@staticmethod
|
||||
def commit_guilty_changes():
|
||||
# just commit for now
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_crash_backtraces(classes=None, guilty_id=None, machine_id=None, build=None, most_recent=None, record_id=None):
|
||||
q = db.session.query(Record.payload, Record.id)
|
||||
# Short circuit if we know the record ID
|
||||
if record_id:
|
||||
q = q.filter(Record.id == record_id)
|
||||
return q.first()
|
||||
if build:
|
||||
q = q.filter(Record.build == build)
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
if guilty_id:
|
||||
q = q.filter(Record.guilty_id == guilty_id)
|
||||
if machine_id:
|
||||
q = q.filter(Record.machine_id == machine_id)
|
||||
if most_recent:
|
||||
interval_sec = 24 * 60 * 60 * int(most_recent)
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def reset_processed_records(classes=None, id=None):
|
||||
q = db.session.query(Record)
|
||||
if not classes:
|
||||
classes = ['org.clearlinux/crash/clr']
|
||||
q = q.filter(Record.classification.in_(classes))
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
if id:
|
||||
q = q.filter(Record.id == id)
|
||||
records = q.all()
|
||||
for r in records:
|
||||
r.processed = False
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_machine_ids_for_guilty(id, most_recent=None):
|
||||
q = db.session.query(Record.build, Record.machine_id, db.func.count(Record.id).label('total'), Record.guilty_id)
|
||||
q = q.filter(Record.guilty_id == id)
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
q = q.filter(Record.build.op('~')('^[0-9][0-9]+$'))
|
||||
q = q.group_by(Record.build, Record.machine_id, Record.guilty_id)
|
||||
q = q.order_by(desc(cast(Record.build, db.Integer)), desc('total'))
|
||||
if most_recent:
|
||||
interval_sec = 24 * 60 * 60 * int(most_recent)
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def get_update_msgs():
|
||||
q = db.session.query(Record.payload)
|
||||
q = q.filter(Record.classification == "org.clearlinux/swupd-client/update")
|
||||
|
||||
sec_2_weeks = 24 * 60 * 60 * 7
|
||||
current_time = time()
|
||||
time_2_weeks_ago = current_time - sec_2_weeks
|
||||
|
||||
# query for records created in that last 2 weeks
|
||||
q = q.filter(Record.timestamp_client > time_2_weeks_ago)
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_swupd_msgs(most_recent=None):
|
||||
q = db.session.query(Record.timestamp_client, Record.machine_id, Record.payload)
|
||||
q = q.filter(Record.classification.like('org.clearlinux/swupd-client/%'))
|
||||
|
||||
if most_recent:
|
||||
interval_sec = 24 * 60 * 60 * int(most_recent)
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
|
||||
q = q.order_by(desc(Record.timestamp_client))
|
||||
return q
|
||||
|
||||
@staticmethod
|
||||
def get_heartbeat_msgs(most_recent=None):
|
||||
# These two expressions are SQL CASE conditional expressions, later
|
||||
# used within count(distinct ...) aggregates for the query.
|
||||
internal_expr = case([(Record.external == False, Record.machine_id), ]).label('internal_count')
|
||||
external_expr = case([(Record.external == True, Record.machine_id), ]).label('external_count')
|
||||
|
||||
q = db.session.query(Record.build, db.func.count(db.distinct(internal_expr)), db.func.count(db.distinct(external_expr)))
|
||||
q = q.filter(Record.classification == "org.clearlinux/heartbeat/ping")
|
||||
q = q.filter(Record.system_name == 'clear-linux-os')
|
||||
q = q.group_by(Record.build)
|
||||
|
||||
if most_recent:
|
||||
interval_sec = 24 * 60 * 60 * int(most_recent)
|
||||
current_time = time()
|
||||
sec_in_past = current_time - interval_sec
|
||||
q = q.filter(Record.timestamp_client > sec_in_past)
|
||||
|
||||
q = q.order_by(cast(Record.build, db.Integer))
|
||||
return q.all()
|
||||
|
||||
|
||||
class GuiltyBlacklist(db.Model):
|
||||
__tablename__ = 'guilty_blacklisted'
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
function = db.Column(db.String)
|
||||
module = db.Column(db.String)
|
||||
|
||||
def __init__(self, func, mod):
|
||||
self.function = func
|
||||
self.module = mod
|
||||
|
||||
def __repr__(self):
|
||||
return "<GuiltyBlacklist(id='{}', guilty='{}:{}')>".format(self.id, self.function, self.module)
|
||||
|
||||
def __str__(self):
|
||||
return str(self.to_dict())
|
||||
|
||||
def to_dict(self):
|
||||
guilty = {
|
||||
'function': self.function,
|
||||
'module': self.module
|
||||
}
|
||||
return guilty
|
||||
|
||||
@staticmethod
|
||||
def add(func, mod):
|
||||
try:
|
||||
g = GuiltyBlacklist(func, mod)
|
||||
db.session.add(g)
|
||||
db.session.commit()
|
||||
return g
|
||||
except:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def remove(func, mod):
|
||||
q = db.session.query(GuiltyBlacklist)
|
||||
q = q.filter_by(function=func, module=mod)
|
||||
entry = q.first()
|
||||
db.session.delete(entry)
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_guilties():
|
||||
q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module)
|
||||
q = q.order_by(GuiltyBlacklist.function)
|
||||
return q.all()
|
||||
|
||||
@staticmethod
|
||||
def exists(func, mod):
|
||||
q = db.session.query(GuiltyBlacklist.function, GuiltyBlacklist.module)
|
||||
q = q.filter_by(function=func, module=mod)
|
||||
return len(q.all()) != 0 and True or False
|
||||
|
||||
@staticmethod
|
||||
def update(to_add, to_remove):
|
||||
try:
|
||||
for i in to_add:
|
||||
if not GuiltyBlacklist.exists(i[0], i[1]):
|
||||
g = GuiltyBlacklist(i[0], i[1])
|
||||
db.session.add(g)
|
||||
for i in to_remove:
|
||||
if GuiltyBlacklist.exists(i[0], i[1]):
|
||||
q = db.session.query(GuiltyBlacklist)
|
||||
q = q.filter_by(function=i[0], module=i[1])
|
||||
entry = q.first()
|
||||
db.session.delete(entry)
|
||||
db.session.commit()
|
||||
except:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
# vi: ts=4 et sw=4 sts=4
|
||||
Reference in New Issue
Block a user