From 950ca69070e258375096eb5208de693dd4943cc1 Mon Sep 17 00:00:00 2001 From: Alex Jaramillo Date: Mon, 23 Mar 2020 10:25:22 -0700 Subject: [PATCH] Replace crash for load more Replace crash pagination by load more crashes logic. Signed-off-by: Alex Jaramillo --- processing/process.py | 1 + processing/processing/main.py | 4 + services/processing/Dockerfile | 1 + services/testing.env | 2 + services/webapp/requirements.txt | 1 + telemetryui/telemetryui/cache.py | 44 +++++ .../telemetryui/templates/crashes.html | 118 ++++-------- .../telemetryui/templates/crashes_list.html | 2 +- .../telemetryui/templates/records.html | 175 +++++++++--------- telemetryui/telemetryui/views.py | 47 +---- utils/shared_utils/crash.py | 100 +++++++++- 11 files changed, 289 insertions(+), 206 deletions(-) create mode 100644 telemetryui/telemetryui/cache.py diff --git a/processing/process.py b/processing/process.py index f4e4d35..7e40f0f 100755 --- a/processing/process.py +++ b/processing/process.py @@ -14,6 +14,7 @@ from processing.main import ( from processing import crash + class GlobalParams(luigi.Config): db_name = os.environ['POSTGRES_DB'] diff --git a/processing/processing/main.py b/processing/processing/main.py index dbb80a2..e651382 100644 --- a/processing/processing/main.py +++ b/processing/processing/main.py @@ -40,6 +40,7 @@ UPDATE_GUILTY = """ UPDATE records SET guilty_id = %s WHERE id = %s """ + class GuilyBlacklist(object): def __init__(self, blacklisted): @@ -91,6 +92,9 @@ def process_crashes(cur, crashes, schema="public", debug=False): gid = row[0] ### Update record table with guilty_id cur.execute(UPDATE_GUILTY, (gid, rid,)) + else: + print("# function or module is None for record id {}".format(rid)) + ### Update latest processed record table with record_id cur.execute(UPDATE_PROCESSED_RECORD, (rid,)) print("#### Updating last processed id to {}\n".format(rid)) diff --git a/services/processing/Dockerfile b/services/processing/Dockerfile index a944ac9..b544874 100644 --- a/services/processing/Dockerfile +++ b/services/processing/Dockerfile @@ -8,6 +8,7 @@ RUN swupd bundle-add python3-basic RUN mkdir -p /srv/processing COPY ./processing . COPY ./services/processing/. . +COPY ./utils/shared_utils/crash.py ./processing/crash.py RUN pip3 install -r requirements.txt RUN groupadd -r appuser && useradd -r -g appuser appuser RUN chown -R appuser:appuser /srv/processing diff --git a/services/testing.env b/services/testing.env index 8d1ee0d..16dbc42 100644 --- a/services/testing.env +++ b/services/testing.env @@ -8,3 +8,5 @@ PGDATA=/var/lib/postgresql/data/pgdata REDIS_HOSTNAME=redis REDIS_PORT=6379 REDIS_PASSWD= +# Luigi log level +PROCESSING_LOG_LEVEL=DEBUG diff --git a/services/webapp/requirements.txt b/services/webapp/requirements.txt index 0d45703..e1b5387 100644 --- a/services/webapp/requirements.txt +++ b/services/webapp/requirements.txt @@ -16,3 +16,4 @@ six==1.12.0 SQLAlchemy==1.3.5 Werkzeug==0.15.5 WTForms==2.2.1 +cxxfilt diff --git a/telemetryui/telemetryui/cache.py b/telemetryui/telemetryui/cache.py new file mode 100644 index 0000000..938ec54 --- /dev/null +++ b/telemetryui/telemetryui/cache.py @@ -0,0 +1,44 @@ +# Copyright (C) 2015-2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import ast +import redis +from . import app + +REDIS_HOSTNAME = app.config.get('REDIS_HOSTNAME', 'localhost') +REDIS_PORT = app.config.get('REDIS_PORT', 6379) +REDIS_PASSWD = app.config.get('REDIS_PASSWD', None) + + +def get_cached_data(varname, expiration, funct, *args, **kwargs): + try: + redis_client = redis.StrictRedis(decode_responses=True, + host=REDIS_HOSTNAME, + port=REDIS_PORT, + password=REDIS_PASSWD,); + # Try to get data from redis first + ret = redis_client.get(varname) + if ret is not None: + # Convert to original type if successful + ret = ast.literal_eval(ret) + else: + # If nothing was found, query the database + ret = funct(*args, **kwargs) + # Convert to string representation and cache via redis + redis_client.set(varname, repr(ret), ex=expiration) + except redis.exceptions.ConnectionError as e: + print("%s Redis probably isn't running?" % str(e)) + # If we can't connect to redis, just query directly + ret = funct(*args, **kwargs) + return ret + + +def uncache_data(varname): + try: + redis_client = redis.StrictRedis(decode_responses=True); + redis_client.delete(varname) + except redis.exceptions.ConnectionError as e: + print("%s Redis probably isn't running?" % str(e)) + return + +# vi: ts=4 et sw=4 sts=4 diff --git a/telemetryui/telemetryui/templates/crashes.html b/telemetryui/telemetryui/templates/crashes.html index e02395b..d968683 100644 --- a/telemetryui/telemetryui/templates/crashes.html +++ b/telemetryui/telemetryui/templates/crashes.html @@ -1,18 +1,8 @@ {#- - # Copyright 2015-2017 Intel Corporation - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. + # Copyright (C) 2015-2020 Intel Corporation + # SPDX-License-Identifier: Apache-2.0 -#} + {% extends "base.html" %} {%- block page_title %} @@ -24,8 +14,6 @@

Crash stats

Top Crashes

- {% set CRASH_PAGE_SIZE = 10 %} - {%- with messages = get_flashed_messages(with_categories=true) %} {%- if messages %} {%- for category, m in messages %} @@ -37,7 +25,7 @@
{%- if guilties %} - +
@@ -49,67 +37,18 @@ {%- endfor %} - - {%- set maxtotal = guilties[0].total %} - {%- for g in guilties %} - - - - - - {%- for c in g.builds %} - {%- for b in builds %} - {%- if b[0] == c[0] and c[1] != "0" %} - - - {%- elif b[0] == c[0] %} - - {%- endif %} - {%- endfor %} - {%- endfor %} - - - {%- endfor %} + + {% include "crashes_list.html" %}
Top crashes in the last 7 days, with frequency per build
#Comment
{{ loop.index + ((page - 1) * CRASH_PAGE_SIZE) }} - {{ g.guilty|truncate(30, True) }} - - - - {%- set width = g.total / maxtotal * 100 %} - - - - - - - {%- set width = c[1] / b[1] * 100 %} - - - - - -
-
{% if g.comment %}{{ g.comment|truncate(30, True) }}{% endif %}
-
-
- - {%- if pages != 0 %} - - {%- endif %} + +
+ +
{%- else %} {# guilties #} @@ -157,7 +96,7 @@
{% endblock %} - {#- # vi: ft=jinja ts=8 et sw=4 sts=4 #} diff --git a/telemetryui/telemetryui/templates/crashes_list.html b/telemetryui/telemetryui/templates/crashes_list.html index 55d3dad..4e7e54d 100644 --- a/telemetryui/telemetryui/templates/crashes_list.html +++ b/telemetryui/telemetryui/templates/crashes_list.html @@ -6,7 +6,7 @@ {%- set maxtotal = guilties[0].total %} {%- for g in guilties %} -{{ loop.index + ((page - 1) * CRASH_PAGE_SIZE) }} +{{ loop.index + page_offset }} {{ g.guilty|truncate(30, True) }} diff --git a/telemetryui/telemetryui/templates/records.html b/telemetryui/telemetryui/templates/records.html index b506e30..87b3251 100644 --- a/telemetryui/telemetryui/templates/records.html +++ b/telemetryui/telemetryui/templates/records.html @@ -2,125 +2,126 @@ # Copyright (C) 2015-2020 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#} + {% extends "base.html" %} {% block content %} -
-
- - -
-
- -
+
+
+ +
+
+ +
+
- - - - - - - - - - - - +
SourceMachine IDAgeSeverityClassificationOSPayloadMore
+ + + + + + + + + + + {% include "records_list.html" %} - -
SourceMachine IDAgeSeverityClassificationOSPayloadMore
+ + - -
- + +
+ + + - -