diff --git a/README.md b/README.md index 0134a40..c9e00b0 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ that exposes several views to visualize the telemetry data. The `telemetryui` app also provides a REST API to perform queries on the data. The Flask apps have several dependencies listed [here](services/collector/requirements.txt) -and [here](services/telemetryui/requirements.txt). The testing infrastructure -is described by [this](docker-compose.yaml) docker-compose.yaml and production by -[this](docker-compose.prod.yaml) docker-compose.prod.yaml +and [here](deployments/services/telemetryui/requirements.txt). The testing infrastructure +is described by [this](deployments/docker-compose.yaml) docker-compose.yaml and production by +[this](deployments/docker-compose.prod.yaml) docker-compose.prod.yaml ## Security considerations @@ -49,14 +49,14 @@ the build step will fail. vi services/production.env # build production environment -sudo -E docker-compose --file ./docker-compose.prod.yaml build --force-rm +sudo -E docker-compose --file ./deployments/docker-compose.prod.yaml build --force-rm ``` Once the images are build successfully the environment can be started using: ``` # start environment on the background -sudo -E docker-compose --file ./docker-compose.prod.yaml up --detach +sudo -E docker-compose --file ./deployments/docker-compose.prod.yaml up --detach ``` ### Deploying as systemd service @@ -72,8 +72,8 @@ After=docker.service [Service] Restart=always WorkingDirectory=/srv/telemetrics-backend -ExecStart=/usr/local/bin/docker-compose --file docker-compose.prod.yaml up -ExecStop=/usr/local/bin/docker-compose --file docker-compose.prod.yaml down -v +ExecStart=/usr/local/bin/docker-compose --file deployments/docker-compose.prod.yaml up +ExecStop=/usr/local/bin/docker-compose --file deployments/docker-compose.prod.yaml down -v [Install] WantedBy=multi-user.target @@ -230,10 +230,10 @@ started, this applies for both production and testing configurations. ``` # Build -sudo -E docker-compose build --force-rm +sudo -E docker-compose --file deployments/docker-compose.yaml build --force-rm # Launch -sudo -E docker-compose up +sudo -E docker-compose --file deployments/docker-compose.yaml up ``` ## Software License diff --git a/docker-compose.prod.yaml b/deployments/docker-compose.prod.yaml similarity index 85% rename from docker-compose.prod.yaml rename to deployments/docker-compose.prod.yaml index 2e5b427..d83ff66 100644 --- a/docker-compose.prod.yaml +++ b/deployments/docker-compose.prod.yaml @@ -21,8 +21,8 @@ services: - frontend restart: always build: - context: . - dockerfile: ./services/nginx/Dockerfile + context: .. + dockerfile: ./deployments/services/nginx/Dockerfile volumes: - "telemetry-socket-volume:/var/run/uwsgi/" - "telemetry-nginx-logs:/var/log/nginx" @@ -47,8 +47,8 @@ services: - "telemetry-uwsgi-logs:/var/www/collector/log" restart: always build: - context: . - dockerfile: ./services/collector/Dockerfile + context: .. + dockerfile: ./deployments/services/collector/Dockerfile env_file: - services/production.env depends_on: ["db", "redis"] @@ -63,8 +63,8 @@ services: - "telemetry-uwsgi-logs:/var/www/webapp/log" restart: always build: - context: . - dockerfile: ./services/webapp/Dockerfile + context: .. + dockerfile: ./deployments/services/webapp/Dockerfile env_file: - ./services/production.env depends_on: ["db", "redis"] @@ -75,8 +75,8 @@ services: networks: - backend build: - context: . - dockerfile: ./services/processing/Dockerfile + context: .. + dockerfile: ./deployments/services/processing/Dockerfile env_file: - ./services/production.env depends_on: ["db", "migrate"] @@ -92,7 +92,8 @@ services: FLASK_APP: /var/www/webapp/run.py working_dir: /var/www/webapp depends_on: ["db"] - command: /var/www/webapp/migrate_entrypoint.sh + restart: on-failure + command: flask db upgrade volumes: telemetry-socket-volume: diff --git a/docker-compose.yaml b/deployments/docker-compose.yaml similarity index 81% rename from docker-compose.yaml rename to deployments/docker-compose.yaml index 4ffe1f9..bc0121a 100644 --- a/docker-compose.yaml +++ b/deployments/docker-compose.yaml @@ -14,8 +14,8 @@ services: volumes: - "telemetry-socket-volume:/var/run/uwsgi/" build: - context: . - dockerfile: ./services/nginx/Dockerfile + context: .. + dockerfile: ./deployments/services/nginx/Dockerfile ports: - 80:80 - 443:443 @@ -34,8 +34,8 @@ services: volumes: - "telemetry-socket-volume:/var/www/collector/socket" build: - context: . - dockerfile: ./services/collector/Dockerfile + context: .. + dockerfile: ./deployments/services/collector/Dockerfile env_file: - services/testing.env depends_on: ["db", "redis"] @@ -48,8 +48,8 @@ services: volumes: - "telemetry-socket-volume:/var/www/webapp/socket" build: - context: . - dockerfile: ./services/webapp/Dockerfile + context: .. + dockerfile: ./deployments/services/webapp/Dockerfile env_file: - ./services/testing.env depends_on: ["db", "redis"] @@ -60,8 +60,8 @@ services: networks: - backend build: - context: . - dockerfile: ./services/processing/Dockerfile + context: .. + dockerfile: ./deployments/services/processing/Dockerfile env_file: - ./services/testing.env depends_on: ["db", "migrate"] @@ -77,7 +77,8 @@ services: FLASK_APP: /var/www/webapp/run.py working_dir: /var/www/webapp depends_on: ["db"] - command: /var/www/webapp/migrate_entrypoint.sh + restart: on-failure + command: flask db upgrade volumes: telemetry-socket-volume: diff --git a/services/collector/.env b/deployments/services/collector/.env similarity index 100% rename from services/collector/.env rename to deployments/services/collector/.env diff --git a/services/collector/Dockerfile b/deployments/services/collector/Dockerfile similarity index 81% rename from services/collector/Dockerfile rename to deployments/services/collector/Dockerfile index d90c752..c1e151c 100644 --- a/services/collector/Dockerfile +++ b/deployments/services/collector/Dockerfile @@ -4,8 +4,8 @@ WORKDIR /var/www/collector RUN swupd bundle-add python3-basic uwsgi RUN mkdir -p /var/www/collector/log /var/www/collector/socket -COPY ./services/collector/requirements.txt . -COPY ./services/collector/collector.ini . +COPY ./deployments/services/collector/requirements.txt . +COPY ./deployments/services/collector/collector.ini . COPY ./collector/collector /var/www/collector/collector COPY ./utils/shared_utils/model.py /var/www/collector/collector/ RUN pip3 install -r requirements.txt diff --git a/services/collector/collector.ini b/deployments/services/collector/collector.ini similarity index 100% rename from services/collector/collector.ini rename to deployments/services/collector/collector.ini diff --git a/services/collector/requirements.txt b/deployments/services/collector/requirements.txt similarity index 100% rename from services/collector/requirements.txt rename to deployments/services/collector/requirements.txt diff --git a/services/nginx/Dockerfile b/deployments/services/nginx/Dockerfile similarity index 60% rename from services/nginx/Dockerfile rename to deployments/services/nginx/Dockerfile index 96886af..44441cf 100644 --- a/services/nginx/Dockerfile +++ b/deployments/services/nginx/Dockerfile @@ -4,9 +4,9 @@ FROM clearlinux:latest RUN swupd bundle-add nginx RUN useradd nginx -COPY services/nginx/nginx.conf /etc/nginx/nginx.conf -COPY services/nginx/sites.conf /etc/nginx/conf.d/sites.conf -COPY services/nginx/*.pem /etc/nginx/ssl/ +COPY deployments/services/nginx/nginx.conf /etc/nginx/nginx.conf +COPY deployments/services/nginx/sites.conf /etc/nginx/conf.d/sites.conf +COPY deployments/services/nginx/*.pem /etc/nginx/ssl/ COPY telemetryui/telemetryui/static /var/www/static RUN chown -R nginx:nginx /var/www/static diff --git a/services/nginx/nginx.conf b/deployments/services/nginx/nginx.conf similarity index 100% rename from services/nginx/nginx.conf rename to deployments/services/nginx/nginx.conf diff --git a/services/nginx/sites.conf b/deployments/services/nginx/sites.conf similarity index 100% rename from services/nginx/sites.conf rename to deployments/services/nginx/sites.conf diff --git a/services/nginx/telemetry.cert.pem b/deployments/services/nginx/telemetry.cert.pem similarity index 100% rename from services/nginx/telemetry.cert.pem rename to deployments/services/nginx/telemetry.cert.pem diff --git a/services/nginx/telemetry.key.pem b/deployments/services/nginx/telemetry.key.pem similarity index 100% rename from services/nginx/telemetry.key.pem rename to deployments/services/nginx/telemetry.key.pem diff --git a/services/processing/Dockerfile b/deployments/services/processing/Dockerfile similarity index 91% rename from services/processing/Dockerfile rename to deployments/services/processing/Dockerfile index b544874..c9b9b8e 100644 --- a/services/processing/Dockerfile +++ b/deployments/services/processing/Dockerfile @@ -7,7 +7,7 @@ WORKDIR /srv/processing RUN swupd bundle-add python3-basic RUN mkdir -p /srv/processing COPY ./processing . -COPY ./services/processing/. . +COPY ./deployments/services/processing/. . COPY ./utils/shared_utils/crash.py ./processing/crash.py RUN pip3 install -r requirements.txt RUN groupadd -r appuser && useradd -r -g appuser appuser diff --git a/services/processing/entrypoint.py b/deployments/services/processing/entrypoint.py similarity index 100% rename from services/processing/entrypoint.py rename to deployments/services/processing/entrypoint.py diff --git a/services/processing/requirements.txt b/deployments/services/processing/requirements.txt similarity index 100% rename from services/processing/requirements.txt rename to deployments/services/processing/requirements.txt diff --git a/services/processing/run.sh b/deployments/services/processing/run.sh similarity index 100% rename from services/processing/run.sh rename to deployments/services/processing/run.sh diff --git a/services/production.env b/deployments/services/production.env similarity index 100% rename from services/production.env rename to deployments/services/production.env diff --git a/services/redis/.env b/deployments/services/redis/.env similarity index 100% rename from services/redis/.env rename to deployments/services/redis/.env diff --git a/services/testing.env b/deployments/services/testing.env similarity index 100% rename from services/testing.env rename to deployments/services/testing.env diff --git a/services/webapp/.dockerignore b/deployments/services/webapp/.dockerignore similarity index 100% rename from services/webapp/.dockerignore rename to deployments/services/webapp/.dockerignore diff --git a/services/webapp/.env b/deployments/services/webapp/.env similarity index 100% rename from services/webapp/.env rename to deployments/services/webapp/.env diff --git a/services/webapp/Dockerfile b/deployments/services/webapp/Dockerfile similarity index 85% rename from services/webapp/Dockerfile rename to deployments/services/webapp/Dockerfile index 3a2af6d..6ca9b99 100644 --- a/services/webapp/Dockerfile +++ b/deployments/services/webapp/Dockerfile @@ -6,7 +6,7 @@ WORKDIR /var/www/webapp RUN swupd bundle-add python3-basic uwsgi RUN mkdir -p /var/www/webapp/uwsgi-spool /var/www/webapp/log /var/www/webapp/socket -COPY ./services/webapp/. . +COPY ./deployments/services/webapp/requirements.txt ./deployments/services/webapp/webapp.ini ./ COPY ./telemetryui/. /var/www/webapp COPY ./utils/shared_utils/crash.py ./utils/shared_utils/model.py /var/www/webapp/telemetryui/ RUN pip3 install -r requirements.txt diff --git a/services/webapp/requirements.txt b/deployments/services/webapp/requirements.txt similarity index 100% rename from services/webapp/requirements.txt rename to deployments/services/webapp/requirements.txt diff --git a/services/webapp/webapp.ini b/deployments/services/webapp/webapp.ini similarity index 100% rename from services/webapp/webapp.ini rename to deployments/services/webapp/webapp.ini diff --git a/processing/process.py b/processing/process.py index 7e40f0f..fd7fa80 100755 --- a/processing/process.py +++ b/processing/process.py @@ -37,11 +37,11 @@ class ProcessCrashes(luigi.Task): cur = db.cursor() crash.GUILTY_BLACKLIST = init_guilty_blacklist(cur) last_id = get_latest_id(cur) - print("#### Start processing after record id: {}".format(last_id)) + print("# Start processing after record id: {}".format(last_id)) crashes = get_crashes(cur, last_id) - print("#### Will process [{}] crashes".format(len(crashes))) + print("# Will process [{}] crashes".format(len(crashes))) process_crashes(cur, crashes) - print("#### Done processing\n") + print("# Done processing\n") cur.close() db.close() self.is_complete = True diff --git a/processing/processing/main.py b/processing/processing/main.py index e651382..7d412d9 100644 --- a/processing/processing/main.py +++ b/processing/processing/main.py @@ -78,6 +78,7 @@ def process_crashes(cur, crashes, schema="public", debug=False): insert_guilty = INSERT_GUILTY.format(schema) for rid, backtrace, _ in crashes: if backtrace is None: + print("## backtrace record {} is None".format(rid)) continue function, module = parse_crash(backtrace) if function is not None and module is not None: @@ -93,11 +94,11 @@ def process_crashes(cur, crashes, schema="public", debug=False): ### Update record table with guilty_id cur.execute(UPDATE_GUILTY, (gid, rid,)) else: - print("# function or module is None for record id {}".format(rid)) + print("## function or module is None for record id {}".format(rid)) ### Update latest processed record table with record_id cur.execute(UPDATE_PROCESSED_RECORD, (rid,)) - print("#### Updating last processed id to {}\n".format(rid)) + print("## Updating last processed id to {}\n".format(rid)) def connect(conf): diff --git a/services/webapp/entrypoint.py b/services/webapp/entrypoint.py deleted file mode 100644 index b3a25ca..0000000 --- a/services/webapp/entrypoint.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2020 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import os -import psycopg2 - -CONN = "dbname='{}' user='{}' host='{}' password='{}'" - -def main(): - db_host = os.environ['POSTGRES_HOSTNAME'] - db_user = os.environ['POSTGRES_USER'] - db_passwd = os.environ['POSTGRES_PASSWORD'] - try: - conn = psycopg2.connect(CONN.format('telemetry', db_user, db_host, db_passwd)) - conn.close() - except psycopg2.OperationalError as op_error: - print("could not connect to server: Connection refused") - exit(1) - -if __name__ == '__main__': - main() diff --git a/services/webapp/migrate_entrypoint.sh b/services/webapp/migrate_entrypoint.sh deleted file mode 100755 index aea4ff9..0000000 --- a/services/webapp/migrate_entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright (C) 2020 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -set -e - -until python3 /var/www/webapp/entrypoint.py; do - >&2 echo "Postgres is unavailable - sleeping" - sleep 1 -done - -flask db upgrade - -exit 0 diff --git a/telemetryui/telemetryui/views.py b/telemetryui/telemetryui/views.py index bf3a104..2944167 100644 --- a/telemetryui/telemetryui/views.py +++ b/telemetryui/telemetryui/views.py @@ -113,6 +113,7 @@ def crashes_page(offset=0): @app.route('/crashes/', methods=['GET', 'POST']) def crashes(filter=None): form = forms.GuiltyDetailsForm() + backtrace_classes = crash.get_backtrace_classes() if request.method == 'POST': if form.validate_on_submit() is False: for e in form.comment.errors: @@ -150,9 +151,9 @@ def crashes(filter=None): uncache_data(GUILTY_CACHE_KEY) return redirect(endpoint) - backtrace_classes = crash.get_backtrace_classes() tmp = get_cached_data(GUILTY_CACHE_KEY, 600, Record.get_top_crash_guilties, classes=backtrace_classes) + tmp = Record.get_top_crash_guilties(classes=backtrace_classes) if filter: guilties = crash.guilty_list_for_build(tmp, filter)