From 13752fa9ca2f3377667ed37b58051f25c1edb84f Mon Sep 17 00:00:00 2001
From: Otavio Pontes <otavio.pontes@intel.com>
Date: Fri, 3 Apr 2020 13:32:25 -0700
Subject: [PATCH] actions: Validate if all tests are going to be executed

Validate if we are really calling make check for all tests in
test/functional directory.

Signed-off-by: Otavio Pontes <otavio.pontes@intel.com>
---
 .github/workflows/ccpp.yml            | 260 ++++++++++++++++----------
 scripts/github_actions/run_check.bash |  15 ++
 2 files changed, 173 insertions(+), 102 deletions(-)
 create mode 100755 scripts/github_actions/run_check.bash

diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml
index e2d47b30..4c11dd9e 100644
--- a/.github/workflows/ccpp.yml
+++ b/.github/workflows/ccpp.yml
@@ -37,17 +37,6 @@ jobs:
     - name: run distcheck
       run:  make distcheck
 
-    - name: Validate bats tests
-      run:  |
-             TESTS=$(for i in $(seq $NUM_JOBS); do scripts/github_actions/filter_bats_list.bash $i $NUM_JOBS; done|wc -l)
-             TESTS_CI=$(find test/functional/ -name "*.bats" -path "*only_in_ci*" | wc -l)
-             TESTS=$((TESTS+TESTS_CI))
-             TOTAL=$(find test/functional/ -name "*.bats" | wc -l)
-             if [ "$TOTAL" -ne "$TESTS" ]; then
-                 echo "Error in filter_bats_test.bash script"
-                 exit 1
-             fi
-
     - name: print status
       if: failure()
       run: cat test-suite.log
@@ -67,18 +56,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -99,18 +88,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -131,18 +120,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -163,18 +152,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -195,18 +184,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -227,18 +216,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -259,18 +248,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -291,18 +280,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -323,18 +312,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -355,18 +344,18 @@ jobs:
         path: dependencies
         key: swupd-dependencies
 
-    - name: build_dep
-      run:  scripts/github_actions/build_ci_dependencies.bash
-
     - name: build
-      run:  scripts/github_actions/build_ci.bash
+      run:  |
+            scripts/github_actions/build_ci_dependencies.bash
+            scripts/github_actions/build_ci.bash
 
     - name: run check
-      run: |
-            FILES="$(scripts/github_actions/filter_bats_list.bash $JOB_NUMBER $NUM_JOBS)"
-            NUM_FILES="$(echo $FILES | tr ' ' '\n' | wc -l)"
-            echo "Running $NUM_FILES tests"
-            env TESTS="$(echo $FILES)" make -e -j2 check
+      run:  scripts/github_actions/run_check.bash "$JOB_NUMBER" "$NUM_JOBS"
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-${{ env.JOB_NUMBER }}
+        path: job-${{ env.JOB_NUMBER }}
 
     - name: print status
       if: failure()
@@ -395,11 +384,18 @@ jobs:
       run: |
             FILES="$(find test/functional/only_in_ci_slow/ -name "*.bats")"
             env TESTS="$(echo $FILES)" make -e -j10 check
+            echo "$FILES" >> job-only-ci
 
     - name: run check system
       run: |
             FILES="$(find test/functional/only_in_ci_system/ -name "*.bats")"
             env TESTS="$(echo $FILES)" make -e -j1 check
+            echo "$FILES" >> job-only-ci
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: job-only-ci
+        path: job-only-ci
 
     - name: print status
       if: failure()
@@ -409,5 +405,65 @@ jobs:
     runs-on: ubuntu-latest
     needs: [unit_and_style, test_job1, test_job2, test_job3, test_job4, test_job5, test_job6, test_job7, test_job8, test_job9, test_job10, test_only_ci]
     steps:
+    - uses: actions/checkout@v1
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-1
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-2
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-3
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-4
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-5
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-6
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-7
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-8
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-9
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-10
+         path: jobs/
+
+    - uses: actions/download-artifact@v1
+      with:
+         name: job-only-ci
+         path: jobs/
+
     - name: Successful
-      run:  echo "Successful"
+      run:  |
+            find test/functional/ -name "*.bats" | sort -u > all_tests
+            cat jobs/job-* | sort -u > run_tests
+            diff -u all_tests run_tests
diff --git a/scripts/github_actions/run_check.bash b/scripts/github_actions/run_check.bash
new file mode 100755
index 00000000..af4167c1
--- /dev/null
+++ b/scripts/github_actions/run_check.bash
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+JOB_NUMBER="$1"
+NUM_JOBS="$2"
+
+FILES=$(scripts/github_actions/filter_bats_list.bash "$JOB_NUMBER" "$NUM_JOBS")
+NUM_FILES=$(echo "$FILES" | tr ' ' '\n' | wc -l)
+echo "Running $NUM_FILES tests"
+# shellcheck disable=SC2116
+# shellcheck disable=SC2086
+# Weird, but we really need this
+env TESTS="$(echo $FILES)" make -e -j2 check
+
+echo "$FILES" > job-"$JOB_NUMBER"