Release v1.10.2

This release improves reliability of both the crash and oops probes.

* The crash probe will now reprocess a core file if it encounters
  missing symbols when creating backtraces the first time. This
  improves telemetry when run on Clear Linux OS, which downloads debuginfo
  on-the-fly as needed.

* The oops parsing will now capture "unreliable" stack frames for kernel
  oopses using the same notation as the kernel ("? "). Previously, the
  parsing code ignored the unreliable flag, so those frames appeared to be
  trustworthy frames.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([telemetrics-client], [1.9.0], [https://clearlinux.org/])
+AC_INIT([telemetrics-client], [1.10.2], [https://clearlinux.org/])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([1.14 -Wall -Werror -Wno-extra-portability foreign subdir-objects])
 AM_SILENT_RULES([yes])
@@ -137,3 +137,27 @@ AC_CONFIG_COMMANDS([config dir],[mkdir -p src/data; touch src/data/.dirstamp])
 AC_CONFIG_FILES([Makefile])
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
 AC_OUTPUT
+
+AC_MSG_NOTICE([
+
+telemetrics-client $VERSION
+===========================
+
+prefix:                 $prefix
+exec_prefix:            $exec_prefix
+libdir:                 $libdir
+bindir:                 $bindir
+datarootdir:            $datarootdir
+
+compiler:               $CC
+cflags:                 $CFLAGS
+ldflags:                $LDFLAGS
+
+unitdir:                $unitpath
+tmpfilesdir:            $tmpfilespath
+sysctldir:              $sysctlpath
+systemconfdir:          $confpath
+socketdir:              $socketpath
+loglevel:               $loglevel
+logtype:                $logtype
+])

src/probes/crash_probe.c

@@ -176,6 +176,55 @@ static int temp_core_file(void)
         return tmp;
 }
 
+/* There are a number of initialization routines required to initialize the Elf
+ * and Dwfl objects used by libdwfl to later process a core file. The argument
+ * e_core is the address of an Elf pointer declared by the caller, and core_fd
+ * is the open file descriptor for the core file.
+ */
+static int prepare_corefile(Elf **e_core, int core_fd)
+{
+        // Cleanup previous corefile processing if needed
+        if (d_core) {
+                dwfl_end(d_core);
+        }
+        if (*e_core) {
+                elf_end(*e_core);
+        }
+
+        // Boilerplate initialization for elfutils (libdwfl)
+        if (!(*e_core = elf_begin(core_fd, ELF_C_READ, NULL))) {
+                tm_elf_err("Failed to get file descriptor for ELF core file");
+                goto fail;
+        }
+
+        if (!(d_core = dwfl_begin(&cb))) {
+                tm_dwfl_err("Failed to start new libdwfl session");
+                goto fail;
+        }
+
+        int core_modules = 0;
+        core_modules = dwfl_core_file_report(d_core, *e_core, NULL);
+        if (core_modules == -1) {
+                tm_dwfl_err("Failed to report modules for ELF core file");
+                goto fail;
+        }
+
+        if (dwfl_report_end(d_core, NULL, NULL) != 0) {
+                tm_dwfl_err("Failed to finish reporting modules");
+                goto fail;
+        }
+
+        if ((core_for_pid = dwfl_core_file_attach(d_core, *e_core)) < 0) {
+                tm_dwfl_err("Failed to prepare libdwfl session for thread"
+                            " iteration");
+                goto fail;
+        }
+
+        return 0;
+fail:
+        return -1;
+}
+
 /* This callback is invoked for every frame in a thread. From a Dwfl_Frame, we
  * are able to extract the program counter (PC), and from that, the procedure
  * name via a Dwfl_Module.
@@ -305,6 +354,41 @@ fail:
         return false;
 }
 
+/* This is the entry point for libdwfl to unwind the backtrace from the core
+ * file. All data necessary for processing is referenced by d_core (a *Dwfl),
+ * which must be initialized prior to calling this function.  The callback
+ * function for processing each core thread is passed as the second argument to
+ * dwfl_getthreads(). The backtrace argument is the address of a pointer to a
+ * GString object declared by the caller which stores the backtrace data as a
+ * string.
+ */
+static int process_corefile(GString **backtrace)
+{
+        if (*backtrace) {
+                g_string_free(*backtrace, TRUE);
+        }
+        *backtrace = g_string_new(NULL);
+
+        if (dwfl_getthreads(d_core, thread_cb, backtrace) != DWARF_CB_OK) {
+                /* When errors occur during the unwinding, we reach this point.
+                 * If an error string is set for the particular error, send an
+                 * "error" record to capture at least a partial backtrace if
+                 * some frames were processed.
+                 */
+                if (errorstr != NULL) {
+                        telem_log(LOG_ERR, "%s", errorstr);
+                        g_string_append_printf(header, "Error: %s", errorstr);
+                        g_string_prepend(*backtrace, header->str);
+                        send_data(backtrace, error_severity, error_class);
+                }
+                goto fail;
+        }
+
+        return 0;
+fail:
+        return -1;
+}
+
 static bool in_clr_build(gchar *fullpath)
 {
         /*
@@ -373,7 +457,6 @@ static void free_glib_strings(void)
 int main(int argc, char **argv)
 {
         Elf *e_core = NULL;
-        int core_modules = 0;
         int ret = EXIT_FAILURE;
         int core_fd = STDIN_FILENO;
         GString *backtrace = NULL;
@@ -468,30 +551,7 @@ int main(int argc, char **argv)
 
         elf_version(EV_CURRENT);
 
-        if (!(e_core = elf_begin(core_fd, ELF_C_READ, NULL))) {
-                tm_elf_err("Failed to get file descriptor for ELF core file");
-                goto fail;
-        }
-
-        if (!(d_core = dwfl_begin(&cb))) {
-                tm_dwfl_err("Failed to start new libdwfl session");
-                goto fail;
-        }
-
-        core_modules = dwfl_core_file_report(d_core, e_core, NULL);
-        if (core_modules == -1) {
-                tm_dwfl_err("Failed to report modules for ELF core file");
-                goto fail;
-        }
-
-        if (dwfl_report_end(d_core, NULL, NULL) != 0) {
-                tm_dwfl_err("Failed to finish reporting modules");
-                goto fail;
-        }
-
-        if ((core_for_pid = dwfl_core_file_attach(d_core, e_core)) < 0) {
-                tm_dwfl_err("Failed to prepare libdwfl session for thread"
-                            " iteration");
+        if (prepare_corefile(&e_core, core_fd) < 0) {
                 goto fail;
         }
 
@@ -504,17 +564,27 @@ int main(int argc, char **argv)
                 g_string_append_printf(header, "Signal: %d\n", signal_num);
         }
 
-        backtrace = g_string_new(NULL);
-        if (dwfl_getthreads(d_core, thread_cb, &backtrace) != DWARF_CB_OK) {
-                if (errorstr != NULL) {
-                        telem_log(LOG_ERR, "%s", errorstr);
-                        g_string_append_printf(header, "Error: %s", errorstr);
-                        g_string_prepend(backtrace, header->str);
-                        send_data(&backtrace, error_severity, error_class);
-                }
+        if (process_corefile(&backtrace) < 0) {
                 goto fail;
         }
 
+        /* On Clear Linux OS, missing symbols may appear if automatic debuginfo
+         * downloads are still in flight. So if any missing symbols appear on
+         * the first run (indicated by the presence of "??? - ["), wait 10
+         * seconds and try again.
+         */
+        if (strstr(backtrace->str, "??? - [")) {
+                sleep(10);
+
+                if (prepare_corefile(&e_core, core_fd) < 0) {
+                        goto fail;
+                }
+
+                if (process_corefile(&backtrace) < 0) {
+                        goto fail;
+                }
+        }
+
         g_string_prepend(backtrace, header->str);
 
         if (!send_data(&backtrace, default_severity, clr_class)) {

src/probes/oops_parser.c

@@ -477,13 +477,6 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
                 start++;
         }
 
-        if (*start && *start == '?') {
-                start++;
-                if (*start && isspace(*start)) {
-                        start++;
-                }
-        }
-
         offset_ptr = strchr(start, '+');
         end = offset_ptr ? offset_ptr : (start + strlen(start));
         frame->function = strndup(start, (size_t)(end - start));

tests/check_probes.c

@@ -100,23 +100,23 @@ START_TEST(watchdog_payload)
         ck_assert(strstr(pl->str, "#1 dump_stack"));
         ck_assert(strstr(pl->str, "#2 warn_slowpath_common"));
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
-        ck_assert(strstr(pl->str, "#4 __queue_work"));
+        ck_assert(strstr(pl->str, "#4 ? __queue_work"));
         ck_assert(strstr(pl->str, "#5 dev_watchdog"));
-        ck_assert(strstr(pl->str, "#6 dev_deactivate_queue.constprop.30"));
+        ck_assert(strstr(pl->str, "#6 ? dev_deactivate_queue.constprop.30"));
         ck_assert(strstr(pl->str, "#7 call_timer_fn"));
-        ck_assert(strstr(pl->str, "#8 dev_deactivate_queue.constprop.30"));
+        ck_assert(strstr(pl->str, "#8 ? dev_deactivate_queue.constprop.30"));
         ck_assert(strstr(pl->str, "#9 run_timer_softirq"));
         ck_assert(strstr(pl->str, "#10 __do_softirq"));
         ck_assert(strstr(pl->str, "#11 irq_exit"));
         ck_assert(strstr(pl->str, "#12 smp_apic_timer_interrupt"));
         ck_assert(strstr(pl->str, "#13 apic_timer_interrupt"));
-        ck_assert(strstr(pl->str, "#14 __hrtimer_start_range_ns"));
-        ck_assert(strstr(pl->str, "#15 cpuidle_enter_state"));
-        ck_assert(strstr(pl->str, "#16 cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#14 ? __hrtimer_start_range_ns"));
+        ck_assert(strstr(pl->str, "#15 ? cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#16 ? cpuidle_enter_state"));
         ck_assert(strstr(pl->str, "#17 cpuidle_idle_call"));
         ck_assert(strstr(pl->str, "#18 arch_cpu_idle"));
         ck_assert(strstr(pl->str, "#19 cpu_startup_entry"));
-        ck_assert(strstr(pl->str, "#20 clockevents_register_device"));
+        ck_assert(strstr(pl->str, "#20 ? clockevents_register_device"));
         ck_assert(strstr(pl->str, "#21 start_secondary"));
 
         g_string_free(pl, true);
@@ -141,8 +141,8 @@ START_TEST(alsa_bug_payload)
         ck_assert(strstr(pl->str, "Tainted : Not tainted"));
 
         ck_assert(strstr(pl->str, "#1 warn_on_slowpath"));
-        ck_assert(strstr(pl->str, "#2 vsnprintf"));
-        ck_assert(strstr(pl->str, "#3 _spin_unlock_irqrestore"));
+        ck_assert(strstr(pl->str, "#2 ? vsnprintf"));
+        ck_assert(strstr(pl->str, "#3 ? _spin_unlock_irqrestore"));
         ck_assert(strstr(pl->str, "#10 start_kernel"));
         g_string_free(pl, true);
 }
@@ -173,14 +173,14 @@ START_TEST(warning_payload)
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
         ck_assert(strstr(pl->str, "#4 preempt_notifier_register"));
         ck_assert(strstr(pl->str, "#5 VBoxHost_RTThreadCtxHooksRegister"));
-        ck_assert(strstr(pl->str, "#6 update_curr"));
-        ck_assert(strstr(pl->str, "#7 supdrvIOCtlFast"));
-        ck_assert(strstr(pl->str, "#8 VBoxDrvLinuxIOCtl_4_3_28"));
-        ck_assert(strstr(pl->str, "#9 put_prev_entity"));
-        ck_assert(strstr(pl->str, "#10 do_vfs_ioctl"));
-        ck_assert(strstr(pl->str, "#11 pick_next_task_rt"));
-        ck_assert(strstr(pl->str, "#12 SyS_ioctl"));
-        ck_assert(strstr(pl->str, "#13 entry_SYSCALL_64_fastpath"));
+        ck_assert(strstr(pl->str, "#6 ? update_curr"));
+        ck_assert(strstr(pl->str, "#7 ? supdrvIOCtlFast"));
+        ck_assert(strstr(pl->str, "#8 ? VBoxDrvLinuxIOCtl_4_3_28"));
+        ck_assert(strstr(pl->str, "#9 ? put_prev_entity"));
+        ck_assert(strstr(pl->str, "#10 ? do_vfs_ioctl"));
+        ck_assert(strstr(pl->str, "#11 ? pick_next_task_rt"));
+        ck_assert(strstr(pl->str, "#12 ? SyS_ioctl"));
+        ck_assert(strstr(pl->str, "#13 ? entry_SYSCALL_64_fastpath"));
 
         g_string_free(pl, true);
 }
@@ -208,16 +208,16 @@ START_TEST(warn_on_payload)
         ck_assert(strstr(pl->str, "#2 warn_slowpath_common"));
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
         ck_assert(strstr(pl->str, "#4 dev_watchdog"));
-        ck_assert(strstr(pl->str, "#5 qdisc_rcu_free"));
+        ck_assert(strstr(pl->str, "#5 ? qdisc_rcu_free"));
         ck_assert(strstr(pl->str, "#6 call_timer_fn"));
-        ck_assert(strstr(pl->str, "#7 qdisc_rcu_free"));
+        ck_assert(strstr(pl->str, "#7 ? qdisc_rcu_free"));
         ck_assert(strstr(pl->str, "#8 run_timer_softirq"));
         ck_assert(strstr(pl->str, "#9 __do_softirq"));
         ck_assert(strstr(pl->str, "#10 irq_exit"));
         ck_assert(strstr(pl->str, "#11 smp_apic_timer_interrupt"));
         ck_assert(strstr(pl->str, "#12 apic_timer_interrupt"));
-        ck_assert(strstr(pl->str, "#13 cpuidle_enter_state"));
-        ck_assert(strstr(pl->str, "#14 cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#13 ? cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#14 ? cpuidle_enter_state"));
         ck_assert(strstr(pl->str, "#15 cpuidle_enter"));
         ck_assert(strstr(pl->str, "#16 cpu_startup_entry"));
         ck_assert(strstr(pl->str, "#17 start_secondary"));
@@ -285,20 +285,20 @@ START_TEST(sysctl2_payload)
         ck_assert(strstr(pl->str, "Kernel Version : 2.6.32.53-bigmem-5-openvz #28"));
         ck_assert(strstr(pl->str, "Tainted : P"));
 
-        ck_assert(strstr(pl->str, "#1 set_fail"));
-        ck_assert(strstr(pl->str, "#2 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#3 sysctl_check_lookup"));
-        ck_assert(strstr(pl->str, "#4 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#5 sysctl_check_lookup"));
-        ck_assert(strstr(pl->str, "#6 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#7 __kmalloc"));
-        ck_assert(strstr(pl->str, "#8 __register_sysctl_paths"));
-        ck_assert(strstr(pl->str, "#9 proc_register"));
-        ck_assert(strstr(pl->str, "#10 register_net_sysctl_table"));
-        ck_assert(strstr(pl->str, "#11 nf_conntrack_net_init"));
-        ck_assert(strstr(pl->str, "#12 setup_net"));
-        ck_assert(strstr(pl->str, "#13 copy_net_ns"));
-        ck_assert(strstr(pl->str, "#14 create_new_namespaces"));
+        ck_assert(strstr(pl->str, "#1 ? set_fail"));
+        ck_assert(strstr(pl->str, "#2 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#3 ? sysctl_check_lookup"));
+        ck_assert(strstr(pl->str, "#4 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#5 ? sysctl_check_lookup"));
+        ck_assert(strstr(pl->str, "#6 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#7 ? __kmalloc"));
+        ck_assert(strstr(pl->str, "#8 ? __register_sysctl_paths"));
+        ck_assert(strstr(pl->str, "#9 ? proc_register"));
+        ck_assert(strstr(pl->str, "#10 ? register_net_sysctl_table"));
+        ck_assert(strstr(pl->str, "#11 ? nf_conntrack_net_init"));
+        ck_assert(strstr(pl->str, "#12 ? setup_net"));
+        ck_assert(strstr(pl->str, "#13 ? copy_net_ns"));
+        ck_assert(strstr(pl->str, "#14 ? create_new_namespaces"));
 
         g_string_free(pl, true);
 }
@@ -327,7 +327,7 @@ START_TEST(softlockup_payload)
         ck_assert(strstr(pl->str, "#2 btrfs_tree_read_lock"));
         ck_assert(strstr(pl->str, "#3 btrfs_read_lock_root_node"));
         ck_assert(strstr(pl->str, "#4 btrfs_search_slot"));
-        ck_assert(strstr(pl->str, "#5 crypto_shash_update"));
+        ck_assert(strstr(pl->str, "#5 ? crypto_shash_update"));
         ck_assert(strstr(pl->str, "#6 btrfs_lookup_xattr"));
         ck_assert(strstr(pl->str, "#7 __btrfs_getxattr"));
         ck_assert(strstr(pl->str, "#8 btrfs_getxattr"));
@@ -356,19 +356,19 @@ START_TEST(rtnl_payload)
 
         ck_assert(strstr(pl->str, "Tainted : Not tainted"));
 
-        ck_assert(strstr(pl->str, "#1 dump_stack"));
-        ck_assert(strstr(pl->str, "#2 bond_set_rx_mode"));
-        ck_assert(strstr(pl->str, "#3 __dev_mc_add"));
-        ck_assert(strstr(pl->str, "#4 igmp6_group_added"));
-        ck_assert(strstr(pl->str, "#5 sock_def_readable"));
-        ck_assert(strstr(pl->str, "#6 ipv6_dev_mc_inc"));
-        ck_assert(strstr(pl->str, "#7 ipv6_sock_mc_join"));
-        ck_assert(strstr(pl->str, "#8 do_ipv6_setsockopt.isra.6"));
-        ck_assert(strstr(pl->str, "#9 pipe_write"));
-        ck_assert(strstr(pl->str, "#10 SYSC_sendto"));
-        ck_assert(strstr(pl->str, "#11 ipv6_setsockopt"));
-        ck_assert(strstr(pl->str, "#12 SyS_setsockopt"));
-        ck_assert(strstr(pl->str, "#13 system_call_fastpath"));
+        ck_assert(strstr(pl->str, "#1 ? dump_stack"));
+        ck_assert(strstr(pl->str, "#2 ? bond_set_rx_mode"));
+        ck_assert(strstr(pl->str, "#3 ? __dev_mc_add"));
+        ck_assert(strstr(pl->str, "#4 ? igmp6_group_added"));
+        ck_assert(strstr(pl->str, "#5 ? sock_def_readable"));
+        ck_assert(strstr(pl->str, "#6 ? ipv6_dev_mc_inc"));
+        ck_assert(strstr(pl->str, "#7 ? ipv6_sock_mc_join"));
+        ck_assert(strstr(pl->str, "#8 ? do_ipv6_setsockopt.isra.6"));
+        ck_assert(strstr(pl->str, "#9 ? pipe_write"));
+        ck_assert(strstr(pl->str, "#10 ? SYSC_sendto"));
+        ck_assert(strstr(pl->str, "#11 ? ipv6_setsockopt"));
+        ck_assert(strstr(pl->str, "#12 ? SyS_setsockopt"));
+        ck_assert(strstr(pl->str, "#13 ? system_call_fastpath"));
 
         g_string_free(pl, true);
 }
@@ -454,49 +454,49 @@ START_TEST(irq_payload)
         ck_assert(strstr(pl->str, "#1 dump_stack"));
         ck_assert(strstr(pl->str, "#2 __report_bad_irq"));
         ck_assert(strstr(pl->str, "#3 note_interrupt"));
-        ck_assert(strstr(pl->str, "#4 __do_softirq"));
+        ck_assert(strstr(pl->str, "#4 ? __do_softirq"));
         ck_assert(strstr(pl->str, "#5 handle_irq_event_percpu"));
         ck_assert(strstr(pl->str, "#6 handle_irq_event"));
-        ck_assert(strstr(pl->str, "#7 handle_nested_irq"));
+        ck_assert(strstr(pl->str, "#7 ? handle_nested_irq"));
         ck_assert(strstr(pl->str, "#8 handle_level_irq"));
-        ck_assert(strstr(pl->str, "#9 do_IRQ"));
-        ck_assert(strstr(pl->str, "#10 common_interrupt"));
-        ck_assert(strstr(pl->str, "#11 __do_softirq"));
-        ck_assert(strstr(pl->str, "#12 handle_irq"));
-        ck_assert(strstr(pl->str, "#13 irq_exit"));
-        ck_assert(strstr(pl->str, "#14 do_IRQ"));
-        ck_assert(strstr(pl->str, "#15 pci_write"));
-        ck_assert(strstr(pl->str, "#16 common_interrupt"));
-        ck_assert(strstr(pl->str, "#17 audit_make_tree"));
-        ck_assert(strstr(pl->str, "#18 usbdev_ioctl"));
-        ck_assert(strstr(pl->str, "#19 _raw_spin_unlock_irqrestore"));
-        ck_assert(strstr(pl->str, "#20 __setup_irq"));
-        ck_assert(strstr(pl->str, "#21 kmem_cache_alloc_trace"));
-        ck_assert(strstr(pl->str, "#22 vsnprintf"));
-        ck_assert(strstr(pl->str, "#23 request_threaded_irq"));
-        ck_assert(strstr(pl->str, "#24 usb_hcd_platform_shutdown"));
-        ck_assert(strstr(pl->str, "#25 request_threaded_irq"));
-        ck_assert(strstr(pl->str, "#26 usb_add_hcd"));
-        ck_assert(strstr(pl->str, "#27 usb_hcd_pci_probe"));
-        ck_assert(strstr(pl->str, "#28 rpm_resume"));
-        ck_assert(strstr(pl->str, "#29 pci_device_probe"));
-        ck_assert(strstr(pl->str, "#30 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#31 pci_match_device"));
-        ck_assert(strstr(pl->str, "#32 __driver_attach"));
-        ck_assert(strstr(pl->str, "#33 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#34 bus_for_each_dev"));
-        ck_assert(strstr(pl->str, "#35 driver_attach"));
-        ck_assert(strstr(pl->str, "#36 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#37 bus_add_driver"));
-        ck_assert(strstr(pl->str, "#38 pci_match_id"));
-        ck_assert(strstr(pl->str, "#39 driver_register"));
-        ck_assert(strstr(pl->str, "#40 ohci_hcd_mod_init"));
-        ck_assert(strstr(pl->str, "#41 ohci_hcd_mod_init"));
-        ck_assert(strstr(pl->str, "#42 __pci_register_driver"));
-        ck_assert(strstr(pl->str, "#43 uhci_hcd_init"));
-        ck_assert(strstr(pl->str, "#44 do_one_initcall"));
-        ck_assert(strstr(pl->str, "#45 kernel_init_freeable"));
-        ck_assert(strstr(pl->str, "#46 do_early_param"));
+        ck_assert(strstr(pl->str, "#9 ? do_IRQ"));
+        ck_assert(strstr(pl->str, "#10 ? common_interrupt"));
+        ck_assert(strstr(pl->str, "#11 ? __do_softirq"));
+        ck_assert(strstr(pl->str, "#12 ? handle_irq"));
+        ck_assert(strstr(pl->str, "#13 ? irq_exit"));
+        ck_assert(strstr(pl->str, "#14 ? do_IRQ"));
+        ck_assert(strstr(pl->str, "#15 ? pci_write"));
+        ck_assert(strstr(pl->str, "#16 ? common_interrupt"));
+        ck_assert(strstr(pl->str, "#17 ? audit_make_tree"));
+        ck_assert(strstr(pl->str, "#18 ? usbdev_ioctl"));
+        ck_assert(strstr(pl->str, "#19 ? _raw_spin_unlock_irqrestore"));
+        ck_assert(strstr(pl->str, "#20 ? __setup_irq"));
+        ck_assert(strstr(pl->str, "#21 ? kmem_cache_alloc_trace"));
+        ck_assert(strstr(pl->str, "#22 ? vsnprintf"));
+        ck_assert(strstr(pl->str, "#23 ? request_threaded_irq"));
+        ck_assert(strstr(pl->str, "#24 ? usb_hcd_platform_shutdown"));
+        ck_assert(strstr(pl->str, "#25 ? request_threaded_irq"));
+        ck_assert(strstr(pl->str, "#26 ? usb_add_hcd"));
+        ck_assert(strstr(pl->str, "#27 ? usb_hcd_pci_probe"));
+        ck_assert(strstr(pl->str, "#28 ? rpm_resume"));
+        ck_assert(strstr(pl->str, "#29 ? pci_device_probe"));
+        ck_assert(strstr(pl->str, "#30 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#31 ? pci_match_device"));
+        ck_assert(strstr(pl->str, "#32 ? __driver_attach"));
+        ck_assert(strstr(pl->str, "#33 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#34 ? bus_for_each_dev"));
+        ck_assert(strstr(pl->str, "#35 ? driver_attach"));
+        ck_assert(strstr(pl->str, "#36 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#37 ? bus_add_driver"));
+        ck_assert(strstr(pl->str, "#38 ? pci_match_id"));
+        ck_assert(strstr(pl->str, "#39 ? driver_register"));
+        ck_assert(strstr(pl->str, "#40 ? ohci_hcd_mod_init"));
+        ck_assert(strstr(pl->str, "#41 ? ohci_hcd_mod_init"));
+        ck_assert(strstr(pl->str, "#42 ? __pci_register_driver"));
+        ck_assert(strstr(pl->str, "#43 ? uhci_hcd_init"));
+        ck_assert(strstr(pl->str, "#44 ? do_one_initcall"));
+        ck_assert(strstr(pl->str, "#45 ? kernel_init_freeable"));
+        ck_assert(strstr(pl->str, "#46 ? do_early_param"));
 
         g_string_free(pl, true);
 
@@ -521,12 +521,12 @@ START_TEST(general_protection_fault_payload)
         ck_assert(strstr(pl->str, "Modules : tun 8021q garp mrp bridge stp llc bonding ipt_REJECT xt_comment xt_multiport"));
         ck_assert(strstr(pl->str, "lpc_ich iw_cm mfd_core nfsd auth_rpcgss wmi nfs_acl lockd ipmi_si ipmi_devintf"));
 
-        ck_assert(strstr(pl->str, "#1 alloc_pipe_info"));
+        ck_assert(strstr(pl->str, "#1 ? alloc_pipe_info"));
         ck_assert(strstr(pl->str, "#2 alloc_pipe_info"));
         ck_assert(strstr(pl->str, "#3 create_pipe_files"));
         ck_assert(strstr(pl->str, "#4 __do_pipe_flags"));
         ck_assert(strstr(pl->str, "#5 SyS_pipe2"));
-        ck_assert(strstr(pl->str, "#6 __audit_syscall_exit"));
+        ck_assert(strstr(pl->str, "#6 ? __audit_syscall_exit"));
         ck_assert(strstr(pl->str, "#7 SyS_pipe"));
         ck_assert(strstr(pl->str, "#8 system_call_fastpath"));
 
@@ -594,29 +594,29 @@ START_TEST(bug_kernel_handle_payload)
         ck_assert(strstr(pl->str, "Modules : nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE be2iscsi"));
         ck_assert(strstr(pl->str, "drm_kms_helper firewire_ohci drm firewire_core tg3 crc_itu_t i2c_core video usb_storage sunrpc"));
 
-        ck_assert(strstr(pl->str, "#1 _nv007312rm"));
-        ck_assert(strstr(pl->str, "#2 _nv007847rm"));
-        ck_assert(strstr(pl->str, "#3 _nv004049rm"));
-        ck_assert(strstr(pl->str, "#4 _nv004049rm"));
-        ck_assert(strstr(pl->str, "#5 _nv010019rm"));
-        ck_assert(strstr(pl->str, "#6 _nv014983rm"));
-        ck_assert(strstr(pl->str, "#7 _nv001097rm"));
-        ck_assert(strstr(pl->str, "#8 rm_init_adapter"));
-        ck_assert(strstr(pl->str, "#9 nv_kern_open"));
-        ck_assert(strstr(pl->str, "#10 chrdev_open"));
-        ck_assert(strstr(pl->str, "#11 do_dentry_open"));
-        ck_assert(strstr(pl->str, "#12 cdev_put"));
-        ck_assert(strstr(pl->str, "#13 finish_open"));
-        ck_assert(strstr(pl->str, "#14 do_last"));
-        ck_assert(strstr(pl->str, "#15 inode_permission"));
-        ck_assert(strstr(pl->str, "#16 link_path_walk"));
-        ck_assert(strstr(pl->str, "#17 path_openat"));
-        ck_assert(strstr(pl->str, "#18 do_filp_open"));
-        ck_assert(strstr(pl->str, "#19 __alloc_fd"));
-        ck_assert(strstr(pl->str, "#20 do_sys_open"));
-        ck_assert(strstr(pl->str, "#21 __audit_syscall_entry"));
-        ck_assert(strstr(pl->str, "#22 sys_open"));
-        ck_assert(strstr(pl->str, "#23 system_call_fastpath"));
+        ck_assert(strstr(pl->str, "#1 ? _nv007312rm"));
+        ck_assert(strstr(pl->str, "#2 ? _nv007847rm"));
+        ck_assert(strstr(pl->str, "#3 ? _nv004049rm"));
+        ck_assert(strstr(pl->str, "#4 ? _nv004049rm"));
+        ck_assert(strstr(pl->str, "#5 ? _nv010019rm"));
+        ck_assert(strstr(pl->str, "#6 ? _nv014983rm"));
+        ck_assert(strstr(pl->str, "#7 ? _nv001097rm"));
+        ck_assert(strstr(pl->str, "#8 ? rm_init_adapter"));
+        ck_assert(strstr(pl->str, "#9 ? nv_kern_open"));
+        ck_assert(strstr(pl->str, "#10 ? chrdev_open"));
+        ck_assert(strstr(pl->str, "#11 ? do_dentry_open"));
+        ck_assert(strstr(pl->str, "#12 ? cdev_put"));
+        ck_assert(strstr(pl->str, "#13 ? finish_open"));
+        ck_assert(strstr(pl->str, "#14 ? do_last"));
+        ck_assert(strstr(pl->str, "#15 ? inode_permission"));
+        ck_assert(strstr(pl->str, "#16 ? link_path_walk"));
+        ck_assert(strstr(pl->str, "#17 ? path_openat"));
+        ck_assert(strstr(pl->str, "#18 ? do_filp_open"));
+        ck_assert(strstr(pl->str, "#19 ? __alloc_fd"));
+        ck_assert(strstr(pl->str, "#20 ? do_sys_open"));
+        ck_assert(strstr(pl->str, "#21 ? __audit_syscall_entry"));
+        ck_assert(strstr(pl->str, "#22 ? sys_open"));
+        ck_assert(strstr(pl->str, "#23 ? system_call_fastpath"));
 
         g_string_free(pl, true);