dlan17/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge tag 'trace-v5.16' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace

Browse Source 
Pull tracing updates from Steven Rostedt:

 - kprobes: Restructured stack unwinder to show properly on x86 when a
   stack dump happens from a kretprobe callback.

 - Fix to bootconfig parsing

 - Have tracefs allow owner and group permissions by default (only
   denying others). There's been pressure to allow non root to tracefs
   in a controlled fashion, and using groups is probably the safest.

 - Bootconfig memory managament updates.

 - Bootconfig clean up to have the tools directory be less dependent on
   changes in the kernel tree.

 - Allow perf to be traced by function tracer.

 - Rewrite of function graph tracer to be a callback from the function
   tracer instead of having its own trampoline (this change will happen
   on an arch by arch basis, and currently only x86_64 implements it).

 - Allow multiple direct trampolines (bpf hooks to functions) be batched
   together in one synchronization.

 - Allow histogram triggers to add variables that can perform
   calculations against the event's fields.

 - Use the linker to determine architecture callbacks from the ftrace
   trampoline to allow for proper parameter prototypes and prevent
   warnings from the compiler.

 - Extend histogram triggers to key off of variables.

 - Have trace recursion use bit magic to determine preempt context over
   if branches.

 - Have trace recursion disable preemption as all use cases do anyway.

 - Added testing for verification of tracing utilities.

 - Various small clean ups and fixes.

* tag 'trace-v5.16' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace: (101 commits)
  tracing/histogram: Fix semicolon.cocci warnings
  tracing/histogram: Fix documentation inline emphasis warning
  tracing: Increase PERF_MAX_TRACE_SIZE to handle Sentinel1 and docker together
  tracing: Show size of requested perf buffer
  bootconfig: Initialize ret in xbc_parse_tree()
  ftrace: do CPU checking after preemption disabled
  ftrace: disable preemption when recursion locked
  tracing/histogram: Document expression arithmetic and constants
  tracing/histogram: Optimize division by a power of 2
  tracing/histogram: Covert expr to const if both operands are constants
  tracing/histogram: Simplify handling of .sym-offset in expressions
  tracing: Fix operator precedence for hist triggers expression
  tracing: Add division and multiplication support for hist triggers
  tracing: Add support for creating hist trigger variables from literal
  selftests/ftrace: Stop tracing while reading the trace file by default
  MAINTAINERS: Update KPROBES and TRACING entries
  test_kprobes: Move it from kernel/ to lib/
  docs, kprobes: Remove invalid URL and add new reference
  samples/kretprobes: Fix return value if register_kretprobe() failed
  lib/bootconfig: Fix the xbc_get_info kerneldoc
  ...
This commit is contained in:
Linus Torvalds
2021-11-01 20:05:19 -07:00
parent d54f486035 feea69ec12
commit 79ef0c0014
135 changed files with 3012 additions and 1529 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/Makefile
-1
View File
@@ -85,7 +85,6 @@ obj-$(CONFIG_PID_NS) += pid_namespace.o
obj-$(CONFIG_IKCONFIG) += configs.o
obj-$(CONFIG_IKHEADERS) += kheaders.o
obj-$(CONFIG_SMP) += stop_machine.o
obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o
obj-$(CONFIG_AUDIT) += audit.o auditfilter.o
obj-$(CONFIG_AUDITSYSCALL) += auditsc.o audit_watch.o audit_fsnotify.o audit_tree.o
obj-$(CONFIG_GCOV_KERNEL) += gcov/
kernel/events/internal.h
+1 -6
View File
@@ -205,12 +205,7 @@ DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)
static inline int get_recursion_context(int *recursion)
{
unsigned int pc = preempt_count();
unsigned char rctx = 0;
rctx += !!(pc & (NMI_MASK));
rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
unsigned char rctx = interrupt_context_level();
if (recursion[rctx])
return -1;
kernel/kprobes.c
+282 -229
View File
File diff suppressed because it is too large Load Diff
kernel/livepatch/patch.c
+6 -6
View File
@@ -49,14 +49,15 @@ static void notrace klp_ftrace_handler(unsigned long ip,
ops = container_of(fops, struct klp_ops, fops);
/*
* The ftrace_test_recursion_trylock() will disable preemption,
* which is required for the variant of synchronize_rcu() that is
* used to allow patching functions where RCU is not watching.
* See klp_synchronize_transition() for more details.
*/
bit = ftrace_test_recursion_trylock(ip, parent_ip);
if (WARN_ON_ONCE(bit < 0))
return;
/*
* A variant of synchronize_rcu() is used to allow patching functions
* where RCU is not watching, see klp_synchronize_transition().
*/
preempt_disable_notrace();
func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
stack_node);
@@ -120,7 +121,6 @@ static void notrace klp_ftrace_handler(unsigned long ip,
klp_arch_set_pc(fregs, (unsigned long)func->new_func);
unlock:
preempt_enable_notrace();
ftrace_test_recursion_unlock(bit);
}
kernel/test_kprobes.c
-313
View File
@@ -1,313 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* test_kprobes.c - simple sanity test for *probes
*
* Copyright IBM Corp. 2008
*/
#define pr_fmt(fmt) "Kprobe smoke test: " fmt
#include <linux/kernel.h>
#include <linux/kprobes.h>
#include <linux/random.h>
#define div_factor 3
static u32 rand1, preh_val, posth_val;
static int errors, handler_errors, num_tests;
static u32 (*target)(u32 value);
static u32 (*target2)(u32 value);
static noinline u32 kprobe_target(u32 value)
{
return (value / div_factor);
}
static int kp_pre_handler(struct kprobe *p, struct pt_regs *regs)
{
if (preemptible()) {
handler_errors++;
pr_err("pre-handler is preemptible\n");
}
preh_val = (rand1 / div_factor);
return 0;
}
static void kp_post_handler(struct kprobe *p, struct pt_regs *regs,
unsigned long flags)
{
if (preemptible()) {
handler_errors++;
pr_err("post-handler is preemptible\n");
}
if (preh_val != (rand1 / div_factor)) {
handler_errors++;
pr_err("incorrect value in post_handler\n");
}
posth_val = preh_val + div_factor;
}
static struct kprobe kp = {
.symbol_name = "kprobe_target",
.pre_handler = kp_pre_handler,
.post_handler = kp_post_handler
};
static int test_kprobe(void)
{
int ret;
ret = register_kprobe(&kp);
if (ret < 0) {
pr_err("register_kprobe returned %d\n", ret);
return ret;
}
ret = target(rand1);
unregister_kprobe(&kp);
if (preh_val == 0) {
pr_err("kprobe pre_handler not called\n");
handler_errors++;
}
if (posth_val == 0) {
pr_err("kprobe post_handler not called\n");
handler_errors++;
}
return 0;
}
static noinline u32 kprobe_target2(u32 value)
{
return (value / div_factor) + 1;
}
static int kp_pre_handler2(struct kprobe *p, struct pt_regs *regs)
{
preh_val = (rand1 / div_factor) + 1;
return 0;
}
static void kp_post_handler2(struct kprobe *p, struct pt_regs *regs,
unsigned long flags)
{
if (preh_val != (rand1 / div_factor) + 1) {
handler_errors++;
pr_err("incorrect value in post_handler2\n");
}
posth_val = preh_val + div_factor;
}
static struct kprobe kp2 = {
.symbol_name = "kprobe_target2",
.pre_handler = kp_pre_handler2,
.post_handler = kp_post_handler2
};
static int test_kprobes(void)
{
int ret;
struct kprobe *kps[2] = {&kp, &kp2};
/* addr and flags should be cleard for reusing kprobe. */
kp.addr = NULL;
kp.flags = 0;
ret = register_kprobes(kps, 2);
if (ret < 0) {
pr_err("register_kprobes returned %d\n", ret);
return ret;
}
preh_val = 0;
posth_val = 0;
ret = target(rand1);
if (preh_val == 0) {
pr_err("kprobe pre_handler not called\n");
handler_errors++;
}
if (posth_val == 0) {
pr_err("kprobe post_handler not called\n");
handler_errors++;
}
preh_val = 0;
posth_val = 0;
ret = target2(rand1);
if (preh_val == 0) {
pr_err("kprobe pre_handler2 not called\n");
handler_errors++;
}
if (posth_val == 0) {
pr_err("kprobe post_handler2 not called\n");
handler_errors++;
}
unregister_kprobes(kps, 2);
return 0;
}
#ifdef CONFIG_KRETPROBES
static u32 krph_val;
static int entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
{
if (preemptible()) {
handler_errors++;
pr_err("kretprobe entry handler is preemptible\n");
}
krph_val = (rand1 / div_factor);
return 0;
}
static int return_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
{
unsigned long ret = regs_return_value(regs);
if (preemptible()) {
handler_errors++;
pr_err("kretprobe return handler is preemptible\n");
}
if (ret != (rand1 / div_factor)) {
handler_errors++;
pr_err("incorrect value in kretprobe handler\n");
}
if (krph_val == 0) {
handler_errors++;
pr_err("call to kretprobe entry handler failed\n");
}
krph_val = rand1;
return 0;
}
static struct kretprobe rp = {
.handler = return_handler,
.entry_handler = entry_handler,
.kp.symbol_name = "kprobe_target"
};
static int test_kretprobe(void)
{
int ret;
ret = register_kretprobe(&rp);
if (ret < 0) {
pr_err("register_kretprobe returned %d\n", ret);
return ret;
}
ret = target(rand1);
unregister_kretprobe(&rp);
if (krph_val != rand1) {
pr_err("kretprobe handler not called\n");
handler_errors++;
}
return 0;
}
static int return_handler2(struct kretprobe_instance *ri, struct pt_regs *regs)
{
unsigned long ret = regs_return_value(regs);
if (ret != (rand1 / div_factor) + 1) {
handler_errors++;
pr_err("incorrect value in kretprobe handler2\n");
}
if (krph_val == 0) {
handler_errors++;
pr_err("call to kretprobe entry handler failed\n");
}
krph_val = rand1;
return 0;
}
static struct kretprobe rp2 = {
.handler = return_handler2,
.entry_handler = entry_handler,
.kp.symbol_name = "kprobe_target2"
};
static int test_kretprobes(void)
{
int ret;
struct kretprobe *rps[2] = {&rp, &rp2};
/* addr and flags should be cleard for reusing kprobe. */
rp.kp.addr = NULL;
rp.kp.flags = 0;
ret = register_kretprobes(rps, 2);
if (ret < 0) {
pr_err("register_kretprobe returned %d\n", ret);
return ret;
}
krph_val = 0;
ret = target(rand1);
if (krph_val != rand1) {
pr_err("kretprobe handler not called\n");
handler_errors++;
}
krph_val = 0;
ret = target2(rand1);
if (krph_val != rand1) {
pr_err("kretprobe handler2 not called\n");
handler_errors++;
}
unregister_kretprobes(rps, 2);
return 0;
}
#endif /* CONFIG_KRETPROBES */
int init_test_probes(void)
{
int ret;
target = kprobe_target;
target2 = kprobe_target2;
do {
rand1 = prandom_u32();
} while (rand1 <= div_factor);
pr_info("started\n");
num_tests++;
ret = test_kprobe();
if (ret < 0)
errors++;
num_tests++;
ret = test_kprobes();
if (ret < 0)
errors++;
#ifdef CONFIG_KRETPROBES
num_tests++;
ret = test_kretprobe();
if (ret < 0)
errors++;
num_tests++;
ret = test_kretprobes();
if (ret < 0)
errors++;
#endif /* CONFIG_KRETPROBES */
if (errors)
pr_err("BUG: %d out of %d tests failed\n", errors, num_tests);
else if (handler_errors)
pr_err("BUG: %d error(s) running handlers\n", handler_errors);
else
pr_info("passed successfully\n");
return 0;
}
kernel/trace/Makefile
+1
View File
@@ -47,6 +47,7 @@ obj-$(CONFIG_TRACING) += trace_output.o
obj-$(CONFIG_TRACING) += trace_seq.o
obj-$(CONFIG_TRACING) += trace_stat.o
obj-$(CONFIG_TRACING) += trace_printk.o
obj-$(CONFIG_TRACING) += pid_list.o
obj-$(CONFIG_TRACING_MAP) += tracing_map.o
obj-$(CONFIG_PREEMPTIRQ_DELAY_TEST) += preemptirq_delay_test.o
obj-$(CONFIG_SYNTH_EVENT_GEN_TEST) += synth_event_gen_test.o
kernel/trace/fgraph.c
+4 -2
View File
@@ -115,6 +115,7 @@ int function_graph_enter(unsigned long ret, unsigned long func,
{
struct ftrace_graph_ent trace;
#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
/*
* Skip graph tracing if the return location is served by direct trampoline,
* since call sequence and return addresses are unpredictable anyway.
@@ -124,6 +125,7 @@ int function_graph_enter(unsigned long ret, unsigned long func,
if (ftrace_direct_func_count &&
ftrace_find_rec_direct(ret - MCOUNT_INSN_SIZE))
return -EBUSY;
#endif
trace.func = func;
trace.depth = ++current->curr_ret_depth;
@@ -333,10 +335,10 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
#endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
static struct ftrace_ops graph_ops = {
.func = ftrace_stub,
.func = ftrace_graph_func,
.flags = FTRACE_OPS_FL_INITIALIZED |
FTRACE_OPS_FL_PID |
FTRACE_OPS_FL_STUB,
FTRACE_OPS_GRAPH_STUB,
#ifdef FTRACE_GRAPH_TRAMP_ADDR
.trampoline = FTRACE_GRAPH_TRAMP_ADDR,
/* trampoline_size is only needed for dynamically allocated tramps */
kernel/trace/ftrace.c
+283 -65
View File
@@ -119,14 +119,9 @@ struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
struct ftrace_ops global_ops;
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs);
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
/* Defined by vmlinux.lds.h see the commment above arch_ftrace_ops_list_func for details */
void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs);
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
@@ -581,7 +576,7 @@ static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
static int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
{
struct ftrace_profile_page *pg;
int functions;
@@ -988,8 +983,9 @@ static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
}
}
entry = tracefs_create_file("function_profile_enabled", 0644,
d_tracer, NULL, &ftrace_profile_fops);
entry = tracefs_create_file("function_profile_enabled",
TRACE_MODE_WRITE, d_tracer, NULL,
&ftrace_profile_fops);
if (!entry)
pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
}
@@ -2394,6 +2390,39 @@ unsigned long ftrace_find_rec_direct(unsigned long ip)
return entry->direct;
}
static struct ftrace_func_entry*
ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
struct ftrace_hash **free_hash)
{
struct ftrace_func_entry *entry;
if (ftrace_hash_empty(direct_functions) ||
direct_functions->count > 2 * (1 << direct_functions->size_bits)) {
struct ftrace_hash *new_hash;
int size = ftrace_hash_empty(direct_functions) ? 0 :
direct_functions->count + 1;
if (size < 32)
size = 32;
new_hash = dup_hash(direct_functions, size);
if (!new_hash)
return NULL;
*free_hash = direct_functions;
direct_functions = new_hash;
}
entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
return NULL;
entry->ip = ip;
entry->direct = addr;
__add_hash_entry(direct_functions, entry);
return entry;
}
static void call_direct_funcs(unsigned long ip, unsigned long pip,
struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
@@ -5110,39 +5139,16 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr)
}
ret = -ENOMEM;
if (ftrace_hash_empty(direct_functions) ||
direct_functions->count > 2 * (1 << direct_functions->size_bits)) {
struct ftrace_hash *new_hash;
int size = ftrace_hash_empty(direct_functions) ? 0 :
direct_functions->count + 1;
if (size < 32)
size = 32;
new_hash = dup_hash(direct_functions, size);
if (!new_hash)
goto out_unlock;
free_hash = direct_functions;
direct_functions = new_hash;
}
entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
goto out_unlock;
direct = ftrace_find_direct_func(addr);
if (!direct) {
direct = ftrace_alloc_direct_func(addr);
if (!direct) {
kfree(entry);
if (!direct)
goto out_unlock;
}
}
entry->ip = ip;
entry->direct = addr;
__add_hash_entry(direct_functions, entry);
entry = ftrace_add_rec_direct(ip, addr, &free_hash);
if (!entry)
goto out_unlock;
ret = ftrace_set_filter_ip(&direct_ops, ip, 0, 0);
if (ret)
@@ -5395,6 +5401,216 @@ int modify_ftrace_direct(unsigned long ip,
return ret;
}
EXPORT_SYMBOL_GPL(modify_ftrace_direct);
#define MULTI_FLAGS (FTRACE_OPS_FL_IPMODIFY | FTRACE_OPS_FL_DIRECT | \
FTRACE_OPS_FL_SAVE_REGS)
static int check_direct_multi(struct ftrace_ops *ops)
{
if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
return -EINVAL;
if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS)
return -EINVAL;
return 0;
}
static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr)
{
struct ftrace_func_entry *entry, *del;
int size, i;
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
del = __ftrace_lookup_ip(direct_functions, entry->ip);
if (del && del->direct == addr) {
remove_hash_entry(direct_functions, del);
kfree(del);
}
}
}
}
/**
* register_ftrace_direct_multi - Call a custom trampoline directly
* for multiple functions registered in @ops
* @ops: The address of the struct ftrace_ops object
* @addr: The address of the trampoline to call at @ops functions
*
* This is used to connect a direct calls to @addr from the nop locations
* of the functions registered in @ops (with by ftrace_set_filter_ip
* function).
*
* The location that it calls (@addr) must be able to handle a direct call,
* and save the parameters of the function being traced, and restore them
* (or inject new ones if needed), before returning.
*
* Returns:
* 0 on success
* -EINVAL - The @ops object was already registered with this call or
* when there are no functions in @ops object.
* -EBUSY - Another direct function is already attached (there can be only one)
* -ENODEV - @ip does not point to a ftrace nop location (or not supported)
* -ENOMEM - There was an allocation failure.
*/
int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
{
struct ftrace_hash *hash, *free_hash = NULL;
struct ftrace_func_entry *entry, *new;
int err = -EBUSY, size, i;
if (ops->func || ops->trampoline)
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
return -EINVAL;
if (ops->flags & FTRACE_OPS_FL_ENABLED)
return -EINVAL;
hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(hash))
return -EINVAL;
mutex_lock(&direct_mutex);
/* Make sure requested entries are not already registered.. */
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
if (ftrace_find_rec_direct(entry->ip))
goto out_unlock;
}
}
/* ... and insert them to direct_functions hash. */
err = -ENOMEM;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
new = ftrace_add_rec_direct(entry->ip, addr, &free_hash);
if (!new)
goto out_remove;
entry->direct = addr;
}
}
ops->func = call_direct_funcs;
ops->flags = MULTI_FLAGS;
ops->trampoline = FTRACE_REGS_ADDR;
err = register_ftrace_function(ops);
out_remove:
if (err)
remove_direct_functions_hash(hash, addr);
out_unlock:
mutex_unlock(&direct_mutex);
if (free_hash) {
synchronize_rcu_tasks();
free_ftrace_hash(free_hash);
}
return err;
}
EXPORT_SYMBOL_GPL(register_ftrace_direct_multi);
/**
* unregister_ftrace_direct_multi - Remove calls to custom trampoline
* previously registered by register_ftrace_direct_multi for @ops object.
* @ops: The address of the struct ftrace_ops object
*
* This is used to remove a direct calls to @addr from the nop locations
* of the functions registered in @ops (with by ftrace_set_filter_ip
* function).
*
* Returns:
* 0 on success
* -EINVAL - The @ops object was not properly registered.
*/
int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
{
struct ftrace_hash *hash = ops->func_hash->filter_hash;
int err;
if (check_direct_multi(ops))
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EINVAL;
mutex_lock(&direct_mutex);
err = unregister_ftrace_function(ops);
remove_direct_functions_hash(hash, addr);
mutex_unlock(&direct_mutex);
return err;
}
EXPORT_SYMBOL_GPL(unregister_ftrace_direct_multi);
/**
* modify_ftrace_direct_multi - Modify an existing direct 'multi' call
* to call something else
* @ops: The address of the struct ftrace_ops object
* @addr: The address of the new trampoline to call at @ops functions
*
* This is used to unregister currently registered direct caller and
* register new one @addr on functions registered in @ops object.
*
* Note there's window between ftrace_shutdown and ftrace_startup calls
* where there will be no callbacks called.
*
* Returns: zero on success. Non zero on error, which includes:
* -EINVAL - The @ops object was not properly registered.
*/
int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
{
struct ftrace_hash *hash;
struct ftrace_func_entry *entry, *iter;
static struct ftrace_ops tmp_ops = {
.func = ftrace_stub,
.flags = FTRACE_OPS_FL_STUB,
};
int i, size;
int err;
if (check_direct_multi(ops))
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EINVAL;
mutex_lock(&direct_mutex);
/* Enable the tmp_ops to have the same functions as the direct ops */
ftrace_ops_init(&tmp_ops);
tmp_ops.func_hash = ops->func_hash;
err = register_ftrace_function(&tmp_ops);
if (err)
goto out_direct;
/*
* Now the ftrace_ops_list_func() is called to do the direct callers.
* We can safely change the direct functions attached to each entry.
*/
mutex_lock(&ftrace_lock);
hash = ops->func_hash->filter_hash;
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(iter, &hash->buckets[i], hlist) {
entry = __ftrace_lookup_ip(direct_functions, iter->ip);
if (!entry)
continue;
entry->direct = addr;
}
}
/* Removing the tmp_ops will add the updated direct callers to the functions */
unregister_ftrace_function(&tmp_ops);
mutex_unlock(&ftrace_lock);
out_direct:
mutex_unlock(&direct_mutex);
return err;
}
EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi);
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
/**
@@ -6109,10 +6325,10 @@ void ftrace_create_filter_files(struct ftrace_ops *ops,
struct dentry *parent)
{
trace_create_file("set_ftrace_filter", 0644, parent,
trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent,
ops, &ftrace_filter_fops);
trace_create_file("set_ftrace_notrace", 0644, parent,
trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent,
ops, &ftrace_notrace_fops);
}
@@ -6139,19 +6355,19 @@ void ftrace_destroy_filter_files(struct ftrace_ops *ops)
static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
{
trace_create_file("available_filter_functions", 0444,
trace_create_file("available_filter_functions", TRACE_MODE_READ,
d_tracer, NULL, &ftrace_avail_fops);
trace_create_file("enabled_functions", 0444,
trace_create_file("enabled_functions", TRACE_MODE_READ,
d_tracer, NULL, &ftrace_enabled_fops);
ftrace_create_filter_files(&global_ops, d_tracer);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
trace_create_file("set_graph_function", 0644, d_tracer,
trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer,
NULL,
&ftrace_graph_fops);
trace_create_file("set_graph_notrace", 0644, d_tracer,
trace_create_file("set_graph_notrace", TRACE_MODE_WRITE, d_tracer,
NULL,
&ftrace_graph_notrace_fops);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
@@ -6846,6 +7062,11 @@ void __init ftrace_free_init_mem(void)
ftrace_free_mem(NULL, start, end);
}
int __init __weak ftrace_dyn_arch_init(void)
{
return 0;
}
void __init ftrace_init(void)
{
extern unsigned long __start_mcount_loc[];
@@ -6977,16 +7198,15 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op;
int bit;
/*
* The ftrace_test_and_set_recursion() will disable preemption,
* which is required since some of the ops may be dynamically
* allocated, they must be freed after a synchronize_rcu().
*/
bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
if (bit < 0)
return;
/*
* Some of the ops may be dynamically allocated,
* they must be freed after a synchronize_rcu().
*/
preempt_disable_notrace();
do_for_each_ftrace_op(op, ftrace_ops_list) {
/* Stub functions don't need to be called nor tested */
if (op->flags & FTRACE_OPS_FL_STUB)
@@ -7010,7 +7230,6 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
}
} while_for_each_ftrace_op(op);
out:
preempt_enable_notrace();
trace_clear_recursion(bit);
}
@@ -7026,21 +7245,23 @@ out:
* Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
* An architecture can pass partial regs with ftrace_ops and still
* set the ARCH_SUPPORTS_FTRACE_OPS.
*
* In vmlinux.lds.h, ftrace_ops_list_func() is defined to be
* arch_ftrace_ops_list_func.
*/
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
{
__ftrace_ops_list_func(ip, parent_ip, NULL, fregs);
}
NOKPROBE_SYMBOL(ftrace_ops_list_func);
#else
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
{
__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
}
NOKPROBE_SYMBOL(ftrace_ops_no_ops);
#endif
NOKPROBE_SYMBOL(arch_ftrace_ops_list_func);
/*
* If there's only one function registered but it does not support
@@ -7056,12 +7277,9 @@ static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
if (bit < 0)
return;
preempt_disable_notrace();
if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching())
op->func(ip, parent_ip, op, fregs);
preempt_enable_notrace();
trace_clear_recursion(bit);
}
NOKPROBE_SYMBOL(ftrace_ops_assist_func);
@@ -7184,10 +7402,10 @@ static void clear_ftrace_pids(struct trace_array *tr, int type)
synchronize_rcu();
if ((type & TRACE_PIDS) && pid_list)
trace_free_pid_list(pid_list);
trace_pid_list_free(pid_list);
if ((type & TRACE_NO_PIDS) && no_pid_list)
trace_free_pid_list(no_pid_list);
trace_pid_list_free(no_pid_list);
}
void ftrace_clear_pids(struct trace_array *tr)
@@ -7428,7 +7646,7 @@ pid_write(struct file *filp, const char __user *ubuf,
if (filtered_pids) {
synchronize_rcu();
trace_free_pid_list(filtered_pids);
trace_pid_list_free(filtered_pids);
} else if (pid_list && !other_pids) {
/* Register a probe to set whether to ignore the tracing of a task */
register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
@@ -7494,10 +7712,10 @@ static const struct file_operations ftrace_no_pid_fops = {
void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
{
trace_create_file("set_ftrace_pid", 0644, d_tracer,
trace_create_file("set_ftrace_pid", TRACE_MODE_WRITE, d_tracer,
tr, &ftrace_pid_fops);
trace_create_file("set_ftrace_notrace_pid", 0644, d_tracer,
tr, &ftrace_no_pid_fops);
trace_create_file("set_ftrace_notrace_pid", TRACE_MODE_WRITE,
d_tracer, tr, &ftrace_no_pid_fops);
}
void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
kernel/trace/pid_list.c
+495
View File
@@ -0,0 +1,495 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org>
*/
#include <linux/spinlock.h>
#include <linux/irq_work.h>
#include <linux/slab.h>
#include "trace.h"
/* See pid_list.h for details */
static inline union lower_chunk *get_lower_chunk(struct trace_pid_list *pid_list)
{
union lower_chunk *chunk;
lockdep_assert_held(&pid_list->lock);
if (!pid_list->lower_list)
return NULL;
chunk = pid_list->lower_list;
pid_list->lower_list = chunk->next;
pid_list->free_lower_chunks--;
WARN_ON_ONCE(pid_list->free_lower_chunks < 0);
chunk->next = NULL;
/*
* If a refill needs to happen, it can not happen here
* as the scheduler run queue locks are held.
*/
if (pid_list->free_lower_chunks <= CHUNK_REALLOC)
irq_work_queue(&pid_list->refill_irqwork);
return chunk;
}
static inline union upper_chunk *get_upper_chunk(struct trace_pid_list *pid_list)
{
union upper_chunk *chunk;
lockdep_assert_held(&pid_list->lock);
if (!pid_list->upper_list)
return NULL;
chunk = pid_list->upper_list;
pid_list->upper_list = chunk->next;
pid_list->free_upper_chunks--;
WARN_ON_ONCE(pid_list->free_upper_chunks < 0);
chunk->next = NULL;
/*
* If a refill needs to happen, it can not happen here
* as the scheduler run queue locks are held.
*/
if (pid_list->free_upper_chunks <= CHUNK_REALLOC)
irq_work_queue(&pid_list->refill_irqwork);
return chunk;
}
static inline void put_lower_chunk(struct trace_pid_list *pid_list,
union lower_chunk *chunk)
{
lockdep_assert_held(&pid_list->lock);
chunk->next = pid_list->lower_list;
pid_list->lower_list = chunk;
pid_list->free_lower_chunks++;
}
static inline void put_upper_chunk(struct trace_pid_list *pid_list,
union upper_chunk *chunk)
{
lockdep_assert_held(&pid_list->lock);
chunk->next = pid_list->upper_list;
pid_list->upper_list = chunk;
pid_list->free_upper_chunks++;
}
static inline bool upper_empty(union upper_chunk *chunk)
{
/*
* If chunk->data has no lower chunks, it will be the same
* as a zeroed bitmask. Use find_first_bit() to test it
* and if it doesn't find any bits set, then the array
* is empty.
*/
int bit = find_first_bit((unsigned long *)chunk->data,
sizeof(chunk->data) * 8);
return bit >= sizeof(chunk->data) * 8;
}
static inline int pid_split(unsigned int pid, unsigned int *upper1,
unsigned int *upper2, unsigned int *lower)
{
/* MAX_PID should cover all pids */
BUILD_BUG_ON(MAX_PID < PID_MAX_LIMIT);
/* In case a bad pid is passed in, then fail */
if (unlikely(pid >= MAX_PID))
return -1;
*upper1 = (pid >> UPPER1_SHIFT) & UPPER_MASK;
*upper2 = (pid >> UPPER2_SHIFT) & UPPER_MASK;
*lower = pid & LOWER_MASK;
return 0;
}
static inline unsigned int pid_join(unsigned int upper1,
unsigned int upper2, unsigned int lower)
{
return ((upper1 & UPPER_MASK) << UPPER1_SHIFT) |
((upper2 & UPPER_MASK) << UPPER2_SHIFT) |
(lower & LOWER_MASK);
}
/**
* trace_pid_list_is_set - test if the pid is set in the list
* @pid_list: The pid list to test
* @pid: The pid to to see if set in the list.
*
* Tests if @pid is is set in the @pid_list. This is usually called
* from the scheduler when a task is scheduled. Its pid is checked
* if it should be traced or not.
*
* Return true if the pid is in the list, false otherwise.
*/
bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)
{
union upper_chunk *upper_chunk;
union lower_chunk *lower_chunk;
unsigned long flags;
unsigned int upper1;
unsigned int upper2;
unsigned int lower;
bool ret = false;
if (!pid_list)
return false;
if (pid_split(pid, &upper1, &upper2, &lower) < 0)
return false;
raw_spin_lock_irqsave(&pid_list->lock, flags);
upper_chunk = pid_list->upper[upper1];
if (upper_chunk) {
lower_chunk = upper_chunk->data[upper2];
if (lower_chunk)
ret = test_bit(lower, lower_chunk->data);
}
raw_spin_unlock_irqrestore(&pid_list->lock, flags);
return ret;
}
/**
* trace_pid_list_set - add a pid to the list
* @pid_list: The pid list to add the @pid to.
* @pid: The pid to add.
*
* Adds @pid to @pid_list. This is usually done explicitly by a user
* adding a task to be traced, or indirectly by the fork function
* when children should be traced and a task's pid is in the list.
*
* Return 0 on success, negative otherwise.
*/
int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)
{
union upper_chunk *upper_chunk;
union lower_chunk *lower_chunk;
unsigned long flags;
unsigned int upper1;
unsigned int upper2;
unsigned int lower;
int ret;
if (!pid_list)
return -ENODEV;
if (pid_split(pid, &upper1, &upper2, &lower) < 0)
return -EINVAL;
raw_spin_lock_irqsave(&pid_list->lock, flags);
upper_chunk = pid_list->upper[upper1];
if (!upper_chunk) {
upper_chunk = get_upper_chunk(pid_list);
if (!upper_chunk) {
ret = -ENOMEM;
goto out;
}
pid_list->upper[upper1] = upper_chunk;
}
lower_chunk = upper_chunk->data[upper2];
if (!lower_chunk) {
lower_chunk = get_lower_chunk(pid_list);
if (!lower_chunk) {
ret = -ENOMEM;
goto out;
}
upper_chunk->data[upper2] = lower_chunk;
}
set_bit(lower, lower_chunk->data);
ret = 0;
out:
raw_spin_unlock_irqrestore(&pid_list->lock, flags);
return ret;
}
/**
* trace_pid_list_clear - remove a pid from the list
* @pid_list: The pid list to remove the @pid from.
* @pid: The pid to remove.
*
* Removes @pid from @pid_list. This is usually done explicitly by a user
* removing tasks from tracing, or indirectly by the exit function
* when a task that is set to be traced exits.
*
* Return 0 on success, negative otherwise.
*/
int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)
{
union upper_chunk *upper_chunk;
union lower_chunk *lower_chunk;
unsigned long flags;
unsigned int upper1;
unsigned int upper2;
unsigned int lower;
if (!pid_list)
return -ENODEV;
if (pid_split(pid, &upper1, &upper2, &lower) < 0)
return -EINVAL;
raw_spin_lock_irqsave(&pid_list->lock, flags);
upper_chunk = pid_list->upper[upper1];
if (!upper_chunk)
goto out;
lower_chunk = upper_chunk->data[upper2];
if (!lower_chunk)
goto out;
clear_bit(lower, lower_chunk->data);
/* if there's no more bits set, add it to the free list */
if (find_first_bit(lower_chunk->data, LOWER_MAX) >= LOWER_MAX) {
put_lower_chunk(pid_list, lower_chunk);
upper_chunk->data[upper2] = NULL;
if (upper_empty(upper_chunk)) {
put_upper_chunk(pid_list, upper_chunk);
pid_list->upper[upper1] = NULL;
}
}
out:
raw_spin_unlock_irqrestore(&pid_list->lock, flags);
return 0;
}
/**
* trace_pid_list_next - return the next pid in the list
* @pid_list: The pid list to examine.
* @pid: The pid to start from
* @next: The pointer to place the pid that is set starting from @pid.
*
* Looks for the next consecutive pid that is in @pid_list starting
* at the pid specified by @pid. If one is set (including @pid), then
* that pid is placed into @next.
*
* Return 0 when a pid is found, -1 if there are no more pids included.
*/
int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
unsigned int *next)
{
union upper_chunk *upper_chunk;
union lower_chunk *lower_chunk;
unsigned long flags;
unsigned int upper1;
unsigned int upper2;
unsigned int lower;
if (!pid_list)
return -ENODEV;
if (pid_split(pid, &upper1, &upper2, &lower) < 0)
return -EINVAL;
raw_spin_lock_irqsave(&pid_list->lock, flags);
for (; upper1 <= UPPER_MASK; upper1++, upper2 = 0) {
upper_chunk = pid_list->upper[upper1];
if (!upper_chunk)
continue;
for (; upper2 <= UPPER_MASK; upper2++, lower = 0) {
lower_chunk = upper_chunk->data[upper2];
if (!lower_chunk)
continue;
lower = find_next_bit(lower_chunk->data, LOWER_MAX,
lower);
if (lower < LOWER_MAX)
goto found;
}
}
found:
raw_spin_unlock_irqrestore(&pid_list->lock, flags);
if (upper1 > UPPER_MASK)
return -1;
*next = pid_join(upper1, upper2, lower);
return 0;
}
/**
* trace_pid_list_first - return the first pid in the list
* @pid_list: The pid list to examine.
* @pid: The pointer to place the pid first found pid that is set.
*
* Looks for the first pid that is set in @pid_list, and places it
* into @pid if found.
*
* Return 0 when a pid is found, -1 if there are no pids set.
*/
int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)
{
return trace_pid_list_next(pid_list, 0, pid);
}
static void pid_list_refill_irq(struct irq_work *iwork)
{
struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,
refill_irqwork);
union upper_chunk *upper = NULL;
union lower_chunk *lower = NULL;
union upper_chunk **upper_next = &upper;
union lower_chunk **lower_next = &lower;
int upper_count;
int lower_count;
int ucnt = 0;
int lcnt = 0;
again:
raw_spin_lock(&pid_list->lock);
upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;
lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;
raw_spin_unlock(&pid_list->lock);
if (upper_count <= 0 && lower_count <= 0)
return;
while (upper_count-- > 0) {
union upper_chunk *chunk;
chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
if (!chunk)
break;
*upper_next = chunk;
upper_next = &chunk->next;
ucnt++;
}
while (lower_count-- > 0) {
union lower_chunk *chunk;
chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
if (!chunk)
break;
*lower_next = chunk;
lower_next = &chunk->next;
lcnt++;
}
raw_spin_lock(&pid_list->lock);
if (upper) {
*upper_next = pid_list->upper_list;
pid_list->upper_list = upper;
pid_list->free_upper_chunks += ucnt;
}
if (lower) {
*lower_next = pid_list->lower_list;
pid_list->lower_list = lower;
pid_list->free_lower_chunks += lcnt;
}
raw_spin_unlock(&pid_list->lock);
/*
* On success of allocating all the chunks, both counters
* will be less than zero. If they are not, then an allocation
* failed, and we should not try again.
*/
if (upper_count >= 0 || lower_count >= 0)
return;
/*
* When the locks were released, free chunks could have
* been used and allocation needs to be done again. Might as
* well allocate it now.
*/
goto again;
}
/**
* trace_pid_list_alloc - create a new pid_list
*
* Allocates a new pid_list to store pids into.
*
* Returns the pid_list on success, NULL otherwise.
*/
struct trace_pid_list *trace_pid_list_alloc(void)
{
struct trace_pid_list *pid_list;
int i;
/* According to linux/thread.h, pids can be no bigger that 30 bits */
WARN_ON_ONCE(pid_max > (1 << 30));
pid_list = kzalloc(sizeof(*pid_list), GFP_KERNEL);
if (!pid_list)
return NULL;
init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);
raw_spin_lock_init(&pid_list->lock);
for (i = 0; i < CHUNK_ALLOC; i++) {
union upper_chunk *chunk;
chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
if (!chunk)
break;
chunk->next = pid_list->upper_list;
pid_list->upper_list = chunk;
pid_list->free_upper_chunks++;
}
for (i = 0; i < CHUNK_ALLOC; i++) {
union lower_chunk *chunk;
chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
if (!chunk)
break;
chunk->next = pid_list->lower_list;
pid_list->lower_list = chunk;
pid_list->free_lower_chunks++;
}
return pid_list;
}
/**
* trace_pid_list_free - Frees an allocated pid_list.
*
* Frees the memory for a pid_list that was allocated.
*/
void trace_pid_list_free(struct trace_pid_list *pid_list)
{
union upper_chunk *upper;
union lower_chunk *lower;
int i, j;
if (!pid_list)
return;
irq_work_sync(&pid_list->refill_irqwork);
while (pid_list->lower_list) {
union lower_chunk *chunk;
chunk = pid_list->lower_list;
pid_list->lower_list = pid_list->lower_list->next;
kfree(chunk);
}
while (pid_list->upper_list) {
union upper_chunk *chunk;
chunk = pid_list->upper_list;
pid_list->upper_list = pid_list->upper_list->next;
kfree(chunk);
}
for (i = 0; i < UPPER1_SIZE; i++) {
upper = pid_list->upper[i];
if (upper) {
for (j = 0; j < UPPER2_SIZE; j++) {
lower = upper->data[j];
kfree(lower);
}
kfree(upper);
}
}
kfree(pid_list);
}
kernel/trace/pid_list.h
+88
View File
@@ -0,0 +1,88 @@
// SPDX-License-Identifier: GPL-2.0
/* Do not include this file directly. */
#ifndef _TRACE_INTERNAL_PID_LIST_H
#define _TRACE_INTERNAL_PID_LIST_H
/*
* In order to keep track of what pids to trace, a tree is created much
* like page tables are used. This creates a sparse bit map, where
* the tree is filled in when needed. A PID is at most 30 bits (see
* linux/thread.h), and is broken up into 3 sections based on the bit map
* of the bits. The 8 MSB is the "upper1" section. The next 8 MSB is the
* "upper2" section and the 14 LSB is the "lower" section.
*
* A trace_pid_list structure holds the "upper1" section, in an
* array of 256 pointers (1 or 2K in size) to "upper_chunk" unions, where
* each has an array of 256 pointers (1 or 2K in size) to the "lower_chunk"
* structures, where each has an array of size 2K bytes representing a bitmask
* of the 14 LSB of the PID (256 * 8 = 2048)
*
* When a trace_pid_list is allocated, it includes the 256 pointer array
* of the upper1 unions. Then a "cache" of upper and lower is allocated
* where these will be assigned as needed.
*
* When a bit is set in the pid_list bitmask, the pid to use has
* the 8 MSB masked, and this is used to index the array in the
* pid_list to find the next upper union. If the element is NULL,
* then one is retrieved from the upper_list cache. If none is
* available, then -ENOMEM is returned.
*
* The next 8 MSB is used to index into the "upper2" section. If this
* element is NULL, then it is retrieved from the lower_list cache.
* Again, if one is not available -ENOMEM is returned.
*
* Finally the 14 LSB of the PID is used to set the bit in the 16384
* bitmask (made up of 2K bytes).
*
* When the second upper section or the lower section has their last
* bit cleared, they are added back to the free list to be reused
* when needed.
*/
#define UPPER_BITS 8
#define UPPER_MAX (1 << UPPER_BITS)
#define UPPER1_SIZE (1 << UPPER_BITS)
#define UPPER2_SIZE (1 << UPPER_BITS)
#define LOWER_BITS 14
#define LOWER_MAX (1 << LOWER_BITS)
#define LOWER_SIZE (LOWER_MAX / BITS_PER_LONG)
#define UPPER1_SHIFT (LOWER_BITS + UPPER_BITS)
#define UPPER2_SHIFT LOWER_BITS
#define LOWER_MASK (LOWER_MAX - 1)
#define UPPER_MASK (UPPER_MAX - 1)
/* According to linux/thread.h pids can not be bigger than or equal to 1 << 30 */
#define MAX_PID (1 << 30)
/* Just keep 6 chunks of both upper and lower in the cache on alloc */
#define CHUNK_ALLOC 6
/* Have 2 chunks free, trigger a refill of the cache */
#define CHUNK_REALLOC 2
union lower_chunk {
union lower_chunk *next;
unsigned long data[LOWER_SIZE]; // 2K in size
};
union upper_chunk {
union upper_chunk *next;
union lower_chunk *data[UPPER2_SIZE]; // 1 or 2K in size
};
struct trace_pid_list {
raw_spinlock_t lock;
struct irq_work refill_irqwork;
union upper_chunk *upper[UPPER1_SIZE]; // 1 or 2K in size
union upper_chunk *upper_list;
union lower_chunk *lower_list;
int free_upper_chunks;
int free_lower_chunks;
};
#endif /* _TRACE_INTERNAL_PID_LIST_H */
kernel/trace/ring_buffer.c
+2 -7
View File
@@ -3167,14 +3167,9 @@ static __always_inline int
trace_recursive_lock(struct ring_buffer_per_cpu *cpu_buffer)
{
unsigned int val = cpu_buffer->current_context;
unsigned long pc = preempt_count();
int bit;
int bit = interrupt_context_level();
if (!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
bit = RB_CTX_NORMAL;
else
bit = pc & NMI_MASK ? RB_CTX_NMI :
pc & HARDIRQ_MASK ? RB_CTX_IRQ : RB_CTX_SOFTIRQ;
bit = RB_CTX_NORMAL - bit;
if (unlikely(val & (1 << (bit + cpu_buffer->nest)))) {
/*
kernel/trace/trace.c
+64 -87
View File
@@ -512,12 +512,6 @@ int call_filter_check_discard(struct trace_event_call *call, void *rec,
return 0;
}
void trace_free_pid_list(struct trace_pid_list *pid_list)
{
vfree(pid_list->pids);
kfree(pid_list);
}
/**
* trace_find_filtered_pid - check if a pid exists in a filtered_pid list
* @filtered_pids: The list of pids to check
@@ -528,14 +522,7 @@ void trace_free_pid_list(struct trace_pid_list *pid_list)
bool
trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
{
/*
* If pid_max changed after filtered_pids was created, we
* by default ignore all pids greater than the previous pid_max.
*/
if (search_pid >= filtered_pids->pid_max)
return false;
return test_bit(search_pid, filtered_pids->pids);
return trace_pid_list_is_set(filtered_pids, search_pid);
}
/**
@@ -592,15 +579,11 @@ void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
return;
}
/* Sorry, but we don't support pid_max changing after setting */
if (task->pid >= pid_list->pid_max)
return;
/* "self" is set for forks, and NULL for exits */
if (self)
set_bit(task->pid, pid_list->pids);
trace_pid_list_set(pid_list, task->pid);
else
clear_bit(task->pid, pid_list->pids);
trace_pid_list_clear(pid_list, task->pid);
}
/**
@@ -617,18 +600,19 @@ void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
*/
void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
{
unsigned long pid = (unsigned long)v;
long pid = (unsigned long)v;
unsigned int next;
(*pos)++;
/* pid already is +1 of the actual previous bit */
pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
if (trace_pid_list_next(pid_list, pid, &next) < 0)
return NULL;
pid = next;
/* Return pid + 1 to allow zero to be represented */
if (pid < pid_list->pid_max)
return (void *)(pid + 1);
return NULL;
return (void *)(pid + 1);
}
/**
@@ -645,12 +629,14 @@ void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
{
unsigned long pid;
unsigned int first;
loff_t l = 0;
pid = find_first_bit(pid_list->pids, pid_list->pid_max);
if (pid >= pid_list->pid_max)
if (trace_pid_list_first(pid_list, &first) < 0)
return NULL;
pid = first;
/* Return pid + 1 so that zero can be the exit value */
for (pid++; pid && l < *pos;
pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
@@ -686,7 +672,7 @@ int trace_pid_write(struct trace_pid_list *filtered_pids,
unsigned long val;
int nr_pids = 0;
ssize_t read = 0;
ssize_t ret = 0;
ssize_t ret;
loff_t pos;
pid_t pid;
@@ -699,34 +685,23 @@ int trace_pid_write(struct trace_pid_list *filtered_pids,
* the user. If the operation fails, then the current list is
* not modified.
*/
pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
pid_list = trace_pid_list_alloc();
if (!pid_list) {
trace_parser_put(&parser);
return -ENOMEM;
}
pid_list->pid_max = READ_ONCE(pid_max);
/* Only truncating will shrink pid_max */
if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
pid_list->pid_max = filtered_pids->pid_max;
pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
if (!pid_list->pids) {
trace_parser_put(&parser);
kfree(pid_list);
return -ENOMEM;
}
if (filtered_pids) {
/* copy the current bits to the new max */
for_each_set_bit(pid, filtered_pids->pids,
filtered_pids->pid_max) {
set_bit(pid, pid_list->pids);
ret = trace_pid_list_first(filtered_pids, &pid);
while (!ret) {
trace_pid_list_set(pid_list, pid);
ret = trace_pid_list_next(filtered_pids, pid + 1, &pid);
nr_pids++;
}
}
ret = 0;
while (cnt > 0) {
pos = 0;
@@ -742,12 +717,13 @@ int trace_pid_write(struct trace_pid_list *filtered_pids,
ret = -EINVAL;
if (kstrtoul(parser.buffer, 0, &val))
break;
if (val >= pid_list->pid_max)
break;
pid = (pid_t)val;
set_bit(pid, pid_list->pids);
if (trace_pid_list_set(pid_list, pid) < 0) {
ret = -1;
break;
}
nr_pids++;
trace_parser_clear(&parser);
@@ -756,13 +732,13 @@ int trace_pid_write(struct trace_pid_list *filtered_pids,
trace_parser_put(&parser);
if (ret < 0) {
trace_free_pid_list(pid_list);
trace_pid_list_free(pid_list);
return ret;
}
if (!nr_pids) {
/* Cleared the list of pids */
trace_free_pid_list(pid_list);
trace_pid_list_free(pid_list);
read = ret;
pid_list = NULL;
}
@@ -1714,7 +1690,8 @@ static void trace_create_maxlat_file(struct trace_array *tr,
{
INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
tr->d_max_latency = trace_create_file("tracing_max_latency",
TRACE_MODE_WRITE,
d_tracer, &tr->max_latency,
&tracing_max_lat_fops);
}
@@ -1748,8 +1725,8 @@ void latency_fsnotify(struct trace_array *tr)
|| defined(CONFIG_OSNOISE_TRACER)
#define trace_create_maxlat_file(tr, d_tracer) \
trace_create_file("tracing_max_latency", 0644, d_tracer, \
&tr->max_latency, &tracing_max_lat_fops)
trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \
d_tracer, &tr->max_latency, &tracing_max_lat_fops)
#else
#define trace_create_maxlat_file(tr, d_tracer) do { } while (0)
@@ -6077,7 +6054,7 @@ trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
static void trace_create_eval_file(struct dentry *d_tracer)
{
trace_create_file("eval_map", 0444, d_tracer,
trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
NULL, &tracing_eval_map_fops);
}
@@ -8590,27 +8567,27 @@ tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
}
/* per cpu trace_pipe */
trace_create_cpu_file("trace_pipe", 0444, d_cpu,
trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
tr, cpu, &tracing_pipe_fops);
/* per cpu trace */
trace_create_cpu_file("trace", 0644, d_cpu,
trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
tr, cpu, &tracing_fops);
trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
tr, cpu, &tracing_buffers_fops);
trace_create_cpu_file("stats", 0444, d_cpu,
trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
tr, cpu, &tracing_stats_fops);
trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu,
tr, cpu, &tracing_entries_fops);
#ifdef CONFIG_TRACER_SNAPSHOT
trace_create_cpu_file("snapshot", 0644, d_cpu,
trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
tr, cpu, &snapshot_fops);
trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
tr, cpu, &snapshot_raw_fops);
#endif
}
@@ -8816,8 +8793,8 @@ create_trace_option_file(struct trace_array *tr,
topt->opt = opt;
topt->tr = tr;
topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
&trace_options_fops);
topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
t_options, topt, &trace_options_fops);
}
@@ -8892,7 +8869,7 @@ create_trace_option_core_file(struct trace_array *tr,
if (!t_options)
return NULL;
return trace_create_file(option, 0644, t_options,
return trace_create_file(option, TRACE_MODE_WRITE, t_options,
(void *)&tr->trace_flags_index[index],
&trace_options_core_fops);
}
@@ -9417,28 +9394,28 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
struct trace_event_file *file;
int cpu;
trace_create_file("available_tracers", 0444, d_tracer,
trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
tr, &show_traces_fops);
trace_create_file("current_tracer", 0644, d_tracer,
trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer,
tr, &set_tracer_fops);
trace_create_file("tracing_cpumask", 0644, d_tracer,
trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer,
tr, &tracing_cpumask_fops);
trace_create_file("trace_options", 0644, d_tracer,
trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
tr, &tracing_iter_fops);
trace_create_file("trace", 0644, d_tracer,
trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
tr, &tracing_fops);
trace_create_file("trace_pipe", 0444, d_tracer,
trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
tr, &tracing_pipe_fops);
trace_create_file("buffer_size_kb", 0644, d_tracer,
trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer,
tr, &tracing_entries_fops);
trace_create_file("buffer_total_size_kb", 0444, d_tracer,
trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
tr, &tracing_total_entries_fops);
trace_create_file("free_buffer", 0200, d_tracer,
@@ -9449,25 +9426,25 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
file = __find_event_file(tr, "ftrace", "print");
if (file && file->dir)
trace_create_file("trigger", 0644, file->dir, file,
&event_trigger_fops);
trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
file, &event_trigger_fops);
tr->trace_marker_file = file;
trace_create_file("trace_marker_raw", 0220, d_tracer,
tr, &tracing_mark_raw_fops);
trace_create_file("trace_clock", 0644, d_tracer, tr,
trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr,
&trace_clock_fops);
trace_create_file("tracing_on", 0644, d_tracer,
trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
tr, &rb_simple_fops);
trace_create_file("timestamp_mode", 0444, d_tracer, tr,
trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
&trace_time_stamp_mode_fops);
tr->buffer_percent = 50;
trace_create_file("buffer_percent", 0444, d_tracer,
trace_create_file("buffer_percent", TRACE_MODE_READ, d_tracer,
tr, &buffer_percent_fops);
create_trace_options_dir(tr);
@@ -9478,11 +9455,11 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
MEM_FAIL(1, "Could not allocate function filter files");
#ifdef CONFIG_TRACER_SNAPSHOT
trace_create_file("snapshot", 0644, d_tracer,
trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
tr, &snapshot_fops);
#endif
trace_create_file("error_log", 0644, d_tracer,
trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
tr, &tracing_err_log_fops);
for_each_tracing_cpu(cpu)
@@ -9675,19 +9652,19 @@ static __init int tracer_init_tracefs(void)
init_tracer_tracefs(&global_trace, NULL);
ftrace_init_tracefs_toplevel(&global_trace, NULL);
trace_create_file("tracing_thresh", 0644, NULL,
trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
&global_trace, &tracing_thresh_fops);
trace_create_file("README", 0444, NULL,
trace_create_file("README", TRACE_MODE_READ, NULL,
NULL, &tracing_readme_fops);
trace_create_file("saved_cmdlines", 0444, NULL,
trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
NULL, &tracing_saved_cmdlines_fops);
trace_create_file("saved_cmdlines_size", 0644, NULL,
trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
NULL, &tracing_saved_cmdlines_size_fops);
trace_create_file("saved_tgids", 0444, NULL,
trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
NULL, &tracing_saved_tgids_fops);
trace_eval_init();
@@ -9699,7 +9676,7 @@ static __init int tracer_init_tracefs(void)
#endif
#ifdef CONFIG_DYNAMIC_FTRACE
trace_create_file("dyn_ftrace_total_info", 0444, NULL,
trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
NULL, &tracing_dyn_info_fops);
#endif
kernel/trace/trace.h
+14 -5
View File
@@ -22,11 +22,16 @@
#include <linux/ctype.h>
#include <linux/once_lite.h>
#include "pid_list.h"
#ifdef CONFIG_FTRACE_SYSCALLS
#include <asm/unistd.h> /* For NR_SYSCALLS */
#include <asm/syscall.h> /* some archs define it here */
#endif
#define TRACE_MODE_WRITE 0640
#define TRACE_MODE_READ 0440
enum trace_type {
__TRACE_FIRST_TYPE = 0,
@@ -188,10 +193,14 @@ struct trace_options {
struct trace_option_dentry *topts;
};
struct trace_pid_list {
int pid_max;
unsigned long *pids;
};
struct trace_pid_list *trace_pid_list_alloc(void);
void trace_pid_list_free(struct trace_pid_list *pid_list);
bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid);
int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid);
int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid);
int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid);
int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
unsigned int *next);
enum {
TRACE_PIDS = BIT(0),
@@ -881,7 +890,7 @@ static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
* is set, and called by an interrupt handler, we still
* want to trace it.
*/
if (in_irq())
if (in_hardirq())
trace_recursion_set(TRACE_IRQ_BIT);
else
trace_recursion_clear(TRACE_IRQ_BIT);
kernel/trace/trace_boot.c
+4
View File
@@ -430,6 +430,8 @@ trace_boot_init_histograms(struct trace_event_file *file,
/* All digit started node should be instances. */
if (trace_boot_compose_hist_cmd(node, buf, size) == 0) {
tmp = kstrdup(buf, GFP_KERNEL);
if (!tmp)
return;
if (trigger_process_regex(file, buf) < 0)
pr_err("Failed to apply hist trigger: %s\n", tmp);
kfree(tmp);
@@ -439,6 +441,8 @@ trace_boot_init_histograms(struct trace_event_file *file,
if (xbc_node_find_subkey(hnode, "keys")) {
if (trace_boot_compose_hist_cmd(hnode, buf, size) == 0) {
tmp = kstrdup(buf, GFP_KERNEL);
if (!tmp)
return;
if (trigger_process_regex(file, buf) < 0)
pr_err("Failed to apply hist trigger: %s\n", tmp);
kfree(tmp);
kernel/trace/trace_dynevent.c
+1 -1
View File
@@ -262,7 +262,7 @@ static __init int init_dynamic_event(void)
if (ret)
return 0;
entry = tracefs_create_file("dynamic_events", 0644, NULL,
entry = tracefs_create_file("dynamic_events", TRACE_MODE_WRITE, NULL,
NULL, &dynamic_events_ops);
/* Event list interface */
kernel/trace/trace_event_perf.c
+5 -4
View File
@@ -400,7 +400,8 @@ void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp)
BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
"perf buffer not large enough"))
"perf buffer not large enough, wanted %d, have %d",
size, PERF_MAX_TRACE_SIZE))
return NULL;
*rctxp = rctx = perf_swevent_get_recursion_context();
@@ -441,13 +442,13 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
if (!rcu_is_watching())
return;
if ((unsigned long)ops->private != smp_processor_id())
return;
bit = ftrace_test_recursion_trylock(ip, parent_ip);
if (bit < 0)
return;
if ((unsigned long)ops->private != smp_processor_id())
goto out;
event = container_of(ops, struct perf_event, ftrace_ops);
/*
kernel/trace/trace_events.c
+25 -23
View File
@@ -885,10 +885,10 @@ static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
tracepoint_synchronize_unregister();
if ((type & TRACE_PIDS) && pid_list)
trace_free_pid_list(pid_list);
trace_pid_list_free(pid_list);
if ((type & TRACE_NO_PIDS) && no_pid_list)
trace_free_pid_list(no_pid_list);
trace_pid_list_free(no_pid_list);
}
static void ftrace_clear_event_pids(struct trace_array *tr, int type)
@@ -1967,7 +1967,7 @@ event_pid_write(struct file *filp, const char __user *ubuf,
if (filtered_pids) {
tracepoint_synchronize_unregister();
trace_free_pid_list(filtered_pids);
trace_pid_list_free(filtered_pids);
} else if (pid_list && !other_pids) {
register_pid_events(tr);
}
@@ -2312,7 +2312,8 @@ event_subsystem_dir(struct trace_array *tr, const char *name,
/* the ftrace system is special, do not create enable or filter files */
if (strcmp(name, "ftrace") != 0) {
entry = tracefs_create_file("filter", 0644, dir->entry, dir,
entry = tracefs_create_file("filter", TRACE_MODE_WRITE,
dir->entry, dir,
&ftrace_subsystem_filter_fops);
if (!entry) {
kfree(system->filter);
@@ -2320,7 +2321,7 @@ event_subsystem_dir(struct trace_array *tr, const char *name,
pr_warn("Could not create tracefs '%s/filter' entry\n", name);
}
trace_create_file("enable", 0644, dir->entry, dir,
trace_create_file("enable", TRACE_MODE_WRITE, dir->entry, dir,
&ftrace_system_enable_fops);
}
@@ -2402,12 +2403,12 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file)
}
if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
trace_create_file("enable", 0644, file->dir, file,
trace_create_file("enable", TRACE_MODE_WRITE, file->dir, file,
&ftrace_enable_fops);
#ifdef CONFIG_PERF_EVENTS
if (call->event.type && call->class->reg)
trace_create_file("id", 0444, file->dir,
trace_create_file("id", TRACE_MODE_READ, file->dir,
(void *)(long)call->event.type,
&ftrace_event_id_fops);
#endif
@@ -2423,22 +2424,22 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file)
* triggers or filters.
*/
if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
trace_create_file("filter", 0644, file->dir, file,
&ftrace_event_filter_fops);
trace_create_file("filter", TRACE_MODE_WRITE, file->dir,
file, &ftrace_event_filter_fops);
trace_create_file("trigger", 0644, file->dir, file,
&event_trigger_fops);
trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
file, &event_trigger_fops);
}
#ifdef CONFIG_HIST_TRIGGERS
trace_create_file("hist", 0444, file->dir, file,
trace_create_file("hist", TRACE_MODE_READ, file->dir, file,
&event_hist_fops);
#endif
#ifdef CONFIG_HIST_TRIGGERS_DEBUG
trace_create_file("hist_debug", 0444, file->dir, file,
trace_create_file("hist_debug", TRACE_MODE_READ, file->dir, file,
&event_hist_debug_fops);
#endif
trace_create_file("format", 0444, file->dir, call,
trace_create_file("format", TRACE_MODE_READ, file->dir, call,
&ftrace_event_format_fops);
#ifdef CONFIG_TRACE_EVENT_INJECT
@@ -3433,7 +3434,7 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
struct dentry *d_events;
struct dentry *entry;
entry = tracefs_create_file("set_event", 0644, parent,
entry = tracefs_create_file("set_event", TRACE_MODE_WRITE, parent,
tr, &ftrace_set_event_fops);
if (!entry) {
pr_warn("Could not create tracefs 'set_event' entry\n");
@@ -3446,7 +3447,7 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
return -ENOMEM;
}
entry = trace_create_file("enable", 0644, d_events,
entry = trace_create_file("enable", TRACE_MODE_WRITE, d_events,
tr, &ftrace_tr_enable_fops);
if (!entry) {
pr_warn("Could not create tracefs 'enable' entry\n");
@@ -3455,24 +3456,25 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
/* There are not as crucial, just warn if they are not created */
entry = tracefs_create_file("set_event_pid", 0644, parent,
entry = tracefs_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
tr, &ftrace_set_event_pid_fops);
if (!entry)
pr_warn("Could not create tracefs 'set_event_pid' entry\n");
entry = tracefs_create_file("set_event_notrace_pid", 0644, parent,
tr, &ftrace_set_event_notrace_pid_fops);
entry = tracefs_create_file("set_event_notrace_pid",
TRACE_MODE_WRITE, parent, tr,
&ftrace_set_event_notrace_pid_fops);
if (!entry)
pr_warn("Could not create tracefs 'set_event_notrace_pid' entry\n");
/* ring buffer internal formats */
entry = trace_create_file("header_page", 0444, d_events,
entry = trace_create_file("header_page", TRACE_MODE_READ, d_events,
ring_buffer_print_page_header,
&ftrace_show_header_fops);
if (!entry)
pr_warn("Could not create tracefs 'header_page' entry\n");
entry = trace_create_file("header_event", 0444, d_events,
entry = trace_create_file("header_event", TRACE_MODE_READ, d_events,
ring_buffer_print_entry_header,
&ftrace_show_header_fops);
if (!entry)
@@ -3689,8 +3691,8 @@ __init int event_trace_init(void)
if (!tr)
return -ENODEV;
entry = tracefs_create_file("available_events", 0444, NULL,
tr, &ftrace_avail_fops);
entry = tracefs_create_file("available_events", TRACE_MODE_READ,
NULL, tr, &ftrace_avail_fops);
if (!entry)
pr_warn("Could not create tracefs 'available_events' entry\n");
kernel/trace/trace_events_hist.c
+329 -86
View File
@@ -66,7 +66,9 @@
C(EMPTY_SORT_FIELD, "Empty sort field"), \
C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \
C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \
C(INVALID_STR_OPERAND, "String type can not be an operand in expression"),
C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \
C(EXPECT_NUMBER, "Expecting numeric literal"), \
C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"),
#undef C
#define C(a, b) HIST_ERR_##a
@@ -89,12 +91,15 @@ typedef u64 (*hist_field_fn_t) (struct hist_field *field,
#define HIST_FIELD_OPERANDS_MAX 2
#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
#define HIST_ACTIONS_MAX 8
#define HIST_CONST_DIGITS_MAX 21
enum field_op_id {
FIELD_OP_NONE,
FIELD_OP_PLUS,
FIELD_OP_MINUS,
FIELD_OP_UNARY_MINUS,
FIELD_OP_DIV,
FIELD_OP_MULT,
};
/*
@@ -152,6 +157,9 @@ struct hist_field {
bool read_once;
unsigned int var_str_idx;
/* Numeric literals are represented as u64 */
u64 constant;
};
static u64 hist_field_none(struct hist_field *field,
@@ -163,6 +171,15 @@ static u64 hist_field_none(struct hist_field *field,
return 0;
}
static u64 hist_field_const(struct hist_field *field,
struct tracing_map_elt *elt,
struct trace_buffer *buffer,
struct ring_buffer_event *rbe,
void *event)
{
return field->constant;
}
static u64 hist_field_counter(struct hist_field *field,
struct tracing_map_elt *elt,
struct trace_buffer *buffer,
@@ -271,6 +288,44 @@ static u64 hist_field_minus(struct hist_field *hist_field,
return val1 - val2;
}
static u64 hist_field_div(struct hist_field *hist_field,
struct tracing_map_elt *elt,
struct trace_buffer *buffer,
struct ring_buffer_event *rbe,
void *event)
{
struct hist_field *operand1 = hist_field->operands[0];
struct hist_field *operand2 = hist_field->operands[1];
u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
/* Return -1 for the undefined case */
if (!val2)
return -1;
/* Use shift if the divisor is a power of 2 */
if (!(val2 & (val2 - 1)))
return val1 >> __ffs64(val2);
return div64_u64(val1, val2);
}
static u64 hist_field_mult(struct hist_field *hist_field,
struct tracing_map_elt *elt,
struct trace_buffer *buffer,
struct ring_buffer_event *rbe,
void *event)
{
struct hist_field *operand1 = hist_field->operands[0];
struct hist_field *operand2 = hist_field->operands[1];
u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
return val1 * val2;
}
static u64 hist_field_unary_minus(struct hist_field *hist_field,
struct tracing_map_elt *elt,
struct trace_buffer *buffer,
@@ -341,6 +396,7 @@ enum hist_field_flags {
HIST_FIELD_FL_CPU = 1 << 15,
HIST_FIELD_FL_ALIAS = 1 << 16,
HIST_FIELD_FL_BUCKET = 1 << 17,
HIST_FIELD_FL_CONST = 1 << 18,
};
struct var_defs {
@@ -1516,6 +1572,12 @@ static void expr_field_str(struct hist_field *field, char *expr)
{
if (field->flags & HIST_FIELD_FL_VAR_REF)
strcat(expr, "$");
else if (field->flags & HIST_FIELD_FL_CONST) {
char str[HIST_CONST_DIGITS_MAX];
snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
strcat(expr, str);
}
strcat(expr, hist_field_name(field, 0));
@@ -1571,6 +1633,12 @@ static char *expr_str(struct hist_field *field, unsigned int level)
case FIELD_OP_PLUS:
strcat(expr, "+");
break;
case FIELD_OP_DIV:
strcat(expr, "/");
break;
case FIELD_OP_MULT:
strcat(expr, "*");
break;
default:
kfree(expr);
return NULL;
@@ -1581,34 +1649,92 @@ static char *expr_str(struct hist_field *field, unsigned int level)
return expr;
}
static int contains_operator(char *str)
/*
* If field_op != FIELD_OP_NONE, *sep points to the root operator
* of the expression tree to be evaluated.
*/
static int contains_operator(char *str, char **sep)
{
enum field_op_id field_op = FIELD_OP_NONE;
char *op;
char *minus_op, *plus_op, *div_op, *mult_op;
op = strpbrk(str, "+-");
if (!op)
return FIELD_OP_NONE;
switch (*op) {
case '-':
/*
* Report the last occurrence of the operators first, so that the
* expression is evaluated left to right. This is important since
* subtraction and division are not associative.
*
* e.g
* 64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
* 14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
*/
/*
* First, find lower precedence addition and subtraction
* since the expression will be evaluated recursively.
*/
minus_op = strrchr(str, '-');
if (minus_op) {
/*
* Unfortunately, the modifier ".sym-offset"
* can confuse things.
* Unary minus is not supported in sub-expressions. If
* present, it is always the next root operator.
*/
if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
return FIELD_OP_NONE;
if (*str == '-')
if (minus_op == str) {
field_op = FIELD_OP_UNARY_MINUS;
else
field_op = FIELD_OP_MINUS;
break;
case '+':
field_op = FIELD_OP_PLUS;
break;
default:
break;
goto out;
}
field_op = FIELD_OP_MINUS;
}
plus_op = strrchr(str, '+');
if (plus_op || minus_op) {
/*
* For operators of the same precedence use to rightmost as the
* root, so that the expression is evaluated left to right.
*/
if (plus_op > minus_op)
field_op = FIELD_OP_PLUS;
goto out;
}
/*
* Multiplication and division have higher precedence than addition and
* subtraction.
*/
div_op = strrchr(str, '/');
if (div_op)
field_op = FIELD_OP_DIV;
mult_op = strrchr(str, '*');
/*
* For operators of the same precedence use to rightmost as the
* root, so that the expression is evaluated left to right.
*/
if (mult_op > div_op)
field_op = FIELD_OP_MULT;
out:
if (sep) {
switch (field_op) {
case FIELD_OP_UNARY_MINUS:
case FIELD_OP_MINUS:
*sep = minus_op;
break;
case FIELD_OP_PLUS:
*sep = plus_op;
break;
case FIELD_OP_DIV:
*sep = div_op;
break;
case FIELD_OP_MULT:
*sep = mult_op;
break;
case FIELD_OP_NONE:
default:
*sep = NULL;
break;
}
}
return field_op;
@@ -1689,6 +1815,15 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
goto out;
}
if (flags & HIST_FIELD_FL_CONST) {
hist_field->fn = hist_field_const;
hist_field->size = sizeof(u64);
hist_field->type = kstrdup("u64", GFP_KERNEL);
if (!hist_field->type)
goto free;
goto out;
}
if (flags & HIST_FIELD_FL_STACKTRACE) {
hist_field->fn = hist_field_none;
goto out;
@@ -1925,7 +2060,7 @@ static char *field_name_from_var(struct hist_trigger_data *hist_data,
if (strcmp(var_name, name) == 0) {
field = hist_data->attrs->var_defs.expr[i];
if (contains_operator(field) || is_var_ref(field))
if (contains_operator(field, NULL) || is_var_ref(field))
continue;
return field;
}
@@ -2002,7 +2137,11 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
*flags |= HIST_FIELD_FL_HEX;
else if (strcmp(modifier, "sym") == 0)
*flags |= HIST_FIELD_FL_SYM;
else if (strcmp(modifier, "sym-offset") == 0)
/*
* 'sym-offset' occurrences in the trigger string are modified
* to 'symXoffset' to simplify arithmetic expression parsing.
*/
else if (strcmp(modifier, "symXoffset") == 0)
*flags |= HIST_FIELD_FL_SYM_OFFSET;
else if ((strcmp(modifier, "execname") == 0) &&
(strcmp(field_name, "common_pid") == 0))
@@ -2090,6 +2229,29 @@ static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
return alias;
}
static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
char *str, char *var_name,
unsigned long *flags)
{
struct trace_array *tr = hist_data->event_file->tr;
struct hist_field *field = NULL;
u64 constant;
if (kstrtoull(str, 0, &constant)) {
hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
return NULL;
}
*flags |= HIST_FIELD_FL_CONST;
field = create_hist_field(hist_data, NULL, *flags, var_name);
if (!field)
return NULL;
field->constant = constant;
return field;
}
static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
struct trace_event_file *file, char *str,
unsigned long *flags, char *var_name)
@@ -2100,6 +2262,15 @@ static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
unsigned long buckets = 0;
int ret = 0;
if (isdigit(str[0])) {
hist_field = parse_const(hist_data, str, var_name, flags);
if (!hist_field) {
ret = -EINVAL;
goto out;
}
return hist_field;
}
s = strchr(str, '.');
if (s) {
s = strchr(++s, '.');
@@ -2156,21 +2327,24 @@ static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
struct trace_event_file *file,
char *str, unsigned long flags,
char *var_name, unsigned int level);
char *var_name, unsigned int *n_subexprs);
static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
struct trace_event_file *file,
char *str, unsigned long flags,
char *var_name, unsigned int level)
char *var_name, unsigned int *n_subexprs)
{
struct hist_field *operand1, *expr = NULL;
unsigned long operand_flags;
int ret = 0;
char *s;
/* Unary minus operator, increment n_subexprs */
++*n_subexprs;
/* we support only -(xxx) i.e. explicit parens required */
if (level > 3) {
if (*n_subexprs > 3) {
hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
ret = -EINVAL;
goto free;
@@ -2187,8 +2361,16 @@ static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
}
s = strrchr(str, ')');
if (s)
if (s) {
/* unary minus not supported in sub-expressions */
if (*(s+1) != '\0') {
hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
errpos(str));
ret = -EINVAL;
goto free;
}
*s = '\0';
}
else {
ret = -EINVAL; /* no closing ')' */
goto free;
@@ -2202,7 +2384,7 @@ static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
}
operand_flags = 0;
operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
if (IS_ERR(operand1)) {
ret = PTR_ERR(operand1);
goto free;
@@ -2233,9 +2415,15 @@ static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
return ERR_PTR(ret);
}
/*
* If the operands are var refs, return pointers the
* variable(s) referenced in var1 and var2, else NULL.
*/
static int check_expr_operands(struct trace_array *tr,
struct hist_field *operand1,
struct hist_field *operand2)
struct hist_field *operand2,
struct hist_field **var1,
struct hist_field **var2)
{
unsigned long operand1_flags = operand1->flags;
unsigned long operand2_flags = operand2->flags;
@@ -2248,6 +2436,7 @@ static int check_expr_operands(struct trace_array *tr,
if (!var)
return -EINVAL;
operand1_flags = var->flags;
*var1 = var;
}
if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
@@ -2258,6 +2447,7 @@ static int check_expr_operands(struct trace_array *tr,
if (!var)
return -EINVAL;
operand2_flags = var->flags;
*var2 = var;
}
if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
@@ -2272,44 +2462,46 @@ static int check_expr_operands(struct trace_array *tr,
static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
struct trace_event_file *file,
char *str, unsigned long flags,
char *var_name, unsigned int level)
char *var_name, unsigned int *n_subexprs)
{
struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
unsigned long operand_flags;
struct hist_field *var1 = NULL, *var2 = NULL;
unsigned long operand_flags, operand2_flags;
int field_op, ret = -EINVAL;
char *sep, *operand1_str;
hist_field_fn_t op_fn;
bool combine_consts;
if (level > 3) {
if (*n_subexprs > 3) {
hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
return ERR_PTR(-EINVAL);
}
field_op = contains_operator(str);
field_op = contains_operator(str, &sep);
if (field_op == FIELD_OP_NONE)
return parse_atom(hist_data, file, str, &flags, var_name);
if (field_op == FIELD_OP_UNARY_MINUS)
return parse_unary(hist_data, file, str, flags, var_name, ++level);
return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
switch (field_op) {
case FIELD_OP_MINUS:
sep = "-";
break;
case FIELD_OP_PLUS:
sep = "+";
break;
default:
/* Binary operator found, increment n_subexprs */
++*n_subexprs;
/* Split the expression string at the root operator */
if (!sep)
goto free;
}
*sep = '\0';
operand1_str = str;
str = sep+1;
operand1_str = strsep(&str, sep);
if (!operand1_str || !str)
goto free;
operand_flags = 0;
operand1 = parse_atom(hist_data, file, operand1_str,
&operand_flags, NULL);
/* LHS of string is an expression e.g. a+b in a+b+c */
operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
if (IS_ERR(operand1)) {
ret = PTR_ERR(operand1);
operand1 = NULL;
@@ -2321,9 +2513,9 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
goto free;
}
/* rest of string could be another expression e.g. b+c in a+b+c */
/* RHS of string is another expression e.g. c in a+b+c */
operand_flags = 0;
operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
if (IS_ERR(operand2)) {
ret = PTR_ERR(operand2);
operand2 = NULL;
@@ -2335,11 +2527,38 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
goto free;
}
ret = check_expr_operands(file->tr, operand1, operand2);
switch (field_op) {
case FIELD_OP_MINUS:
op_fn = hist_field_minus;
break;
case FIELD_OP_PLUS:
op_fn = hist_field_plus;
break;
case FIELD_OP_DIV:
op_fn = hist_field_div;
break;
case FIELD_OP_MULT:
op_fn = hist_field_mult;
break;
default:
ret = -EINVAL;
goto free;
}
ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
if (ret)
goto free;
flags |= HIST_FIELD_FL_EXPR;
operand_flags = var1 ? var1->flags : operand1->flags;
operand2_flags = var2 ? var2->flags : operand2->flags;
/*
* If both operands are constant, the expression can be
* collapsed to a single constant.
*/
combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
flags |= operand1->flags &
(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
@@ -2356,31 +2575,43 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
expr->operands[0] = operand1;
expr->operands[1] = operand2;
/* The operand sizes should be the same, so just pick one */
expr->size = operand1->size;
if (combine_consts) {
if (var1)
expr->operands[0] = var1;
if (var2)
expr->operands[1] = var2;
expr->operator = field_op;
expr->name = expr_str(expr, 0);
expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
if (!expr->type) {
ret = -ENOMEM;
goto free;
}
expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
switch (field_op) {
case FIELD_OP_MINUS:
expr->fn = hist_field_minus;
break;
case FIELD_OP_PLUS:
expr->fn = hist_field_plus;
break;
default:
ret = -EINVAL;
goto free;
expr->operands[0] = NULL;
expr->operands[1] = NULL;
/*
* var refs won't be destroyed immediately
* See: destroy_hist_field()
*/
destroy_hist_field(operand2, 0);
destroy_hist_field(operand1, 0);
expr->name = expr_str(expr, 0);
} else {
expr->fn = op_fn;
/* The operand sizes should be the same, so just pick one */
expr->size = operand1->size;
expr->operator = field_op;
expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
if (!expr->type) {
ret = -ENOMEM;
goto free;
}
expr->name = expr_str(expr, 0);
}
return expr;
free:
free:
destroy_hist_field(operand1, 0);
destroy_hist_field(operand2, 0);
destroy_hist_field(expr, 0);
@@ -3751,9 +3982,9 @@ static int __create_val_field(struct hist_trigger_data *hist_data,
unsigned long flags)
{
struct hist_field *hist_field;
int ret = 0;
int ret = 0, n_subexprs = 0;
hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
if (IS_ERR(hist_field)) {
ret = PTR_ERR(hist_field);
goto out;
@@ -3894,7 +4125,7 @@ static int create_key_field(struct hist_trigger_data *hist_data,
struct hist_field *hist_field = NULL;
unsigned long flags = 0;
unsigned int key_size;
int ret = 0;
int ret = 0, n_subexprs = 0;
if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
return -EINVAL;
@@ -3907,7 +4138,7 @@ static int create_key_field(struct hist_trigger_data *hist_data,
hist_field = create_hist_field(hist_data, NULL, flags, NULL);
} else {
hist_field = parse_expr(hist_data, file, field_str, flags,
NULL, 0);
NULL, &n_subexprs);
if (IS_ERR(hist_field)) {
ret = PTR_ERR(hist_field);
goto out;
@@ -4706,7 +4937,6 @@ static void hist_trigger_stacktrace_print(struct seq_file *m,
unsigned long *stacktrace_entries,
unsigned int max_entries)
{
char str[KSYM_SYMBOL_LEN];
unsigned int spaces = 8;
unsigned int i;
@@ -4715,8 +4945,7 @@ static void hist_trigger_stacktrace_print(struct seq_file *m,
return;
seq_printf(m, "%*c", 1 + spaces, ' ');
sprint_symbol(str, stacktrace_entries[i]);
seq_printf(m, "%s\n", str);
seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
}
}
@@ -4726,7 +4955,6 @@ static void hist_trigger_print_key(struct seq_file *m,
struct tracing_map_elt *elt)
{
struct hist_field *key_field;
char str[KSYM_SYMBOL_LEN];
bool multiline = false;
const char *field_name;
unsigned int i;
@@ -4747,14 +4975,12 @@ static void hist_trigger_print_key(struct seq_file *m,
seq_printf(m, "%s: %llx", field_name, uval);
} else if (key_field->flags & HIST_FIELD_FL_SYM) {
uval = *(u64 *)(key + key_field->offset);
sprint_symbol_no_offset(str, uval);
seq_printf(m, "%s: [%llx] %-45s", field_name,
uval, str);
seq_printf(m, "%s: [%llx] %-45ps", field_name,
uval, (void *)(uintptr_t)uval);
} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
uval = *(u64 *)(key + key_field->offset);
sprint_symbol(str, uval);
seq_printf(m, "%s: [%llx] %-55s", field_name,
uval, str);
seq_printf(m, "%s: [%llx] %-55pS", field_name,
uval, (void *)(uintptr_t)uval);
} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
struct hist_elt_data *elt_data = elt->private_data;
char *comm;
@@ -4950,6 +5176,8 @@ static void hist_field_debug_show_flags(struct seq_file *m,
if (flags & HIST_FIELD_FL_ALIAS)
seq_puts(m, " HIST_FIELD_FL_ALIAS\n");
else if (flags & HIST_FIELD_FL_CONST)
seq_puts(m, " HIST_FIELD_FL_CONST\n");
}
static int hist_field_debug_show(struct seq_file *m,
@@ -4971,6 +5199,9 @@ static int hist_field_debug_show(struct seq_file *m,
field->var.idx);
}
if (field->flags & HIST_FIELD_FL_CONST)
seq_printf(m, " constant: %llu\n", field->constant);
if (field->flags & HIST_FIELD_FL_ALIAS)
seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
field->var_ref_idx);
@@ -5213,6 +5444,8 @@ static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
if (hist_field->flags & HIST_FIELD_FL_CPU)
seq_puts(m, "common_cpu");
else if (hist_field->flags & HIST_FIELD_FL_CONST)
seq_printf(m, "%llu", hist_field->constant);
else if (field_name) {
if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
hist_field->flags & HIST_FIELD_FL_ALIAS)
@@ -5795,7 +6028,7 @@ static int event_hist_trigger_func(struct event_command *cmd_ops,
struct synth_event *se;
const char *se_name;
bool remove = false;
char *trigger, *p;
char *trigger, *p, *start;
int ret = 0;
lockdep_assert_held(&event_mutex);
@@ -5843,6 +6076,16 @@ static int event_hist_trigger_func(struct event_command *cmd_ops,
trigger = strstrip(trigger);
}
/*
* To simplify arithmetic expression parsing, replace occurrences of
* '.sym-offset' modifier with '.symXoffset'
*/
start = strstr(trigger, ".sym-offset");
while (start) {
*(start + 4) = 'X';
start = strstr(start + 11, ".sym-offset");
}
attrs = parse_hist_trigger_attrs(file->tr, trigger);
if (IS_ERR(attrs))
return PTR_ERR(attrs);
kernel/trace/trace_events_synth.c
+2 -2
View File
@@ -2227,8 +2227,8 @@ static __init int trace_events_synth_init(void)
if (err)
goto err;
entry = tracefs_create_file("synthetic_events", 0644, NULL,
NULL, &synth_events_fops);
entry = tracefs_create_file("synthetic_events", TRACE_MODE_WRITE,
NULL, NULL, &synth_events_fops);
if (!entry) {
err = -ENODEV;
goto err;
kernel/trace/trace_functions.c
-5
View File
@@ -186,7 +186,6 @@ function_trace_call(unsigned long ip, unsigned long parent_ip,
return;
trace_ctx = tracing_gen_ctx();
preempt_disable_notrace();
cpu = smp_processor_id();
data = per_cpu_ptr(tr->array_buffer.data, cpu);
@@ -194,7 +193,6 @@ function_trace_call(unsigned long ip, unsigned long parent_ip,
trace_function(tr, ip, parent_ip, trace_ctx);
ftrace_test_recursion_unlock(bit);
preempt_enable_notrace();
}
#ifdef CONFIG_UNWINDER_ORC
@@ -298,8 +296,6 @@ function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
if (bit < 0)
return;
preempt_disable_notrace();
cpu = smp_processor_id();
data = per_cpu_ptr(tr->array_buffer.data, cpu);
if (atomic_read(&data->disabled))
@@ -324,7 +320,6 @@ function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
out:
ftrace_test_recursion_unlock(bit);
preempt_enable_notrace();
}
static void
kernel/trace/trace_functions_graph.c
+2 -2
View File
@@ -120,7 +120,7 @@ static inline int ftrace_graph_ignore_irqs(void)
if (!ftrace_graph_skip_irqs || trace_recursion_test(TRACE_IRQ_BIT))
return 0;
return in_irq();
return in_hardirq();
}
int trace_graph_entry(struct ftrace_graph_ent *trace)
@@ -1340,7 +1340,7 @@ static __init int init_graph_tracefs(void)
if (ret)
return 0;
trace_create_file("max_graph_depth", 0644, NULL,
trace_create_file("max_graph_depth", TRACE_MODE_WRITE, NULL,
NULL, &graph_depth_fops);
return 0;
kernel/trace/trace_hwlat.c
+5 -5
View File
@@ -79,8 +79,8 @@ struct hwlat_kthread_data {
int nmi_cpu;
};
struct hwlat_kthread_data hwlat_single_cpu_data;
DEFINE_PER_CPU(struct hwlat_kthread_data, hwlat_per_cpu_data);
static struct hwlat_kthread_data hwlat_single_cpu_data;
static DEFINE_PER_CPU(struct hwlat_kthread_data, hwlat_per_cpu_data);
/* Tells NMIs to call back to the hwlat tracer to record timestamps */
bool trace_hwlat_callback_enabled;
@@ -782,21 +782,21 @@ static int init_tracefs(void)
if (!top_dir)
return -ENOMEM;
hwlat_sample_window = tracefs_create_file("window", 0640,
hwlat_sample_window = tracefs_create_file("window", TRACE_MODE_WRITE,
top_dir,
&hwlat_window,
&trace_min_max_fops);
if (!hwlat_sample_window)
goto err;
hwlat_sample_width = tracefs_create_file("width", 0644,
hwlat_sample_width = tracefs_create_file("width", TRACE_MODE_WRITE,
top_dir,
&hwlat_width,
&trace_min_max_fops);
if (!hwlat_sample_width)
goto err;
hwlat_thread_mode = trace_create_file("mode", 0644,
hwlat_thread_mode = trace_create_file("mode", TRACE_MODE_WRITE,
top_dir,
NULL,
&thread_mode_fops);
kernel/trace/trace_kprobe.c
+5 -5
View File
@@ -97,7 +97,7 @@ static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk
static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
{
return !!(kprobe_gone(&tk->rp.kp));
return kprobe_gone(&tk->rp.kp);
}
static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
@@ -1925,16 +1925,16 @@ static __init int init_kprobe_trace(void)
if (ret)
return 0;
entry = tracefs_create_file("kprobe_events", 0644, NULL,
NULL, &kprobe_events_ops);
entry = tracefs_create_file("kprobe_events", TRACE_MODE_WRITE,
NULL, NULL, &kprobe_events_ops);
/* Event list interface */
if (!entry)
pr_warn("Could not create tracefs 'kprobe_events' entry\n");
/* Profile interface */
entry = tracefs_create_file("kprobe_profile", 0444, NULL,
NULL, &kprobe_profile_ops);
entry = tracefs_create_file("kprobe_profile", TRACE_MODE_READ,
NULL, NULL, &kprobe_profile_ops);
if (!entry)
pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
kernel/trace/trace_osnoise.c
+20 -19
View File
@@ -294,19 +294,19 @@ static void print_osnoise_headers(struct seq_file *s)
seq_puts(s, "# _-----=> irqs-off\n");
seq_puts(s, "# / _----=> need-resched\n");
seq_puts(s, "# | / _---=> hardirq/softirq\n");
seq_puts(s, "# || / _--=> preempt-depth ");
seq_puts(s, " MAX\n");
seq_puts(s, "# || / ");
seq_puts(s, "# || / _--=> preempt-depth\n");
seq_puts(s, "# ||| / _-=> migrate-disable ");
seq_puts(s, " MAX\n");
seq_puts(s, "# |||| / delay ");
seq_puts(s, " SINGLE Interference counters:\n");
seq_puts(s, "# |||| RUNTIME ");
seq_puts(s, "# ||||| RUNTIME ");
seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
seq_puts(s, "# TASK-PID CPU# |||| TIMESTAMP IN US ");
seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
seq_puts(s, "# | | | |||| | | ");
seq_puts(s, "# | | | ||||| | | ");
seq_puts(s, " | | | | | | | |\n");
}
#endif /* CONFIG_PREEMPT_RT */
@@ -378,11 +378,12 @@ static void print_timerlat_headers(struct seq_file *s)
seq_puts(s, "# / _----=> need-resched\n");
seq_puts(s, "# | / _---=> hardirq/softirq\n");
seq_puts(s, "# || / _--=> preempt-depth\n");
seq_puts(s, "# || /\n");
seq_puts(s, "# |||| ACTIVATION\n");
seq_puts(s, "# TASK-PID CPU# |||| TIMESTAMP ID ");
seq_puts(s, " CONTEXT LATENCY\n");
seq_puts(s, "# | | | |||| | | ");
seq_puts(s, "# ||| / _-=> migrate-disable\n");
seq_puts(s, "# |||| / delay\n");
seq_puts(s, "# ||||| ACTIVATION\n");
seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
seq_puts(s, " CONTEXT LATENCY\n");
seq_puts(s, "# | | | ||||| | | ");
seq_puts(s, " | |\n");
}
#endif /* CONFIG_PREEMPT_RT */
@@ -1856,38 +1857,38 @@ static int init_tracefs(void)
if (!top_dir)
return 0;
tmp = tracefs_create_file("period_us", 0640, top_dir,
tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
&osnoise_period, &trace_min_max_fops);
if (!tmp)
goto err;
tmp = tracefs_create_file("runtime_us", 0644, top_dir,
tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
&osnoise_runtime, &trace_min_max_fops);
if (!tmp)
goto err;
tmp = tracefs_create_file("stop_tracing_us", 0640, top_dir,
tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
&osnoise_stop_tracing_in, &trace_min_max_fops);
if (!tmp)
goto err;
tmp = tracefs_create_file("stop_tracing_total_us", 0640, top_dir,
tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
&osnoise_stop_tracing_total, &trace_min_max_fops);
if (!tmp)
goto err;
tmp = trace_create_file("cpus", 0644, top_dir, NULL, &cpus_fops);
tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
if (!tmp)
goto err;
#ifdef CONFIG_TIMERLAT_TRACER
#ifdef CONFIG_STACKTRACE
tmp = tracefs_create_file("print_stack", 0640, top_dir,
tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
&osnoise_print_stack, &trace_min_max_fops);
if (!tmp)
goto err;
#endif
tmp = tracefs_create_file("timerlat_period_us", 0640, top_dir,
tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
&timerlat_period, &trace_min_max_fops);
if (!tmp)
goto err;
kernel/trace/trace_output.c
+4 -13
View File
@@ -8,6 +8,7 @@
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/ftrace.h>
#include <linux/kprobes.h>
#include <linux/sched/clock.h>
#include <linux/sched/mm.h>
@@ -346,22 +347,12 @@ int trace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...)
}
EXPORT_SYMBOL_GPL(trace_output_call);
#ifdef CONFIG_KRETPROBES
static inline const char *kretprobed(const char *name)
static inline const char *kretprobed(const char *name, unsigned long addr)
{
static const char tramp_name[] = "kretprobe_trampoline";
int size = sizeof(tramp_name);
if (strncmp(tramp_name, name, size) == 0)
if (is_kretprobe_trampoline(addr))
return "[unknown/kretprobe'd]";
return name;
}
#else
static inline const char *kretprobed(const char *name)
{
return name;
}
#endif /* CONFIG_KRETPROBES */
void
trace_seq_print_sym(struct trace_seq *s, unsigned long address, bool offset)
@@ -374,7 +365,7 @@ trace_seq_print_sym(struct trace_seq *s, unsigned long address, bool offset)
sprint_symbol(str, address);
else
kallsyms_lookup(address, NULL, NULL, NULL, str);
name = kretprobed(str);
name = kretprobed(str, address);
if (name && strlen(name)) {
trace_seq_puts(s, name);
kernel/trace/trace_printk.c
+1 -1
View File
@@ -384,7 +384,7 @@ static __init int init_trace_printk_function_export(void)
if (ret)
return 0;
trace_create_file("printk_formats", 0444, NULL,
trace_create_file("printk_formats", TRACE_MODE_READ, NULL,
NULL, &ftrace_formats_fops);
return 0;
kernel/trace/trace_recursion_record.c
+2 -2
View File
@@ -226,8 +226,8 @@ __init static int create_recursed_functions(void)
{
struct dentry *dentry;
dentry = trace_create_file("recursed_functions", 0644, NULL, NULL,
&recursed_functions_fops);
dentry = trace_create_file("recursed_functions", TRACE_MODE_WRITE,
NULL, NULL, &recursed_functions_fops);
if (!dentry)
pr_warn("WARNING: Failed to create recursed_functions\n");
return 0;
kernel/trace/trace_selftest.c
+91 -1
View File
@@ -287,6 +287,40 @@ static int trace_selftest_ops(struct trace_array *tr, int cnt)
if (trace_selftest_test_probe3_cnt != 4)
goto out_free;
/* Remove trace function from probe 3 */
func1_name = "!" __stringify(DYN_FTRACE_TEST_NAME);
len1 = strlen(func1_name);
ftrace_set_filter(&test_probe3, func1_name, len1, 0);
DYN_FTRACE_TEST_NAME();
print_counts();
if (trace_selftest_test_probe1_cnt != 3)
goto out_free;
if (trace_selftest_test_probe2_cnt != 2)
goto out_free;
if (trace_selftest_test_probe3_cnt != 4)
goto out_free;
if (cnt > 1) {
if (trace_selftest_test_global_cnt == 0)
goto out_free;
}
if (trace_selftest_test_dyn_cnt == 0)
goto out_free;
DYN_FTRACE_TEST_NAME2();
print_counts();
if (trace_selftest_test_probe1_cnt != 3)
goto out_free;
if (trace_selftest_test_probe2_cnt != 3)
goto out_free;
if (trace_selftest_test_probe3_cnt != 5)
goto out_free;
ret = 0;
out_free:
unregister_ftrace_function(dyn_ops);
@@ -750,6 +784,12 @@ static struct fgraph_ops fgraph_ops __initdata = {
.retfunc = &trace_graph_return,
};
#if defined(CONFIG_DYNAMIC_FTRACE) && \
defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS)
#define TEST_DIRECT_TRAMP
noinline __noclone static void trace_direct_tramp(void) { }
#endif
/*
* Pretty much the same than for the function tracer from which the selftest
* has been borrowed.
@@ -760,6 +800,7 @@ trace_selftest_startup_function_graph(struct tracer *trace,
{
int ret;
unsigned long count;
char *func_name __maybe_unused;
#ifdef CONFIG_DYNAMIC_FTRACE
if (ftrace_filter_param) {
@@ -808,8 +849,57 @@ trace_selftest_startup_function_graph(struct tracer *trace,
goto out;
}
/* Don't test dynamic tracing, the function tracer already did */
#ifdef TEST_DIRECT_TRAMP
tracing_reset_online_cpus(&tr->array_buffer);
set_graph_array(tr);
/*
* Some archs *cough*PowerPC*cough* add characters to the
* start of the function names. We simply put a '*' to
* accommodate them.
*/
func_name = "*" __stringify(DYN_FTRACE_TEST_NAME);
ftrace_set_global_filter(func_name, strlen(func_name), 1);
/*
* Register direct function together with graph tracer
* and make sure we get graph trace.
*/
ret = register_ftrace_direct((unsigned long) DYN_FTRACE_TEST_NAME,
(unsigned long) trace_direct_tramp);
if (ret)
goto out;
ret = register_ftrace_graph(&fgraph_ops);
if (ret) {
warn_failed_init_tracer(trace, ret);
goto out;
}
DYN_FTRACE_TEST_NAME();
count = 0;
tracing_stop();
/* check the trace buffer */
ret = trace_test_buffer(&tr->array_buffer, &count);
unregister_ftrace_graph(&fgraph_ops);
ret = unregister_ftrace_direct((unsigned long) DYN_FTRACE_TEST_NAME,
(unsigned long) trace_direct_tramp);
if (ret)
goto out;
tracing_start();
if (!ret && !count) {
ret = -1;
goto out;
}
#endif
/* Don't test dynamic tracing, the function tracer already did */
out:
/* Stop it if we failed */
if (ret)
kernel/trace/trace_stack.c
+3 -3
View File
@@ -559,14 +559,14 @@ static __init int stack_trace_init(void)
if (ret)
return 0;
trace_create_file("stack_max_size", 0644, NULL,
trace_create_file("stack_max_size", TRACE_MODE_WRITE, NULL,
&stack_trace_max_size, &stack_max_size_fops);
trace_create_file("stack_trace", 0444, NULL,
trace_create_file("stack_trace", TRACE_MODE_READ, NULL,
NULL, &stack_trace_fops);
#ifdef CONFIG_DYNAMIC_FTRACE
trace_create_file("stack_trace_filter", 0644, NULL,
trace_create_file("stack_trace_filter", TRACE_MODE_WRITE, NULL,
&trace_ops, &stack_trace_filter_fops);
#endif
kernel/trace/trace_stat.c
+3 -3
View File
@@ -297,9 +297,9 @@ static int init_stat_file(struct stat_session *session)
if (!stat_dir && (ret = tracing_stat_init()))
return ret;
session->file = tracefs_create_file(session->ts->name, 0644,
stat_dir,
session, &tracing_stat_fops);
session->file = tracefs_create_file(session->ts->name, TRACE_MODE_WRITE,
stat_dir, session,
&tracing_stat_fops);
if (!session->file)
return -ENOMEM;
return 0;
kernel/trace/trace_uprobe.c
+2 -2
View File
@@ -1655,10 +1655,10 @@ static __init int init_uprobe_trace(void)
if (ret)
return 0;
trace_create_file("uprobe_events", 0644, NULL,
trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL,
NULL, &uprobe_events_ops);
/* Profile interface */
trace_create_file("uprobe_profile", 0444, NULL,
trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL,
NULL, &uprobe_profile_ops);
return 0;
}
kernel/trace/tracing_map.c
+23 -17
View File
@@ -834,29 +834,35 @@ int tracing_map_init(struct tracing_map *map)
return err;
}
static int cmp_entries_dup(const struct tracing_map_sort_entry **a,
const struct tracing_map_sort_entry **b)
static int cmp_entries_dup(const void *A, const void *B)
{
const struct tracing_map_sort_entry *a, *b;
int ret = 0;
if (memcmp((*a)->key, (*b)->key, (*a)->elt->map->key_size))
a = *(const struct tracing_map_sort_entry **)A;
b = *(const struct tracing_map_sort_entry **)B;
if (memcmp(a->key, b->key, a->elt->map->key_size))
ret = 1;
return ret;
}
static int cmp_entries_sum(const struct tracing_map_sort_entry **a,
const struct tracing_map_sort_entry **b)
static int cmp_entries_sum(const void *A, const void *B)
{
const struct tracing_map_elt *elt_a, *elt_b;
const struct tracing_map_sort_entry *a, *b;
struct tracing_map_sort_key *sort_key;
struct tracing_map_field *field;
tracing_map_cmp_fn_t cmp_fn;
void *val_a, *val_b;
int ret = 0;
elt_a = (*a)->elt;
elt_b = (*b)->elt;
a = *(const struct tracing_map_sort_entry **)A;
b = *(const struct tracing_map_sort_entry **)B;
elt_a = a->elt;
elt_b = b->elt;
sort_key = &elt_a->map->sort_key;
@@ -873,18 +879,21 @@ static int cmp_entries_sum(const struct tracing_map_sort_entry **a,
return ret;
}
static int cmp_entries_key(const struct tracing_map_sort_entry **a,
const struct tracing_map_sort_entry **b)
static int cmp_entries_key(const void *A, const void *B)
{
const struct tracing_map_elt *elt_a, *elt_b;
const struct tracing_map_sort_entry *a, *b;
struct tracing_map_sort_key *sort_key;
struct tracing_map_field *field;
tracing_map_cmp_fn_t cmp_fn;
void *val_a, *val_b;
int ret = 0;
elt_a = (*a)->elt;
elt_b = (*b)->elt;
a = *(const struct tracing_map_sort_entry **)A;
b = *(const struct tracing_map_sort_entry **)B;
elt_a = a->elt;
elt_b = b->elt;
sort_key = &elt_a->map->sort_key;
@@ -989,10 +998,8 @@ static void sort_secondary(struct tracing_map *map,
struct tracing_map_sort_key *primary_key,
struct tracing_map_sort_key *secondary_key)
{
int (*primary_fn)(const struct tracing_map_sort_entry **,
const struct tracing_map_sort_entry **);
int (*secondary_fn)(const struct tracing_map_sort_entry **,
const struct tracing_map_sort_entry **);
int (*primary_fn)(const void *, const void *);
int (*secondary_fn)(const void *, const void *);
unsigned i, start = 0, n_sub = 1;
if (is_key(map, primary_key->field_idx))
@@ -1061,8 +1068,7 @@ int tracing_map_sort_entries(struct tracing_map *map,
unsigned int n_sort_keys,
struct tracing_map_sort_entry ***sort_entries)
{
int (*cmp_entries_fn)(const struct tracing_map_sort_entry **,
const struct tracing_map_sort_entry **);
int (*cmp_entries_fn)(const void *, const void *);
struct tracing_map_sort_entry *sort_entry, **entries;
int i, n_entries, ret;