mirror of
				https://github.com/torvalds/linux.git
				synced 2025-11-04 10:40:15 +02:00 
			
		
		
		
	jump label, locking/static_keys: Update docs
Signed-off-by: Jason Baron <jbaron@akamai.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: benh@kernel.crashing.org Cc: bp@alien8.de Cc: davem@davemloft.net Cc: ddaney@caviumnetworks.com Cc: heiko.carstens@de.ibm.com Cc: linux-kernel@vger.kernel.org Cc: liuj97@gmail.com Cc: luto@amacapital.net Cc: michael@ellerman.id.au Cc: rabin@rab.in Cc: ralf@linux-mips.org Cc: rostedt@goodmis.org Cc: vbabka@suse.cz Cc: will.deacon@arm.com Link: http://lkml.kernel.org/r/6b50f2f6423a2244f37f4b1d2d6c211b9dcdf4f8.1438227999.git.jbaron@akamai.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
		
							parent
							
								
									2bf9e0ab08
								
							
						
					
					
						commit
						412758cb26
					
				
					 2 changed files with 99 additions and 69 deletions
				
			
		| 
						 | 
					@ -1,7 +1,22 @@
 | 
				
			||||||
			Static Keys
 | 
								Static Keys
 | 
				
			||||||
			-----------
 | 
								-----------
 | 
				
			||||||
 | 
					
 | 
				
			||||||
By: Jason Baron <jbaron@redhat.com>
 | 
					DEPRECATED API:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					The use of 'struct static_key' directly, is now DEPRECATED. In addition
 | 
				
			||||||
 | 
					static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					struct static_key false = STATIC_KEY_INIT_FALSE;
 | 
				
			||||||
 | 
					struct static_key true = STATIC_KEY_INIT_TRUE;
 | 
				
			||||||
 | 
					static_key_true()
 | 
				
			||||||
 | 
					static_key_false()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					The updated API replacements are:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					DEFINE_STATIC_KEY_TRUE(key);
 | 
				
			||||||
 | 
					DEFINE_STATIC_KEY_FALSE(key);
 | 
				
			||||||
 | 
					static_key_likely()
 | 
				
			||||||
 | 
					statick_key_unlikely()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
0) Abstract
 | 
					0) Abstract
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -9,22 +24,22 @@ Static keys allows the inclusion of seldom used features in
 | 
				
			||||||
performance-sensitive fast-path kernel code, via a GCC feature and a code
 | 
					performance-sensitive fast-path kernel code, via a GCC feature and a code
 | 
				
			||||||
patching technique. A quick example:
 | 
					patching technique. A quick example:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	struct static_key key = STATIC_KEY_INIT_FALSE;
 | 
						DEFINE_STATIC_KEY_FALSE(key);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	...
 | 
						...
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        if (static_key_false(&key))
 | 
					        if (static_branch_unlikely(&key))
 | 
				
			||||||
                do unlikely code
 | 
					                do unlikely code
 | 
				
			||||||
        else
 | 
					        else
 | 
				
			||||||
                do likely code
 | 
					                do likely code
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	...
 | 
						...
 | 
				
			||||||
	static_key_slow_inc();
 | 
						static_branch_enable(&key);
 | 
				
			||||||
	...
 | 
						...
 | 
				
			||||||
	static_key_slow_inc();
 | 
						static_branch_disable(&key);
 | 
				
			||||||
	...
 | 
						...
 | 
				
			||||||
 | 
					
 | 
				
			||||||
The static_key_false() branch will be generated into the code with as little
 | 
					The static_branch_unlikely() branch will be generated into the code with as little
 | 
				
			||||||
impact to the likely code path as possible.
 | 
					impact to the likely code path as possible.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -56,7 +71,7 @@ the branch site to change the branch direction.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
For example, if we have a simple branch that is disabled by default:
 | 
					For example, if we have a simple branch that is disabled by default:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	if (static_key_false(&key))
 | 
						if (static_branch_unlikely(&key))
 | 
				
			||||||
		printk("I am the true branch\n");
 | 
							printk("I am the true branch\n");
 | 
				
			||||||
 | 
					
 | 
				
			||||||
Thus, by default the 'printk' will not be emitted. And the code generated will
 | 
					Thus, by default the 'printk' will not be emitted. And the code generated will
 | 
				
			||||||
| 
						 | 
					@ -75,68 +90,55 @@ the basis for the static keys facility.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
In order to make use of this optimization you must first define a key:
 | 
					In order to make use of this optimization you must first define a key:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	struct static_key key;
 | 
						DEFINE_STATIC_KEY_TRUE(key);
 | 
				
			||||||
 | 
					 | 
				
			||||||
Which is initialized as:
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
	struct static_key key = STATIC_KEY_INIT_TRUE;
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
or:
 | 
					or:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	struct static_key key = STATIC_KEY_INIT_FALSE;
 | 
						DEFINE_STATIC_KEY_FALSE(key);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
If the key is not initialized, it is default false. The 'struct static_key',
 | 
					
 | 
				
			||||||
must be a 'global'. That is, it can't be allocated on the stack or dynamically
 | 
					The key must be global, that is, it can't be allocated on the stack or dynamically
 | 
				
			||||||
allocated at run-time.
 | 
					allocated at run-time.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
The key is then used in code as:
 | 
					The key is then used in code as:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        if (static_key_false(&key))
 | 
					        if (static_branch_unlikely(&key))
 | 
				
			||||||
                do unlikely code
 | 
					                do unlikely code
 | 
				
			||||||
        else
 | 
					        else
 | 
				
			||||||
                do likely code
 | 
					                do likely code
 | 
				
			||||||
 | 
					
 | 
				
			||||||
Or:
 | 
					Or:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        if (static_key_true(&key))
 | 
					        if (static_branch_likely(&key))
 | 
				
			||||||
                do likely code
 | 
					                do likely code
 | 
				
			||||||
        else
 | 
					        else
 | 
				
			||||||
                do unlikely code
 | 
					                do unlikely code
 | 
				
			||||||
 | 
					
 | 
				
			||||||
A key that is initialized via 'STATIC_KEY_INIT_FALSE', must be used in a
 | 
					Keys defined via DEFINE_STATIC_KEY_TRUE(), or DEFINE_STATIC_KEY_FALSE, may
 | 
				
			||||||
'static_key_false()' construct. Likewise, a key initialized via
 | 
					be used in either static_branch_likely() or static_branch_unlikely()
 | 
				
			||||||
'STATIC_KEY_INIT_TRUE' must be used in a 'static_key_true()' construct. A
 | 
					statemnts.
 | 
				
			||||||
single key can be used in many branches, but all the branches must match the
 | 
					 | 
				
			||||||
way that the key has been initialized.
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
The branch(es) can then be switched via:
 | 
					Branch(es) can be set true via:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	static_key_slow_inc(&key);
 | 
					static_branch_enable(&key);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					or false via:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					static_branch_disable(&key);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					The branch(es) can then be switched via reference counts:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						static_branch_inc(&key);
 | 
				
			||||||
	...
 | 
						...
 | 
				
			||||||
	static_key_slow_dec(&key);
 | 
						static_branch_dec(&key);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
Thus, 'static_key_slow_inc()' means 'make the branch true', and
 | 
					Thus, 'static_branch_inc()' means 'make the branch true', and
 | 
				
			||||||
'static_key_slow_dec()' means 'make the branch false' with appropriate
 | 
					'static_branch_dec()' means 'make the branch false' with appropriate
 | 
				
			||||||
reference counting. For example, if the key is initialized true, a
 | 
					reference counting. For example, if the key is initialized true, a
 | 
				
			||||||
static_key_slow_dec(), will switch the branch to false. And a subsequent
 | 
					static_branch_dec(), will switch the branch to false. And a subsequent
 | 
				
			||||||
static_key_slow_inc(), will change the branch back to true. Likewise, if the
 | 
					static_branch_inc(), will change the branch back to true. Likewise, if the
 | 
				
			||||||
key is initialized false, a 'static_key_slow_inc()', will change the branch to
 | 
					key is initialized false, a 'static_branch_inc()', will change the branch to
 | 
				
			||||||
true. And then a 'static_key_slow_dec()', will again make the branch false.
 | 
					true. And then a 'static_branch_dec()', will again make the branch false.
 | 
				
			||||||
 | 
					 | 
				
			||||||
An example usage in the kernel is the implementation of tracepoints:
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
        static inline void trace_##name(proto)                          \
 | 
					 | 
				
			||||||
        {                                                               \
 | 
					 | 
				
			||||||
                if (static_key_false(&__tracepoint_##name.key))		\
 | 
					 | 
				
			||||||
                        __DO_TRACE(&__tracepoint_##name,                \
 | 
					 | 
				
			||||||
                                TP_PROTO(data_proto),                   \
 | 
					 | 
				
			||||||
                                TP_ARGS(data_args),                     \
 | 
					 | 
				
			||||||
                                TP_CONDITION(cond));                    \
 | 
					 | 
				
			||||||
        }
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
Tracepoints are disabled by default, and can be placed in performance critical
 | 
					 | 
				
			||||||
pieces of the kernel. Thus, by using a static key, the tracepoints can have
 | 
					 | 
				
			||||||
absolutely minimal impact when not in use.
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
4) Architecture level code patching interface, 'jump labels'
 | 
					4) Architecture level code patching interface, 'jump labels'
 | 
				
			||||||
| 
						 | 
					@ -150,9 +152,12 @@ simply fall back to a traditional, load, test, and jump sequence.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
* #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h
 | 
					* #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h
 | 
				
			||||||
 | 
					
 | 
				
			||||||
* __always_inline bool arch_static_branch(struct static_key *key), see:
 | 
					* __always_inline bool arch_static_branch(struct static_key *key, bool branch), see:
 | 
				
			||||||
					arch/x86/include/asm/jump_label.h
 | 
										arch/x86/include/asm/jump_label.h
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					* __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch),
 | 
				
			||||||
 | 
										see: arch/x86/include/asm/jump_label.h
 | 
				
			||||||
 | 
					
 | 
				
			||||||
* void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type),
 | 
					* void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type),
 | 
				
			||||||
					see: arch/x86/kernel/jump_label.c
 | 
										see: arch/x86/kernel/jump_label.c
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -173,7 +178,7 @@ SYSCALL_DEFINE0(getppid)
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
        int pid;
 | 
					        int pid;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
+       if (static_key_false(&key))
 | 
					+       if (static_branch_unlikely(&key))
 | 
				
			||||||
+               printk("I am the true branch\n");
 | 
					+               printk("I am the true branch\n");
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        rcu_read_lock();
 | 
					        rcu_read_lock();
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -7,17 +7,52 @@
 | 
				
			||||||
 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
 | 
					 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
 | 
				
			||||||
 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
 | 
					 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
 | 
				
			||||||
 *
 | 
					 *
 | 
				
			||||||
 | 
					 * DEPRECATED API:
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * The use of 'struct static_key' directly, is now DEPRECATED. In addition
 | 
				
			||||||
 | 
					 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * struct static_key false = STATIC_KEY_INIT_FALSE;
 | 
				
			||||||
 | 
					 * struct static_key true = STATIC_KEY_INIT_TRUE;
 | 
				
			||||||
 | 
					 * static_key_true()
 | 
				
			||||||
 | 
					 * static_key_false()
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * The updated API replacements are:
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * DEFINE_STATIC_KEY_TRUE(key);
 | 
				
			||||||
 | 
					 * DEFINE_STATIC_KEY_FALSE(key);
 | 
				
			||||||
 | 
					 * static_key_likely()
 | 
				
			||||||
 | 
					 * statick_key_unlikely()
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 * Jump labels provide an interface to generate dynamic branches using
 | 
					 * Jump labels provide an interface to generate dynamic branches using
 | 
				
			||||||
 * self-modifying code. Assuming toolchain and architecture support, the result
 | 
					 * self-modifying code. Assuming toolchain and architecture support, if we
 | 
				
			||||||
 * of a "if (static_key_false(&key))" statement is an unconditional branch (which
 | 
					 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
 | 
				
			||||||
 * defaults to false - and the true block is placed out of line).
 | 
					 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
 | 
				
			||||||
 | 
					 * (which defaults to false - and the true block is placed out of line).
 | 
				
			||||||
 | 
					 * Similarly, we can define an initially true key via
 | 
				
			||||||
 | 
					 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
 | 
				
			||||||
 | 
					 * "if (static_branch_unlikely(&key))", in which case we will generate an
 | 
				
			||||||
 | 
					 * unconditional branch to the out-of-line true branch. Keys that are
 | 
				
			||||||
 | 
					 * initially true or false can be using in both static_branch_unlikely()
 | 
				
			||||||
 | 
					 * and static_branch_likely() statements.
 | 
				
			||||||
 *
 | 
					 *
 | 
				
			||||||
 * However at runtime we can change the branch target using
 | 
					 * At runtime we can change the branch target by setting the key
 | 
				
			||||||
 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
 | 
					 * to true via a call to static_branch_enable(), or false using
 | 
				
			||||||
 * object, and for as long as there are references all branches referring to
 | 
					 * static_branch_disable(). If the direction of the branch is switched by
 | 
				
			||||||
 * that particular key will point to the (out of line) true block.
 | 
					 * these calls then we run-time modify the branch target via a
 | 
				
			||||||
 | 
					 * no-op -> jump or jump -> no-op conversion. For example, for an
 | 
				
			||||||
 | 
					 * initially false key that is used in an "if (static_branch_unlikely(&key))"
 | 
				
			||||||
 | 
					 * statement, setting the key to true requires us to patch in a jump
 | 
				
			||||||
 | 
					 * to the out-of-line of true branch.
 | 
				
			||||||
 *
 | 
					 *
 | 
				
			||||||
 * Since this relies on modifying code, the static_key_slow_{inc,dec}() functions
 | 
					 * In addtion to static_branch_{enable,disable}, we can also reference count
 | 
				
			||||||
 | 
					 * the key or branch direction via static_branch_{inc,dec}. Thus,
 | 
				
			||||||
 | 
					 * static_branch_inc() can be thought of as a 'make more true' and
 | 
				
			||||||
 | 
					 * static_branch_dec() as a 'make more false'. The inc()/dec()
 | 
				
			||||||
 | 
					 * interface is meant to be used exclusively from the inc()/dec() for a given
 | 
				
			||||||
 | 
					 * key.
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Since this relies on modifying code, the branch modifying functions
 | 
				
			||||||
 * must be considered absolute slow paths (machine wide synchronization etc.).
 | 
					 * must be considered absolute slow paths (machine wide synchronization etc.).
 | 
				
			||||||
 * OTOH, since the affected branches are unconditional, their runtime overhead
 | 
					 * OTOH, since the affected branches are unconditional, their runtime overhead
 | 
				
			||||||
 * will be absolutely minimal, esp. in the default (off) case where the total
 | 
					 * will be absolutely minimal, esp. in the default (off) case where the total
 | 
				
			||||||
| 
						 | 
					@ -29,20 +64,10 @@
 | 
				
			||||||
 * cause significant performance degradation. Struct static_key_deferred and
 | 
					 * cause significant performance degradation. Struct static_key_deferred and
 | 
				
			||||||
 * static_key_slow_dec_deferred() provide for this.
 | 
					 * static_key_slow_dec_deferred() provide for this.
 | 
				
			||||||
 *
 | 
					 *
 | 
				
			||||||
 * Lacking toolchain and or architecture support, jump labels fall back to a simple
 | 
					 * Lacking toolchain and or architecture support, static keys fall back to a
 | 
				
			||||||
 * conditional branch.
 | 
					 * simple conditional branch.
 | 
				
			||||||
 *
 | 
					 *
 | 
				
			||||||
 * struct static_key my_key = STATIC_KEY_INIT_TRUE;
 | 
					 * Additional babbling in: Documentation/static-keys.txt
 | 
				
			||||||
 *
 | 
					 | 
				
			||||||
 *   if (static_key_true(&my_key)) {
 | 
					 | 
				
			||||||
 *   }
 | 
					 | 
				
			||||||
 *
 | 
					 | 
				
			||||||
 * will result in the true case being in-line and starts the key with a single
 | 
					 | 
				
			||||||
 * reference. Mixing static_key_true() and static_key_false() on the same key is not
 | 
					 | 
				
			||||||
 * allowed.
 | 
					 | 
				
			||||||
 *
 | 
					 | 
				
			||||||
 * Not initializing the key (static data is initialized to 0s anyway) is the
 | 
					 | 
				
			||||||
 * same as using STATIC_KEY_INIT_FALSE.
 | 
					 | 
				
			||||||
 */
 | 
					 */
 | 
				
			||||||
 | 
					
 | 
				
			||||||
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
 | 
					#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
		Loading…
	
		Reference in a new issue