forked from mirrors/linux
531bb52a86
This is boot code and thus Spectre-safe: we run this _way_ before userspace comes along to have a chance to poison our branch predictor. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Arjan van de Ven <arjan@linux.intel.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Borislav Petkov <bp@suse.de> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: David Woodhouse <dwmw2@infradead.org> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Tom Lendacky <thomas.lendacky@amd.com> Signed-off-by: Ingo Molnar <mingo@kernel.org>
159 lines
4.3 KiB
ArmAsm
159 lines
4.3 KiB
ArmAsm
/*
|
|
* AMD Memory Encryption Support
|
|
*
|
|
* Copyright (C) 2016 Advanced Micro Devices, Inc.
|
|
*
|
|
* Author: Tom Lendacky <thomas.lendacky@amd.com>
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/page.h>
|
|
#include <asm/processor-flags.h>
|
|
#include <asm/msr-index.h>
|
|
#include <asm/nospec-branch.h>
|
|
|
|
.text
|
|
.code64
|
|
ENTRY(sme_encrypt_execute)
|
|
|
|
/*
|
|
* Entry parameters:
|
|
* RDI - virtual address for the encrypted mapping
|
|
* RSI - virtual address for the decrypted mapping
|
|
* RDX - length to encrypt
|
|
* RCX - virtual address of the encryption workarea, including:
|
|
* - stack page (PAGE_SIZE)
|
|
* - encryption routine page (PAGE_SIZE)
|
|
* - intermediate copy buffer (PMD_PAGE_SIZE)
|
|
* R8 - physcial address of the pagetables to use for encryption
|
|
*/
|
|
|
|
push %rbp
|
|
movq %rsp, %rbp /* RBP now has original stack pointer */
|
|
|
|
/* Set up a one page stack in the non-encrypted memory area */
|
|
movq %rcx, %rax /* Workarea stack page */
|
|
leaq PAGE_SIZE(%rax), %rsp /* Set new stack pointer */
|
|
addq $PAGE_SIZE, %rax /* Workarea encryption routine */
|
|
|
|
push %r12
|
|
movq %rdi, %r10 /* Encrypted area */
|
|
movq %rsi, %r11 /* Decrypted area */
|
|
movq %rdx, %r12 /* Area length */
|
|
|
|
/* Copy encryption routine into the workarea */
|
|
movq %rax, %rdi /* Workarea encryption routine */
|
|
leaq __enc_copy(%rip), %rsi /* Encryption routine */
|
|
movq $(.L__enc_copy_end - __enc_copy), %rcx /* Encryption routine length */
|
|
rep movsb
|
|
|
|
/* Setup registers for call */
|
|
movq %r10, %rdi /* Encrypted area */
|
|
movq %r11, %rsi /* Decrypted area */
|
|
movq %r8, %rdx /* Pagetables used for encryption */
|
|
movq %r12, %rcx /* Area length */
|
|
movq %rax, %r8 /* Workarea encryption routine */
|
|
addq $PAGE_SIZE, %r8 /* Workarea intermediate copy buffer */
|
|
|
|
ANNOTATE_RETPOLINE_SAFE
|
|
call *%rax /* Call the encryption routine */
|
|
|
|
pop %r12
|
|
|
|
movq %rbp, %rsp /* Restore original stack pointer */
|
|
pop %rbp
|
|
|
|
ret
|
|
ENDPROC(sme_encrypt_execute)
|
|
|
|
ENTRY(__enc_copy)
|
|
/*
|
|
* Routine used to encrypt memory in place.
|
|
* This routine must be run outside of the kernel proper since
|
|
* the kernel will be encrypted during the process. So this
|
|
* routine is defined here and then copied to an area outside
|
|
* of the kernel where it will remain and run decrypted
|
|
* during execution.
|
|
*
|
|
* On entry the registers must be:
|
|
* RDI - virtual address for the encrypted mapping
|
|
* RSI - virtual address for the decrypted mapping
|
|
* RDX - address of the pagetables to use for encryption
|
|
* RCX - length of area
|
|
* R8 - intermediate copy buffer
|
|
*
|
|
* RAX - points to this routine
|
|
*
|
|
* The area will be encrypted by copying from the non-encrypted
|
|
* memory space to an intermediate buffer and then copying from the
|
|
* intermediate buffer back to the encrypted memory space. The physical
|
|
* addresses of the two mappings are the same which results in the area
|
|
* being encrypted "in place".
|
|
*/
|
|
/* Enable the new page tables */
|
|
mov %rdx, %cr3
|
|
|
|
/* Flush any global TLBs */
|
|
mov %cr4, %rdx
|
|
andq $~X86_CR4_PGE, %rdx
|
|
mov %rdx, %cr4
|
|
orq $X86_CR4_PGE, %rdx
|
|
mov %rdx, %cr4
|
|
|
|
push %r15
|
|
push %r12
|
|
|
|
movq %rcx, %r9 /* Save area length */
|
|
movq %rdi, %r10 /* Save encrypted area address */
|
|
movq %rsi, %r11 /* Save decrypted area address */
|
|
|
|
/* Set the PAT register PA5 entry to write-protect */
|
|
movl $MSR_IA32_CR_PAT, %ecx
|
|
rdmsr
|
|
mov %rdx, %r15 /* Save original PAT value */
|
|
andl $0xffff00ff, %edx /* Clear PA5 */
|
|
orl $0x00000500, %edx /* Set PA5 to WP */
|
|
wrmsr
|
|
|
|
wbinvd /* Invalidate any cache entries */
|
|
|
|
/* Copy/encrypt up to 2MB at a time */
|
|
movq $PMD_PAGE_SIZE, %r12
|
|
1:
|
|
cmpq %r12, %r9
|
|
jnb 2f
|
|
movq %r9, %r12
|
|
|
|
2:
|
|
movq %r11, %rsi /* Source - decrypted area */
|
|
movq %r8, %rdi /* Dest - intermediate copy buffer */
|
|
movq %r12, %rcx
|
|
rep movsb
|
|
|
|
movq %r8, %rsi /* Source - intermediate copy buffer */
|
|
movq %r10, %rdi /* Dest - encrypted area */
|
|
movq %r12, %rcx
|
|
rep movsb
|
|
|
|
addq %r12, %r11
|
|
addq %r12, %r10
|
|
subq %r12, %r9 /* Kernel length decrement */
|
|
jnz 1b /* Kernel length not zero? */
|
|
|
|
/* Restore PAT register */
|
|
movl $MSR_IA32_CR_PAT, %ecx
|
|
rdmsr
|
|
mov %r15, %rdx /* Restore original PAT value */
|
|
wrmsr
|
|
|
|
pop %r12
|
|
pop %r15
|
|
|
|
ret
|
|
.L__enc_copy_end:
|
|
ENDPROC(__enc_copy)
|