1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#define __pa(x) ((unsigned long)(x))
17#define __va(x) ((void *)((unsigned long)(x)))
18
19
20#undef CONFIG_PAGE_TABLE_ISOLATION
21
22#include "error.h"
23#include "misc.h"
24
25
26#include <linux/pgtable.h>
27#include <asm/cmpxchg.h>
28#include <asm/trap_pf.h>
29#include <asm/trapnr.h>
30#include <asm/init.h>
31
32#undef __PAGE_OFFSET
33#define __PAGE_OFFSET __PAGE_OFFSET_BASE
34#include "../../mm/ident_map.c"
35
36#define _SETUP
37#include <asm/setup.h>
38#undef _SETUP
39
40extern unsigned long get_cmd_line_ptr(void);
41
42
43pteval_t __default_kernel_pte_mask __read_mostly = ~0;
44
45
46struct alloc_pgt_data {
47 unsigned char *pgt_buf;
48 unsigned long pgt_buf_size;
49 unsigned long pgt_buf_offset;
50};
51
52
53
54
55
56
57static void *alloc_pgt_page(void *context)
58{
59 struct alloc_pgt_data *pages = (struct alloc_pgt_data *)context;
60 unsigned char *entry;
61
62
63 if (pages->pgt_buf_offset >= pages->pgt_buf_size) {
64 debug_putstr("out of pgt_buf in " __FILE__ "!?\n");
65 debug_putaddr(pages->pgt_buf_offset);
66 debug_putaddr(pages->pgt_buf_size);
67 return NULL;
68 }
69
70 entry = pages->pgt_buf + pages->pgt_buf_offset;
71 pages->pgt_buf_offset += PAGE_SIZE;
72
73 return entry;
74}
75
76
77static struct alloc_pgt_data pgt_data;
78
79
80static unsigned long top_level_pgt;
81
82phys_addr_t physical_mask = (1ULL << __PHYSICAL_MASK_SHIFT) - 1;
83
84
85
86
87
88static struct x86_mapping_info mapping_info;
89
90
91
92
93static void add_identity_map(unsigned long start, unsigned long end)
94{
95 int ret;
96
97
98 start = round_down(start, PMD_SIZE);
99 end = round_up(end, PMD_SIZE);
100 if (start >= end)
101 return;
102
103
104 ret = kernel_ident_mapping_init(&mapping_info, (pgd_t *)top_level_pgt, start, end);
105 if (ret)
106 error("Error: kernel_ident_mapping_init() failed\n");
107}
108
109
110void initialize_identity_maps(void *rmode)
111{
112 unsigned long cmdline;
113
114
115 physical_mask &= ~sme_me_mask;
116
117
118 mapping_info.alloc_pgt_page = alloc_pgt_page;
119 mapping_info.context = &pgt_data;
120 mapping_info.page_flag = __PAGE_KERNEL_LARGE_EXEC | sme_me_mask;
121 mapping_info.kernpg_flag = _KERNPG_TABLE;
122
123
124
125
126
127
128 pgt_data.pgt_buf_offset = 0;
129
130
131
132
133
134
135
136
137
138
139
140
141 top_level_pgt = read_cr3_pa();
142 if (p4d_offset((pgd_t *)top_level_pgt, 0) == (p4d_t *)_pgtable) {
143 pgt_data.pgt_buf = _pgtable + BOOT_INIT_PGT_SIZE;
144 pgt_data.pgt_buf_size = BOOT_PGT_SIZE - BOOT_INIT_PGT_SIZE;
145 memset(pgt_data.pgt_buf, 0, pgt_data.pgt_buf_size);
146 } else {
147 pgt_data.pgt_buf = _pgtable;
148 pgt_data.pgt_buf_size = BOOT_PGT_SIZE;
149 memset(pgt_data.pgt_buf, 0, pgt_data.pgt_buf_size);
150 top_level_pgt = (unsigned long)alloc_pgt_page(&pgt_data);
151 }
152
153
154
155
156
157
158
159
160 add_identity_map((unsigned long)_head, (unsigned long)_end);
161 boot_params = rmode;
162 add_identity_map((unsigned long)boot_params, (unsigned long)(boot_params + 1));
163 cmdline = get_cmd_line_ptr();
164 add_identity_map(cmdline, cmdline + COMMAND_LINE_SIZE);
165
166
167 sev_verify_cbit(top_level_pgt);
168 write_cr3(top_level_pgt);
169}
170
171static pte_t *split_large_pmd(struct x86_mapping_info *info,
172 pmd_t *pmdp, unsigned long __address)
173{
174 unsigned long page_flags;
175 unsigned long address;
176 pte_t *pte;
177 pmd_t pmd;
178 int i;
179
180 pte = (pte_t *)info->alloc_pgt_page(info->context);
181 if (!pte)
182 return NULL;
183
184 address = __address & PMD_MASK;
185
186 page_flags = info->page_flag & ~_PAGE_PSE;
187
188
189 for (i = 0; i < PTRS_PER_PMD; i++) {
190 set_pte(&pte[i], __pte(address | page_flags));
191 address += PAGE_SIZE;
192 }
193
194
195
196
197
198
199
200
201
202
203 pmd = __pmd((unsigned long)pte | info->kernpg_flag);
204 set_pmd(pmdp, pmd);
205
206 write_cr3(top_level_pgt);
207
208 return pte + pte_index(__address);
209}
210
211static void clflush_page(unsigned long address)
212{
213 unsigned int flush_size;
214 char *cl, *start, *end;
215
216
217
218
219
220 flush_size = 64;
221 start = (char *)(address & PAGE_MASK);
222 end = start + PAGE_SIZE;
223
224
225
226
227
228 asm volatile("mfence" : : : "memory");
229
230 for (cl = start; cl != end; cl += flush_size)
231 clflush(cl);
232}
233
234static int set_clr_page_flags(struct x86_mapping_info *info,
235 unsigned long address,
236 pteval_t set, pteval_t clr)
237{
238 pgd_t *pgdp = (pgd_t *)top_level_pgt;
239 p4d_t *p4dp;
240 pud_t *pudp;
241 pmd_t *pmdp;
242 pte_t *ptep, pte;
243
244
245
246
247
248
249
250
251
252
253
254 asm volatile("mov %[address], %%r9"
255 :: [address] "g" (*(unsigned long *)address)
256 : "r9", "memory");
257
258
259
260
261
262 p4dp = p4d_offset(pgdp, address);
263 pudp = pud_offset(p4dp, address);
264 pmdp = pmd_offset(pudp, address);
265
266 if (pmd_large(*pmdp))
267 ptep = split_large_pmd(info, pmdp, address);
268 else
269 ptep = pte_offset_kernel(pmdp, address);
270
271 if (!ptep)
272 return -ENOMEM;
273
274
275
276
277
278 if ((set | clr) & _PAGE_ENC)
279 clflush_page(address);
280
281
282 pte = *ptep;
283 pte = pte_set_flags(pte, set);
284 pte = pte_clear_flags(pte, clr);
285 set_pte(ptep, pte);
286
287
288 write_cr3(top_level_pgt);
289
290 return 0;
291}
292
293int set_page_decrypted(unsigned long address)
294{
295 return set_clr_page_flags(&mapping_info, address, 0, _PAGE_ENC);
296}
297
298int set_page_encrypted(unsigned long address)
299{
300 return set_clr_page_flags(&mapping_info, address, _PAGE_ENC, 0);
301}
302
303int set_page_non_present(unsigned long address)
304{
305 return set_clr_page_flags(&mapping_info, address, 0, _PAGE_PRESENT);
306}
307
308static void do_pf_error(const char *msg, unsigned long error_code,
309 unsigned long address, unsigned long ip)
310{
311 error_putstr(msg);
312
313 error_putstr("\nError Code: ");
314 error_puthex(error_code);
315 error_putstr("\nCR2: 0x");
316 error_puthex(address);
317 error_putstr("\nRIP relative to _head: 0x");
318 error_puthex(ip - (unsigned long)_head);
319 error_putstr("\n");
320
321 error("Stopping.\n");
322}
323
324void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code)
325{
326 unsigned long address = native_read_cr2();
327 unsigned long end;
328 bool ghcb_fault;
329
330 ghcb_fault = sev_es_check_ghcb_fault(address);
331
332 address &= PMD_MASK;
333 end = address + PMD_SIZE;
334
335
336
337
338
339
340
341 if (error_code & (X86_PF_PROT | X86_PF_USER | X86_PF_RSVD))
342 do_pf_error("Unexpected page-fault:", error_code, address, regs->ip);
343 else if (ghcb_fault)
344 do_pf_error("Page-fault on GHCB page:", error_code, address, regs->ip);
345
346
347
348
349
350 add_identity_map(address, end);
351}
352