· 8 years ago · Jan 13, 2018, 03:02 PM
1diff --git a/sys/cpu/x86_64/include/asmacros.h b/sys/cpu/x86_64/include/asmacros.h
2index 9cf250452..376fb497c 100644
3--- a/sys/cpu/x86_64/include/asmacros.h
4+++ b/sys/cpu/x86_64/include/asmacros.h
5@@ -141,7 +141,188 @@
6 .type __CONCAT(X,name),@function; __CONCAT(X,name):
7
8 /*
9- * Macros to create and destroy a trap frame.
10+ * stack frame macro support - supports mmu isolation, swapgs, and
11+ * stack frame pushing and popping.
12+ */
13+
14+/*
15+ * Kernel pmap isolation to work-around the massive Intel mmu bug
16+ * that allows kernel memory to be sussed out due to speculative memory
17+ * reads and instruction execution creating timing differences that can
18+ * be detected by userland. e.g. force speculative read, speculatively
19+ * execute a cmp/branch sequence, detect timing. Iterate cmp $values
20+ * to suss-out content of speculatively read kernel memory.
21+ *
22+ * We do this by creating a trampoline area for all user->kernel and
23+ * kernel->user transitions. The trampoline area allows us to limit
24+ * the reach the kernel map in the isolated version of the user pmap
25+ * to JUST the trampoline area (for all cpus), tss, and vector area.
26+ *
27+ * It is very important that these transitions not access any memory
28+ * outside of the trampoline page while the isolated user process pmap
29+ * is active in %cr3.
30+ *
31+ * The trampoline does not add much overhead when pmap isolation is
32+ * disabled, so we just run with it regardless. Of course, when pmap
33+ * isolation is enabled, the %cr3 loads add 150-250ns to every system
34+ * call as well as (without PCID) smash the TLB.
35+ *
36+ * KMMUENTER - Executed by the trampoline when a user->kernel transition
37+ * is detected. The stack pointer points into the pcpu
38+ * trampoline space and is available for register save/restore.
39+ * Other registers have not yet been saved. %gs points at
40+ * the kernel pcpu structure.
41+ *
42+ * Caller has already determined that a transition is in
43+ * progress and has already issued the swapgs. hwtf indicates
44+ * how much hardware has already pushed.
45+ *
46+ * KMMUEXIT - Executed when a kernel->user transition is made. The stack
47+ * pointer points into the pcpu trampoline space and we are
48+ * almost ready to iretq. %gs still points at the kernel pcpu
49+ * structure.
50+ *
51+ * Caller has already determined that a transition is in
52+ * progress. hwtf indicates how much hardware has already
53+ * pushed.
54+ */
55+#define KMMUENTER_TFRIP \
56+ subq $TR_RIP, %rsp ; \
57+ movq %r10, TR_R10(%rsp) ; \
58+ movq %r11, TR_R11(%rsp) ; \
59+ testq $PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; \
60+ je 40f ; \
61+ movq PCPU(trampoline)+TR_PCB_CR3,%r10 ; \
62+ movq %r10,%cr3 ; \
63+40: \
64+ movq %rsp, %r10 ; /* trampoline rsp */ \
65+ movq PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */ \
66+ movq TR_SS(%r10), %r11 ; \
67+ pushq %r11 ; \
68+ movq TR_RSP(%r10), %r11 ; \
69+ pushq %r11 ; \
70+ movq TR_RFLAGS(%r10), %r11 ; \
71+ pushq %r11 ; \
72+ movq TR_CS(%r10), %r11 ; \
73+ pushq %r11 ; \
74+ movq TR_RIP(%r10), %r11 ; \
75+ pushq %r11 ; \
76+ movq TR_R11(%r10), %r11 ; \
77+ movq TR_R10(%r10), %r10 \
78+
79+#define KMMUENTER_TFERR \
80+ subq $TR_ERR, %rsp ; \
81+ movq %r10, TR_R10(%rsp) ; \
82+ movq %r11, TR_R11(%rsp) ; \
83+ testq $PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; \
84+ je 40f ; \
85+ movq PCPU(trampoline)+TR_PCB_CR3,%r10 ; \
86+ movq %r10,%cr3 ; \
87+40: \
88+ movq %rsp, %r10 ; /* trampoline rsp */ \
89+ movq PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */ \
90+ movq TR_SS(%r10), %r11 ; \
91+ pushq %r11 ; \
92+ movq TR_RSP(%r10), %r11 ; \
93+ pushq %r11 ; \
94+ movq TR_RFLAGS(%r10), %r11 ; \
95+ pushq %r11 ; \
96+ movq TR_CS(%r10), %r11 ; \
97+ pushq %r11 ; \
98+ movq TR_RIP(%r10), %r11 ; \
99+ pushq %r11 ; \
100+ movq TR_ERR(%r10), %r11 ; \
101+ pushq %r11 ; \
102+ movq TR_R11(%r10), %r11 ; \
103+ movq TR_R10(%r10), %r10 \
104+
105+#define KMMUENTER_TFERR_SAVECR2 \
106+ subq $TR_ERR, %rsp ; \
107+ movq %r10, TR_R10(%rsp) ; \
108+ movq %r11, TR_R11(%rsp) ; \
109+ movq %cr2, %r10 ; \
110+ movq %r10, PCPU(trampoline)+TR_CR2 ; \
111+ testq $PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; \
112+ je 40f ; \
113+ movq PCPU(trampoline)+TR_PCB_CR3,%r10 ; \
114+ movq %r10,%cr3 ; \
115+40: \
116+ movq %rsp, %r10 ; /* trampoline rsp */ \
117+ movq PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */ \
118+ movq TR_SS(%r10), %r11 ; \
119+ pushq %r11 ; \
120+ movq TR_RSP(%r10), %r11 ; \
121+ pushq %r11 ; \
122+ movq TR_RFLAGS(%r10), %r11 ; \
123+ pushq %r11 ; \
124+ movq TR_CS(%r10), %r11 ; \
125+ pushq %r11 ; \
126+ movq TR_RIP(%r10), %r11 ; \
127+ pushq %r11 ; \
128+ movq TR_ERR(%r10), %r11 ; \
129+ pushq %r11 ; \
130+ movq TR_R11(%r10), %r11 ; \
131+ movq TR_R10(%r10), %r10 \
132+
133+/*
134+ * Set %cr3 if necessary on syscall entry. No registers may be
135+ * disturbed.
136+ */
137+#define KMMUENTER_SYSCALL \
138+ testq $PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; \
139+ je 40f ; \
140+ pushq %r10 ; \
141+ movq PCPU(trampoline)+TR_PCB_CR3,%r10 ; \
142+ movq %r10,%cr3 ; \
143+ popq %r10 ; \
144+40: \
145+
146+/*
147+ * We are positioned at the base of the trapframe. Advance the trapframe
148+ * and handle MMU isolation. MMU isolation requires us to copy the
149+ * hardware frame to the trampoline area before setting %cr3 to the
150+ * isolated map. We then set the %rsp for iretq to TR_RIP in the
151+ * trampoline area (after restoring the register we saved in TR_ERR).
152+ */
153+#define KMMUEXIT \
154+ addq $TF_RIP,%rsp ; \
155+ testq $PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; \
156+ je 40f ; \
157+ movq %r11, PCPU(trampoline)+TR_ERR ; /* save in TR_ERR */ \
158+ popq %r11 ; /* copy %rip */ \
159+ movq %r11, PCPU(trampoline)+TR_RIP ; \
160+ popq %r11 ; /* copy %cs */ \
161+ movq %r11, PCPU(trampoline)+TR_CS ; \
162+ popq %r11 ; /* copy %rflags */ \
163+ movq %r11, PCPU(trampoline)+TR_RFLAGS ; \
164+ popq %r11 ; /* copy %rsp */ \
165+ movq %r11, PCPU(trampoline)+TR_RSP ; \
166+ popq %r11 ; /* copy %ss */ \
167+ movq %r11, PCPU(trampoline)+TR_SS ; \
168+ movq %gs:0,%r11 ; \
169+ addq $GD_TRAMPOLINE+TR_ERR,%r11 ; \
170+ movq %r11,%rsp ; \
171+ movq PCPU(trampoline)+TR_PCB_CR3_ISO,%r11 ; \
172+ movq %r11,%cr3 ; \
173+ popq %r11 ; /* positioned at TR_RIP after this */ \
174+40: \
175+
176+/*
177+ * Warning: user stack pointer already loaded into %rsp at this
178+ * point. We still have the kernel %gs.
179+ */
180+#define KMMUEXIT_SYSCALL \
181+ testq $PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; \
182+ je 40f ; \
183+ movq %r10, PCPU(trampoline)+TR_R10 ; \
184+ movq PCPU(trampoline)+TR_PCB_CR3_ISO,%r10 ; \
185+ movq %r10,%cr3 ; \
186+ movq PCPU(trampoline)+TR_R10, %r10 ; \
187+40: \
188+
189+/*
190+ * Macros to create and destroy a trap frame. rsp has already been shifted
191+ * to the base of the trapframe in the thread structure.
192 */
193 #define PUSH_FRAME_REGS \
194 movq %rdi,TF_RDI(%rsp) ; \
195@@ -160,19 +341,56 @@
196 movq %r14,TF_R14(%rsp) ; \
197 movq %r15,TF_R15(%rsp)
198
199-#define PUSH_FRAME \
200- subq $TF_RIP,%rsp ; /* extend hardware frame to trapframe */ \
201- testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
202- jz 1f ; /* Yes, dont swapgs again */ \
203- swapgs ; \
204+/*
205+ * PUSH_FRAME is the first thing executed upon interrupt entry. We are
206+ * responsible for swapgs execution and the KMMUENTER dispatch.
207+ */
208+#define PUSH_FRAME_TFRIP \
209+ testb $SEL_RPL_MASK,TF_CS-TF_RIP(%rsp) ; /* from userland? */ \
210+ jz 1f ; \
211+ swapgs ; /* from userland */ \
212+ KMMUENTER_TFRIP ; /* from userland */ \
213 1: \
214- PUSH_FRAME_REGS \
215+ subq $TF_RIP,%rsp ; \
216+ PUSH_FRAME_REGS \
217
218+#define PUSH_FRAME_TFERR \
219+ testb $SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) ; /* from userland? */ \
220+ jz 1f ; \
221+ swapgs ; /* from userland */ \
222+ KMMUENTER_TFERR ; /* from userland */ \
223+1: \
224+ subq $TF_ERR,%rsp ; \
225+ PUSH_FRAME_REGS \
226+
227+#define PUSH_FRAME_TFERR_SAVECR2 \
228+ testb $SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) ; \
229+ jz 1f ; \
230+ swapgs ; /* from userland */ \
231+ KMMUENTER_TFERR_SAVECR2 ;/* from userland */ \
232+ subq $TF_ERR,%rsp ; \
233+ PUSH_FRAME_REGS ; \
234+ movq PCPU(trampoline)+TR_CR2, %r10 ; \
235+ jmp 2f ; \
236+1: \
237+ subq $TF_ERR,%rsp ; \
238+ PUSH_FRAME_REGS ; \
239+ movq %cr2, %r10 ; \
240+2: \
241+ movq %r10, TF_ADDR(%rsp)
242+
243+/*
244+ * Called when the iretq in doreti_iret faults. XXX
245+ */
246 #define PUSH_FRAME_NOSWAP \
247- subq $TF_RIP,%rsp ; /* extend hardware frame to trapframe */ \
248+ KMMUENTER_TFRIP ; \
249 PUSH_FRAME_REGS \
250
251-#define POP_FRAME \
252+/*
253+ * POP_FRAME is issued just prior to the iretq, or just prior to a
254+ * jmp doreti_iret. These must be passed in to the macro.
255+ */
256+#define POP_FRAME(lastinsn) \
257 movq TF_RDI(%rsp),%rdi ; \
258 movq TF_RSI(%rsp),%rsi ; \
259 movq TF_RDX(%rsp),%rdx ; \
260@@ -188,11 +406,16 @@
261 movq TF_R13(%rsp),%r13 ; \
262 movq TF_R14(%rsp),%r14 ; \
263 movq TF_R15(%rsp),%r15 ; \
264- testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
265- jz 1f ; /* keep kernel GS.base */ \
266 cli ; \
267- swapgs ; \
268-1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno, tf_xflags */
269+ testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* return to user? */ \
270+ jz 1f ; \
271+ KMMUEXIT ; /* return to user */ \
272+ swapgs ; /* return to user */ \
273+ jmp 2f ; \
274+1: \
275+ addq $TF_RIP,%rsp ; /* setup for iretq */ \
276+2: \
277+ lastinsn
278
279 /*
280 * Access per-CPU data.
281diff --git a/sys/cpu/x86_64/include/frame.h b/sys/cpu/x86_64/include/frame.h
282index eb1072046..1b15f50e8 100644
283--- a/sys/cpu/x86_64/include/frame.h
284+++ b/sys/cpu/x86_64/include/frame.h
285@@ -118,6 +118,34 @@ struct intrframe {
286 register_t if_ss;
287 };
288
289+/*
290+ * The trampframe is placed at the top of the trampoline page and
291+ * contains all the information needed to trampoline into and out
292+ * of the isolated user pmap.
293+ */
294+struct trampframe {
295+ register_t tr_unused01;
296+ register_t tr_cr2;
297+ register_t tr_r10;
298+ register_t tr_r11;
299+ register_t tr_err;
300+ register_t tr_rip;
301+ register_t tr_cs;
302+ register_t tr_rflags;
303+ register_t tr_rsp;
304+ register_t tr_ss;
305+
306+ /*
307+ * Top of hw stack in TSS is &tr_pcb_rsp (first push is tr_ss).
308+ * Make sure this is at least 16-byte aligned, so be sure the
309+ * fields below are in multiples of 16 bytes.
310+ */
311+ register_t tr_pcb_rsp; /* hw frame tramp top of stack */
312+ register_t tr_pcb_flags; /* copy of pcb control flags */
313+ register_t tr_pcb_cr3_iso; /* copy of isolated pml4e */
314+ register_t tr_pcb_cr3; /* copy of primary pml4e */
315+};
316+
317 int kdb_trap(int, int, struct trapframe *);
318
319 #endif /* _CPU_FRAME_H_ */
320diff --git a/sys/cpu/x86_64/include/segments.h b/sys/cpu/x86_64/include/segments.h
321index 4976d5aba..0aa64af91 100644
322--- a/sys/cpu/x86_64/include/segments.h
323+++ b/sys/cpu/x86_64/include/segments.h
324@@ -151,30 +151,10 @@ struct savetls {
325 struct tls_info info[2];
326 };
327
328-/* is memory segment descriptor pointer ? */
329-#define ISMEMSDP(s) ((s->d_type) >= SDT_MEMRO && (s->d_type) <= SDT_MEMERAC)
330-
331-/* is 286 gate descriptor pointer ? */
332-#define IS286GDP(s) (((s->d_type) >= SDT_SYS286CGT \
333- && (s->d_type) < SDT_SYS286TGT))
334-
335-/* is 386 gate descriptor pointer ? */
336-#define IS386GDP(s) (((s->d_type) >= SDT_SYS386CGT \
337- && (s->d_type) < SDT_SYS386TGT))
338-
339-/* is gate descriptor pointer ? */
340-#define ISGDP(s) (IS286GDP(s) || IS386GDP(s))
341-
342-/* is segment descriptor pointer ? */
343-#define ISSDP(s) (ISMEMSDP(s) || !ISGDP(s))
344-
345-/* is system segment descriptor pointer ? */
346-#define ISSYSSDP(s) (!ISMEMSDP(s) && !ISGDP(s))
347-
348 /*
349 * Software definitions are in this convenient format,
350 * which are translated into inconvenient segment descriptors
351- * when needed to be used by the 386 hardware
352+ * when needed to be used by the x86 hardware.
353 */
354
355 struct soft_segment_descriptor {
356@@ -254,11 +234,11 @@ struct region_descriptor {
357 #ifndef LOCORE
358
359 #ifdef _KERNEL
360-extern struct user_segment_descriptor gdt[];
361 extern struct soft_segment_descriptor gdt_segs[];
362 extern struct gate_descriptor idt_arr[MAXCPU][NIDT];
363 extern struct region_descriptor r_idt_arr[];
364-extern struct mtx dt_lock;
365+extern struct region_descriptor r_gdt;
366+extern struct user_segment_descriptor gdt[NGDT * MAXCPU];
367
368 void lgdt(struct region_descriptor *rdp);
369 void sdtossd(struct user_segment_descriptor *sdp,
370diff --git a/sys/platform/pc64/apic/apic_vector.s b/sys/platform/pc64/apic/apic_vector.s
371index 60fea63f9..c7ef77e63 100644
372--- a/sys/platform/pc64/apic/apic_vector.s
373+++ b/sys/platform/pc64/apic/apic_vector.s
374@@ -34,8 +34,8 @@
375
376 #define MPLOCKED lock ;
377
378-#define APIC_PUSH_FRAME \
379- PUSH_FRAME ; /* 15 regs + space for 5 extras */ \
380+#define APIC_PUSH_FRAME_TFRIP \
381+ PUSH_FRAME_TFRIP ; /* 15 regs + space for 5 extras */ \
382 movq $0,TF_XFLAGS(%rsp) ; \
383 movq $0,TF_TRAPNO(%rsp) ; \
384 movq $0,TF_ADDR(%rsp) ; \
385@@ -48,8 +48,8 @@
386 * segment register being changed (e.g. by procfs), which is why syscalls
387 * have to use doreti.
388 */
389-#define APIC_POP_FRAME \
390- POP_FRAME ; \
391+#define APIC_POP_FRAME(lastinsn) \
392+ POP_FRAME(lastinsn) \
393
394 #define IOAPICADDR(irq_num) \
395 CNAME(ioapic_irqs) + IOAPIC_IRQI_SIZE * (irq_num) + IOAPIC_IRQI_ADDR
396@@ -118,7 +118,7 @@
397 .text ; \
398 SUPERALIGN_TEXT ; \
399 IDTVEC(ioapic_intr##irq_num) ; \
400- APIC_PUSH_FRAME ; \
401+ APIC_PUSH_FRAME_TFRIP ; \
402 FAKE_MCOUNT(TF_RIP(%rsp)) ; \
403 MASK_LEVEL_IRQ(irq_num) ; \
404 movq lapic, %rax ; \
405@@ -172,12 +172,11 @@ IDTVEC(ioapic_intr##irq_num) ; \
406 SUPERALIGN_TEXT
407 .globl Xspuriousint
408 Xspuriousint:
409- APIC_PUSH_FRAME
410+ APIC_PUSH_FRAME_TFRIP
411 /* No EOI cycle used here */
412 FAKE_MCOUNT(TF_RIP(%rsp))
413 MEXITCOUNT
414- APIC_POP_FRAME
415- jmp doreti_iret
416+ APIC_POP_FRAME(jmp doreti_iret)
417
418 /*
419 * Handle TLB shootdowns.
420@@ -188,7 +187,7 @@ Xspuriousint:
421 SUPERALIGN_TEXT
422 .globl Xinvltlb
423 Xinvltlb:
424- APIC_PUSH_FRAME
425+ APIC_PUSH_FRAME_TFRIP
426 movq lapic, %rax
427 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
428 FAKE_MCOUNT(TF_RIP(%rsp))
429@@ -213,7 +212,7 @@ Xinvltlb:
430 SUPERALIGN_TEXT
431 .globl Xsniff
432 Xsniff:
433- APIC_PUSH_FRAME
434+ APIC_PUSH_FRAME_TFRIP
435 movq lapic, %rax
436 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
437 FAKE_MCOUNT(TF_RIP(%rsp))
438@@ -223,8 +222,7 @@ Xsniff:
439 movq TF_RSP(%rsp),%rax
440 movq %rax,PCPU(sample_sp)
441 MEXITCOUNT
442- APIC_POP_FRAME
443- jmp doreti_iret
444+ APIC_POP_FRAME(jmp doreti_iret)
445
446 /*
447 * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
448@@ -240,7 +238,7 @@ Xsniff:
449 SUPERALIGN_TEXT
450 .globl Xcpustop
451 Xcpustop:
452- APIC_PUSH_FRAME
453+ APIC_PUSH_FRAME_TFRIP
454 movq lapic, %rax
455 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
456
457@@ -338,7 +336,7 @@ Xcpustop:
458 SUPERALIGN_TEXT
459 .globl Xipiq
460 Xipiq:
461- APIC_PUSH_FRAME
462+ APIC_PUSH_FRAME_TFRIP
463 movq lapic, %rax
464 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
465 FAKE_MCOUNT(TF_RIP(%rsp))
466@@ -364,14 +362,13 @@ Xipiq:
467 1:
468 orl $RQF_IPIQ,PCPU(reqflags)
469 MEXITCOUNT
470- APIC_POP_FRAME
471- jmp doreti_iret
472+ APIC_POP_FRAME(jmp doreti_iret)
473
474 .text
475 SUPERALIGN_TEXT
476 .globl Xtimer
477 Xtimer:
478- APIC_PUSH_FRAME
479+ APIC_PUSH_FRAME_TFRIP
480 movq lapic, %rax
481 movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
482 FAKE_MCOUNT(TF_RIP(%rsp))
483@@ -404,8 +401,7 @@ Xtimer:
484 1:
485 orl $RQF_TIMER,PCPU(reqflags)
486 MEXITCOUNT
487- APIC_POP_FRAME
488- jmp doreti_iret
489+ APIC_POP_FRAME(jmp doreti_iret)
490
491 MCOUNT_LABEL(bintr)
492 INTR_HANDLER(0)
493diff --git a/sys/platform/pc64/icu/icu_vector.s b/sys/platform/pc64/icu/icu_vector.s
494index f0a9d4e37..d2c8b3bc2 100644
495--- a/sys/platform/pc64/icu/icu_vector.s
496+++ b/sys/platform/pc64/icu/icu_vector.s
497@@ -85,7 +85,7 @@
498 * Macro helpers
499 */
500 #define ICU_PUSH_FRAME \
501- PUSH_FRAME ; /* 15 regs + space for 5 extras */ \
502+ PUSH_FRAME_TFRIP ; /* 15 regs + space for 5 extras */ \
503 movl $0,TF_XFLAGS(%rsp) ; \
504 movl $0,TF_TRAPNO(%rsp) ; \
505 movl $0,TF_ADDR(%rsp) ; \
506diff --git a/sys/platform/pc64/include/globaldata.h b/sys/platform/pc64/include/globaldata.h
507index 376a4348c..fcfdcb5c4 100644
508--- a/sys/platform/pc64/include/globaldata.h
509+++ b/sys/platform/pc64/include/globaldata.h
510@@ -68,7 +68,6 @@ struct mdglobaldata {
511 struct user_segment_descriptor gd_common_tssd;
512 struct user_segment_descriptor *gd_tss_gdt;
513 struct thread *gd_npxthread;
514- struct x86_64tss gd_common_tss;
515 union savefpu gd_savefpu; /* fast bcopy/zero temp fpu save area */
516 int gd_fpu_lock; /* fast bcopy/zero cpu lock */
517 int gd_xinvaltlb; /* reentrancy check invaltlb routine */
518@@ -80,18 +79,18 @@ struct mdglobaldata {
519 u_int gd_unused002;
520 u_int gd_unused003;
521 u_int gd_ss_eflags;
522- pt_entry_t *gd_cunused0;
523- pt_entry_t *gd_cunused1;
524- pt_entry_t *gd_cunused2;
525- pt_entry_t *gd_cunused3;
526+ long gd_lunused0;
527+ long gd_lunused1;
528+ long gd_lunused2;
529+ long gd_lunusde3;
530 caddr_t gd_aunused0;
531 caddr_t gd_aunused1;
532 caddr_t gd_aunused2;
533 struct pv_entry *gd_newpv;
534 u_int gd_acpi_id;
535 u_int gd_apic_id;
536- register_t gd_scratch_rsp;
537- register_t unused004;
538+ register_t gd_unused004;
539+ register_t gd_unused005;
540 register_t gd_user_fs; /* current user fs in MSR */
541 register_t gd_user_gs; /* current user gs in MSR */
542 cpumask_t gd_unused006;
543@@ -109,23 +108,50 @@ struct mdglobaldata {
544 * This is the upper (0xff800000) address space layout that is per-cpu.
545 * It is setup in locore.s and pmap.c for the BSP and in mp_machdep.c for
546 * each AP. genassym helps export this to the assembler code.
547+ *
548+ * Most of the major elements in struct privatespace must be
549+ * PAGE_SIZE aligned.
550 */
551 struct privatespace {
552- /* JG TODO: fix comments describing layout */
553- /* page 0 - data page */
554+ /*
555+ * page 0 - data page
556+ */
557 struct mdglobaldata mdglobaldata;
558 char __filler0[MDGLOBALDATA_PAD];
559
560- /* page 1..4 - CPAGE1,CPAGE2,CPAGE3,PPAGE1 (unused) */
561- char unused1[PAGE_SIZE];
562+ /*
563+ * page 1 - Unused (unmapped)
564+ */
565 char unused2[PAGE_SIZE];
566- char unused3[PAGE_SIZE];
567- char unused4[PAGE_SIZE];
568+
569+ /*
570+ * page 2 - Trampoline page. Put the trampoline and common_tss
571+ * in the same page to make them easier to isolate
572+ * from the rest of the kernel map. See x86_64/pmap.c
573+ *
574+ * rsp0 points into trampoline. Interrupts are always
575+ * disabled for this case but leave reserved1[]
576+ * reserved just in case.
577+ */
578+ char reserved1[PAGE_SIZE -
579+ sizeof(struct trampframe) -
580+ sizeof(uint64_t) -
581+ sizeof(struct x86_64tss)];
582+ struct trampframe trampoline;
583+ uint64_t reserved1b; /* 16-byte-align trampoline */
584+ struct x86_64tss common_tss;
585+
586+ /*
587+ * page 3, 4 - Double fault stack
588+ */
589+ char dblstack[PAGE_SIZE * 2];
590
591 /* page 5..4+UPAGES - idle stack (UPAGES pages) */
592 char idlestack[UPAGES * PAGE_SIZE];
593-};
594+} __packed;
595+
596 #define mdcpu ((struct mdglobaldata *)_get_mycpu())
597+#define pscpu ((struct privatespace *)_get_mycpu())
598
599 #endif
600
601diff --git a/sys/platform/pc64/include/pcb.h b/sys/platform/pc64/include/pcb.h
602index 3ae8c25cb..6264e32ca 100644
603--- a/sys/platform/pc64/include/pcb.h
604+++ b/sys/platform/pc64/include/pcb.h
605@@ -48,8 +48,9 @@
606 #include <machine/npx.h>
607
608 struct pcb {
609- register_t padxx[8];
610- register_t pcb_cr3;
611+ register_t padxx[7];
612+ register_t pcb_cr3_iso; /* isolated U (+minimal K) PML4e */
613+ register_t pcb_cr3; /* U+K PML4e */
614 register_t pcb_r15;
615 register_t pcb_r14;
616 register_t pcb_r13;
617@@ -83,10 +84,11 @@ struct pcb {
618 struct pcb_ext *pcb_ext; /* optional pcb extension */
619 };
620
621-#define PCB_DBREGS 0x02 /* process using debug registers */
622-#define PCB_FPUINITDONE 0x08 /* fpu state is initialized */
623-#define FP_SOFTFP 0x01 /* process using software fltng pnt emulator */
624-#define FP_VIRTFP 0x04 /* virtual kernel wants exception */
625+#define PCB_DBREGS 0x00000002 /* process using debug registers */
626+#define PCB_FPUINITDONE 0x00000008 /* fpu state is initialized */
627+#define PCB_ISOMMU 0x00000010 /* isolated mmu context active */
628+#define FP_SOFTFP 0x01 /* process using soft flt emulator */
629+#define FP_VIRTFP 0x04 /* vkernel wants exception */
630
631 #ifdef _KERNEL
632 void savectx(struct pcb *);
633diff --git a/sys/platform/pc64/include/pmap.h b/sys/platform/pc64/include/pmap.h
634index b423e8998..5d657a909 100644
635--- a/sys/platform/pc64/include/pmap.h
636+++ b/sys/platform/pc64/include/pmap.h
637@@ -75,9 +75,14 @@
638 ((unsigned long)(l1) << PAGE_SHIFT))
639
640 /*
641+ * NKPML4E is the number of PML4E slots used for KVM. Each slot represents
642+ * 512GB of KVM. A number between 1 and 128 may be specified. To support
643+ * the maximum machine configuration of 64TB we recommend around
644+ * 16 slots (8TB of KVM).
645+ *
646 * NOTE: We no longer hardwire NKPT, it is calculated in create_pagetables()
647 */
648-#define NKPML4E 1 /* number of kernel PML4 slots */
649+#define NKPML4E 16
650 /* NKPDPE defined in vmparam.h */
651
652 /*
653@@ -97,14 +102,11 @@
654 #define NUPTE_USER ((vm_pindex_t)NPTEPG * NPDEPG * NPDPEPG * NUPDP_USER)
655
656 /*
657- * Number of 512G dmap PML4 slots (max ~254 or so but don't go over 64,
658- * which gives us 32TB of ram). Because we cache free, empty pmaps the
659- * initialization overhead is minimal.
660- *
661- * It should be possible to bump this up to 255 (but not 256), which would
662- * be able to address a maximum of ~127TB of physical ram.
663+ * Number of 512G dmap PML4 slots. There are 512 slots of which 256 are
664+ * used by the kernel. Of those 256 we allow up to 128 to be used by the
665+ * DMAP (for 64TB of ram), leaving 128 for the kernel and other incidentals.
666 */
667-#define NDMPML4E 64
668+#define NDMPML4E 128
669
670 /*
671 * The *PML4I values control the layout of virtual memory. Each PML4
672@@ -112,9 +114,17 @@
673 */
674 #define PML4PML4I (NPML4EPG/2) /* Index of recursive pml4 mapping */
675
676-#define KPML4I (NPML4EPG-1) /* Top 512GB for KVM */
677+#define KPML4I (NPML4EPG-NKPML4E) /* Start of KVM */
678 #define DMPML4I (KPML4I-NDMPML4E) /* Next 512GBxN down for dmap */
679
680+/*
681+ * Make sure the kernel map and DMAP don't overflow the 256 PDP entries
682+ * we have available. Minus one for the PML4PML4I.
683+ */
684+#if NKPML4E + NDMPML4E >= 255
685+#error "NKPML4E or NDMPML4E is too large"
686+#endif
687+
688 /*
689 * The location of KERNBASE in the last PD of the kernel's KVM (KPML4I)
690 * space. Each PD represents 1GB. The kernel must be placed here
691@@ -128,7 +138,7 @@
692 * in the future or 16MB of space. Each PD represents 2MB so
693 * use NPDEPG-8 to place the per-CPU data.
694 */
695-#define MPPML4I KPML4I
696+#define MPPML4I (KPML4I + NKPML4E - 1)
697 #define MPPDPI KPDPI
698 #define MPPTDI (NPDEPG-8)
699
700@@ -271,23 +281,26 @@ RB_PROTOTYPE2(pv_entry_rb_tree, pv_entry, pv_entry,
701 #define PROTECTION_CODES_SIZE 8
702 #define PAT_INDEX_SIZE 8
703
704-#define PM_PLACEMARKS 16 /* 4 per level x 4 levels */
705-#define PM_PLACEMARKS_SHIFT 4 /* 1 << 4 == 16 */
706+#define PM_PLACEMARKS 64 /* 16 @ 4 zones */
707 #define PM_NOPLACEMARK ((vm_pindex_t)-1)
708 #define PM_PLACEMARK_WAKEUP ((vm_pindex_t)0x8000000000000000LLU)
709
710 struct pmap {
711 pml4_entry_t *pm_pml4; /* KVA of level 4 page table */
712+ pml4_entry_t *pm_pml4_iso; /* (isolated version) */
713 struct pv_entry *pm_pmlpv; /* PV entry for pml4 */
714+ struct pv_entry *pm_pmlpv_iso; /* (isolated version) */
715 TAILQ_ENTRY(pmap) pm_pmnode; /* list of pmaps */
716 RB_HEAD(pv_entry_rb_tree, pv_entry) pm_pvroot;
717 int pm_count; /* reference count */
718 cpulock_t pm_active_lock; /* interlock */
719 cpumask_t pm_active; /* active on cpus */
720 int pm_flags;
721+ uint32_t pm_softhold;
722 struct pmap_statistics pm_stats; /* pmap statistics */
723 struct spinlock pm_spin;
724- struct pv_entry *pm_pvhint; /* pv_entry lookup hint */
725+ struct pv_entry *pm_pvhint_pt; /* pv_entry lookup hint */
726+ struct pv_entry *pm_pvhint_pte; /* pv_entry lookup hint */
727 vm_pindex_t pm_placemarks[PM_PLACEMARKS];
728 long pm_invgen;
729 uint64_t pmap_bits[PG_BITS_SIZE];
730diff --git a/sys/platform/pc64/vmm/vmx.c b/sys/platform/pc64/vmm/vmx.c
731index 7eaeb10f0..ffa8558c6 100644
732--- a/sys/platform/pc64/vmm/vmx.c
733+++ b/sys/platform/pc64/vmm/vmx.c
734@@ -939,7 +939,7 @@ vmx_vminit(struct vmm_guest_options *options)
735 */
736 gd = mycpu;
737 ERROR_IF(vmwrite(VMCS_HOST_GS_BASE, (uint64_t)gd));
738- ERROR_IF(vmwrite(VMCS_HOST_TR_BASE, (uint64_t)&gd->gd_prvspace->mdglobaldata.gd_common_tss));
739+ ERROR_IF(vmwrite(VMCS_HOST_TR_BASE, (uint64_t)&gd->gd_prvspace->common_tss));
740
741 ERROR_IF(vmwrite(VMCS_HOST_GDTR_BASE, (uint64_t)&gdt[gd->gd_cpuid * NGDT]));
742 ERROR_IF(vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)r_idt_arr[gd->gd_cpuid].rd_base));
743@@ -1070,7 +1070,7 @@ vmx_handle_cpu_migration(void)
744
745 /* Host related registers */
746 ERROR_IF(vmwrite(VMCS_HOST_GS_BASE, (uint64_t) gd)); /* mycpu points to %gs:0 */
747- ERROR_IF(vmwrite(VMCS_HOST_TR_BASE, (uint64_t) &gd->gd_prvspace->mdglobaldata.gd_common_tss));
748+ ERROR_IF(vmwrite(VMCS_HOST_TR_BASE, (uint64_t) &gd->gd_prvspace->common_tss));
749
750 ERROR_IF(vmwrite(VMCS_HOST_GDTR_BASE, (uint64_t) &gdt[gd->gd_cpuid * NGDT]));
751 ERROR_IF(vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t) r_idt_arr[gd->gd_cpuid].rd_base));
752diff --git a/sys/platform/pc64/x86_64/exception.S b/sys/platform/pc64/x86_64/exception.S
753index 116b33786..a229cad09 100644
754--- a/sys/platform/pc64/x86_64/exception.S
755+++ b/sys/platform/pc64/x86_64/exception.S
756@@ -61,7 +61,6 @@
757 * us for the use of the swapgs instruction. We cannot be interrupted
758 * until the GS.base value is correct. For most traps, we automatically
759 * then enable interrupts if the interrupted context had them enabled.
760- * This is equivalent to the i386 port's use of SDT_SYS386TGT.
761 *
762 * The cpu will push a certain amount of state onto the kernel stack for
763 * the current process. See x86_64/include/frame.h.
764@@ -82,27 +81,27 @@ MCOUNT_LABEL(user)
765 MCOUNT_LABEL(btrap)
766
767 /*
768- * Interrupts are enabled for all traps, otherwise horrible livelocks
769- * can occur with the smp_invltlb and cpusync ode.
770+ * Interrupts must be disabled for all traps, otherwise horrible %gs
771+ * issues will occur.
772 */
773-#if 0
774-#define TRAP_NOEN(a) \
775- subq $TF_RIP,%rsp; \
776- movq $0,TF_XFLAGS(%rsp) ; \
777- movq $(a),TF_TRAPNO(%rsp) ; \
778- movq $0,TF_ADDR(%rsp) ; \
779- movq $0,TF_ERR(%rsp) ; \
780- jmp alltraps_noen
781-#endif
782
783 /* Regular traps; The cpu does not supply tf_err for these. */
784 #define TRAP(a) \
785- subq $TF_RIP,%rsp; \
786- movq $0,TF_XFLAGS(%rsp) ; \
787- movq $(a),TF_TRAPNO(%rsp) ; \
788- movq $0,TF_ADDR(%rsp) ; \
789- movq $0,TF_ERR(%rsp) ; \
790+ PUSH_FRAME_TFRIP ; \
791+ movq $0,TF_XFLAGS(%rsp) ; \
792+ movq $(a),TF_TRAPNO(%rsp) ; \
793+ movq $0,TF_ADDR(%rsp) ; \
794+ movq $0,TF_ERR(%rsp) ; \
795+ jmp alltraps
796+
797+/* This group of traps have tf_err already pushed by the cpu */
798+#define TRAP_ERR(a) \
799+ PUSH_FRAME_TFERR ; \
800+ movq $(a),TF_TRAPNO(%rsp) ; \
801+ movq $0,TF_ADDR(%rsp) ; \
802+ movq $0,TF_XFLAGS(%rsp) ; \
803 jmp alltraps
804+
805 IDTVEC(dbg)
806 TRAP(T_TRCTRAP)
807 IDTVEC(bpt)
808@@ -128,13 +127,6 @@ IDTVEC(fpu)
809 IDTVEC(xmm)
810 TRAP(T_XMMFLT)
811
812-/* This group of traps have tf_err already pushed by the cpu */
813-#define TRAP_ERR(a) \
814- subq $TF_ERR,%rsp; \
815- movq $(a),TF_TRAPNO(%rsp) ; \
816- movq $0,TF_ADDR(%rsp) ; \
817- movq $0,TF_XFLAGS(%rsp) ; \
818- jmp alltraps
819 IDTVEC(tss)
820 TRAP_ERR(T_TSSFLT)
821 IDTVEC(missing)
822@@ -147,21 +139,16 @@ IDTVEC(align)
823 /*
824 * alltraps entry point. Use swapgs if this is the first time in the
825 * kernel from userland. Reenable interrupts if they were enabled
826- * before the trap. This approximates SDT_SYS386TGT on the i386 port.
827+ * before the trap.
828+ *
829+ * WARNING! %gs not available until after our swapgs code
830 */
831-
832 SUPERALIGN_TEXT
833 .globl alltraps
834 .type alltraps,@function
835 alltraps:
836- /* Fixup %gs if coming from userland */
837- testb $SEL_RPL_MASK,TF_CS(%rsp)
838- jz alltraps_testi
839- swapgs
840-alltraps_testi:
841- testq $PSL_I,TF_RFLAGS(%rsp)
842- jz alltraps_pushregs
843- sti
844+
845+#if 0
846 alltraps_pushregs:
847 movq %rdi,TF_RDI(%rsp)
848 alltraps_pushregs_no_rdi:
849@@ -179,6 +166,8 @@ alltraps_pushregs_no_rdi:
850 movq %r13,TF_R13(%rsp)
851 movq %r14,TF_R14(%rsp)
852 movq %r15,TF_R15(%rsp)
853+#endif
854+ sti
855 FAKE_MCOUNT(TF_RIP(%rsp))
856 .globl calltrap
857 .type calltrap,@function
858@@ -189,67 +178,27 @@ calltrap:
859 MEXITCOUNT
860 jmp doreti /* Handle any pending ASTs */
861
862- /*
863- * alltraps_noen entry point. Unlike alltraps above, we want to
864- * leave the interrupts disabled. This corresponds to
865- * SDT_SYS386IGT on the i386 port.
866- */
867- SUPERALIGN_TEXT
868- .globl alltraps_noen
869- .type alltraps_noen,@function
870-alltraps_noen:
871- /* Fixup %gs if coming from userland */
872- testb $SEL_RPL_MASK,TF_CS(%rsp)
873- jz alltraps_pushregs
874- swapgs
875- jmp alltraps_pushregs
876-
877 IDTVEC(dblfault)
878- subq $TF_ERR,%rsp
879+ PUSH_FRAME_TFERR
880 movq $T_DOUBLEFLT,TF_TRAPNO(%rsp)
881 movq $0,TF_ADDR(%rsp)
882- movq $0,TF_ERR(%rsp)
883 movq $0,TF_XFLAGS(%rsp)
884- movq %rdi,TF_RDI(%rsp)
885- movq %rsi,TF_RSI(%rsp)
886- movq %rdx,TF_RDX(%rsp)
887- movq %rcx,TF_RCX(%rsp)
888- movq %r8,TF_R8(%rsp)
889- movq %r9,TF_R9(%rsp)
890- movq %rax,TF_RAX(%rsp)
891- movq %rbx,TF_RBX(%rsp)
892- movq %rbp,TF_RBP(%rsp)
893- movq %r10,TF_R10(%rsp)
894- movq %r11,TF_R11(%rsp)
895- movq %r12,TF_R12(%rsp)
896- movq %r13,TF_R13(%rsp)
897- movq %r14,TF_R14(%rsp)
898- movq %r15,TF_R15(%rsp)
899- testb $SEL_RPL_MASK,TF_CS(%rsp)
900- jz 1f
901- swapgs
902-1: movq %rsp, %rdi
903+
904 cld
905+ movq %rsp, %rdi
906 call dblfault_handler
907 2: hlt
908 jmp 2b
909
910+ /*
911+ * We need to save the contents of %cr2 before PUSH_FRAME* messes
912+ * with %cr3.
913+ */
914 IDTVEC(page)
915- subq $TF_ERR,%rsp
916+ PUSH_FRAME_TFERR_SAVECR2
917 movq $T_PAGEFLT,TF_TRAPNO(%rsp)
918- /* Fixup %gs if coming from userland */
919- testb $SEL_RPL_MASK,TF_CS(%rsp)
920- jz 1f
921- swapgs
922-1:
923- movq %rdi,TF_RDI(%rsp) /* free up a GP register */
924- movq %cr2,%rdi /* preserve %cr2 before .. */
925- movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */
926 movq $0,TF_XFLAGS(%rsp)
927- testq $PSL_I,TF_RFLAGS(%rsp)
928- jz alltraps_pushregs_no_rdi
929- sti
930- jmp alltraps_pushregs_no_rdi
931+ jmp alltraps
932
933 /*
934 * We have to special-case this one. If we get a trap in doreti() at
935@@ -258,48 +207,44 @@ IDTVEC(page)
936 * XXX linux has a trap handler for their equivalent of load_gs().
937 */
938 IDTVEC(prot)
939- subq $TF_ERR,%rsp
940+ pushq %r10
941+ leaq doreti_iret(%rip),%r10
942+ cmpq %r10,TF_RIP-TF_ERR+8(%rsp) /* +8 due to pushq */
943+ jne prot_normal
944+ testb $SEL_RPL_MASK,TF_CS-TF_ERR+8(%rsp) /* +8 due to pushq */
945+ jnz prot_normal
946+ swapgs /* doreti_iret fault from kernel mode */
947+prot_normal:
948+ popq %r10
949+ PUSH_FRAME_TFERR
950 movq $T_PROTFLT,TF_TRAPNO(%rsp)
951 movq $0,TF_ADDR(%rsp)
952 movq $0,TF_XFLAGS(%rsp)
953- movq %rdi,TF_RDI(%rsp) /* free up a GP register */
954-
955- /*
956- * Fixup %gs if coming from userland. Handle the special case where
957- * %fs faults in doreti at the iretq instruction itself.
958- */
959- leaq doreti_iret(%rip),%rdi
960- cmpq %rdi,TF_RIP(%rsp) /* special iretq fault case */
961- je 2f
962- testb $SEL_RPL_MASK,TF_CS(%rsp) /* check if from userland */
963- jz 1f
964-2:
965- swapgs
966-1:
967- testq $PSL_I,TF_RFLAGS(%rsp)
968- jz alltraps_pushregs_no_rdi
969- sti
970- jmp alltraps_pushregs_no_rdi
971+ jmp alltraps
972
973 /*
974 * Fast syscall entry point. We enter here with just our new %cs/%ss set,
975 * and the new privilige level. We are still running on the old user stack
976 * pointer. We have to juggle a few things around to find our stack etc.
977 * swapgs gives us access to our PCPU space only.
978+ *
979+ * We use GD_TRAMPOLINE+TR_R10
980 */
981 IDTVEC(fast_syscall)
982- swapgs
983- movq %rsp,PCPU(scratch_rsp)
984- movq PCPU(common_tss) + TSS_RSP0, %rsp
985+ swapgs /* get kernel %gs */
986+ movq %rsp,PCPU(trampoline)+TR_R10 /* save user %rsp */
987+ movq PCPU(common_tss)+TSS_RSP0,%rsp
988+ KMMUENTER_SYSCALL
989+ movq PCPU(trampoline)+TR_PCB_RSP,%rsp
990+
991 /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
992 subq $TF_SIZE,%rsp
993 /* defer TF_RSP till we have a spare register */
994 movq %r11,TF_RFLAGS(%rsp)
995 movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */
996- movq PCPU(scratch_rsp),%r11 /* %r11 already saved */
997+ movq PCPU(trampoline)+TR_R10,%r11 /* %r11 already saved */
998 movq %r11,TF_RSP(%rsp) /* user stack pointer */
999 orl $RQF_QUICKRET,PCPU(reqflags)
1000- sti
1001 movq $KUDSEL,TF_SS(%rsp)
1002 movq $KUCSEL,TF_CS(%rsp)
1003 movq $2,TF_ERR(%rsp)
1004@@ -318,6 +263,7 @@ IDTVEC(fast_syscall)
1005 movq %r13,TF_R13(%rsp) /* C preserved */
1006 movq %r14,TF_R14(%rsp) /* C preserved */
1007 movq %r15,TF_R15(%rsp) /* C preserved */
1008+ sti
1009 FAKE_MCOUNT(TF_RIP(%rsp))
1010 movq %rsp, %rdi
1011 call syscall2
1012@@ -338,8 +284,10 @@ IDTVEC(fast_syscall)
1013 movq TF_RFLAGS(%rsp),%r11
1014 movq TF_RIP(%rsp),%rcx
1015 movq TF_RSP(%rsp),%rsp
1016+ KMMUEXIT_SYSCALL
1017 swapgs
1018 sysretq
1019+
1020 /*
1021 * Normal slow / full iret
1022 */
1023@@ -378,64 +326,19 @@ IDTVEC(fast_syscall32)
1024 */
1025
1026 IDTVEC(nmi)
1027- subq $TF_RIP,%rsp
1028- movq $(T_NMI),TF_TRAPNO(%rsp)
1029+ PUSH_FRAME_TFRIP
1030+ movq $0,TF_XFLAGS(%rsp)
1031+ movq $T_NMI,TF_TRAPNO(%rsp)
1032 movq $0,TF_ADDR(%rsp)
1033 movq $0,TF_ERR(%rsp)
1034- movq $0,TF_XFLAGS(%rsp)
1035- movq %rdi,TF_RDI(%rsp)
1036- movq %rsi,TF_RSI(%rsp)
1037- movq %rdx,TF_RDX(%rsp)
1038- movq %rcx,TF_RCX(%rsp)
1039- movq %r8,TF_R8(%rsp)
1040- movq %r9,TF_R9(%rsp)
1041- movq %rax,TF_RAX(%rsp)
1042- movq %rbx,TF_RBX(%rsp)
1043- movq %rbp,TF_RBP(%rsp)
1044- movq %r10,TF_R10(%rsp)
1045- movq %r11,TF_R11(%rsp)
1046- movq %r12,TF_R12(%rsp)
1047- movq %r13,TF_R13(%rsp)
1048- movq %r14,TF_R14(%rsp)
1049- movq %r15,TF_R15(%rsp)
1050- xorl %ebx,%ebx
1051- testb $SEL_RPL_MASK,TF_CS(%rsp)
1052- jnz nmi_needswapgs /* we came from userland */
1053- movl $MSR_GSBASE,%ecx
1054- rdmsr
1055- cmpl $VM_MAX_USER_ADDRESS >> 32,%edx
1056- jae nmi_calltrap /* GS.base holds a kernel VA */
1057-nmi_needswapgs:
1058- incl %ebx
1059- swapgs
1060-/* Note: this label is also used by ddb and gdb: */
1061-nmi_calltrap:
1062+
1063 FAKE_MCOUNT(TF_RIP(%rsp))
1064 cld
1065 movq %rsp, %rdi
1066 call trap
1067 MEXITCOUNT
1068- testl %ebx,%ebx
1069- jz nmi_restoreregs
1070- swapgs
1071-nmi_restoreregs:
1072- movq TF_RDI(%rsp),%rdi
1073- movq TF_RSI(%rsp),%rsi
1074- movq TF_RDX(%rsp),%rdx
1075- movq TF_RCX(%rsp),%rcx
1076- movq TF_R8(%rsp),%r8
1077- movq TF_R9(%rsp),%r9
1078- movq TF_RAX(%rsp),%rax
1079- movq TF_RBX(%rsp),%rbx
1080- movq TF_RBP(%rsp),%rbp
1081- movq TF_R10(%rsp),%r10
1082- movq TF_R11(%rsp),%r11
1083- movq TF_R12(%rsp),%r12
1084- movq TF_R13(%rsp),%r13
1085- movq TF_R14(%rsp),%r14
1086- movq TF_R15(%rsp),%r15
1087- addq $TF_RIP,%rsp
1088- iretq
1089+
1090+ POP_FRAME(jmp doreti_iret)
1091
1092 /*
1093 * This function is what cpu_heavy_restore jumps to after a new process
1094diff --git a/sys/platform/pc64/x86_64/genassym.c b/sys/platform/pc64/x86_64/genassym.c
1095index 0ae3f04d0..8f29ec5a4 100644
1096--- a/sys/platform/pc64/x86_64/genassym.c
1097+++ b/sys/platform/pc64/x86_64/genassym.c
1098@@ -115,6 +115,7 @@ ASSYM(GD_CPUMASK_SIMPLE, offsetof(struct mdglobaldata, mi.gd_cpumask_simple));
1099 ASSYM(GD_CPUMASK_OFFSET, offsetof(struct mdglobaldata, mi.gd_cpumask_offset));
1100 ASSYM(GD_IRESERVED, offsetof(struct mdglobaldata, mi.gd_ireserved[0]));
1101
1102+ASSYM(PCB_CR3_ISO, offsetof(struct pcb, pcb_cr3_iso));
1103 ASSYM(PCB_CR3, offsetof(struct pcb, pcb_cr3));
1104 ASSYM(PCB_R15, offsetof(struct pcb, pcb_r15));
1105 ASSYM(PCB_R14, offsetof(struct pcb, pcb_r14));
1106@@ -133,7 +134,10 @@ ASSYM(PCB_DR2, offsetof(struct pcb, pcb_dr2));
1107 ASSYM(PCB_DR3, offsetof(struct pcb, pcb_dr3));
1108 ASSYM(PCB_DR6, offsetof(struct pcb, pcb_dr6));
1109 ASSYM(PCB_DR7, offsetof(struct pcb, pcb_dr7));
1110+
1111 ASSYM(PCB_DBREGS, PCB_DBREGS);
1112+ASSYM(PCB_ISOMMU, PCB_ISOMMU);
1113+
1114 ASSYM(PCB_EXT, offsetof(struct pcb, pcb_ext));
1115 ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags));
1116 ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault));
1117@@ -200,14 +204,28 @@ ASSYM(FIRST_SOFTINT, FIRST_SOFTINT);
1118 ASSYM(MDGLOBALDATA_BASEALLOC_PAGES, MDGLOBALDATA_BASEALLOC_PAGES);
1119
1120 ASSYM(GD_PRIVATE_TSS, offsetof(struct mdglobaldata, gd_private_tss));
1121-ASSYM(GD_SCRATCH_RSP, offsetof(struct mdglobaldata, gd_scratch_rsp));
1122+ASSYM(GD_COMMON_TSS, offsetof(struct privatespace, common_tss));
1123+ASSYM(GD_TRAMPOLINE, offsetof(struct privatespace, trampoline));
1124 ASSYM(GD_USER_FS, offsetof(struct mdglobaldata, gd_user_fs));
1125 ASSYM(GD_USER_GS, offsetof(struct mdglobaldata, gd_user_gs));
1126 ASSYM(GD_INTR_NESTING_LEVEL, offsetof(struct mdglobaldata, mi.gd_intr_nesting_level));
1127
1128+ASSYM(TR_CR2, offsetof(struct trampframe, tr_cr2));
1129+ASSYM(TR_R10, offsetof(struct trampframe, tr_r10));
1130+ASSYM(TR_R11, offsetof(struct trampframe, tr_r11));
1131+ASSYM(TR_ERR, offsetof(struct trampframe, tr_err));
1132+ASSYM(TR_RIP, offsetof(struct trampframe, tr_rip));
1133+ASSYM(TR_CS, offsetof(struct trampframe, tr_cs));
1134+ASSYM(TR_RFLAGS, offsetof(struct trampframe, tr_rflags));
1135+ASSYM(TR_RSP, offsetof(struct trampframe, tr_rsp));
1136+ASSYM(TR_SS, offsetof(struct trampframe, tr_ss));
1137+ASSYM(TR_PCB_RSP, offsetof(struct trampframe, tr_pcb_rsp));
1138+ASSYM(TR_PCB_FLAGS, offsetof(struct trampframe, tr_pcb_flags));
1139+ASSYM(TR_PCB_CR3_ISO, offsetof(struct trampframe, tr_pcb_cr3_iso));
1140+ASSYM(TR_PCB_CR3, offsetof(struct trampframe, tr_pcb_cr3));
1141+
1142 ASSYM(GD_IPENDING, offsetof(struct mdglobaldata, gd_ipending));
1143 ASSYM(GD_SPENDING, offsetof(struct mdglobaldata, gd_spending));
1144-ASSYM(GD_COMMON_TSS, offsetof(struct mdglobaldata, gd_common_tss));
1145 ASSYM(GD_COMMON_TSSD, offsetof(struct mdglobaldata, gd_common_tssd));
1146 ASSYM(GD_TSS_GDT, offsetof(struct mdglobaldata, gd_tss_gdt));
1147 ASSYM(GD_NPXTHREAD, offsetof(struct mdglobaldata, gd_npxthread));
1148diff --git a/sys/platform/pc64/x86_64/global.s b/sys/platform/pc64/x86_64/global.s
1149index ce612a549..d1f40980d 100644
1150--- a/sys/platform/pc64/x86_64/global.s
1151+++ b/sys/platform/pc64/x86_64/global.s
1152@@ -49,7 +49,9 @@
1153 * Define layout of the global data. On SMP this lives in
1154 * the per-cpu address space, otherwise it's in the data segment.
1155 */
1156+ .globl gd_trampoline
1157 .globl gd_curthread, gd_npxthread, gd_reqflags, gd_common_tss
1158+ .set gd_trampoline,globaldata + GD_TRAMPOLINE
1159 .set gd_curthread,globaldata + GD_CURTHREAD
1160 .set gd_npxthread,globaldata + GD_NPXTHREAD
1161 .set gd_reqflags,globaldata + GD_REQFLAGS
1162@@ -77,7 +79,6 @@
1163 .globl gd_ss_eflags, gd_intr_nesting_level
1164 .globl gd_spending, gd_ipending
1165 .globl gd_cnt, gd_private_tss
1166- .globl gd_scratch_rsp
1167 .globl gd_user_fs, gd_user_gs
1168 .globl gd_sample_pc
1169 .globl gd_sample_sp
1170@@ -94,7 +95,6 @@
1171 .set gd_ipending,globaldata + GD_IPENDING
1172 .set gd_spending,globaldata + GD_SPENDING
1173 .set gd_cnt,globaldata + GD_CNT
1174- .set gd_scratch_rsp,globaldata + GD_SCRATCH_RSP
1175 .set gd_user_fs,globaldata + GD_USER_FS
1176 .set gd_user_gs,globaldata + GD_USER_GS
1177 .set gd_sample_pc,globaldata + GD_SAMPLE_PC
1178diff --git a/sys/platform/pc64/x86_64/ipl.s b/sys/platform/pc64/x86_64/ipl.s
1179index a6d609333..cd32c272a 100644
1180--- a/sys/platform/pc64/x86_64/ipl.s
1181+++ b/sys/platform/pc64/x86_64/ipl.s
1182@@ -206,7 +206,7 @@ doreti_next:
1183 .globl doreti_iret
1184 .globl doreti_syscall_ret
1185 doreti_syscall_ret:
1186- POP_FRAME /* registers and %gs (+cli) */
1187+ POP_FRAME() /* registers and %gs (+cli) */
1188 /* WARNING: special global doreti_iret is also used by exception.S */
1189 doreti_iret:
1190 iretq
1191diff --git a/sys/platform/pc64/x86_64/machdep.c b/sys/platform/pc64/x86_64/machdep.c
1192index ff0255685..f6fa619a1 100644
1193--- a/sys/platform/pc64/x86_64/machdep.c
1194+++ b/sys/platform/pc64/x86_64/machdep.c
1195@@ -2,7 +2,7 @@
1196 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
1197 * Copyright (c) 1992 Terrence R. Lambert.
1198 * Copyright (c) 2003 Peter Wemm.
1199- * Copyright (c) 2008 The DragonFly Project.
1200+ * Copyright (c) 2008-2017 The DragonFly Project.
1201 * All rights reserved.
1202 *
1203 * This code is derived from software contributed to Berkeley by
1204@@ -253,41 +253,50 @@ int imcr_present = 0;
1205 int naps = 0; /* # of Applications processors */
1206
1207 u_int base_memory;
1208-struct mtx dt_lock; /* lock for GDT and LDT */
1209
1210 static int
1211 sysctl_hw_physmem(SYSCTL_HANDLER_ARGS)
1212 {
1213 u_long pmem = ctob(physmem);
1214+ int error;
1215+
1216+ error = sysctl_handle_long(oidp, &pmem, 0, req);
1217
1218- int error = sysctl_handle_long(oidp, &pmem, 0, req);
1219 return (error);
1220 }
1221
1222 SYSCTL_PROC(_hw, HW_PHYSMEM, physmem, CTLTYPE_ULONG|CTLFLAG_RD,
1223- 0, 0, sysctl_hw_physmem, "LU", "Total system memory in bytes (number of pages * page size)");
1224+ 0, 0, sysctl_hw_physmem, "LU",
1225+ "Total system memory in bytes (number of pages * page size)");
1226
1227 static int
1228 sysctl_hw_usermem(SYSCTL_HANDLER_ARGS)
1229 {
1230- int error = sysctl_handle_int(oidp, 0,
1231- ctob(physmem - vmstats.v_wire_count), req);
1232+ u_long usermem = ctob(physmem - vmstats.v_wire_count);
1233+ int error;
1234+
1235+ error = sysctl_handle_long(oidp, &usermem, 0, req);
1236+
1237 return (error);
1238 }
1239
1240-SYSCTL_PROC(_hw, HW_USERMEM, usermem, CTLTYPE_INT|CTLFLAG_RD,
1241- 0, 0, sysctl_hw_usermem, "IU", "");
1242+SYSCTL_PROC(_hw, HW_USERMEM, usermem, CTLTYPE_ULONG|CTLFLAG_RD,
1243+ 0, 0, sysctl_hw_usermem, "LU", "");
1244
1245 static int
1246 sysctl_hw_availpages(SYSCTL_HANDLER_ARGS)
1247 {
1248- int error = sysctl_handle_int(oidp, 0,
1249- x86_64_btop(avail_end - avail_start), req);
1250+ int error;
1251+ u_long availpages;
1252+
1253+ availpages = x86_64_btop(avail_end - avail_start);
1254+ error = sysctl_handle_long(oidp, &availpages, 0, req);
1255+
1256 return (error);
1257 }
1258
1259-SYSCTL_PROC(_hw, OID_AUTO, availpages, CTLTYPE_INT|CTLFLAG_RD,
1260- 0, 0, sysctl_hw_availpages, "I", "");
1261+SYSCTL_PROC(_hw, OID_AUTO, availpages, CTLTYPE_ULONG|CTLFLAG_RD,
1262+ 0, 0, sysctl_hw_availpages, "LU", "");
1263
1264 vm_paddr_t Maxmem;
1265 vm_paddr_t Realmem;
1266@@ -418,15 +427,21 @@ again:
1267 }
1268
1269 /*
1270- * Do not allow the sizeof(struct buf) * nbuf to exceed half of
1271+ * Do not allow the sizeof(struct buf) * nbuf to exceed 1/4 of
1272 * the valloc space which is just the virtual_end - virtual_start
1273- * section. We use valloc() to allocate the buf header array.
1274+ * section. This is typically ~2GB regardless of the amount of
1275+ * memory, so we use 500MB as a metric.
1276+ *
1277+ * This is because we use valloc() to allocate the buf header array.
1278+ *
1279+ * NOTE: buffer space in bytes is limited by vfs.*bufspace sysctls.
1280 */
1281- if (nbuf > (virtual_end - virtual_start) / sizeof(struct buf) / 2) {
1282+ if (nbuf > (virtual_end - virtual_start) / sizeof(struct buf) / 4) {
1283 nbuf = (virtual_end - virtual_start) /
1284 sizeof(struct buf) / 2;
1285- kprintf("Warning: nbufs capped at %ld due to valloc "
1286- "considerations\n", nbuf);
1287+ kprintf("Warning: nbufs capped at %ld due to "
1288+ "valloc considerations\n",
1289+ nbuf);
1290 }
1291
1292 nswbuf_mem = lmax(lmin(nbuf / 32, 512), 8);
1293@@ -850,7 +865,7 @@ sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code)
1294 regs->tf_rip -= SZSIGCODE_EXTRA_BYTES;
1295
1296 /*
1297- * i386 abi specifies that the direction flag must be cleared
1298+ * x86 abi specifies that the direction flag must be cleared
1299 * on function entry
1300 */
1301 regs->tf_rflags &= ~(PSL_T | PSL_D);
1302@@ -991,7 +1006,7 @@ sys_sigreturn(struct sigreturn_args *uap)
1303 */
1304 if (!EFL_SECURE(rflags & ~PSL_RF, regs->tf_rflags & ~PSL_RF)) {
1305 kprintf("sigreturn: rflags = 0x%lx\n", (long)rflags);
1306- return(EINVAL);
1307+ return(EINVAL);
1308 }
1309
1310 /*
1311@@ -1054,7 +1069,7 @@ cpu_halt(void)
1312 *
1313 * The main loop is entered with a critical section held, we must release
1314 * the critical section before doing anything else. lwkt_switch() will
1315- * check for pending interrupts due to entering and exiting its own
1316+ * check for pending interrupts due to entering and exiting its own
1317 * critical section.
1318 *
1319 * NOTE: On an SMP system we rely on a scheduler IPI to wake a HLTed cpu up.
1320@@ -1306,9 +1321,8 @@ exec_setregs(u_long entry, u_long stack, u_long ps_strings)
1321 struct pcb *pcb = td->td_pcb;
1322 struct trapframe *regs = lp->lwp_md.md_regs;
1323
1324- /* was i386_user_cleanup() in NetBSD */
1325 user_ldt_free(pcb);
1326-
1327+
1328 clear_quickret();
1329 bzero((char *)regs, sizeof(struct trapframe));
1330 regs->tf_rip = entry;
1331@@ -1360,7 +1374,7 @@ exec_setregs(u_long entry, u_long stack, u_long ps_strings)
1332
1333 /*
1334 * NOTE: The MSR values must be correct so we can return to
1335- * userland. gd_user_fs/gs must be correct so the switch
1336+ * userland. gd_user_fs/gs must be correct so the switch
1337 * code knows what the current MSR values are.
1338 */
1339 pcb->pcb_fsbase = 0; /* Values loaded from PCB on switch */
1340@@ -1411,7 +1425,7 @@ SYSCTL_INT(_machdep, CPU_DISRTCSET, disable_rtc_set,
1341 CTLFLAG_RW, &disable_rtc_set, 0, "");
1342
1343 #if 0 /* JG */
1344-SYSCTL_STRUCT(_machdep, CPU_BOOTINFO, bootinfo,
1345+SYSCTL_STRUCT(_machdep, CPU_BOOTINFO, bootinfo,
1346 CTLFLAG_RD, &bootinfo, bootinfo, "");
1347 #endif
1348
1349@@ -1439,7 +1453,7 @@ SYSCTL_PROC(_machdep, OID_AUTO, efi_map, CTLTYPE_OPAQUE|CTLFLAG_RD, NULL, 0,
1350 efi_map_sysctl_handler, "S,efi_map_header", "Raw EFI Memory Map");
1351
1352 /*
1353- * Initialize 386 and configure to run kernel
1354+ * Initialize x86 and configure to run kernel
1355 */
1356
1357 /*
1358@@ -1473,7 +1487,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1359 0, /* segment descriptor present */
1360 0, /* long */
1361 0, /* default 32 vs 16 bit size */
1362- 0 /* limit granularity (byte/page units)*/ },
1363+ 0 /* limit granularity (byte/page units)*/ },
1364 /* GCODE_SEL 1 Code Descriptor for kernel */
1365 { 0x0, /* segment base address */
1366 0xfffff, /* length - all address space */
1367@@ -1482,7 +1496,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1368 1, /* segment descriptor present */
1369 1, /* long */
1370 0, /* default 32 vs 16 bit size */
1371- 1 /* limit granularity (byte/page units)*/ },
1372+ 1 /* limit granularity (byte/page units)*/ },
1373 /* GDATA_SEL 2 Data Descriptor for kernel */
1374 { 0x0, /* segment base address */
1375 0xfffff, /* length - all address space */
1376@@ -1491,7 +1505,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1377 1, /* segment descriptor present */
1378 1, /* long */
1379 0, /* default 32 vs 16 bit size */
1380- 1 /* limit granularity (byte/page units)*/ },
1381+ 1 /* limit granularity (byte/page units)*/ },
1382 /* GUCODE32_SEL 3 32 bit Code Descriptor for user */
1383 { 0x0, /* segment base address */
1384 0xfffff, /* length - all address space */
1385@@ -1500,7 +1514,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1386 1, /* segment descriptor present */
1387 0, /* long */
1388 1, /* default 32 vs 16 bit size */
1389- 1 /* limit granularity (byte/page units)*/ },
1390+ 1 /* limit granularity (byte/page units)*/ },
1391 /* GUDATA_SEL 4 32/64 bit Data Descriptor for user */
1392 { 0x0, /* segment base address */
1393 0xfffff, /* length - all address space */
1394@@ -1509,7 +1523,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1395 1, /* segment descriptor present */
1396 0, /* long */
1397 1, /* default 32 vs 16 bit size */
1398- 1 /* limit granularity (byte/page units)*/ },
1399+ 1 /* limit granularity (byte/page units)*/ },
1400 /* GUCODE_SEL 5 64 bit Code Descriptor for user */
1401 { 0x0, /* segment base address */
1402 0xfffff, /* length - all address space */
1403@@ -1518,7 +1532,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1404 1, /* segment descriptor present */
1405 1, /* long */
1406 0, /* default 32 vs 16 bit size */
1407- 1 /* limit granularity (byte/page units)*/ },
1408+ 1 /* limit granularity (byte/page units)*/ },
1409 /* GPROC0_SEL 6 Proc 0 Tss Descriptor */
1410 {
1411 0x0, /* segment base address */
1412@@ -1528,7 +1542,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1413 1, /* segment descriptor present */
1414 0, /* long */
1415 0, /* unused - default 32 vs 16 bit size */
1416- 0 /* limit granularity (byte/page units)*/ },
1417+ 0 /* limit granularity (byte/page units)*/ },
1418 /* Actually, the TSS is a system descriptor which is double size */
1419 { 0x0, /* segment base address */
1420 0x0, /* length */
1421@@ -1537,7 +1551,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1422 0, /* segment descriptor present */
1423 0, /* long */
1424 0, /* default 32 vs 16 bit size */
1425- 0 /* limit granularity (byte/page units)*/ },
1426+ 0 /* limit granularity (byte/page units)*/ },
1427 /* GUGS32_SEL 8 32 bit GS Descriptor for user */
1428 { 0x0, /* segment base address */
1429 0xfffff, /* length - all address space */
1430@@ -1546,7 +1560,7 @@ struct soft_segment_descriptor gdt_segs[] = {
1431 1, /* segment descriptor present */
1432 0, /* long */
1433 1, /* default 32 vs 16 bit size */
1434- 1 /* limit granularity (byte/page units)*/ },
1435+ 1 /* limit granularity (byte/page units)*/ },
1436 };
1437
1438 void
1439@@ -1843,11 +1857,27 @@ efi_fb_init_vaddr(int direct_map)
1440 }
1441 }
1442
1443+static u_int
1444+efifb_color_depth(struct efi_fb *efifb)
1445+{
1446+ uint32_t mask;
1447+ u_int depth;
1448+
1449+ mask = efifb->fb_mask_red | efifb->fb_mask_green |
1450+ efifb->fb_mask_blue | efifb->fb_mask_reserved;
1451+ if (mask == 0)
1452+ return (0);
1453+ for (depth = 1; mask != 1; depth++)
1454+ mask >>= 1;
1455+ return (depth);
1456+}
1457+
1458 int
1459 probe_efi_fb(int early)
1460 {
1461 struct efi_fb *efifb;
1462 caddr_t kmdp;
1463+ u_int depth;
1464
1465 if (have_efi_framebuffer) {
1466 if (!early &&
1467@@ -1865,13 +1895,21 @@ probe_efi_fb(int early)
1468 if (efifb == NULL)
1469 return 1;
1470
1471+ depth = efifb_color_depth(efifb);
1472+ /*
1473+ * Our bootloader should already notice, when we won't be able to
1474+ * use the UEFI framebuffer.
1475+ */
1476+ if (depth != 24 && depth != 32)
1477+ return 1;
1478+
1479 have_efi_framebuffer = 1;
1480
1481 efi_fb_info.is_vga_boot_display = 1;
1482 efi_fb_info.width = efifb->fb_width;
1483 efi_fb_info.height = efifb->fb_height;
1484- efi_fb_info.stride = efifb->fb_stride * 4;
1485- efi_fb_info.depth = 32;
1486+ efi_fb_info.depth = depth;
1487+ efi_fb_info.stride = efifb->fb_stride * (depth / 8);
1488 efi_fb_info.paddr = efifb->fb_addr;
1489 if (early) {
1490 efi_fb_info.vaddr = 0;
1491@@ -1999,6 +2037,9 @@ getmemsize(caddr_t kmdp, u_int64_t first)
1492
1493 /*
1494 * Align anything else used in the validation loop.
1495+ *
1496+ * Also make sure that our 2MB kernel text+data+bss mappings
1497+ * do not overlap potentially allocatable space.
1498 */
1499 first = (first + PHYSMAP_ALIGN_MASK) & ~PHYSMAP_ALIGN_MASK;
1500
1501@@ -2024,10 +2065,14 @@ getmemsize(caddr_t kmdp, u_int64_t first)
1502 * Validate the physical memory. The physical memory segments
1503 * have already been aligned to PHYSMAP_ALIGN which is a multiple
1504 * of PAGE_SIZE.
1505+ *
1506+ * We no longer perform an exhaustive memory test. Instead we
1507+ * simply test the first and last word in each physmap[]
1508+ * segment.
1509 */
1510 for (i = 0; i <= physmap_idx; i += 2) {
1511 vm_paddr_t end;
1512- vm_paddr_t incr = PHYSMAP_ALIGN;
1513+ vm_paddr_t incr;
1514
1515 end = physmap[i + 1];
1516
1517@@ -2036,43 +2081,58 @@ getmemsize(caddr_t kmdp, u_int64_t first)
1518 volatile uint64_t *ptr = (uint64_t *)CADDR1;
1519 uint64_t tmp;
1520
1521- incr = PHYSMAP_ALIGN;
1522 full = FALSE;
1523
1524 /*
1525- * block out kernel memory as not available.
1526+ * Calculate incr. Just test the first and
1527+ * last page in each physmap[] segment.
1528 */
1529- if (pa >= 0x200000 && pa < first)
1530- goto do_dump_avail;
1531+ if (pa == end - PAGE_SIZE)
1532+ incr = PAGE_SIZE;
1533+ else
1534+ incr = end - pa - PAGE_SIZE;
1535
1536 /*
1537- * block out dcons buffer
1538+ * Make sure we don't skip blacked out areas.
1539 */
1540- if (dcons_addr > 0
1541- && pa >= trunc_page(dcons_addr)
1542- && pa < dcons_addr + dcons_size) {
1543- goto do_dump_avail;
1544+ if (pa < 0x200000 && 0x200000 < end) {
1545+ incr = 0x200000 - pa;
1546+ }
1547+ if (dcons_addr > 0 &&
1548+ pa < dcons_addr &&
1549+ dcons_addr < end) {
1550+ incr = dcons_addr - pa;
1551 }
1552
1553- page_bad = FALSE;
1554+ /*
1555+ * Block out kernel memory as not available.
1556+ */
1557+ if (pa >= 0x200000 && pa < first) {
1558+ incr = first - pa;
1559+ if (pa + incr > end)
1560+ incr = end - pa;
1561+ goto do_dump_avail;
1562+ }
1563
1564 /*
1565- * Always test the first and last block supplied in
1566- * the map entry, but it just takes too long to run
1567- * the test these days and we already have to skip
1568- * pages. Handwave it on PHYSMAP_HANDWAVE boundaries.
1569+ * Block out the dcons buffer if it exists.
1570 */
1571- if (pa != physmap[i]) {
1572- vm_paddr_t bytes = end - pa;
1573- if ((pa & PHYSMAP_HANDWAVE_MASK) == 0 &&
1574- bytes >= PHYSMAP_HANDWAVE + PHYSMAP_ALIGN) {
1575- incr = PHYSMAP_HANDWAVE;
1576- goto handwaved;
1577- }
1578+ if (dcons_addr > 0 &&
1579+ pa >= trunc_page(dcons_addr) &&
1580+ pa < dcons_addr + dcons_size) {
1581+ incr = dcons_addr + dcons_size - pa;
1582+ incr = (incr + PAGE_MASK) &
1583+ ~(vm_paddr_t)PAGE_MASK;
1584+ if (pa + incr > end)
1585+ incr = end - pa;
1586+ goto do_dump_avail;
1587 }
1588
1589+ page_bad = FALSE;
1590+
1591 /*
1592- * map page into kernel: valid, read/write,non-cacheable
1593+ * Map the page non-cacheable for the memory
1594+ * test.
1595 */
1596 *pte = pa |
1597 kernel_pmap.pmap_bits[PG_V_IDX] |
1598@@ -2081,7 +2141,11 @@ getmemsize(caddr_t kmdp, u_int64_t first)
1599 cpu_invlpg(__DEVOLATILE(void *, ptr));
1600 cpu_mfence();
1601
1602+ /*
1603+ * Save original value for restoration later.
1604+ */
1605 tmp = *ptr;
1606+
1607 /*
1608 * Test for alternating 1's and 0's
1609 */
1610@@ -2110,32 +2174,42 @@ getmemsize(caddr_t kmdp, u_int64_t first)
1611 cpu_mfence();
1612 if (*ptr != 0x0)
1613 page_bad = TRUE;
1614+
1615 /*
1616 * Restore original value.
1617 */
1618 *ptr = tmp;
1619-handwaved:
1620
1621 /*
1622 * Adjust array of valid/good pages.
1623 */
1624- if (page_bad == TRUE)
1625+ if (page_bad == TRUE) {
1626+ incr = PAGE_SIZE;
1627 continue;
1628+ }
1629
1630 /*
1631- * If this good page is a continuation of the
1632- * previous set of good pages, then just increase
1633- * the end pointer. Otherwise start a new chunk.
1634- * Note that "end" points one higher than end,
1635- * making the range >= start and < end.
1636- * If we're also doing a speculative memory
1637- * test and we at or past the end, bump up Maxmem
1638- * so that we keep going. The first bad page
1639- * will terminate the loop.
1640+ * Collapse page address into phys_avail[]. Do a
1641+ * continuation of the current phys_avail[] index
1642+ * when possible.
1643 */
1644 if (phys_avail[pa_indx].phys_end == pa) {
1645+ /*
1646+ * Continuation
1647+ */
1648 phys_avail[pa_indx].phys_end += incr;
1649+ } else if (phys_avail[pa_indx].phys_beg ==
1650+ phys_avail[pa_indx].phys_end) {
1651+ /*
1652+ * Current phys_avail is completely empty,
1653+ * reuse the index.
1654+ */
1655+ phys_avail[pa_indx].phys_beg = pa;
1656+ phys_avail[pa_indx].phys_end = pa + incr;
1657 } else {
1658+ /*
1659+ * Allocate next phys_avail index.
1660+ */
1661 ++pa_indx;
1662 if (pa_indx == PHYS_AVAIL_ARRAY_END) {
1663 kprintf(
1664@@ -2148,6 +2222,10 @@ handwaved:
1665 phys_avail[pa_indx].phys_end = pa + incr;
1666 }
1667 physmem += incr / PAGE_SIZE;
1668+
1669+ /*
1670+ * pa available for dumping
1671+ */
1672 do_dump_avail:
1673 if (dump_avail[da_indx].phys_end == pa) {
1674 dump_avail[da_indx].phys_end += incr;
1675@@ -2237,13 +2315,16 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
1676 int metadata_missing, off;
1677 #endif
1678 struct mdglobaldata *gd;
1679+ struct privatespace *ps;
1680 u_int64_t msr;
1681
1682 /*
1683 * Prevent lowering of the ipl if we call tsleep() early.
1684 */
1685 gd = &CPU_prvspace[0]->mdglobaldata;
1686+ ps = (struct privatespace *)gd;
1687 bzero(gd, sizeof(*gd));
1688+ bzero(&ps->common_tss, sizeof(ps->common_tss));
1689
1690 /*
1691 * Note: on both UP and SMP curthread must be set non-NULL
1692@@ -2290,11 +2371,9 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
1693 MachIntrABI = MachIntrABI_ICU;
1694
1695 /*
1696- * start with one cpu. Note: with one cpu, ncpus2_shift, ncpus2_mask,
1697- * and ncpus_fit_mask remain 0.
1698+ * start with one cpu. Note: with one cpu, ncpus_fit_mask remain 0.
1699 */
1700 ncpus = 1;
1701- ncpus2 = 1;
1702 ncpus_fit = 1;
1703 /* Init basic tunables, hz etc */
1704 init_param1();
1705@@ -2303,7 +2382,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
1706 * make gdt memory segments
1707 */
1708 gdt_segs[GPROC0_SEL].ssd_base =
1709- (uintptr_t) &CPU_prvspace[0]->mdglobaldata.gd_common_tss;
1710+ (uintptr_t) &CPU_prvspace[0]->common_tss;
1711
1712 gd->mi.gd_prvspace = CPU_prvspace[0];
1713
1714@@ -2337,7 +2416,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
1715 setidt_global(IDT_DE, &IDTVEC(div), SDT_SYSIGT, SEL_KPL, 0);
1716 setidt_global(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 0);
1717 setidt_global(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 1);
1718- setidt_global(IDT_BP, &IDTVEC(bpt), SDT_SYSIGT, SEL_UPL, 0);
1719+ setidt_global(IDT_BP, &IDTVEC(bpt), SDT_SYSIGT, SEL_UPL, 0);
1720 setidt_global(IDT_OF, &IDTVEC(ofl), SDT_SYSIGT, SEL_KPL, 0);
1721 setidt_global(IDT_BR, &IDTVEC(bnd), SDT_SYSIGT, SEL_KPL, 0);
1722 setidt_global(IDT_UD, &IDTVEC(ill), SDT_SYSIGT, SEL_KPL, 0);
1723@@ -2392,11 +2471,6 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
1724 Debugger("Boot flags requested debugger");
1725 #endif
1726
1727-#if 0 /* JG */
1728- finishidentcpu(); /* Final stage of CPU initialization */
1729- setidt(6, &IDTVEC(ill), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
1730- setidt(13, &IDTVEC(prot), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
1731-#endif
1732 identify_cpu(); /* Final stage of CPU initialization */
1733 initializecpu(0); /* Initialize CPU registers */
1734
1735@@ -2448,20 +2522,27 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
1736 ioapic_enable = 1;
1737 }
1738
1739- /* make an initial tss so cpu can get interrupt stack on syscall! */
1740- gd->gd_common_tss.tss_rsp0 =
1741- (register_t)(thread0.td_kstack +
1742- KSTACK_PAGES * PAGE_SIZE - sizeof(struct pcb));
1743- /* Ensure the stack is aligned to 16 bytes */
1744- gd->gd_common_tss.tss_rsp0 &= ~(register_t)0xF;
1745+ /*
1746+ * TSS entry point for interrupts, traps, and exceptions
1747+ * (sans NMI). This will always go to near the top of the pcpu
1748+ * trampoline area. Hardware-pushed data will be copied into
1749+ * the trap-frame on entry, and (if necessary) returned to the
1750+ * trampoline on exit.
1751+ *
1752+ * We store some pcb data for the trampoline code above the
1753+ * stack the cpu hw pushes into, and arrange things so the
1754+ * address of tr_pcb_rsp is the same as the desired top of
1755+ * stack.
1756+ */
1757+ ps->common_tss.tss_rsp0 = (register_t)&ps->trampoline.tr_pcb_rsp;
1758+ ps->trampoline.tr_pcb_rsp = ps->common_tss.tss_rsp0;
1759
1760 /* double fault stack */
1761- gd->gd_common_tss.tss_ist1 =
1762- (long)&gd->mi.gd_prvspace->idlestack[
1763- sizeof(gd->mi.gd_prvspace->idlestack)];
1764+ ps->common_tss.tss_ist1 = (register_t)ps->dblstack +
1765+ sizeof(ps->dblstack);
1766
1767 /* Set the IO permission bitmap (empty due to tss seg limit) */
1768- gd->gd_common_tss.tss_iobase = sizeof(struct x86_64tss);
1769+ ps->common_tss.tss_iobase = sizeof(struct x86_64tss);
1770
1771 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
1772 gd->gd_tss_gdt = &gdt[GPROC0_SEL];
1773@@ -2527,9 +2608,9 @@ cpu_gdinit(struct mdglobaldata *gd, int cpu)
1774 if (cpu)
1775 gd->mi.gd_curthread = &gd->mi.gd_idlethread;
1776
1777- lwkt_init_thread(&gd->mi.gd_idlethread,
1778- gd->mi.gd_prvspace->idlestack,
1779- sizeof(gd->mi.gd_prvspace->idlestack),
1780+ lwkt_init_thread(&gd->mi.gd_idlethread,
1781+ gd->mi.gd_prvspace->idlestack,
1782+ sizeof(gd->mi.gd_prvspace->idlestack),
1783 0, &gd->mi);
1784 lwkt_set_comm(&gd->mi.gd_idlethread, "idle_%d", cpu);
1785 gd->mi.gd_idlethread.td_switch = cpu_lwkt_switch;
1786@@ -2744,11 +2825,11 @@ set_dbregs(struct lwp *lp, struct dbreg *dbregs)
1787 * carried to decide if it is safe and useful to
1788 * provide access to that capability
1789 */
1790- for (i = 0, mask1 = 0x3<<16, mask2 = 0x2<<16; i < 4;
1791+ for (i = 0, mask1 = 0x3<<16, mask2 = 0x2<<16; i < 4;
1792 i++, mask1 <<= 4, mask2 <<= 4)
1793 if ((dbregs->dr[7] & mask1) == mask2)
1794 return (EINVAL);
1795-
1796+
1797 pcb = lp->lwp_thread->td_pcb;
1798 ucred = lp->lwp_proc->p_ucred;
1799
1800@@ -2818,7 +2899,7 @@ user_dbreg_trap(void)
1801 int nbp; /* number of breakpoints that triggered */
1802 caddr_t addr[4]; /* breakpoint addresses */
1803 int i;
1804-
1805+
1806 dr7 = rdr7();
1807 if ((dr7 & 0xff) == 0) {
1808 /*
1809diff --git a/sys/platform/pc64/x86_64/mp_machdep.c b/sys/platform/pc64/x86_64/mp_machdep.c
1810index 1c8757ffd..adba3ec95 100644
1811--- a/sys/platform/pc64/x86_64/mp_machdep.c
1812+++ b/sys/platform/pc64/x86_64/mp_machdep.c
1813@@ -138,9 +138,7 @@
1814 int current_postcode;
1815
1816 /** XXX FIXME: what system files declare these??? */
1817-extern struct region_descriptor r_gdt;
1818
1819-extern int nkpt;
1820 extern int naps;
1821
1822 int64_t tsc0_offset;
1823@@ -252,8 +250,7 @@ init_secondary(void)
1824
1825 ps = CPU_prvspace[myid];
1826
1827- gdt_segs[GPROC0_SEL].ssd_base =
1828- (long) &ps->mdglobaldata.gd_common_tss;
1829+ gdt_segs[GPROC0_SEL].ssd_base = (long)&ps->common_tss;
1830 ps->mdglobaldata.mi.gd_prvspace = ps;
1831
1832 /* We fill the 32-bit segment descriptors */
1833@@ -286,17 +283,30 @@ init_secondary(void)
1834
1835 md = mdcpu; /* loaded through %gs:0 (mdglobaldata.mi.gd_prvspace)*/
1836
1837- md->gd_common_tss.tss_rsp0 = 0; /* not used until after switch */
1838+ /*
1839+ * TSS entry point for interrupts, traps, and exceptions
1840+ * (sans NMI). This will always go to near the top of the pcpu
1841+ * trampoline area. Hardware-pushed data will be copied into
1842+ * the trap-frame on entry, and (if necessary) returned to the
1843+ * trampoline on exit.
1844+ *
1845+ * We store some pcb data for the trampoline code above the
1846+ * stack the cpu hw pushes into, and arrange things so the
1847+ * address of tr_pcb_rsp is the same as the desired top of
1848+ * stack.
1849+ */
1850+ ps->common_tss.tss_rsp0 = (register_t)&ps->trampoline.tr_pcb_rsp;
1851+ ps->trampoline.tr_pcb_rsp = ps->common_tss.tss_rsp0;
1852+
1853 #if 0 /* JG XXX */
1854- md->gd_common_tss.tss_ioopt = (sizeof md->gd_common_tss) << 16;
1855+ ps->common_tss.tss_ioopt = (sizeof ps->common_tss) << 16;
1856 #endif
1857 md->gd_tss_gdt = &gdt[myid * NGDT + GPROC0_SEL];
1858 md->gd_common_tssd = *md->gd_tss_gdt;
1859
1860 /* double fault stack */
1861- md->gd_common_tss.tss_ist1 =
1862- (long)&md->mi.gd_prvspace->idlestack[
1863- sizeof(md->mi.gd_prvspace->idlestack)];
1864+ ps->common_tss.tss_ist1 = (register_t)ps->dblstack +
1865+ sizeof(ps->dblstack);
1866
1867 ltr(gsel_tss);
1868
1869@@ -500,13 +510,9 @@ start_all_aps(u_int boot_addr)
1870 /* set ncpus to 1 + highest logical cpu. Not all may have come up */
1871 ncpus = x;
1872
1873- /* ncpus2 -- ncpus rounded down to the nearest power of 2 */
1874 for (shift = 0; (1 << shift) <= ncpus; ++shift)
1875 ;
1876 --shift;
1877- ncpus2_shift = shift;
1878- ncpus2 = 1 << shift;
1879- ncpus2_mask = ncpus2 - 1;
1880
1881 /* ncpus_fit -- ncpus rounded up to the nearest power of 2 */
1882 if ((1 << shift) < ncpus)
1883@@ -898,7 +904,7 @@ smp_invltlb(void)
1884 cpumask_t mask;
1885 unsigned long rflags;
1886 #ifdef LOOPRECOVER
1887- uint64_t tsc_base = rdtsc();
1888+ tsc_uclock_t tsc_base = rdtsc();
1889 int repeats = 0;
1890 #endif
1891
1892@@ -1119,7 +1125,7 @@ smp_inval_intr(void)
1893 struct mdglobaldata *md = mdcpu;
1894 cpumask_t cpumask;
1895 #ifdef LOOPRECOVER
1896- uint64_t tsc_base = rdtsc();
1897+ tsc_uclock_t tsc_base = rdtsc();
1898 #endif
1899
1900 #if 0
1901diff --git a/sys/platform/pc64/x86_64/msi_vector.s b/sys/platform/pc64/x86_64/msi_vector.s
1902index a1cc604ba..853b975a8 100644
1903--- a/sys/platform/pc64/x86_64/msi_vector.s
1904+++ b/sys/platform/pc64/x86_64/msi_vector.s
1905@@ -22,7 +22,7 @@
1906 #define IRQ_LIDX(irq_num) ((irq_num) >> 6)
1907
1908 #define MSI_PUSH_FRAME \
1909- PUSH_FRAME ; /* 15 regs + space for 5 extras */ \
1910+ PUSH_FRAME_TFRIP ; /* 15 regs + space for 5 extras */ \
1911 movq $0,TF_XFLAGS(%rsp) ; \
1912 movq $0,TF_TRAPNO(%rsp) ; \
1913 movq $0,TF_ADDR(%rsp) ; \
1914diff --git a/sys/platform/pc64/x86_64/pmap.c b/sys/platform/pc64/x86_64/pmap.c
1915index bf7812926..5964cbec2 100644
1916--- a/sys/platform/pc64/x86_64/pmap.c
1917+++ b/sys/platform/pc64/x86_64/pmap.c
1918@@ -43,10 +43,16 @@
1919 */
1920 /*
1921 * Manage physical address maps for x86-64 systems.
1922+ *
1923+ * Some notes:
1924+ * - The 'M'odified bit is only applicable to terminal PTEs.
1925+ *
1926+ * - The 'U'ser access bit can be set for higher-level PTEs as
1927+ * long as it isn't set for terminal PTEs for pages we don't
1928+ * want user access to.
1929 */
1930
1931 #if 0 /* JG */
1932-#include "opt_disable_pse.h"
1933 #include "opt_pmap.h"
1934 #endif
1935 #include "opt_msgbuf.h"
1936@@ -74,11 +80,11 @@
1937
1938 #include <sys/user.h>
1939 #include <sys/thread2.h>
1940-#include <sys/sysref2.h>
1941 #include <sys/spinlock2.h>
1942 #include <vm/vm_page2.h>
1943
1944 #include <machine/cputypes.h>
1945+#include <machine/cpu.h>
1946 #include <machine/md_var.h>
1947 #include <machine/specialreg.h>
1948 #include <machine/smp.h>
1949@@ -155,6 +161,7 @@
1950 static uint64_t protection_codes[PROTECTION_CODES_SIZE];
1951
1952 struct pmap kernel_pmap;
1953+struct pmap iso_pmap;
1954
1955 MALLOC_DEFINE(M_OBJPMAP, "objpmap", "pmaps associated with VM objects");
1956
1957@@ -169,12 +176,10 @@ vm_offset_t KvaEnd; /* VA end of KVA space (non-inclusive) */
1958 vm_offset_t KvaSize; /* max size of kernel virtual address space */
1959 static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */
1960 //static int pgeflag; /* PG_G or-in */
1961-//static int pseflag; /* PG_PS or-in */
1962 uint64_t PatMsr;
1963
1964 static int ndmpdp;
1965 static vm_paddr_t dmaplimit;
1966-static int nkpt;
1967 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
1968
1969 static pt_entry_t pat_pte_index[PAT_INDEX_SIZE]; /* PAT -> PG_ bits */
1970@@ -184,8 +189,8 @@ static uint64_t KPTbase;
1971 static uint64_t KPTphys;
1972 static uint64_t KPDphys; /* phys addr of kernel level 2 */
1973 static uint64_t KPDbase; /* phys addr of kernel level 2 @ KERNBASE */
1974-uint64_t KPDPphys; /* phys addr of kernel level 3 */
1975-uint64_t KPML4phys; /* phys addr of kernel level 4 */
1976+uint64_t KPDPphys; /* phys addr of kernel level 3 */
1977+uint64_t KPML4phys; /* phys addr of kernel level 4 */
1978
1979 static uint64_t DMPDphys; /* phys addr of direct mapped level 2 */
1980 static uint64_t DMPDPphys; /* phys addr of direct mapped level 3 */
1981@@ -195,7 +200,7 @@ static uint64_t DMPDPphys; /* phys addr of direct mapped level 3 */
1982 */
1983 static vm_zone_t pvzone;
1984 static struct vm_zone pvzone_store;
1985-static int pv_entry_max=0, pv_entry_high_water=0;
1986+static vm_pindex_t pv_entry_max=0, pv_entry_high_water=0;
1987 static int pmap_pagedaemon_waken = 0;
1988 static struct pv_entry *pvinit;
1989
1990@@ -252,12 +257,17 @@ SYSCTL_INT(_machdep, OID_AUTO, pmap_fast_kernel_cpusync, CTLFLAG_RW,
1991 int pmap_dynamic_delete = 0;
1992 SYSCTL_INT(_machdep, OID_AUTO, pmap_dynamic_delete, CTLFLAG_RW,
1993 &pmap_dynamic_delete, 0, "Dynamically delete PT/PD/PDPs");
1994+int pmap_lock_delay = 100;
1995+SYSCTL_INT(_machdep, OID_AUTO, pmap_lock_delay, CTLFLAG_RW,
1996+ &pmap_lock_delay, 0, "Spin loops");
1997+static int isolated_user_pmap = -1;
1998+TUNABLE_INT("machdep.isolated_user_pmap", &isolated_user_pmap);
1999+SYSCTL_INT(_machdep, OID_AUTO, isolated_user_pmap, CTLFLAG_RW,
2000+ &isolated_user_pmap, 0, "Userland pmap isolation");
2001
2002 static int pmap_nx_enable = 0;
2003 /* needs manual TUNABLE in early probe, see below */
2004
2005-#define DISABLE_PSE
2006-
2007 /* Standard user access funtions */
2008 extern int std_copyinstr (const void *udaddr, void *kaddr, size_t len,
2009 size_t *lencopied);
2010@@ -308,7 +318,7 @@ static void pmap_protect_callback(pmap_t pmap, struct pmap_scan_info *info,
2011 pv_entry_t pt_pv, int sharept,
2012 vm_offset_t va, pt_entry_t *ptep, void *arg __unused);
2013
2014-static void i386_protection_init (void);
2015+static void x86_64_protection_init (void);
2016 static void create_pagetables(vm_paddr_t *firstaddr);
2017 static void pmap_remove_all (vm_page_t m);
2018 static boolean_t pmap_testbit (vm_page_t m, int bit);
2019@@ -320,8 +330,6 @@ static void pmap_pinit_defaults(struct pmap *pmap);
2020 static void pv_placemarker_wait(pmap_t pmap, vm_pindex_t *pmark);
2021 static void pv_placemarker_wakeup(pmap_t pmap, vm_pindex_t *pmark);
2022
2023-static unsigned pdir4mb;
2024-
2025 static int
2026 pv_entry_compare(pv_entry_t pv1, pv_entry_t pv2)
2027 {
2028@@ -369,6 +377,41 @@ pmap_page_stats_deleting(vm_page_t m)
2029 }
2030 }
2031
2032+/*
2033+ * This is an ineligent crowbar to prevent heavily threaded programs
2034+ * from creating long live-locks in the pmap code when pmap_mmu_optimize
2035+ * is enabled. Without it a pmap-local page table page can wind up being
2036+ * constantly created and destroyed (without injury, but also without
2037+ * progress) as the optimization tries to switch to the object's shared page
2038+ * table page.
2039+ */
2040+static __inline void
2041+pmap_softwait(pmap_t pmap)
2042+{
2043+ while (pmap->pm_softhold) {
2044+ tsleep_interlock(&pmap->pm_softhold, 0);
2045+ if (pmap->pm_softhold)
2046+ tsleep(&pmap->pm_softhold, PINTERLOCKED, "mmopt", 0);
2047+ }
2048+}
2049+
2050+static __inline void
2051+pmap_softhold(pmap_t pmap)
2052+{
2053+ while (atomic_swap_int(&pmap->pm_softhold, 1) == 1) {
2054+ tsleep_interlock(&pmap->pm_softhold, 0);
2055+ if (atomic_swap_int(&pmap->pm_softhold, 1) == 1)
2056+ tsleep(&pmap->pm_softhold, PINTERLOCKED, "mmopt", 0);
2057+ }
2058+}
2059+
2060+static __inline void
2061+pmap_softdone(pmap_t pmap)
2062+{
2063+ atomic_swap_int(&pmap->pm_softhold, 0);
2064+ wakeup(&pmap->pm_softhold);
2065+}
2066+
2067 /*
2068 * Move the kernel virtual free pointer to the next
2069 * 2MB. This is used to help improve performance
2070@@ -385,23 +428,6 @@ pmap_kmem_choose(vm_offset_t addr)
2071 return newaddr;
2072 }
2073
2074-/*
2075- * pmap_pte_quick:
2076- *
2077- * Super fast pmap_pte routine best used when scanning the pv lists.
2078- * This eliminates many course-grained invltlb calls. Note that many of
2079- * the pv list scans are across different pmaps and it is very wasteful
2080- * to do an entire invltlb when checking a single mapping.
2081- */
2082-static __inline pt_entry_t *pmap_pte(pmap_t pmap, vm_offset_t va);
2083-
2084-static
2085-pt_entry_t *
2086-pmap_pte_quick(pmap_t pmap, vm_offset_t va)
2087-{
2088- return pmap_pte(pmap, va);
2089-}
2090-
2091 /*
2092 * Returns the pindex of a page table entry (representing a terminal page).
2093 * There are NUPTE_TOTAL page table entries possible (a huge number)
2094@@ -487,6 +513,49 @@ pmap_pdp_index(vm_offset_t va)
2095 return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
2096 }
2097
2098+/*
2099+ * Locate the requested pt_entry
2100+ */
2101+static __inline
2102+pv_entry_t
2103+pv_entry_lookup(pmap_t pmap, vm_pindex_t pindex)
2104+{
2105+ pv_entry_t pv;
2106+
2107+ if (pindex < pmap_pt_pindex(0))
2108+ pv = pmap->pm_pvhint_pte;
2109+ else if (pindex < pmap_pd_pindex(0))
2110+ pv = pmap->pm_pvhint_pt;
2111+ else
2112+ pv = NULL;
2113+ cpu_ccfence();
2114+ if (pv == NULL || pv->pv_pmap != pmap) {
2115+ pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot,
2116+ pindex);
2117+ } else if (pv->pv_pindex != pindex) {
2118+ pv = pv_entry_rb_tree_RB_LOOKUP_REL(&pmap->pm_pvroot,
2119+ pindex, pv);
2120+ }
2121+ return pv;
2122+}
2123+
2124+/*
2125+ * pmap_pte_quick:
2126+ *
2127+ * Super fast pmap_pte routine best used when scanning the pv lists.
2128+ * This eliminates many course-grained invltlb calls. Note that many of
2129+ * the pv list scans are across different pmaps and it is very wasteful
2130+ * to do an entire invltlb when checking a single mapping.
2131+ */
2132+static __inline pt_entry_t *pmap_pte(pmap_t pmap, vm_offset_t va);
2133+
2134+static
2135+pt_entry_t *
2136+pmap_pte_quick(pmap_t pmap, vm_offset_t va)
2137+{
2138+ return pmap_pte(pmap, va);
2139+}
2140+
2141 /*
2142 * The placemarker hash must be broken up into four zones so lock
2143 * ordering semantics continue to work (e.g. pte, pt, pd, then pdp).
2144@@ -598,15 +667,19 @@ pmap_pt(pmap_t pmap, vm_offset_t va)
2145 pdp_entry_t *pd;
2146 pv_entry_t pv;
2147 vm_pindex_t pd_pindex;
2148+ vm_paddr_t phys;
2149
2150 if (pmap->pm_flags & PMAP_FLAG_SIMPLE) {
2151 pd_pindex = pmap_pd_pindex(va);
2152- spin_lock(&pmap->pm_spin);
2153+ spin_lock_shared(&pmap->pm_spin);
2154 pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot, pd_pindex);
2155- spin_unlock(&pmap->pm_spin);
2156- if (pv == NULL || pv->pv_m == NULL)
2157+ if (pv == NULL || pv->pv_m == NULL) {
2158+ spin_unlock_shared(&pmap->pm_spin);
2159 return NULL;
2160- return (pmap_pd_to_pt(VM_PAGE_TO_PHYS(pv->pv_m), va));
2161+ }
2162+ phys = VM_PAGE_TO_PHYS(pv->pv_m);
2163+ spin_unlock_shared(&pmap->pm_spin);
2164+ return (pmap_pd_to_pt(phys, va));
2165 } else {
2166 pd = pmap_pd(pmap, va);
2167 if (pd == NULL || (*pd & pmap->pmap_bits[PG_V_IDX]) == 0)
2168@@ -653,7 +726,7 @@ pmap_pte(pmap_t pmap, vm_offset_t va)
2169 * must be in a known associated state (typically by being locked when
2170 * the pmap spinlock isn't held). We allow the race for that case.
2171 *
2172- * NOTE: pm_pvhint is only accessed (read) with the spin-lock held, using
2173+ * NOTE: pm_pvhint* is only accessed (read) with the spin-lock held, using
2174 * cpu_ccfence() to prevent compiler optimizations from reloading the
2175 * field.
2176 */
2177@@ -661,9 +734,12 @@ static __inline
2178 void
2179 pv_cache(pv_entry_t pv, vm_pindex_t pindex)
2180 {
2181- if (pindex >= pmap_pt_pindex(0) && pindex < pmap_pd_pindex(0)) {
2182+ if (pindex < pmap_pt_pindex(0)) {
2183+ if (pv->pv_pmap)
2184+ pv->pv_pmap->pm_pvhint_pte = pv;
2185+ } else if (pindex < pmap_pd_pindex(0)) {
2186 if (pv->pv_pmap)
2187- pv->pv_pmap->pm_pvhint = pv;
2188+ pv->pv_pmap->pm_pvhint_pt = pv;
2189 }
2190 }
2191
2192@@ -697,6 +773,29 @@ vtopte(vm_offset_t va)
2193 return (PTmap + ((va >> PAGE_SHIFT) & mask));
2194 }
2195
2196+/*
2197+ * Returns the physical address translation from va for a user address.
2198+ * (vm_paddr_t)-1 is returned on failure.
2199+ */
2200+vm_paddr_t
2201+uservtophys(vm_offset_t va)
2202+{
2203+ uint64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT +
2204+ NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
2205+ vm_paddr_t pa;
2206+ pt_entry_t pte;
2207+ pmap_t pmap;
2208+
2209+ pmap = vmspace_pmap(mycpu->gd_curthread->td_lwp->lwp_vmspace);
2210+ pa = (vm_paddr_t)-1;
2211+ if (va < VM_MAX_USER_ADDRESS) {
2212+ pte = kreadmem64(PTmap + ((va >> PAGE_SHIFT) & mask));
2213+ if (pte & pmap->pmap_bits[PG_V_IDX])
2214+ pa = (pte & PG_FRAME) | (va & PAGE_MASK);
2215+ }
2216+ return pa;
2217+}
2218+
2219 static uint64_t
2220 allocpages(vm_paddr_t *firstaddr, long n)
2221 {
2222@@ -715,55 +814,83 @@ create_pagetables(vm_paddr_t *firstaddr)
2223 long i; /* must be 64 bits */
2224 long nkpt_base;
2225 long nkpt_phys;
2226+ long nkpd_phys;
2227 int j;
2228
2229 /*
2230 * We are running (mostly) V=P at this point
2231 *
2232- * Calculate NKPT - number of kernel page tables. We have to
2233- * accomodoate prealloction of the vm_page_array, dump bitmap,
2234- * MSGBUF_SIZE, and other stuff. Be generous.
2235- *
2236- * Maxmem is in pages.
2237+ * Calculate how many 1GB PD entries in our PDP pages are needed
2238+ * for the DMAP. This is only allocated if the system does not
2239+ * support 1GB pages. Otherwise ndmpdp is simply a count of
2240+ * the number of 1G terminal entries in our PDP pages are needed.
2241 *
2242- * ndmpdp is the number of 1GB pages we wish to map.
2243+ * NOTE: Maxmem is in pages
2244 */
2245 ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
2246 if (ndmpdp < 4) /* Minimum 4GB of dirmap */
2247 ndmpdp = 4;
2248- KKASSERT(ndmpdp <= NKPDPE * NPDEPG);
2249+ KKASSERT(ndmpdp <= NDMPML4E * NPML4EPG);
2250
2251 /*
2252- * Starting at the beginning of kvm (not KERNBASE).
2253+ * Starting at KERNBASE - map all 2G worth of page table pages.
2254+ * KERNBASE is offset -2G from the end of kvm. This will accomodate
2255+ * all KVM allocations above KERNBASE, including the SYSMAPs below.
2256+ *
2257+ * We do this by allocating 2*512 PT pages. Each PT page can map
2258+ * 2MB, for 2GB total.
2259+ */
2260+ nkpt_base = (NPDPEPG - KPDPI) * NPTEPG; /* typically 2 x 512 */
2261+
2262+ /*
2263+ * Starting at the beginning of kvm (VM_MIN_KERNEL_ADDRESS),
2264+ * Calculate how many page table pages we need to preallocate
2265+ * for early vm_map allocations.
2266+ *
2267+ * A few extra won't hurt, they will get used up in the running
2268+ * system.
2269+ *
2270+ * vm_page array
2271+ * initial pventry's
2272 */
2273 nkpt_phys = (Maxmem * sizeof(struct vm_page) + NBPDR - 1) / NBPDR;
2274 nkpt_phys += (Maxmem * sizeof(struct pv_entry) + NBPDR - 1) / NBPDR;
2275- nkpt_phys += ((nkpt + nkpt + 1 + NKPML4E + NKPDPE + NDMPML4E +
2276- ndmpdp) + 511) / 512;
2277- nkpt_phys += 128;
2278+ nkpt_phys += 128; /* a few extra */
2279
2280 /*
2281- * Starting at KERNBASE - map 2G worth of page table pages.
2282- * KERNBASE is offset -2G from the end of kvm.
2283+ * The highest value nkpd_phys can be set to is
2284+ * NKPDPE - (NPDPEPG - KPDPI) (i.e. NKPDPE - 2).
2285+ *
2286+ * Doing so would cause all PD pages to be pre-populated for
2287+ * a maximal KVM space (approximately 16*512 pages, or 32MB.
2288+ * We can save memory by not doing this.
2289 */
2290- nkpt_base = (NPDPEPG - KPDPI) * NPTEPG; /* typically 2 x 512 */
2291+ nkpd_phys = (nkpt_phys + NPDPEPG - 1) / NPDPEPG;
2292
2293 /*
2294 * Allocate pages
2295+ *
2296+ * Normally NKPML4E=1-16 (1-16 kernel PDP page)
2297+ * Normally NKPDPE= NKPML4E*512-1 (511 min kernel PD pages)
2298+ *
2299+ * Only allocate enough PD pages
2300+ * NOTE: We allocate all kernel PD pages up-front, typically
2301+ * ~511G of KVM, requiring 511 PD pages.
2302 */
2303- KPTbase = allocpages(firstaddr, nkpt_base);
2304- KPTphys = allocpages(firstaddr, nkpt_phys);
2305- KPML4phys = allocpages(firstaddr, 1);
2306- KPDPphys = allocpages(firstaddr, NKPML4E);
2307- KPDphys = allocpages(firstaddr, NKPDPE);
2308+ KPTbase = allocpages(firstaddr, nkpt_base); /* KERNBASE to end */
2309+ KPTphys = allocpages(firstaddr, nkpt_phys); /* KVA start */
2310+ KPML4phys = allocpages(firstaddr, 1); /* recursive PML4 map */
2311+ KPDPphys = allocpages(firstaddr, NKPML4E); /* kernel PDP pages */
2312+ KPDphys = allocpages(firstaddr, nkpd_phys); /* kernel PD pages */
2313
2314 /*
2315- * Calculate the page directory base for KERNBASE,
2316- * that is where we start populating the page table pages.
2317- * Basically this is the end - 2.
2318+ * Alloc PD pages for the area starting at KERNBASE.
2319 */
2320- KPDbase = KPDphys + ((NKPDPE - (NPDPEPG - KPDPI)) << PAGE_SHIFT);
2321+ KPDbase = allocpages(firstaddr, NPDPEPG - KPDPI);
2322
2323+ /*
2324+ * Stuff for our DMAP
2325+ */
2326 DMPDPphys = allocpages(firstaddr, NDMPML4E);
2327 if ((amd_feature & AMDID_PAGE1GB) == 0)
2328 DMPDphys = allocpages(firstaddr, ndmpdp);
2329@@ -818,16 +945,32 @@ create_pagetables(vm_paddr_t *firstaddr)
2330 }
2331
2332 /*
2333- * And connect up the PD to the PDP. The kernel pmap is expected
2334- * to pre-populate all of its PDs. See NKPDPE in vmparam.h.
2335+ * Load PD addresses into the PDP pages for primary KVA space to
2336+ * cover existing page tables. PD's for KERNBASE are handled in
2337+ * the next loop.
2338+ *
2339+ * expected to pre-populate all of its PDs. See NKPDPE in vmparam.h.
2340 */
2341- for (i = 0; i < NKPDPE; i++) {
2342- ((pdp_entry_t *)KPDPphys)[NPDPEPG - NKPDPE + i] =
2343+ for (i = 0; i < nkpd_phys; i++) {
2344+ ((pdp_entry_t *)KPDPphys)[NKPML4E * NPDPEPG - NKPDPE + i] =
2345 KPDphys + (i << PAGE_SHIFT);
2346- ((pdp_entry_t *)KPDPphys)[NPDPEPG - NKPDPE + i] |=
2347+ ((pdp_entry_t *)KPDPphys)[NKPML4E * NPDPEPG - NKPDPE + i] |=
2348+ pmap_bits_default[PG_RW_IDX] |
2349+ pmap_bits_default[PG_V_IDX] |
2350+ pmap_bits_default[PG_A_IDX];
2351+ }
2352+
2353+ /*
2354+ * Load PDs for KERNBASE to the end
2355+ */
2356+ i = (NKPML4E - 1) * NPDPEPG + KPDPI;
2357+ for (j = 0; j < NPDPEPG - KPDPI; ++j) {
2358+ ((pdp_entry_t *)KPDPphys)[i + j] =
2359+ KPDbase + (j << PAGE_SHIFT);
2360+ ((pdp_entry_t *)KPDPphys)[i + j] |=
2361 pmap_bits_default[PG_RW_IDX] |
2362 pmap_bits_default[PG_V_IDX] |
2363- pmap_bits_default[PG_U_IDX];
2364+ pmap_bits_default[PG_A_IDX];
2365 }
2366
2367 /*
2368@@ -838,6 +981,9 @@ create_pagetables(vm_paddr_t *firstaddr)
2369 * entries are set to zero as we allocated enough PD pages
2370 */
2371 if ((amd_feature & AMDID_PAGE1GB) == 0) {
2372+ /*
2373+ * Use 2MB pages
2374+ */
2375 for (i = 0; i < NPDEPG * ndmpdp; i++) {
2376 ((pd_entry_t *)DMPDphys)[i] = i << PDRSHIFT;
2377 ((pd_entry_t *)DMPDphys)[i] |=
2378@@ -857,10 +1003,12 @@ create_pagetables(vm_paddr_t *firstaddr)
2379 (i << PAGE_SHIFT);
2380 ((pdp_entry_t *)DMPDPphys)[i] |=
2381 pmap_bits_default[PG_RW_IDX] |
2382- pmap_bits_default[PG_V_IDX] |
2383- pmap_bits_default[PG_U_IDX];
2384+ pmap_bits_default[PG_V_IDX];
2385 }
2386 } else {
2387+ /*
2388+ * 1GB pages
2389+ */
2390 for (i = 0; i < ndmpdp; i++) {
2391 ((pdp_entry_t *)DMPDPphys)[i] =
2392 (vm_paddr_t)i << PDPSHIFT;
2393@@ -879,7 +1027,7 @@ create_pagetables(vm_paddr_t *firstaddr)
2394 ((pdp_entry_t *)KPML4phys)[PML4PML4I] |=
2395 pmap_bits_default[PG_RW_IDX] |
2396 pmap_bits_default[PG_V_IDX] |
2397- pmap_bits_default[PG_U_IDX];
2398+ pmap_bits_default[PG_A_IDX];
2399
2400 /*
2401 * Connect the Direct Map slots up to the PML4
2402@@ -889,23 +1037,28 @@ create_pagetables(vm_paddr_t *firstaddr)
2403 (DMPDPphys + ((vm_paddr_t)j << PAGE_SHIFT)) |
2404 pmap_bits_default[PG_RW_IDX] |
2405 pmap_bits_default[PG_V_IDX] |
2406- pmap_bits_default[PG_U_IDX];
2407+ pmap_bits_default[PG_A_IDX];
2408 }
2409
2410 /*
2411 * Connect the KVA slot up to the PML4
2412 */
2413- ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
2414- ((pdp_entry_t *)KPML4phys)[KPML4I] |=
2415- pmap_bits_default[PG_RW_IDX] |
2416- pmap_bits_default[PG_V_IDX] |
2417- pmap_bits_default[PG_U_IDX];
2418+ for (j = 0; j < NKPML4E; ++j) {
2419+ ((pdp_entry_t *)KPML4phys)[KPML4I + j] =
2420+ KPDPphys + ((vm_paddr_t)j << PAGE_SHIFT);
2421+ ((pdp_entry_t *)KPML4phys)[KPML4I + j] |=
2422+ pmap_bits_default[PG_RW_IDX] |
2423+ pmap_bits_default[PG_V_IDX] |
2424+ pmap_bits_default[PG_A_IDX];
2425+ }
2426+ cpu_mfence();
2427+ cpu_invltlb();
2428 }
2429
2430 /*
2431 * Bootstrap the system enough to run with virtual memory.
2432 *
2433- * On the i386 this is called after mapping has already been enabled
2434+ * On x86_64 this is called after mapping has already been enabled
2435 * and just syncs the pmap module with what has already been done.
2436 * [We can't call it easily with mapping off since the kernel is not
2437 * mapped with PA == VA, hence we would have to relocate every address
2438@@ -945,7 +1098,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
2439 /*
2440 * Initialize protection array.
2441 */
2442- i386_protection_init();
2443+ x86_64_protection_init();
2444
2445 /*
2446 * The kernel's pmap is statically allocated so we don't have to use
2447@@ -1005,33 +1158,6 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
2448 */
2449 // pgeflag = 0;
2450
2451-/*
2452- * Initialize the 4MB page size flag
2453- */
2454-// pseflag = 0;
2455-/*
2456- * The 4MB page version of the initial
2457- * kernel page mapping.
2458- */
2459- pdir4mb = 0;
2460-
2461-#if !defined(DISABLE_PSE)
2462- if (cpu_feature & CPUID_PSE) {
2463- pt_entry_t ptditmp;
2464- /*
2465- * Note that we have enabled PSE mode
2466- */
2467-// pseflag = kernel_pmap.pmap_bits[PG_PS_IDX];
2468- ptditmp = *(PTmap + x86_64_btop(KERNBASE));
2469- ptditmp &= ~(NBPDR - 1);
2470- ptditmp |= pmap_bits_default[PG_V_IDX] |
2471- pmap_bits_default[PG_RW_IDX] |
2472- pmap_bits_default[PG_PS_IDX] |
2473- pmap_bits_default[PG_U_IDX];
2474-// pgeflag;
2475- pdir4mb = ptditmp;
2476- }
2477-#endif
2478 cpu_invltlb();
2479
2480 /* Initialize the PAT MSR */
2481@@ -1124,30 +1250,28 @@ pmap_set_opt(void)
2482 {
2483 if (cpu_feature & CPUID_PSE) {
2484 load_cr4(rcr4() | CR4_PSE);
2485- if (pdir4mb && mycpu->gd_cpuid == 0) { /* only on BSP */
2486+ if (mycpu->gd_cpuid == 0) /* only on BSP */
2487 cpu_invltlb();
2488- }
2489 }
2490 }
2491
2492 /*
2493- * Initialize the pmap module.
2494- * Called by vm_init, to initialize any structures that the pmap
2495- * system needs to map virtual memory.
2496- * pmap_init has been enhanced to support in a fairly consistant
2497- * way, discontiguous physical memory.
2498+ * Early initialization of the pmap module.
2499+ *
2500+ * Called by vm_init, to initialize any structures that the pmap
2501+ * system needs to map virtual memory. pmap_init has been enhanced to
2502+ * support in a fairly consistant way, discontiguous physical memory.
2503 */
2504 void
2505 pmap_init(void)
2506 {
2507- int i;
2508- int initial_pvs;
2509+ vm_pindex_t initial_pvs;
2510+ vm_pindex_t i;
2511
2512 /*
2513 * Allocate memory for random pmap data structures. Includes the
2514 * pv_head_table.
2515 */
2516-
2517 for (i = 0; i < vm_page_array_size; i++) {
2518 vm_page_t m;
2519
2520@@ -1178,16 +1302,25 @@ pmap_init(void)
2521 * Initialize the address space (zone) for the pv_entries. Set a
2522 * high water mark so that the system can recover from excessive
2523 * numbers of pv entries.
2524+ *
2525+ * Also create the kernel page table template for isolated user
2526+ * pmaps.
2527 */
2528+static void pmap_init_iso_range(vm_offset_t base, size_t bytes);
2529+static void pmap_init2_iso_pmap(void);
2530+#if 0
2531+static void dump_pmap(pmap_t pmap, pt_entry_t pte, int level, vm_offset_t base);
2532+#endif
2533+
2534 void
2535 pmap_init2(void)
2536 {
2537- int shpgperproc = PMAP_SHPGPERPROC;
2538- int entry_max;
2539+ vm_pindex_t shpgperproc = PMAP_SHPGPERPROC;
2540+ vm_pindex_t entry_max;
2541
2542- TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
2543+ TUNABLE_LONG_FETCH("vm.pmap.shpgperproc", &shpgperproc);
2544 pv_entry_max = shpgperproc * maxproc + vm_page_array_size;
2545- TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
2546+ TUNABLE_LONG_FETCH("vm.pmap.pv_entries", &pv_entry_max);
2547 pv_entry_high_water = 9 * (pv_entry_max / 10);
2548
2549 /*
2550@@ -1212,8 +1345,191 @@ pmap_init2(void)
2551 else
2552 pmap_dynamic_delete = 0;
2553 }
2554+
2555+ /*
2556+ * Automatic detection of Intel meltdown bug requiring user/kernel
2557+ * mmap isolation.
2558+ *
2559+ * Currently there are so many Intel cpu's impacted that its better
2560+ * to whitelist future Intel CPUs. Most? AMD cpus are not impacted
2561+ * so the default is off for AMD.
2562+ */
2563+ if (isolated_user_pmap < 0) {
2564+ if (cpu_vendor_id == CPU_VENDOR_INTEL)
2565+ isolated_user_pmap = 1;
2566+ else
2567+ isolated_user_pmap = 0;
2568+ }
2569+ if (isolated_user_pmap) {
2570+ kprintf("machdep.isolated_user_pmap enabled to "
2571+ "protect against (mostly Intel) meltdown bug\n");
2572+ kprintf("system call performance will be impacted\n");
2573+ }
2574+
2575+ pmap_init2_iso_pmap();
2576+}
2577+
2578+/*
2579+ * Create the isolation pmap template. Once created, the template
2580+ * is static and its PML4e entries are used to populate the
2581+ * kernel portion of any isolated user pmaps.
2582+ *
2583+ * Our isolation pmap must contain:
2584+ * (1) trampoline area for all cpus
2585+ * (2) common_tss area for all cpus (its part of the trampoline area now)
2586+ * (3) IDT for all cpus
2587+ * (4) GDT for all cpus
2588+ */
2589+static void
2590+pmap_init2_iso_pmap(void)
2591+{
2592+ int n;
2593+
2594+ if (bootverbose)
2595+ kprintf("Initialize isolation pmap\n");
2596+
2597+ /*
2598+ * Try to use our normal API calls to make this easier. We have
2599+ * to scrap the shadowed kernel PDPs pmap_pinit() creates for our
2600+ * iso_pmap.
2601+ */
2602+ pmap_pinit(&iso_pmap);
2603+ bzero(iso_pmap.pm_pml4, PAGE_SIZE);
2604+
2605+ /*
2606+ * Install areas needed by the cpu and trampoline.
2607+ */
2608+ for (n = 0; n < ncpus; ++n) {
2609+ struct privatespace *ps;
2610+
2611+ ps = CPU_prvspace[n];
2612+ pmap_init_iso_range((vm_offset_t)&ps->trampoline,
2613+ sizeof(ps->trampoline));
2614+ pmap_init_iso_range((vm_offset_t)&ps->common_tss,
2615+ sizeof(ps->common_tss));
2616+ pmap_init_iso_range(r_idt_arr[n].rd_base,
2617+ r_idt_arr[n].rd_limit + 1);
2618+ }
2619+ pmap_init_iso_range((register_t)gdt, sizeof(gdt));
2620+ pmap_init_iso_range((vm_offset_t)(int *)btext,
2621+ (vm_offset_t)(int *)etext -
2622+ (vm_offset_t)(int *)btext);
2623+
2624+#if 0
2625+ kprintf("Dump iso_pmap:\n");
2626+ dump_pmap(&iso_pmap, vtophys(iso_pmap.pm_pml4), 0, 0);
2627+ kprintf("\nDump kernel_pmap:\n");
2628+ dump_pmap(&kernel_pmap, vtophys(kernel_pmap.pm_pml4), 0, 0);
2629+#endif
2630+}
2631+
2632+/*
2633+ * This adds a kernel virtual address range to the isolation pmap.
2634+ */
2635+static void
2636+pmap_init_iso_range(vm_offset_t base, size_t bytes)
2637+{
2638+ pv_entry_t pv;
2639+ pv_entry_t pvp;
2640+ pt_entry_t *ptep;
2641+ pt_entry_t pte;
2642+ vm_offset_t va;
2643+
2644+ if (bootverbose) {
2645+ kprintf("isolate %016jx-%016jx (%zd)\n",
2646+ base, base + bytes, bytes);
2647+ }
2648+ va = base & ~(vm_offset_t)PAGE_MASK;
2649+ while (va < base + bytes) {
2650+ if ((va & PDRMASK) == 0 && va + NBPDR <= base + bytes &&
2651+ (ptep = pmap_pt(&kernel_pmap, va)) != NULL &&
2652+ (*ptep & kernel_pmap.pmap_bits[PG_V_IDX]) &&
2653+ (*ptep & kernel_pmap.pmap_bits[PG_PS_IDX])) {
2654+ /*
2655+ * Use 2MB pages if possible
2656+ */
2657+ pte = *ptep;
2658+ pv = pmap_allocpte(&iso_pmap, pmap_pd_pindex(va), &pvp);
2659+ ptep = pv_pte_lookup(pv, (va >> PDRSHIFT) & 511);
2660+ *ptep = pte;
2661+ va += NBPDR;
2662+ } else {
2663+ /*
2664+ * Otherwise use 4KB pages
2665+ */
2666+ pv = pmap_allocpte(&iso_pmap, pmap_pt_pindex(va), &pvp);
2667+ ptep = pv_pte_lookup(pv, (va >> PAGE_SHIFT) & 511);
2668+ *ptep = vtophys(va) | kernel_pmap.pmap_bits[PG_RW_IDX] |
2669+ kernel_pmap.pmap_bits[PG_V_IDX] |
2670+ kernel_pmap.pmap_bits[PG_A_IDX] |
2671+ kernel_pmap.pmap_bits[PG_M_IDX];
2672+
2673+ va += PAGE_SIZE;
2674+ }
2675+ pv_put(pv);
2676+ pv_put(pvp);
2677+ }
2678 }
2679
2680+#if 0
2681+/*
2682+ * Useful debugging pmap dumper, do not remove (#if 0 when not in use)
2683+ */
2684+static
2685+void
2686+dump_pmap(pmap_t pmap, pt_entry_t pte, int level, vm_offset_t base)
2687+{
2688+ pt_entry_t *ptp;
2689+ vm_offset_t incr;
2690+ int i;
2691+
2692+ switch(level) {
2693+ case 0: /* PML4e page, 512G entries */
2694+ incr = (1LL << 48) / 512;
2695+ break;
2696+ case 1: /* PDP page, 1G entries */
2697+ incr = (1LL << 39) / 512;
2698+ break;
2699+ case 2: /* PD page, 2MB entries */
2700+ incr = (1LL << 30) / 512;
2701+ break;
2702+ case 3: /* PT page, 4KB entries */
2703+ incr = (1LL << 21) / 512;
2704+ break;
2705+ default:
2706+ incr = 0;
2707+ break;
2708+ }
2709+
2710+ if (level == 0)
2711+ kprintf("cr3 %016jx @ va=%016jx\n", pte, base);
2712+ ptp = (void *)PHYS_TO_DMAP(pte & ~(pt_entry_t)PAGE_MASK);
2713+ for (i = 0; i < 512; ++i) {
2714+ if (level == 0 && i == 128)
2715+ base += 0xFFFF000000000000LLU;
2716+ if (ptp[i]) {
2717+ kprintf("%*.*s ", level * 4, level * 4, "");
2718+ if (level == 1 && (ptp[i] & 0x180) == 0x180) {
2719+ kprintf("va=%016jx %3d term %016jx (1GB)\n",
2720+ base, i, ptp[i]);
2721+ } else if (level == 2 && (ptp[i] & 0x180) == 0x180) {
2722+ kprintf("va=%016jx %3d term %016jx (2MB)\n",
2723+ base, i, ptp[i]);
2724+ } else if (level == 3) {
2725+ kprintf("va=%016jx %3d term %016jx\n",
2726+ base, i, ptp[i]);
2727+ } else {
2728+ kprintf("va=%016jx %3d deep %016jx\n",
2729+ base, i, ptp[i]);
2730+ dump_pmap(pmap, ptp[i], level + 1, base);
2731+ }
2732+ }
2733+ base += incr;
2734+ }
2735+}
2736+
2737+#endif
2738+
2739 /*
2740 * Typically used to initialize a fictitious page by vm/device_pager.c
2741 */
2742@@ -1314,7 +1630,19 @@ pmap_extract_done(void *handle)
2743 * fall-through to the real fault code. Does not work with HVM page
2744 * tables.
2745 *
2746- * The returned page, if not NULL, is held (and not busied).
2747+ * if busyp is NULL the returned page, if not NULL, is held (and not busied).
2748+ *
2749+ * If busyp is not NULL and this function sets *busyp non-zero, the returned
2750+ * page is busied (and not held).
2751+ *
2752+ * If busyp is not NULL and this function sets *busyp to zero, the returned
2753+ * page is held (and not busied).
2754+ *
2755+ * If VM_PROT_WRITE is set in prot, and the pte is already writable, the
2756+ * returned page will be dirtied. If the pte is not already writable NULL
2757+ * is returned. In otherwords, if the bit is set and a vm_page_t is returned,
2758+ * any COW will already have happened and that page can be written by the
2759+ * caller.
2760 *
2761 * WARNING! THE RETURNED PAGE IS ONLY HELD AND NOT SUITABLE FOR READING
2762 * OR WRITING AS-IS.
2763@@ -1654,8 +1982,8 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
2764 *
2765 * The page *must* be wired.
2766 */
2767-void
2768-pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count)
2769+static __inline void
2770+_pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count, int doinval)
2771 {
2772 vm_offset_t end_va;
2773 vm_offset_t va;
2774@@ -1675,7 +2003,20 @@ pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count)
2775 atomic_swap_long(ptep, pte);
2776 m++;
2777 }
2778- pmap_invalidate_range(&kernel_pmap, beg_va, end_va);
2779+ if (doinval)
2780+ pmap_invalidate_range(&kernel_pmap, beg_va, end_va);
2781+}
2782+
2783+void
2784+pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count)
2785+{
2786+ _pmap_qenter(beg_va, m, count, 1);
2787+}
2788+
2789+void
2790+pmap_qenter_noinval(vm_offset_t beg_va, vm_page_t *m, int count)
2791+{
2792+ _pmap_qenter(beg_va, m, count, 0);
2793 }
2794
2795 /*
2796@@ -1808,7 +2149,8 @@ pmap_pinit0(struct pmap *pmap)
2797 pmap->pm_pml4 = (pml4_entry_t *)(PTOV_OFFSET + KPML4phys);
2798 pmap->pm_count = 1;
2799 CPUMASK_ASSZERO(pmap->pm_active);
2800- pmap->pm_pvhint = NULL;
2801+ pmap->pm_pvhint_pt = NULL;
2802+ pmap->pm_pvhint_pte = NULL;
2803 RB_INIT(&pmap->pm_pvroot);
2804 spin_init(&pmap->pm_spin, "pmapinit0");
2805 for (i = 0; i < PM_PLACEMARKS; ++i)
2806@@ -1831,7 +2173,8 @@ pmap_pinit_simple(struct pmap *pmap)
2807 */
2808 pmap->pm_count = 1;
2809 CPUMASK_ASSZERO(pmap->pm_active);
2810- pmap->pm_pvhint = NULL;
2811+ pmap->pm_pvhint_pt = NULL;
2812+ pmap->pm_pvhint_pte = NULL;
2813 pmap->pm_flags = PMAP_FLAG_SIMPLE;
2814
2815 pmap_pinit_defaults(pmap);
2816@@ -1871,15 +2214,16 @@ pmap_pinit(struct pmap *pmap)
2817 if (pmap->pm_pml4 == NULL) {
2818 pmap->pm_pml4 =
2819 (pml4_entry_t *)kmem_alloc_pageable(&kernel_map,
2820- PAGE_SIZE,
2821+ PAGE_SIZE * 2,
2822 VM_SUBSYS_PML4);
2823+ pmap->pm_pml4_iso = (void *)((char *)pmap->pm_pml4 + PAGE_SIZE);
2824 }
2825
2826 /*
2827- * Allocate the page directory page, which wires it even though
2828- * it isn't being entered into some higher level page table (it
2829- * being the highest level). If one is already cached we don't
2830- * have to do anything.
2831+ * Allocate the PML4e table, which wires it even though it isn't
2832+ * being entered into some higher level page table (it being the
2833+ * highest level). If one is already cached we don't have to do
2834+ * anything.
2835 */
2836 if ((pv = pmap->pm_pmlpv) == NULL) {
2837 pv = pmap_allocpte(pmap, pmap_pml4_pindex(), NULL);
2838@@ -1896,12 +2240,15 @@ pmap_pinit(struct pmap *pmap)
2839 (DMPDPphys + ((vm_paddr_t)j << PAGE_SHIFT)) |
2840 pmap->pmap_bits[PG_RW_IDX] |
2841 pmap->pmap_bits[PG_V_IDX] |
2842- pmap->pmap_bits[PG_U_IDX];
2843+ pmap->pmap_bits[PG_A_IDX];
2844+ }
2845+ for (j = 0; j < NKPML4E; ++j) {
2846+ pmap->pm_pml4[KPML4I + j] =
2847+ (KPDPphys + ((vm_paddr_t)j << PAGE_SHIFT)) |
2848+ pmap->pmap_bits[PG_RW_IDX] |
2849+ pmap->pmap_bits[PG_V_IDX] |
2850+ pmap->pmap_bits[PG_A_IDX];
2851 }
2852- pmap->pm_pml4[KPML4I] = KPDPphys |
2853- pmap->pmap_bits[PG_RW_IDX] |
2854- pmap->pmap_bits[PG_V_IDX] |
2855- pmap->pmap_bits[PG_U_IDX];
2856
2857 /*
2858 * install self-referential address mapping entry
2859@@ -1909,16 +2256,43 @@ pmap_pinit(struct pmap *pmap)
2860 pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pv->pv_m) |
2861 pmap->pmap_bits[PG_V_IDX] |
2862 pmap->pmap_bits[PG_RW_IDX] |
2863- pmap->pmap_bits[PG_A_IDX] |
2864- pmap->pmap_bits[PG_M_IDX];
2865+ pmap->pmap_bits[PG_A_IDX];
2866 } else {
2867 KKASSERT(pv->pv_m->flags & PG_MAPPED);
2868 KKASSERT(pv->pv_m->flags & PG_WRITEABLE);
2869 }
2870 KKASSERT(pmap->pm_pml4[255] == 0);
2871- KKASSERT(RB_ROOT(&pmap->pm_pvroot) == pv);
2872- KKASSERT(pv->pv_entry.rbe_left == NULL);
2873- KKASSERT(pv->pv_entry.rbe_right == NULL);
2874+
2875+ /*
2876+ * When implementing an isolated userland pmap, a second PML4e table
2877+ * is needed. We use pmap_pml4_pindex() + 1 for convenience, but
2878+ * note that we do not operate on this table using our API functions
2879+ * so handling of the + 1 case is mostly just to prevent implosions.
2880+ *
2881+ * We install an isolated version of the kernel PDPs into this
2882+ * second PML4e table. The pmap code will mirror all user PDPs
2883+ * between the primary and secondary PML4e table.
2884+ */
2885+ if ((pv = pmap->pm_pmlpv_iso) == NULL && isolated_user_pmap &&
2886+ pmap != &iso_pmap) {
2887+ pv = pmap_allocpte(pmap, pmap_pml4_pindex() + 1, NULL);
2888+ pmap->pm_pmlpv_iso = pv;
2889+ pmap_kenter((vm_offset_t)pmap->pm_pml4_iso,
2890+ VM_PAGE_TO_PHYS(pv->pv_m));
2891+ pv_put(pv);
2892+
2893+ /*
2894+ * Install an isolated version of the kernel pmap for
2895+ * user consumption, using PDPs constructed in iso_pmap.
2896+ */
2897+ for (j = 0; j < NKPML4E; ++j) {
2898+ pmap->pm_pml4_iso[KPML4I + j] =
2899+ iso_pmap.pm_pml4[KPML4I + j];
2900+ }
2901+ } else if (pv) {
2902+ KKASSERT(pv->pv_m->flags & PG_MAPPED);
2903+ KKASSERT(pv->pv_m->flags & PG_WRITEABLE);
2904+ }
2905 }
2906
2907 /*
2908@@ -1946,18 +2320,30 @@ pmap_puninit(pmap_t pmap)
2909 KKASSERT(p->flags & (PG_FICTITIOUS|PG_UNMANAGED));
2910 vm_page_unwire(p, 0);
2911 vm_page_flag_clear(p, PG_MAPPED | PG_WRITEABLE);
2912-
2913- /*
2914- * XXX eventually clean out PML4 static entries and
2915- * use vm_page_free_zero()
2916- */
2917 vm_page_free(p);
2918 pmap->pm_pmlpv = NULL;
2919 }
2920+ if ((pv = pmap->pm_pmlpv_iso) != NULL) {
2921+ if (pv_hold_try(pv) == 0)
2922+ pv_lock(pv);
2923+ KKASSERT(pv == pmap->pm_pmlpv_iso);
2924+ p = pmap_remove_pv_page(pv);
2925+ pv_free(pv, NULL);
2926+ pv = NULL; /* safety */
2927+ pmap_kremove((vm_offset_t)pmap->pm_pml4_iso);
2928+ vm_page_busy_wait(p, FALSE, "pgpun");
2929+ KKASSERT(p->flags & (PG_FICTITIOUS|PG_UNMANAGED));
2930+ vm_page_unwire(p, 0);
2931+ vm_page_flag_clear(p, PG_MAPPED | PG_WRITEABLE);
2932+ vm_page_free(p);
2933+ pmap->pm_pmlpv_iso = NULL;
2934+ }
2935 if (pmap->pm_pml4) {
2936 KKASSERT(pmap->pm_pml4 != (void *)(PTOV_OFFSET + KPML4phys));
2937- kmem_free(&kernel_map, (vm_offset_t)pmap->pm_pml4, PAGE_SIZE);
2938+ kmem_free(&kernel_map,
2939+ (vm_offset_t)pmap->pm_pml4, PAGE_SIZE * 2);
2940 pmap->pm_pml4 = NULL;
2941+ pmap->pm_pml4_iso = NULL;
2942 }
2943 KKASSERT(pmap->pm_stats.resident_count == 0);
2944 KKASSERT(pmap->pm_stats.wired_count == 0);
2945@@ -1986,6 +2372,7 @@ pv_entry_t
2946 pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp)
2947 {
2948 pt_entry_t *ptep;
2949+ pt_entry_t *ptep_iso;
2950 pv_entry_t pv;
2951 pv_entry_t pvp;
2952 pt_entry_t v;
2953@@ -2015,9 +2402,13 @@ pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp)
2954 * a pt_pv is not being requested for kernel VAs. The kernel
2955 * pre-wires all higher-level page tables so don't overload managed
2956 * higher-level page tables on top of it!
2957+ *
2958+ * However, its convenient for us to allow the case when creating
2959+ * iso_pmap. This is a bit of a hack but it simplifies iso_pmap
2960+ * a lot.
2961 */
2962 if (ptepindex < pmap_pt_pindex(0)) {
2963- if (ptepindex >= NUPTE_USER) {
2964+ if (ptepindex >= NUPTE_USER && pmap != &iso_pmap) {
2965 /* kernel manages this manually for KVM */
2966 KKASSERT(pvpp == NULL);
2967 } else {
2968@@ -2149,12 +2540,19 @@ pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp)
2969 */
2970 if (pvp) {
2971 v = VM_PAGE_TO_PHYS(m) |
2972- (pmap->pmap_bits[PG_U_IDX] |
2973- pmap->pmap_bits[PG_RW_IDX] |
2974+ (pmap->pmap_bits[PG_RW_IDX] |
2975 pmap->pmap_bits[PG_V_IDX] |
2976- pmap->pmap_bits[PG_A_IDX] |
2977- pmap->pmap_bits[PG_M_IDX]);
2978+ pmap->pmap_bits[PG_A_IDX]);
2979+ if (ptepindex < NUPTE_USER)
2980+ v |= pmap->pmap_bits[PG_U_IDX];
2981+ if (ptepindex < pmap_pt_pindex(0))
2982+ v |= pmap->pmap_bits[PG_M_IDX];
2983+
2984 ptep = pv_pte_lookup(pvp, ptepindex);
2985+ if (pvp == pmap->pm_pmlpv && pmap->pm_pmlpv_iso)
2986+ ptep_iso = pv_pte_lookup(pmap->pm_pmlpv_iso, ptepindex);
2987+ else
2988+ ptep_iso = NULL;
2989 if (*ptep & pmap->pmap_bits[PG_V_IDX]) {
2990 pt_entry_t pte;
2991
2992@@ -2162,7 +2560,12 @@ pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp)
2993 panic("pmap_allocpte: unexpected pte %p/%d",
2994 pvp, (int)ptepindex);
2995 }
2996- pte = pmap_inval_smp(pmap, (vm_offset_t)-1, 1, ptep, v);
2997+ pte = pmap_inval_smp(pmap, (vm_offset_t)-1, 1,
2998+ ptep, v);
2999+ if (ptep_iso) {
3000+ pmap_inval_smp(pmap, (vm_offset_t)-1, 1,
3001+ ptep_iso, v);
3002+ }
3003 if (vm_page_unwire_quick(
3004 PHYS_TO_VM_PAGE(pte & PG_FRAME))) {
3005 panic("pmap_allocpte: shared pgtable "
3006@@ -2172,6 +2575,8 @@ pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp)
3007 pt_entry_t pte;
3008
3009 pte = atomic_swap_long(ptep, v);
3010+ if (ptep_iso)
3011+ atomic_swap_long(ptep_iso, v);
3012 if (pte != 0) {
3013 kprintf("install pgtbl mixup 0x%016jx "
3014 "old/new 0x%016jx/0x%016jx\n",
3015@@ -2189,11 +2594,13 @@ notnew:
3016 KKASSERT(pvp->pv_m != NULL);
3017 ptep = pv_pte_lookup(pvp, ptepindex);
3018 v = VM_PAGE_TO_PHYS(pv->pv_m) |
3019- (pmap->pmap_bits[PG_U_IDX] |
3020- pmap->pmap_bits[PG_RW_IDX] |
3021+ (pmap->pmap_bits[PG_RW_IDX] |
3022 pmap->pmap_bits[PG_V_IDX] |
3023- pmap->pmap_bits[PG_A_IDX] |
3024- pmap->pmap_bits[PG_M_IDX]);
3025+ pmap->pmap_bits[PG_A_IDX]);
3026+ if (ptepindex < NUPTE_USER)
3027+ v |= pmap->pmap_bits[PG_U_IDX];
3028+ if (ptepindex < pmap_pt_pindex(0))
3029+ v |= pmap->pmap_bits[PG_M_IDX];
3030 if (*ptep != v) {
3031 kprintf("mismatched upper level pt %016jx/%016jx\n",
3032 *ptep, v);
3033@@ -2228,6 +2635,7 @@ pmap_allocpte_seg(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp,
3034 vm_object_t object;
3035 pmap_t obpmap;
3036 pmap_t *obpmapp;
3037+ vm_pindex_t *pt_placemark;
3038 vm_offset_t b;
3039 pv_entry_t pte_pv; /* in original or shared pmap */
3040 pv_entry_t pt_pv; /* in original or shared pmap */
3041@@ -2238,6 +2646,7 @@ pmap_allocpte_seg(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp,
3042 pd_entry_t opte; /* contents of *pt */
3043 pd_entry_t npte; /* contents of *pt */
3044 vm_page_t m;
3045+ int softhold;
3046
3047 /*
3048 * Basic tests, require a non-NULL vm_map_entry, require proper
3049@@ -2340,6 +2749,7 @@ pmap_allocpte_seg(pmap_t pmap, vm_pindex_t ptepindex, pv_entry_t *pvpp,
3050 */
3051 pt_pv = NULL;
3052 pte_pv = pmap_allocpte(obpmap, ptepindex, &pt_pv);
3053+ softhold = 0;
3054 retry:
3055 if (ptepindex >= pmap_pt_pindex(0))
3056 xpv = pte_pv;
3057@@ -2352,7 +2762,7 @@ retry:
3058 *
3059 * NOTE: proc_pt_pv can be NULL.
3060 */
3061- proc_pt_pv = pv_get(pmap, pmap_pt_pindex(b), NULL);
3062+ proc_pt_pv = pv_get(pmap, pmap_pt_pindex(b), &pt_placemark);
3063 proc_pd_pv = pmap_allocpte(pmap, pmap_pd_pindex(b), NULL);
3064 #ifdef PMAP_DEBUG2
3065 if (pmap_enter_debug > 0) {
3066@@ -2394,27 +2804,38 @@ retry:
3067 pmap_inval_bulk_t bulk;
3068
3069 if (proc_pt_pv->pv_m->wire_count != 1) {
3070+ /*
3071+ * The page table has a bunch of stuff in it
3072+ * which we have to scrap.
3073+ */
3074+ if (softhold == 0) {
3075+ softhold = 1;
3076+ pmap_softhold(pmap);
3077+ }
3078 pv_put(proc_pd_pv);
3079 pv_put(proc_pt_pv);
3080 pmap_remove(pmap,
3081 va & ~(vm_offset_t)SEG_MASK,
3082 (va + SEG_SIZE) & ~(vm_offset_t)SEG_MASK);
3083- goto retry;
3084+ } else {
3085+ /*
3086+ * The page table is empty and can be destroyed.
3087+ * However, doing so leaves the pt slot unlocked,
3088+ * so we have to loop-up to handle any races until
3089+ * we get a NULL proc_pt_pv and a proper pt_placemark.
3090+ */
3091+ pmap_inval_bulk_init(&bulk, proc_pt_pv->pv_pmap);
3092+ pmap_release_pv(proc_pt_pv, proc_pd_pv, &bulk);
3093+ pmap_inval_bulk_flush(&bulk);
3094+ pv_put(proc_pd_pv);
3095 }
3096-
3097- /*
3098- * The release call will indirectly clean out *pt
3099- */
3100- pmap_inval_bulk_init(&bulk, proc_pt_pv->pv_pmap);
3101- pmap_release_pv(proc_pt_pv, proc_pd_pv, &bulk);
3102- pmap_inval_bulk_flush(&bulk);
3103- proc_pt_pv = NULL;
3104- /* relookup */
3105- pt = pv_pte_lookup(proc_pd_pv, pmap_pt_index(b));
3106+ goto retry;
3107 }
3108
3109 /*
3110- * Handle remaining cases.
3111+ * Handle remaining cases. We are holding pt_placemark to lock
3112+ * the page table page in the primary pmap while we manipulate
3113+ * it.
3114 */
3115 if (*pt == 0) {
3116 atomic_swap_long(pt, npte);
3117@@ -2454,6 +2875,14 @@ retry:
3118 }
3119 }
3120
3121+ if (softhold)
3122+ pmap_softdone(pmap);
3123+
3124+ /*
3125+ * Remove our earmark on the page table page.
3126+ */
3127+ pv_placemarker_wakeup(pmap, pt_placemark);
3128+
3129 /*
3130 * The existing process page table was replaced and must be destroyed
3131 * here.
3132@@ -2464,7 +2893,6 @@ retry:
3133 *pvpp = pt_pv;
3134 else
3135 pv_put(pt_pv);
3136-
3137 return (pte_pv);
3138 }
3139
3140@@ -2516,12 +2944,19 @@ pmap_release(struct pmap *pmap)
3141
3142
3143 /*
3144- * One resident page (the pml4 page) should remain.
3145+ * One resident page (the pml4 page) should remain. Two if
3146+ * the pmap has implemented an isolated userland PML4E table.
3147 * No wired pages should remain.
3148 */
3149+ int expected_res = 0;
3150+
3151+ if ((pmap->pm_flags & PMAP_FLAG_SIMPLE) == 0)
3152+ ++expected_res;
3153+ if (pmap->pm_pmlpv_iso)
3154+ ++expected_res;
3155+
3156 #if 1
3157- if (pmap->pm_stats.resident_count !=
3158- ((pmap->pm_flags & PMAP_FLAG_SIMPLE) ? 0 : 1) ||
3159+ if (pmap->pm_stats.resident_count != expected_res ||
3160 pmap->pm_stats.wired_count != 0) {
3161 kprintf("fatal pmap problem - pmap %p flags %08x "
3162 "rescnt=%jd wirecnt=%jd\n",
3163@@ -2532,8 +2967,7 @@ pmap_release(struct pmap *pmap)
3164 tsleep(pmap, 0, "DEAD", 0);
3165 }
3166 #else
3167- KKASSERT(pmap->pm_stats.resident_count ==
3168- ((pmap->pm_flags & PMAP_FLAG_SIMPLE) ? 0 : 1));
3169+ KKASSERT(pmap->pm_stats.resident_count == expected_res);
3170 KKASSERT(pmap->pm_stats.wired_count == 0);
3171 #endif
3172 }
3173@@ -2591,13 +3025,9 @@ pmap_release_callback(pv_entry_t pv, void *data)
3174 pindex += NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL;
3175 } else if (pv->pv_pindex < pmap_pml4_pindex()) {
3176 /*
3177- * I am PDP, parent is PML4 (there's only one)
3178+ * I am PDP, parent is PML4. We always calculate the
3179+ * normal PML4 here, not the isolated PML4.
3180 */
3181-#if 0
3182- pindex = (pv->pv_pindex - NUPTE_TOTAL - NUPT_TOTAL -
3183- NUPD_TOTAL) >> NPML4EPGSHIFT;
3184- pindex += NUPTE_TOTAL + NUPT_TOTAL + NUPD_TOTAL + NUPDP_TOTAL;
3185-#endif
3186 pindex = pmap_pml4_pindex();
3187 } else {
3188 /*
3189@@ -2671,8 +3101,10 @@ pmap_release_pv(pv_entry_t pv, pv_entry_t pvp, pmap_inval_bulk_t *bulk)
3190 *
3191 * Since we are leaving the top-level pv intact we need
3192 * to break out of what would otherwise be an infinite loop.
3193+ *
3194+ * This covers both the normal and the isolated PML4 page.
3195 */
3196- if (pv->pv_pindex == pmap_pml4_pindex()) {
3197+ if (pv->pv_pindex >= pmap_pml4_pindex()) {
3198 pv_put(pv);
3199 return(-1);
3200 }
3201@@ -2737,9 +3169,13 @@ pmap_remove_pv_pte(pv_entry_t pv, pv_entry_t pvp, pmap_inval_bulk_t *bulk,
3202
3203 KKASSERT(pmap);
3204
3205- if (ptepindex == pmap_pml4_pindex()) {
3206+ if (ptepindex >= pmap_pml4_pindex()) {
3207 /*
3208 * We are the top level PML4E table, there is no parent.
3209+ *
3210+ * This is either the normal or isolated PML4E table.
3211+ * Only the normal is used in regular operation, the isolated
3212+ * is only passed in when breaking down the whole pmap.
3213 */
3214 p = pmap->pm_pmlpv->pv_m;
3215 KKASSERT(pv->pv_m == p); /* debugging */
3216@@ -2752,6 +3188,7 @@ pmap_remove_pv_pte(pv_entry_t pv, pv_entry_t pvp, pmap_inval_bulk_t *bulk,
3217 vm_pindex_t pml4_pindex;
3218 vm_pindex_t pdp_index;
3219 pml4_entry_t *pdp;
3220+ pml4_entry_t *pdp_iso;
3221
3222 pdp_index = ptepindex - pmap_pdp_pindex(0);
3223 if (pvp == NULL) {
3224@@ -2765,6 +3202,16 @@ pmap_remove_pv_pte(pv_entry_t pv, pv_entry_t pvp, pmap_inval_bulk_t *bulk,
3225 KKASSERT((*pdp & pmap->pmap_bits[PG_V_IDX]) != 0);
3226 p = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
3227 pmap_inval_bulk(bulk, (vm_offset_t)-1, pdp, 0);
3228+
3229+ /*
3230+ * Also remove the PDP from the isolated PML4E if the
3231+ * process uses one.
3232+ */
3233+ if (pvp == pmap->pm_pmlpv && pmap->pm_pmlpv_iso) {
3234+ pdp_iso = &pmap->pm_pml4_iso[pdp_index &
3235+ ((1ul << NPML4EPGSHIFT) - 1)];
3236+ pmap_inval_bulk(bulk, (vm_offset_t)-1, pdp_iso, 0);
3237+ }
3238 KKASSERT(pv->pv_m == p); /* debugging */
3239 } else if (ptepindex >= pmap_pd_pindex(0)) {
3240 /*
3241@@ -2971,7 +3418,7 @@ pmap_remove_pv_pte(pv_entry_t pv, pv_entry_t pvp, pmap_inval_bulk_t *bulk,
3242 pvp->pv_m &&
3243 pvp->pv_m->wire_count == 1 &&
3244 (pvp->pv_hold & PV_HOLD_MASK) == 2 &&
3245- pvp->pv_pindex != pmap_pml4_pindex()) {
3246+ pvp->pv_pindex < pmap_pml4_pindex()) {
3247 if (pmap_dynamic_delete == 2)
3248 kprintf("A %jd %08x\n", pvp->pv_pindex, pvp->pv_hold);
3249 if (pmap != &kernel_pmap) {
3250@@ -3026,7 +3473,7 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
3251 vm_offset_t ptppaddr;
3252 vm_page_t nkpg;
3253 pd_entry_t *pt, newpt;
3254- pdp_entry_t newpd;
3255+ pdp_entry_t *pd, newpd;
3256 int update_kernel_vm_end;
3257
3258 /*
3259@@ -3034,11 +3481,15 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
3260 */
3261 if (kernel_vm_end == 0) {
3262 kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
3263- nkpt = 0;
3264- while ((*pmap_pt(&kernel_pmap, kernel_vm_end) & kernel_pmap.pmap_bits[PG_V_IDX]) != 0) {
3265+
3266+ for (;;) {
3267+ pt = pmap_pt(&kernel_pmap, kernel_vm_end);
3268+ if (pt == NULL)
3269+ break;
3270+ if ((*pt & kernel_pmap.pmap_bits[PG_V_IDX]) == 0)
3271+ break;
3272 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) &
3273- ~(PAGE_SIZE * NPTEPG - 1);
3274- nkpt++;
3275+ ~(vm_offset_t)(PAGE_SIZE * NPTEPG - 1);
3276 if (kernel_vm_end - 1 >= kernel_map.max_offset) {
3277 kernel_vm_end = kernel_map.max_offset;
3278 break;
3279@@ -3060,8 +3511,8 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
3280 update_kernel_vm_end = 0;
3281 }
3282
3283- kstart = rounddown2(kstart, PAGE_SIZE * NPTEPG);
3284- kend = roundup2(kend, PAGE_SIZE * NPTEPG);
3285+ kstart = rounddown2(kstart, (vm_offset_t)(PAGE_SIZE * NPTEPG));
3286+ kend = roundup2(kend, (vm_offset_t)(PAGE_SIZE * NPTEPG));
3287
3288 if (kend - 1 >= kernel_map.max_offset)
3289 kend = kernel_map.max_offset;
3290@@ -3069,7 +3520,9 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
3291 while (kstart < kend) {
3292 pt = pmap_pt(&kernel_pmap, kstart);
3293 if (pt == NULL) {
3294- /* We need a new PD entry */
3295+ /*
3296+ * We need a new PD entry
3297+ */
3298 nkpg = vm_page_alloc(NULL, mycpu->gd_rand_incr++,
3299 VM_ALLOC_NORMAL |
3300 VM_ALLOC_SYSTEM |
3301@@ -3080,18 +3533,26 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
3302 }
3303 paddr = VM_PAGE_TO_PHYS(nkpg);
3304 pmap_zero_page(paddr);
3305+ pd = pmap_pd(&kernel_pmap, kstart);
3306+
3307 newpd = (pdp_entry_t)
3308 (paddr |
3309 kernel_pmap.pmap_bits[PG_V_IDX] |
3310 kernel_pmap.pmap_bits[PG_RW_IDX] |
3311- kernel_pmap.pmap_bits[PG_A_IDX] |
3312- kernel_pmap.pmap_bits[PG_M_IDX]);
3313- *pmap_pd(&kernel_pmap, kstart) = newpd;
3314+ kernel_pmap.pmap_bits[PG_A_IDX]);
3315+ atomic_swap_long(pd, newpd);
3316+
3317+#if 0
3318+ kprintf("NEWPD pd=%p pde=%016jx phys=%016jx\n",
3319+ pd, newpd, paddr);
3320+#endif
3321+
3322 continue; /* try again */
3323 }
3324+
3325 if ((*pt & kernel_pmap.pmap_bits[PG_V_IDX]) != 0) {
3326 kstart = (kstart + PAGE_SIZE * NPTEPG) &
3327- ~(PAGE_SIZE * NPTEPG - 1);
3328+ ~(vm_offset_t)(PAGE_SIZE * NPTEPG - 1);
3329 if (kstart - 1 >= kernel_map.max_offset) {
3330 kstart = kernel_map.max_offset;
3331 break;
3332@@ -3117,12 +3578,11 @@ pmap_growkernel(vm_offset_t kstart, vm_offset_t kend)
3333 newpt = (pd_entry_t)(ptppaddr |
3334 kernel_pmap.pmap_bits[PG_V_IDX] |
3335 kernel_pmap.pmap_bits[PG_RW_IDX] |
3336- kernel_pmap.pmap_bits[PG_A_IDX] |
3337- kernel_pmap.pmap_bits[PG_M_IDX]);
3338- atomic_swap_long(pmap_pt(&kernel_pmap, kstart), newpt);
3339+ kernel_pmap.pmap_bits[PG_A_IDX]);
3340+ atomic_swap_long(pt, newpt);
3341
3342 kstart = (kstart + PAGE_SIZE * NPTEPG) &
3343- ~(PAGE_SIZE * NPTEPG - 1);
3344+ ~(vm_offset_t)(PAGE_SIZE * NPTEPG - 1);
3345
3346 if (kstart - 1 >= kernel_map.max_offset) {
3347 kstart = kernel_map.max_offset;
3348@@ -3257,13 +3717,14 @@ static
3349 pv_entry_t
3350 _pv_alloc(pmap_t pmap, vm_pindex_t pindex, int *isnew PMAP_DEBUG_DECL)
3351 {
3352+ struct mdglobaldata *md = mdcpu;
3353 pv_entry_t pv;
3354 pv_entry_t pnew;
3355- struct mdglobaldata *md = mdcpu;
3356+ int pmap_excl = 0;
3357
3358 pnew = NULL;
3359 if (md->gd_newpv) {
3360-#if 0
3361+#if 1
3362 pnew = atomic_swap_ptr((void *)&md->gd_newpv, NULL);
3363 #else
3364 crit_enter();
3365@@ -3275,22 +3736,27 @@ _pv_alloc(pmap_t pmap, vm_pindex_t pindex, int *isnew PMAP_DEBUG_DECL)
3366 if (pnew == NULL)
3367 pnew = zalloc(pvzone);
3368
3369- spin_lock(&pmap->pm_spin);
3370+ spin_lock_shared(&pmap->pm_spin);
3371 for (;;) {
3372 /*
3373 * Shortcut cache
3374 */
3375- pv = pmap->pm_pvhint;
3376- cpu_ccfence();
3377- if (pv == NULL ||
3378- pv->pv_pmap != pmap ||
3379- pv->pv_pindex != pindex) {
3380- pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot,
3381- pindex);
3382- }
3383+ pv = pv_entry_lookup(pmap, pindex);
3384 if (pv == NULL) {
3385 vm_pindex_t *pmark;
3386
3387+ /*
3388+ * Requires exclusive pmap spinlock
3389+ */
3390+ if (pmap_excl == 0) {
3391+ pmap_excl = 1;
3392+ if (!spin_lock_upgrade_try(&pmap->pm_spin)) {
3393+ spin_unlock_shared(&pmap->pm_spin);
3394+ spin_lock(&pmap->pm_spin);
3395+ continue;
3396+ }
3397+ }
3398+
3399 /*
3400 * We need to block if someone is holding our
3401 * placemarker. As long as we determine the
3402@@ -3342,8 +3808,11 @@ _pv_alloc(pmap_t pmap, vm_pindex_t pindex, int *isnew PMAP_DEBUG_DECL)
3403 * we can get the lock, otherwise block and retry.
3404 */
3405 if (__predict_true(_pv_hold_try(pv PMAP_DEBUG_COPY))) {
3406- spin_unlock(&pmap->pm_spin);
3407-#if 0
3408+ if (pmap_excl)
3409+ spin_unlock(&pmap->pm_spin);
3410+ else
3411+ spin_unlock_shared(&pmap->pm_spin);
3412+#if 1
3413 pnew = atomic_swap_ptr((void *)&md->gd_newpv, pnew);
3414 if (pnew)
3415 zfree(pvzone, pnew);
3416@@ -3360,10 +3829,17 @@ _pv_alloc(pmap_t pmap, vm_pindex_t pindex, int *isnew PMAP_DEBUG_DECL)
3417 *isnew = 0;
3418 return(pv);
3419 }
3420- spin_unlock(&pmap->pm_spin);
3421- _pv_lock(pv PMAP_DEBUG_COPY);
3422- pv_put(pv);
3423- spin_lock(&pmap->pm_spin);
3424+ if (pmap_excl) {
3425+ spin_unlock(&pmap->pm_spin);
3426+ _pv_lock(pv PMAP_DEBUG_COPY);
3427+ pv_put(pv);
3428+ spin_lock(&pmap->pm_spin);
3429+ } else {
3430+ spin_unlock_shared(&pmap->pm_spin);
3431+ _pv_lock(pv PMAP_DEBUG_COPY);
3432+ pv_put(pv);
3433+ spin_lock_shared(&pmap->pm_spin);
3434+ }
3435 }
3436 /* NOT REACHED */
3437 }
3438@@ -3376,20 +3852,14 @@ pv_entry_t
3439 _pv_get(pmap_t pmap, vm_pindex_t pindex, vm_pindex_t **pmarkp PMAP_DEBUG_DECL)
3440 {
3441 pv_entry_t pv;
3442+ int pmap_excl = 0;
3443
3444- spin_lock(&pmap->pm_spin);
3445+ spin_lock_shared(&pmap->pm_spin);
3446 for (;;) {
3447 /*
3448 * Shortcut cache
3449 */
3450- pv = pmap->pm_pvhint;
3451- cpu_ccfence();
3452- if (pv == NULL ||
3453- pv->pv_pmap != pmap ||
3454- pv->pv_pindex != pindex) {
3455- pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot,
3456- pindex);
3457- }
3458+ pv = pv_entry_lookup(pmap, pindex);
3459 if (pv == NULL) {
3460 /*
3461 * Block if there is ANY placemarker. If we are to
3462@@ -3404,6 +3874,18 @@ _pv_get(pmap_t pmap, vm_pindex_t pindex, vm_pindex_t **pmarkp PMAP_DEBUG_DECL)
3463 */
3464 vm_pindex_t *pmark;
3465
3466+ /*
3467+ * Requires exclusive pmap spinlock
3468+ */
3469+ if (pmap_excl == 0) {
3470+ pmap_excl = 1;
3471+ if (!spin_lock_upgrade_try(&pmap->pm_spin)) {
3472+ spin_unlock_shared(&pmap->pm_spin);
3473+ spin_lock(&pmap->pm_spin);
3474+ continue;
3475+ }
3476+ }
3477+
3478 pmark = pmap_placemarker_hash(pmap, pindex);
3479
3480 if ((pmarkp && *pmark != PM_NOPLACEMARK) ||
3481@@ -3431,15 +3913,25 @@ _pv_get(pmap_t pmap, vm_pindex_t pindex, vm_pindex_t **pmarkp PMAP_DEBUG_DECL)
3482 }
3483 if (_pv_hold_try(pv PMAP_DEBUG_COPY)) {
3484 pv_cache(pv, pindex);
3485- spin_unlock(&pmap->pm_spin);
3486+ if (pmap_excl)
3487+ spin_unlock(&pmap->pm_spin);
3488+ else
3489+ spin_unlock_shared(&pmap->pm_spin);
3490 KKASSERT(pv->pv_pmap == pmap &&
3491 pv->pv_pindex == pindex);
3492 return(pv);
3493 }
3494- spin_unlock(&pmap->pm_spin);
3495- _pv_lock(pv PMAP_DEBUG_COPY);
3496- pv_put(pv);
3497- spin_lock(&pmap->pm_spin);
3498+ if (pmap_excl) {
3499+ spin_unlock(&pmap->pm_spin);
3500+ _pv_lock(pv PMAP_DEBUG_COPY);
3501+ pv_put(pv);
3502+ spin_lock(&pmap->pm_spin);
3503+ } else {
3504+ spin_unlock_shared(&pmap->pm_spin);
3505+ _pv_lock(pv PMAP_DEBUG_COPY);
3506+ pv_put(pv);
3507+ spin_lock_shared(&pmap->pm_spin);
3508+ }
3509 }
3510 }
3511
3512@@ -3465,14 +3957,7 @@ pv_get_try(pmap_t pmap, vm_pindex_t pindex, vm_pindex_t **pmarkp, int *errorp)
3513
3514 spin_lock_shared(&pmap->pm_spin);
3515
3516- pv = pmap->pm_pvhint;
3517- cpu_ccfence();
3518- if (pv == NULL ||
3519- pv->pv_pmap != pmap ||
3520- pv->pv_pindex != pindex) {
3521- pv = pv_entry_rb_tree_RB_LOOKUP(&pmap->pm_pvroot, pindex);
3522- }
3523-
3524+ pv = pv_entry_lookup(pmap, pindex);
3525 if (pv == NULL) {
3526 vm_pindex_t *pmark;
3527
3528@@ -3642,8 +4127,10 @@ _pv_free(pv_entry_t pv, pv_entry_t pvp PMAP_DEBUG_DECL)
3529 if ((pmap = pv->pv_pmap) != NULL) {
3530 spin_lock(&pmap->pm_spin);
3531 KKASSERT(pv->pv_pmap == pmap);
3532- if (pmap->pm_pvhint == pv)
3533- pmap->pm_pvhint = NULL;
3534+ if (pmap->pm_pvhint_pt == pv)
3535+ pmap->pm_pvhint_pt = NULL;
3536+ if (pmap->pm_pvhint_pte == pv)
3537+ pmap->pm_pvhint_pte = NULL;
3538 pv_entry_rb_tree_RB_REMOVE(&pmap->pm_pvroot, pv);
3539 atomic_add_long(&pmap->pm_stats.resident_count, -1);
3540 pv->pv_pmap = NULL;
3541@@ -4406,7 +4893,7 @@ pmap_remove_callback(pmap_t pmap, struct pmap_scan_info *info,
3542 pt_pv->pv_m &&
3543 pt_pv->pv_m->wire_count == 1 &&
3544 (pt_pv->pv_hold & PV_HOLD_MASK) == 2 &&
3545- pt_pv->pv_pindex != pmap_pml4_pindex()) {
3546+ pt_pv->pv_pindex < pmap_pml4_pindex()) {
3547 if (pmap_dynamic_delete == 2)
3548 kprintf("B %jd %08x\n", pt_pv->pv_pindex, pt_pv->pv_hold);
3549 pv_hold(pt_pv); /* extra hold */
3550@@ -4742,6 +5229,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3551 ptep = vtopte(va);
3552 origpte = *ptep;
3553 } else if (m->flags & (/*PG_FICTITIOUS |*/ PG_UNMANAGED)) { /* XXX */
3554+ pmap_softwait(pmap);
3555 pte_pv = pv_get(pmap, pmap_pte_pindex(va), &pte_placemark);
3556 KKASSERT(pte_pv == NULL);
3557 if (va >= VM_MAX_USER_ADDRESS) {
3558@@ -4758,6 +5246,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3559 (origpte & pmap->pmap_bits[PG_MANAGED_IDX]) == 0,
3560 ("Invalid PTE 0x%016jx @ 0x%016jx\n", origpte, va));
3561 } else {
3562+ pmap_softwait(pmap);
3563 if (va >= VM_MAX_USER_ADDRESS) {
3564 /*
3565 * Kernel map, pv_entry-tracked.
3566@@ -5108,10 +5597,15 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_prot_t prot,
3567 info.mpte = NULL;
3568 info.addr = addr;
3569 info.pmap = pmap;
3570+ info.object = object;
3571
3572+ /*
3573+ * By using the NOLK scan, the callback function must be sure
3574+ * to return -1 if the VM page falls out of the object.
3575+ */
3576 vm_object_hold_shared(object);
3577- vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
3578- pmap_object_init_pt_callback, &info);
3579+ vm_page_rb_tree_RB_SCAN_NOLK(&object->rb_memq, rb_vm_page_scancmp,
3580+ pmap_object_init_pt_callback, &info);
3581 vm_object_drop(object);
3582 }
3583
3584@@ -5121,6 +5615,7 @@ pmap_object_init_pt_callback(vm_page_t p, void *data)
3585 {
3586 struct rb_vm_page_scan_info *info = data;
3587 vm_pindex_t rel_index;
3588+ int hard_busy;
3589
3590 /*
3591 * don't allow an madvise to blow away our really
3592@@ -5137,17 +5632,41 @@ pmap_object_init_pt_callback(vm_page_t p, void *data)
3593 */
3594 if (p->flags & PG_MARKER)
3595 return 0;
3596- if (vm_page_busy_try(p, TRUE))
3597- return 0;
3598+ hard_busy = 0;
3599+again:
3600+ if (hard_busy) {
3601+ if (vm_page_busy_try(p, TRUE))
3602+ return 0;
3603+ } else {
3604+ if (vm_page_sbusy_try(p))
3605+ return 0;
3606+ }
3607 if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
3608 (p->flags & PG_FICTITIOUS) == 0) {
3609- if ((p->queue - p->pc) == PQ_CACHE)
3610+ if ((p->queue - p->pc) == PQ_CACHE) {
3611+ if (hard_busy == 0) {
3612+ vm_page_sbusy_drop(p);
3613+ hard_busy = 1;
3614+ goto again;
3615+ }
3616 vm_page_deactivate(p);
3617+ }
3618 rel_index = p->pindex - info->start_pindex;
3619 pmap_enter_quick(info->pmap,
3620 info->addr + x86_64_ptob(rel_index), p);
3621 }
3622- vm_page_wakeup(p);
3623+ if (hard_busy)
3624+ vm_page_wakeup(p);
3625+ else
3626+ vm_page_sbusy_drop(p);
3627+
3628+ /*
3629+ * We are using an unlocked scan (that is, the scan expects its
3630+ * current element to remain in the tree on return). So we have
3631+ * to check here and abort the scan if it isn't.
3632+ */
3633+ if (p->object != info->object)
3634+ return -1;
3635 lwkt_yield();
3636 return(0);
3637 }
3638@@ -5371,6 +5890,7 @@ pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
3639 * pmap_testbit tests bits in pte's note that the testbit/clearbit
3640 * routines are inline, and a lot of things compile-time evaluate.
3641 */
3642+
3643 static
3644 boolean_t
3645 pmap_testbit(vm_page_t m, int bit)
3646@@ -5391,7 +5911,6 @@ pmap_testbit(vm_page_t m, int bit)
3647 }
3648
3649 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
3650-
3651 #if defined(PMAP_DIAGNOSTIC)
3652 if (pv->pv_pmap == NULL) {
3653 kprintf("Null pmap (tb) at pindex: %"PRIu64"\n",
3654@@ -5692,7 +6211,7 @@ pmap_clear_reference(vm_page_t m)
3655
3656 static
3657 void
3658-i386_protection_init(void)
3659+x86_64_protection_init(void)
3660 {
3661 uint64_t *kp;
3662 int prot;
3663@@ -5982,14 +6501,16 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
3664 {
3665 struct vmspace *oldvm;
3666 struct pmap *pmap;
3667+ thread_t td;
3668
3669 oldvm = lp->lwp_vmspace;
3670
3671 if (oldvm != newvm) {
3672 crit_enter();
3673+ td = curthread;
3674 KKASSERT((newvm->vm_refcnt & VM_REF_DELETED) == 0);
3675 lp->lwp_vmspace = newvm;
3676- if (curthread->td_lwp == lp) {
3677+ if (td->td_lwp == lp) {
3678 pmap = vmspace_pmap(newvm);
3679 ATOMIC_CPUMASK_ORBIT(pmap->pm_active, mycpu->gd_cpuid);
3680 if (pmap->pm_active_lock & CPULOCK_EXCL)
3681@@ -5998,13 +6519,45 @@ pmap_setlwpvm(struct lwp *lp, struct vmspace *newvm)
3682 tlb_flush_count++;
3683 #endif
3684 if (pmap->pmap_bits[TYPE_IDX] == REGULAR_PMAP) {
3685- curthread->td_pcb->pcb_cr3 = vtophys(pmap->pm_pml4);
3686+ td->td_pcb->pcb_cr3 = vtophys(pmap->pm_pml4);
3687+ if (isolated_user_pmap && pmap->pm_pmlpv_iso) {
3688+ td->td_pcb->pcb_cr3_iso =
3689+ vtophys(pmap->pm_pml4_iso);
3690+ td->td_pcb->pcb_flags |= PCB_ISOMMU;
3691+ } else {
3692+ td->td_pcb->pcb_cr3_iso = 0;
3693+ td->td_pcb->pcb_flags &= ~PCB_ISOMMU;
3694+ }
3695 } else if (pmap->pmap_bits[TYPE_IDX] == EPT_PMAP) {
3696- curthread->td_pcb->pcb_cr3 = KPML4phys;
3697+ td->td_pcb->pcb_cr3 = KPML4phys;
3698+ td->td_pcb->pcb_cr3_iso = 0;
3699+ td->td_pcb->pcb_flags &= ~PCB_ISOMMU;
3700 } else {
3701 panic("pmap_setlwpvm: unknown pmap type\n");
3702 }
3703- load_cr3(curthread->td_pcb->pcb_cr3);
3704+
3705+ /*
3706+ * The MMU separation fields needs to be updated.
3707+ * (it can't access the pcb directly from the
3708+ * restricted user pmap).
3709+ */
3710+ {
3711+ struct trampframe *tramp;
3712+
3713+ tramp = &pscpu->trampoline;
3714+ tramp->tr_pcb_cr3 = td->td_pcb->pcb_cr3;
3715+ tramp->tr_pcb_cr3_iso = td->td_pcb->pcb_cr3_iso;
3716+ tramp->tr_pcb_flags = td->td_pcb->pcb_flags;
3717+ tramp->tr_pcb_rsp = (register_t)td->td_pcb;
3718+ /* tr_pcb_rsp doesn't change */
3719+ }
3720+
3721+ /*
3722+ * In kernel-land we always use the normal PML4E
3723+ * so the kernel is fully mapped and can also access
3724+ * user memory.
3725+ */
3726+ load_cr3(td->td_pcb->pcb_cr3);
3727 pmap = vmspace_pmap(oldvm);
3728 ATOMIC_CPUMASK_NANDBIT(pmap->pm_active,
3729 mycpu->gd_cpuid);
3730diff --git a/sys/platform/pc64/x86_64/swtch.s b/sys/platform/pc64/x86_64/swtch.s
3731index 735348769..ee6b3790d 100644
3732--- a/sys/platform/pc64/x86_64/swtch.s
3733+++ b/sys/platform/pc64/x86_64/swtch.s
3734@@ -362,7 +362,13 @@ END(cpu_exit_switch)
3735
3736 ENTRY(cpu_heavy_restore)
3737 movq TD_PCB(%rax),%rdx /* RDX = PCB */
3738- movq %rdx, PCPU(common_tss) + TSS_RSP0
3739+ movq %rdx, PCPU(trampoline)+TR_PCB_RSP
3740+ movq PCB_FLAGS(%rdx), %rcx
3741+ movq %rcx, PCPU(trampoline)+TR_PCB_FLAGS
3742+ movq PCB_CR3_ISO(%rdx), %rcx
3743+ movq %rcx, PCPU(trampoline)+TR_PCB_CR3_ISO
3744+ movq PCB_CR3(%rdx), %rcx
3745+ movq %rcx, PCPU(trampoline)+TR_PCB_CR3
3746 popfq
3747
3748 #if defined(SWTCH_OPTIM_STATS)
3749@@ -459,15 +465,22 @@ ENTRY(cpu_heavy_restore)
3750 jnz 2f
3751 #endif
3752
3753+#if 0
3754 /*
3755- * Going back to the common_tss. We may need to update TSS_RSP0
3756- * which sets the top of the supervisor stack when entering from
3757- * usermode. The PCB is at the top of the stack but we need another
3758- * 16 bytes to take vm86 into account.
3759- */
3760- movq %rdx,%rcx
3761- /*leaq -TF_SIZE(%rdx),%rcx*/
3762- movq %rcx, PCPU(common_tss) + TSS_RSP0
3763+ * Going back to the common_tss. (this was already executed at
3764+ * the top).
3765+ *
3766+ * Set the top of the supervisor stack for the new thread
3767+ * in gd_thread_pcb so the trampoline code can load it into %rsp.
3768+ */
3769+ movq %rdx, PCPU(trampoline)+TR_PCB_RSP
3770+ movq PCB_FLAGS(%rdx), %rcx
3771+ movq %rcx, PCPU(trampoline)+TR_PCB_FLAGS
3772+ movq PCB_CR3_ISO(%rdx), %rcx
3773+ movq %rcx, PCPU(trampoline)+TR_PCB_CR3_ISO
3774+ movq PCB_CR3(%rdx), %rcx
3775+ movq %rcx, PCPU(trampoline)+TR_PCB_CR3
3776+#endif
3777
3778 #if 0 /* JG */
3779 cmpl $0,PCPU(private_tss) /* don't have to reload if */
3780@@ -759,7 +772,7 @@ ENTRY(cpu_kthread_restore)
3781 * rax and rbx come from the switchout code. Call
3782 * lwkt_switch_return(otd).
3783 *
3784- * NOTE: unlike i386, %rsi and %rdi are not call-saved regs.
3785+ * NOTE: unlike i386, the %rsi and %rdi are not call-saved regs.
3786 */
3787 pushq %rax
3788 movq %rbx,%rdi
3789@@ -855,15 +868,6 @@ ENTRY(cpu_lwkt_restore)
3790 je 1f
3791 movq %rcx,%cr3
3792 1:
3793- /*
3794- * Safety, clear RSP0 in the tss so it isn't pointing at the
3795- * previous thread's kstack (if a heavy weight user thread).
3796- * RSP0 should only be used in ring 3 transitions and kernel
3797- * threads run in ring 0 so there should be none.
3798- */
3799- xorq %rdx,%rdx
3800- movq %rdx, PCPU(common_tss) + TSS_RSP0
3801-
3802 /*
3803 * NOTE: %rbx is the previous thread and %rax is the new thread.
3804 * %rbx is retained throughout so we can return it.
3805diff --git a/sys/platform/pc64/x86_64/vm_machdep.c b/sys/platform/pc64/x86_64/vm_machdep.c
3806index 07031e6a6..5c644def7 100644
3807--- a/sys/platform/pc64/x86_64/vm_machdep.c
3808+++ b/sys/platform/pc64/x86_64/vm_machdep.c
3809@@ -88,12 +88,19 @@ void
3810 cpu_fork(struct lwp *lp1, struct lwp *lp2, int flags)
3811 {
3812 struct pcb *pcb2;
3813+ struct pmap *pmap2;
3814
3815 if ((flags & RFPROC) == 0) {
3816 if ((flags & RFMEM) == 0) {
3817- /* unshare user LDT */
3818+ /*
3819+ * Unshare user LDT. > 1 test is MPSAFE. While
3820+ * it can potentially race a 2->1 transition, the
3821+ * worst that happens is that we do an unnecessary
3822+ * ldt replacement.
3823+ */
3824 struct pcb *pcb1 = lp1->lwp_thread->td_pcb;
3825 struct pcb_ldt *pcb_ldt = pcb1->pcb_ldt;
3826+
3827 if (pcb_ldt && pcb_ldt->ldt_refcnt > 1) {
3828 pcb_ldt = user_ldt_alloc(pcb1,pcb_ldt->ldt_len);
3829 user_ldt_free(pcb1);
3830@@ -107,7 +114,7 @@ cpu_fork(struct lwp *lp1, struct lwp *lp2, int flags)
3831 /* Ensure that lp1's pcb is up to date. */
3832 if (mdcpu->gd_npxthread == lp1->lwp_thread)
3833 npxsave(lp1->lwp_thread->td_savefpu);
3834-
3835+
3836 /*
3837 * Copy lp1's PCB. This really only applies to the
3838 * debug registers and FP state, but its faster to just copy the
3839@@ -140,8 +147,16 @@ cpu_fork(struct lwp *lp1, struct lwp *lp2, int flags)
3840 /*
3841 * Set registers for trampoline to user mode. Leave space for the
3842 * return address on stack. These are the kernel mode register values.
3843+ *
3844+ * Set the new pmap CR3. If the new process uses isolated VM spaces,
3845+ * also set the isolated CR3.
3846 */
3847- pcb2->pcb_cr3 = vtophys(vmspace_pmap(lp2->lwp_proc->p_vmspace)->pm_pml4);
3848+ pmap2 = vmspace_pmap(lp2->lwp_proc->p_vmspace);
3849+ pcb2->pcb_cr3 = vtophys(pmap2->pm_pml4);
3850+ if (pcb2->pcb_flags & PCB_ISOMMU)
3851+ pcb2->pcb_cr3_iso = vtophys(pmap2->pm_pml4_iso);
3852+ else
3853+ pcb2->pcb_cr3_iso = 0;
3854 pcb2->pcb_rbx = (unsigned long)fork_return; /* fork_trampoline argument */
3855 pcb2->pcb_rbp = 0;
3856 pcb2->pcb_rsp = (unsigned long)lp2->lwp_md.md_regs - sizeof(void *);
3857@@ -158,7 +173,7 @@ cpu_fork(struct lwp *lp1, struct lwp *lp2, int flags)
3858 /*
3859 * pcb2->pcb_ldt: duplicated below, if necessary.
3860 * pcb2->pcb_savefpu: cloned above.
3861- * pcb2->pcb_flags: cloned above (always 0 here?).
3862+ * pcb2->pcb_flags: cloned above
3863 * pcb2->pcb_onfault: cloned above (always NULL here).
3864 * pcb2->pcb_onfault_sp:cloned above (dont care)
3865 */
3866@@ -171,10 +186,10 @@ cpu_fork(struct lwp *lp1, struct lwp *lp2, int flags)
3867 /* Copy the LDT, if necessary. */
3868 if (pcb2->pcb_ldt != NULL) {
3869 if (flags & RFMEM) {
3870- pcb2->pcb_ldt->ldt_refcnt++;
3871+ atomic_add_int(&pcb2->pcb_ldt->ldt_refcnt, 1);
3872 } else {
3873 pcb2->pcb_ldt = user_ldt_alloc(pcb2,
3874- pcb2->pcb_ldt->ldt_len);
3875+ pcb2->pcb_ldt->ldt_len);
3876 }
3877 }
3878 bcopy(&lp1->lwp_thread->td_tls, &lp2->lwp_thread->td_tls,
3879@@ -264,7 +279,7 @@ cpu_lwp_exit(void)
3880
3881 pcb = td->td_pcb;
3882
3883- /* Some i386 functionality was dropped */
3884+ /* Some x86 functionality was dropped */
3885 KKASSERT(pcb->pcb_ext == NULL);
3886
3887 /*