add support for larger software page sizes on amd64

lib/libc/amd64/string/memcmp.S

@@ -267,7 +267,7 @@ ARCHENTRY(memcmp, baseline)
 	lea		-1(%rsi, %rdx, 1), %r9d
 	xor		%ecx, %eax
 	xor		%r9d, %r8d
-	test		$PAGE_SIZE, %eax	# are they on different pages?
+	test		$PAGE_SIZE_4K, %eax	# are they on different pages?
 	jz		0f
 
 	/* fix up rdi */
@@ -278,7 +278,7 @@ ARCHENTRY(memcmp, baseline)
 	movdqa		%xmm0, -40(%rsp)	# copy to replacement buffer
 	movdqa		%xmm1, -24(%rsp)
 
-0:	test		$PAGE_SIZE, %r8d
+0:	test		$PAGE_SIZE_4K, %r8d
 	jz		0f
 
 	/* fix up rsi */

lib/libc/amd64/string/strcmp.S

@@ -104,7 +104,7 @@ ARCHENTRY(strcmp, baseline)
 	and		$0xf, %eax	# offset from alignment
 	and		$0xf, %edx
 	pxor		%xmm1, %xmm1
-	test		$PAGE_SIZE, %r9d # did the page change?
+	test		$PAGE_SIZE_4K, %r9d # did the page change?
 	jz		0f		# if not, take fast path
 
 	/* heads may cross page boundary, avoid unmapped loads */

lib/libc/amd64/string/strcspn.S

@@ -199,7 +199,7 @@ ARCHENTRY(__strcspn, x86_64_v2)
 	mov		%rdi, %rax		# save original string pointer
 	lea		15(%rdi), %esi		# last byte of the head
 	xor		%edi, %esi
-	test		$PAGE_SIZE, %esi	# does the head cross a page?
+	test		$PAGE_SIZE_4K, %esi	# does the head cross a page?
 	jz		0f
 
 	/* head crosses page: copy to stack to fix up */

lib/libc/amd64/string/strncmp.S

@@ -135,7 +135,7 @@ ARCHENTRY(strncmp, baseline)
 	cmp		$16, %rdx	# end of buffer within the first 32 bytes?
 	jb		.Llt16
 
-	test		$PAGE_SIZE, %r9d # did the page change?
+	test		$PAGE_SIZE_4K, %r9d # did the page change?
 	jz		0f		# if not, take fast path
 
 
@@ -187,7 +187,7 @@ ARCHENTRY(strncmp, baseline)
 	jmp		.Lnormal
 
 	/* buffer ends within the first 16 bytes */
-.Llt16:	test		$PAGE_SIZE, %r9d # did the page change?
+.Llt16:	test		$PAGE_SIZE_4K, %r9d # did the page change?
 	jz		0f		# if not, take fast path
 
 	/* heads may cross page boundary */

lib/libc/amd64/string/strspn.S

@@ -198,7 +198,7 @@ ARCHENTRY(strspn, x86_64_v2)
 	mov		%rdi, %rax		# save original string pointer
 	lea		15(%rdi), %esi		# last byte of the head
 	xor		%edi, %esi
-	test		$PAGE_SIZE, %esi	# does the head cross a page?
+	test		$PAGE_SIZE_4K, %esi	# does the head cross a page?
 	jz		0f
 
 	/* head crosses page: copy to stack to fix up */

lib/libkvm/kvm_amd64.h

@@ -62,9 +62,9 @@ typedef	uint64_t	amd64_pml4e_t;
 
 #ifdef __amd64__
 _Static_assert(NPTEPG == AMD64_NPTEPG, "NPTEPG mismatch");
-_Static_assert(PAGE_SHIFT == AMD64_PAGE_SHIFT, "PAGE_SHIFT mismatch");
-_Static_assert(PAGE_SIZE == AMD64_PAGE_SIZE, "PAGE_SIZE mismatch");
-_Static_assert(PAGE_MASK == AMD64_PAGE_MASK, "PAGE_MASK mismatch");
+_Static_assert(PAGE_SHIFT_4K == AMD64_PAGE_SHIFT, "PAGE_SHIFT mismatch");
+_Static_assert(PAGE_SIZE_4K == AMD64_PAGE_SIZE, "PAGE_SIZE mismatch");
+_Static_assert(PAGE_MASK_4K == AMD64_PAGE_MASK, "PAGE_MASK mismatch");
 _Static_assert(NPDEPG == AMD64_NPDEPG, "NPDEPG mismatch");
 _Static_assert(PDRSHIFT == AMD64_PDRSHIFT, "PDRSHIFT mismatch");
 _Static_assert(NBPDR == AMD64_NBPDR, "NBPDR mismatch");

lib/libthr/arch/amd64/include/pthread_md.h

@@ -40,7 +40,7 @@
 #define	CPU_SPINWAIT		__asm __volatile("pause")
 
 /* For use in _Static_assert to check structs will fit in a page */
-#define	THR_PAGE_SIZE_MIN	PAGE_SIZE
+#define	THR_PAGE_SIZE_MIN	PAGE_SIZE_4K
 
 static __inline struct pthread *
 _get_curthread(void)

sbin/mdconfig/tests/mdconfig_test.sh

@@ -274,22 +274,23 @@ attach_size_rounddown()
 attach_size_rounddown_body()
 {
 	local md
-	local ss=8192
-	local ms=$(($ss + 4096))
-	local ms2=$((2 * $ss + 4096))
+	local pgsz=$(pagesize)
+	local ss=$(($pgsz * 2))
+	local ms=$(($ss + $pgsz))
+	local ms2=$((2 * $ss + $pgsz))
 
-	# Use a sector size that's a likely multiple of PAGE_SIZE, as md(4)
+	# Use a sector size that's a multiple of the kernel page size, as md(4)
 	# expects that for swap MDs.
 	atf_check -s exit:0 -o save:mdconfig.out -e empty \
 	    -x "mdconfig -a -t swap -S $ss -s ${ms}b"
 	md=$(cat mdconfig.out)
-	# 12288 bytes should be rounded down to one sector.
-	check_diskinfo "$md" 8192 1 $ss
+	# one sector plus one page should be rounded down to one sector.
+	check_diskinfo "$md" $ss 1 $ss
 
 	# Resize and verify that the new size was also rounded down.
 	atf_check -s exit:0 -o empty -e empty \
 	    -x "mdconfig -r -u ${md#md} -s ${ms2}b"
-	check_diskinfo "$md" 16384 2 $ss
+	check_diskinfo "$md" $((2 * $ss)) 2 $ss
 }
 attach_size_rounddown_cleanup()
 {

sbin/swapon/swapon.c

@@ -723,8 +723,8 @@ swapon_trim(const char *name)
 	} else
 		errx(1, "%s has an invalid file type", name);
 	/* Trim the device. */
-	ioarg[0] = BBSIZE;
-	ioarg[1] = sz - BBSIZE;
+	ioarg[0] = roundup(BBSIZE, getpagesize());
+	ioarg[1] = sz - ioarg[0];
 	if (ioctl(fd, DIOCGDELETE, ioarg) != 0)
 		warn("ioctl(DIOCGDELETE)");
 

sys/amd64/acpica/acpi_wakeup.c

@@ -307,7 +307,7 @@ static void
 acpi_alloc_wakeup_handler(void **wakeaddr,
     void *wakept_pages[ACPI_WAKEPT_PAGES])
 {
-	vm_page_t wakept_m[ACPI_WAKEPT_PAGES];
+	ptpage_t wakept_m[ACPI_WAKEPT_PAGES];
 	int i;
 
 	*wakeaddr = NULL;
@@ -330,8 +330,7 @@ acpi_alloc_wakeup_handler(void **wakeaddr,
 
 	for (i = 0; i < ACPI_WAKEPT_PAGES - (la57 ? 0 : 1); i++) {
 		wakept_m[i] = pmap_page_alloc_below_4g(true);
-		wakept_pages[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
-		    wakept_m[i]));
+		wakept_pages[i] = pmap_ptpage_va(wakept_m[i]);
 	}
 	if (EVENTHANDLER_REGISTER(power_resume, acpi_stop_beep, NULL,
 	    EVENTHANDLER_PRI_LAST) == NULL) {
@@ -349,7 +348,7 @@ acpi_alloc_wakeup_handler(void **wakeaddr,
 	free(*wakeaddr, M_DEVBUF);
 	for (i = 0; i < ACPI_WAKEPT_PAGES; i++) {
 		if (wakept_m[i] != NULL)
-			vm_page_free(wakept_m[i]);
+			pmap_free_pt_page(NULL, wakept_m[i], false);
 	}
 	*wakeaddr = NULL;
 }

sys/amd64/amd64/efirt_machdep.c

@@ -62,19 +62,27 @@
     1u << EFI_MD_TYPE_RT_CODE | 1u << EFI_MD_TYPE_RT_DATA | \
     1u << EFI_MD_TYPE_FIRMWARE \
 )
+vm_paddr_t pmap_ptpage_pa(ptpage_t ptp);
+void *pmap_ptpage_va(ptpage_t ptp);
+void *pmap_ptpage_pa_to_va(vm_paddr_t);
+ptpage_t pmap_pa_to_ptpage(vm_paddr_t pa);
+ptpage_t pmap_va_to_ptpage(void *p);
 
 static pml5_entry_t *efi_pml5;
 static pml4_entry_t *efi_pml4;
 static vm_object_t obj_1t1_pt;
-static vm_page_t efi_pmltop_page;
+static ptpage_t efi_pmltop_page;
 static vm_pindex_t efi_1t1_idx;
 
 void
 efi_destroy_1t1_map(void)
 {
+
+#if 0
 	struct pctrie_iter pages;
 	vm_page_t m;
 
+	/* CHUQ free these at some point. */
 	if (obj_1t1_pt != NULL) {
 		vm_page_iter_init(&pages, obj_1t1_pt);
 		VM_OBJECT_RLOCK(obj_1t1_pt);
@@ -84,6 +92,7 @@ efi_destroy_1t1_map(void)
 		VM_OBJECT_RUNLOCK(obj_1t1_pt);
 		vm_object_deallocate(obj_1t1_pt);
 	}
+#endif
 
 	obj_1t1_pt = NULL;
 	efi_pml4 = NULL;
@@ -104,12 +113,14 @@ efi_phys_to_kva(vm_paddr_t paddr)
 	return (PHYS_TO_DMAP(paddr));
 }
 
-static vm_page_t
+static ptpage_t
 efi_1t1_page(void)
 {
+	vm_page_t m;
 
-	return (vm_page_grab(obj_1t1_pt, efi_1t1_idx++, VM_ALLOC_NOBUSY |
-	    VM_ALLOC_WIRED | VM_ALLOC_ZERO));
+	/* CHUQ don't worry about freeing this for now */
+	m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
+	return (pmap_pa_to_ptpage(VM_PAGE_TO_PHYS(m)));
 }
 
 static pt_entry_t *
@@ -120,7 +131,7 @@ efi_1t1_pte(vm_offset_t va)
 	pdp_entry_t *pdpe;
 	pd_entry_t *pde;
 	pt_entry_t *pte;
-	vm_page_t m;
+	ptpage_t m;
 	vm_pindex_t pml5_idx, pml4_idx, pdp_idx, pd_idx;
 	vm_paddr_t mphys;
 
@@ -130,48 +141,48 @@ efi_1t1_pte(vm_offset_t va)
 		pml5e = &efi_pml5[pml5_idx];
 		if (*pml5e == 0) {
 			m = efi_1t1_page();
-			mphys = VM_PAGE_TO_PHYS(m);
+			mphys = pmap_ptpage_pa(m);
 			*pml5e = mphys | X86_PG_RW | X86_PG_V;
 		} else {
 			mphys = *pml5e & PG_FRAME;
 		}
-		pml4e = (pml4_entry_t *)PHYS_TO_DMAP(mphys);
+		pml4e = pmap_ptpage_pa_to_va(mphys);
 		pml4e = &pml4e[pml4_idx];
 	} else {
 		pml4e = &efi_pml4[pml4_idx];
 	}
 
 	if (*pml4e == 0) {
 		m = efi_1t1_page();
-		mphys =  VM_PAGE_TO_PHYS(m);
+		mphys =  pmap_ptpage_pa(m);
 		*pml4e = mphys | X86_PG_RW | X86_PG_V;
 	} else {
 		mphys = *pml4e & PG_FRAME;
 	}
 
-	pdpe = (pdp_entry_t *)PHYS_TO_DMAP(mphys);
+	pdpe = pmap_ptpage_pa_to_va(mphys);
 	pdp_idx = pmap_pdpe_index(va);
 	pdpe += pdp_idx;
 	if (*pdpe == 0) {
 		m = efi_1t1_page();
-		mphys =  VM_PAGE_TO_PHYS(m);
+		mphys =  pmap_ptpage_pa(m);
 		*pdpe = mphys | X86_PG_RW | X86_PG_V;
 	} else {
 		mphys = *pdpe & PG_FRAME;
 	}
 
-	pde = (pd_entry_t *)PHYS_TO_DMAP(mphys);
+	pde = pmap_ptpage_pa_to_va(mphys);
 	pd_idx = pmap_pde_index(va);
 	pde += pd_idx;
 	if (*pde == 0) {
 		m = efi_1t1_page();
-		mphys = VM_PAGE_TO_PHYS(m);
+		mphys = pmap_ptpage_pa(m);
 		*pde = mphys | X86_PG_RW | X86_PG_V;
 	} else {
 		mphys = *pde & PG_FRAME;
 	}
 
-	pte = (pt_entry_t *)PHYS_TO_DMAP(mphys);
+	pte = pmap_ptpage_pa_to_va(mphys);
 	pte += pmap_pte_index(va);
 	KASSERT(*pte == 0, ("va %#jx *pt %#jx", va, *pte));
 
@@ -184,11 +195,17 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
 	struct efi_md *p;
 	pt_entry_t *pte;
 	void *pml;
+#if 1
 	vm_page_t m;
+#endif
 	vm_offset_t va;
 	uint64_t idx;
 	int bits, i, mode;
 	bool map_pz = true;
+#if 0 && PAGE_SIZE != PAGE_SIZE_4K
+	vm_paddr_t lastpa = 0;
+	int lastmode = 0;
+#endif
 
 	obj_1t1_pt = vm_pager_allocate(OBJT_PHYS, NULL, ptoa(1 +
 	    NPML4EPG + NPML4EPG * NPDPEPG + NPML4EPG * NPDPEPG * NPDEPG),
@@ -197,7 +214,7 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
 	VM_OBJECT_WLOCK(obj_1t1_pt);
 	efi_pmltop_page = efi_1t1_page();
 	VM_OBJECT_WUNLOCK(obj_1t1_pt);
-	pml = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(efi_pmltop_page));
+	pml = pmap_ptpage_va(efi_pmltop_page);
 	if (la57) {
 		efi_pml5 = pml;
 		pmap_pinit_pml5(efi_pmltop_page);
@@ -206,12 +223,6 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
 		pmap_pinit_pml4(efi_pmltop_page);
 	}
 
-	if ((efi_map_regs & ~EFI_ALLOWED_TYPES_MASK) != 0) {
-		printf("Ignoring the following runtime EFI regions: %#x\n",
-		    efi_map_regs & ~EFI_ALLOWED_TYPES_MASK);
-		efi_map_regs &= EFI_ALLOWED_TYPES_MASK;
-	}
-
 	for (i = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p,
 	    descsz)) {
 		if ((p->md_attr & EFI_MD_ATTR_RT) == 0 &&
@@ -257,17 +268,24 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
 		    X86_PG_V;
 		VM_OBJECT_WLOCK(obj_1t1_pt);
 		for (va = p->md_phys, idx = 0; idx < p->md_pages; idx++,
-		    va += PAGE_SIZE) {
+		    va += EFI_PAGE_SIZE) {
 			pte = efi_1t1_pte(va);
 			pte_store(pte, va | bits);
 
-			m = PHYS_TO_VM_PAGE(va);
-			if (m != NULL && VM_PAGE_TO_PHYS(m) == 0) {
+			m = PHYS_TO_VM_PAGE(trunc_page(va));
+			if (m == NULL)
+				continue;
+			if (VM_PAGE_TO_PHYS(m) == 0) {
 				vm_page_init_page(m, va, -1,
 				    VM_FREEPOOL_DEFAULT);
 				m->order = VM_NFREEORDER + 1; /* invalid */
 				m->pool = VM_NFREEPOOL + 1; /* invalid */
 				pmap_page_set_memattr_noflush(m, mode);
+			} else {
+				KASSERT(m->md.pat_mode == mode,
+				    ("pa 0x%lx idx 0x%lx m %p "
+				    "m->md.pat_mode 0x%x == mode 0x%x",
+				    va, idx, m, m->md.pat_mode, mode));
 			}
 		}
 		VM_OBJECT_WUNLOCK(obj_1t1_pt);
@@ -338,7 +356,7 @@ efi_arch_enter(void)
 	if (pmap_pcid_enabled && !invpcid_works)
 		PCPU_SET(curpmap, NULL);
 
-	cr3 = VM_PAGE_TO_PHYS(efi_pmltop_page);
+	cr3 = pmap_ptpage_pa(efi_pmltop_page);
 	if (pmap_pcid_enabled)
 		cr3 |= pmap_get_pcid(curpmap);
 	load_cr3(cr3);

sys/amd64/amd64/genassym.c

@@ -91,6 +91,7 @@ ASSYM(TDP_CALLCHAIN, TDP_CALLCHAIN);
 ASSYM(TDP_KTHREAD, TDP_KTHREAD);
 
 ASSYM(PAGE_SIZE, PAGE_SIZE);
+ASSYM(PAGE_SIZE_PT, PAGE_SIZE_PT);
 ASSYM(NPTEPG, NPTEPG);
 ASSYM(NPDEPG, NPDEPG);
 ASSYM(addr_P4Tmap, addr_P4Tmap);

sys/amd64/amd64/machdep.c

@@ -1303,6 +1303,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 
 	TSRAW(&thread0, TS_ENTER, __func__, NULL);
 
+	physfree = round_page(physfree);
+
 	kernphys = amd64_loadaddr();
 
 	physfree += kernphys;