--- sys/vm/vm_fault.c.orig +++ sys/vm/vm_fault.c @@ -2227,9 +2227,10 @@ vm_object_t backing_object, dst_object, object, src_object; vm_pindex_t dst_pindex, pindex, src_pindex; vm_prot_t access, prot; - vm_offset_t vaddr; - vm_page_t dst_m; - vm_page_t src_m; + vm_offset_t va; + vm_page_t dst_m, m, src_m; + size_t psize; + int bdry_idx; bool upgrade; upgrade = src_entry == dst_entry; @@ -2250,6 +2251,26 @@ if (!upgrade) access &= ~VM_PROT_WRITE; + if ((bdry_idx = MAP_ENTRY_SPLIT_BOUNDARY_INDEX(dst_entry)) != 0) { + KASSERT(upgrade, + ("%s: copying between largepage mappings", __func__)); + KASSERT((dst_entry->eflags & MAP_ENTRY_USER_WIRED) != 0, + ("%s: largepage mapping isn't wired", __func__)); + + object = dst_entry->object.vm_object; + psize = pagesizes[bdry_idx]; + for (va = dst_entry->start, + pindex = OFF_TO_IDX(dst_entry->offset); + va < dst_entry->end; va += psize, pindex += atop(psize)) { + m = vm_page_lookup_unlocked(object, pindex); + + (void)pmap_enter(dst_map->pmap, va, m, prot, + access | PMAP_ENTER_WIRED | PMAP_ENTER_LARGEPAGE, + bdry_idx); + } + return; + } + src_object = src_entry->object.vm_object; src_pindex = OFF_TO_IDX(src_entry->offset); @@ -2300,9 +2321,8 @@ * with the source object, all of its pages must be dirtied, * regardless of whether they can be written. */ - for (vaddr = dst_entry->start, dst_pindex = 0; - vaddr < dst_entry->end; - vaddr += PAGE_SIZE, dst_pindex++) { + for (va = dst_entry->start, dst_pindex = 0; va < dst_entry->end; + va += PAGE_SIZE, dst_pindex++) { again: /* * Find the page in the source object, and copy it in. @@ -2396,7 +2416,7 @@ */ if (vm_page_all_valid(dst_m)) { VM_OBJECT_WUNLOCK(dst_object); - pmap_enter(dst_map->pmap, vaddr, dst_m, prot, + pmap_enter(dst_map->pmap, va, dst_m, prot, access | (upgrade ? PMAP_ENTER_WIRED : 0), 0); VM_OBJECT_WLOCK(dst_object); } --- sys/vm/vm_map.c.orig +++ sys/vm/vm_map.c @@ -1832,7 +1832,8 @@ vm_map_try_merge_entries(map, prev_entry, new_entry); *res = vm_map_try_merge_entries(map, new_entry, next_entry); - if ((cow & (MAP_PREFAULT | MAP_PREFAULT_PARTIAL)) != 0) { + if ((cow & (MAP_PREFAULT | MAP_PREFAULT_PARTIAL)) != 0 && + (cow & MAP_SPLIT_BOUNDARY_MASK) == 0) { vm_map_pmap_enter(map, start, prot, object, OFF_TO_IDX(offset), end - start, cow & MAP_PREFAULT_PARTIAL); } @@ -3131,7 +3132,8 @@ vm_offset_t useEnd, useStart; if ((entry->eflags & (MAP_ENTRY_IS_SUB_MAP | - MAP_ENTRY_GUARD)) != 0) + MAP_ENTRY_GUARD | MAP_ENTRY_SPLIT_BOUNDARY_MASK)) != + 0) continue; /* --- tests/sys/posixshm/posixshm_test.c.orig +++ tests/sys/posixshm/posixshm_test.c @@ -1661,6 +1661,68 @@ } } +/* + * Test handling of the RO->RW transition for an mlocked largepage mapping. + */ +ATF_TC(largepage_mprotect_wired); +ATF_TC_HEAD(largepage_mprotect_wired, tc) +{ + /* Needed to set rlimit. */ + atf_tc_set_md_var(tc, "require.user", "root"); +} +ATF_TC_BODY(largepage_mprotect_wired, tc) +{ + struct rlimit rl; + char *addr; + size_t ps[MAXPAGESIZES], sz; + u_long max_wired, wired; + int fd, error, pscnt; + + rl.rlim_cur = rl.rlim_max = RLIM_INFINITY; + ATF_REQUIRE_MSG(setrlimit(RLIMIT_MEMLOCK, &rl) == 0, + "setrlimit failed; error=%d", errno); + + sz = sizeof(max_wired); + error = sysctlbyname("vm.max_user_wired", &max_wired, &sz, NULL, 0); + ATF_REQUIRE_MSG(error == 0, + "sysctlbyname(vm.max_user_wired) failed; error=%d", errno); + + sz = sizeof(wired); + error = sysctlbyname("vm.stats.vm.v_user_wire_count", &wired, &sz, NULL, + 0); + ATF_REQUIRE_MSG(error == 0, + "sysctlbyname(vm.stats.vm.v_user_wire_count) failed; error=%d", + errno); + + pscnt = pagesizes(ps); + for (int i = 1; i < pscnt; i++) { + if (ps[i] / ps[0] > max_wired - wired) { + /* Cannot wire past the limit. */ + atf_tc_skip("test would exceed wiring limit"); + } + + fd = shm_open_large(i, SHM_LARGEPAGE_ALLOC_DEFAULT, ps[i]); + + addr = mmap(NULL, ps[i], PROT_READ, MAP_SHARED, fd, 0); + ATF_REQUIRE_MSG(addr != MAP_FAILED, + "mmap(%zu bytes) failed; error=%d", ps[i], errno); + + ATF_REQUIRE_MSG(mlock(addr, ps[i]) == 0, + "mlock failed; error=%d", errno); + + ATF_REQUIRE_MSG(mprotect(addr, ps[i], + PROT_READ | PROT_WRITE) == 0, + "mprotect failed; error=%d", errno); + + memset(addr, 0x41, ps[i]); + ATF_REQUIRE_MSG(addr[0] == 0x41 && addr[ps[i] - 1] == 0x41, + "largepage contents unexpected after mprotect"); + + ATF_REQUIRE(munmap(addr, ps[i]) == 0); + ATF_REQUIRE(close(fd) == 0); + } +} + ATF_TC_WITHOUT_HEAD(largepage_msync); ATF_TC_BODY(largepage_msync, tc) { @@ -2321,6 +2383,7 @@ ATF_TP_ADD_TC(tp, largepage_mlock); ATF_TP_ADD_TC(tp, largepage_msync); ATF_TP_ADD_TC(tp, largepage_mprotect); + ATF_TP_ADD_TC(tp, largepage_mprotect_wired); ATF_TP_ADD_TC(tp, largepage_minherit); ATF_TP_ADD_TC(tp, largepage_pipe); #ifdef __amd64__