4:51 a.m.
Changelog:
V1-->V2:
-Fix the backwards compatibility issues suggested by Dave
-Test with "mod","kmem -f","vm -p" command again.
V2-->V3:
-keep the pml4, upml, last_upml_read and last_pml4_read members in struct
machine_specific like before for extension modules suggested by Dave
-move ptrs_per_pgd to the end of the machine_specific data structure
-rewirte the changelog
-rebase the patchset
-retest it
I found Dave had alread done the first phase of future support for x86_64
5-level page tables(commit 307e7f35f510). when I asked him about the
state of this work, he gave me a more detailed answer and suggestion.
I follow his advice, and do the following job.
1. Refine the original logical:
1) Create some new common function for getting the offset of page table
2) Repace the PML4 and UPML with the common PGD:
machdep->machspec->pml4/upml ==> machdep->pdg
3) Using the PUD in x86_64
2. Add 5-level page tables support for x86_64_k/uvtop()
This patchset is the second phase of the work, As Dave said, we need to be
a manner of determining very early on whether the kernel page tables are
using 5-level and whether each user-space task is using 4- or 5-level page
tables. These will be done after this phase.
About test work:
I have tested this patchset with 4-level and 5-level paging table.
sadump are not be tested.
Dou Liyang (6):
defs.h: Fix the PHYSICAL_PAGE_MASK macro
x86_64: Extract public page table mapping code
x86_64: Unify the page table parsing for 4-level mode
x86_64.c: Add the VMEMMAP support for 5 level page table
x86_64: Add 5-level paging support for x86_64_k/uvtop()
x86_64: Fix the misusage of PGDIR_SHIFT
defs.h | 56 ++---
sadump.c | 7 +-
x86_64.c | 750 +++++++++++++++++++++++++++++++++------------------------------
3 files changed, 418 insertions(+), 395 deletions(-)
--
2.14.3
4:51 a.m.
New subject: [PATCH v3 1/6] defs.h: Fix the PHYSICAL_PAGE_MASK macro
Original crash defines the PHYSICAL_PAGE_MASK as
(~(PAGE_SIZE-1) & (__PHYSICAL_MASK << PAGE_SHIFT))
It moves left PAGE_SHIFT bits for safety, But, as crash expands the physical
bits to 52, this will cause the conversion of phythcal address to virtual
address failed. Remove the PAGE_SHIFT to fix this problem.
Signed-off-by: Dou Liyang <douly.fnst(a)cn.fujitsu.com>
---
defs.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/defs.h b/defs.h
index 92341d2..e0b77ce 100644
--- a/defs.h
+++ b/defs.h
@@ -3405,7 +3405,7 @@ struct arm64_stackframe {
#define __VIRTUAL_MASK ((1UL << __VIRTUAL_MASK_SHIFT) - 1)
#define PAGE_SHIFT 12
#define PAGE_SIZE (1UL << PAGE_SHIFT)
-#define PHYSICAL_PAGE_MASK (~(PAGE_SIZE-1) & (__PHYSICAL_MASK <<
PAGE_SHIFT))
+#define PHYSICAL_PAGE_MASK (~(PAGE_SIZE-1) & __PHYSICAL_MASK )
#define _PAGE_BIT_NX 63
#define _PAGE_PRESENT 0x001
--
2.14.3
4:51 a.m.
New subject: [PATCH v3 2/6] x86_64: Extract public page table mapping code
Currently, page table mapping code is scattered everywhere. Extract public
page table mapping code and prepare for 5-level page. this patch is no
functionality change.
Signed-off-by: Dou Liyang <douly.fnst(a)cn.fujitsu.com>
---
x86_64.c | 410 ++++++++++++++++++++++++++++++---------------------------------
1 file changed, 193 insertions(+), 217 deletions(-)
diff --git a/x86_64.c b/x86_64.c
index 467b5d7..e3a8db7 100644
--- a/x86_64.c
+++ b/x86_64.c
@@ -79,6 +79,11 @@ static void x86_64_calc_phys_base(void);
static int x86_64_is_module_addr(ulong);
static int x86_64_is_kvaddr(ulong);
static int x86_64_is_uvaddr(ulong, struct task_context *);
+static ulong *x86_64_kpgd_offset(ulong, int, int);
+static ulong x86_64_upgd_offset(struct task_context *, ulong, int, int);
+static ulong x86_64_pud_offset(ulong, ulong, int, int);
+static ulong x86_64_pmd_offset(ulong, ulong, int, int);
+static ulong x86_64_pte_offset(ulong, ulong, int, int);
void x86_64_compiler_warning_stub(void);
static void x86_64_init_kernel_pgd(void);
static void x86_64_cpu_pda_init(void);
@@ -1575,6 +1580,153 @@ x86_64_is_uvaddr(ulong addr, struct task_context *tc)
return (addr < USERSPACE_TOP);
}
+/*
+ * Find the kernel pgd entry..
+ * pgd = pgd_offset_k(addr);
+ */
+static ulong *
+x86_64_kpgd_offset(ulong kvaddr, int verbose, int IS_XEN)
+{
+ ulong *pgd;
+
+ FILL_PML4();
+ pgd = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
+ if (verbose) {
+ fprintf(fp, "PGD DIRECTORY: %lx\n", vt->kernel_pgd[0]);
+ if(IS_XEN)
+ fprintf(fp, "PAGE DIRECTORY: %lx [machine]\n", *pgd);
+ else
+ fprintf(fp, "PAGE DIRECTORY: %lx\n", *pgd);
+ }
+
+ return pgd;
+}
+
+/*
+ * Find the user pgd entry..
+ * pgd = pgd_offset(mm, address);
+ */
+static ulong
+x86_64_upgd_offset(struct task_context *tc, ulong uvaddr, int verbose, int IS_XEN)
+{
+ ulong *pgd;
+ ulong pgd_paddr;
+ ulong pgd_pte;
+
+ if (task_mm(tc->task, TRUE))
+ pgd = ULONG_PTR(tt->mm_struct + OFFSET(mm_struct_pgd));
+ else
+ readmem(tc->mm_struct + OFFSET(mm_struct_pgd), KVADDR, &pgd,
+ sizeof(long), "mm_struct pgd", FAULT_ON_ERROR);
+
+ pgd_paddr = x86_64_VTOP((ulong)pgd);
+ FILL_UPML(pgd_paddr, PHYSADDR, PAGESIZE());
+ pgd = ((ulong *)pgd_paddr) + pml4_index(uvaddr);
+ pgd_pte = ULONG(machdep->machspec->pml4 + PAGEOFFSET(pgd));
+ if (verbose) {
+ if(IS_XEN)
+ fprintf(fp, " PGD: %lx => %lx [machine]\n", (ulong)pgd, pgd_pte);
+ else
+ fprintf(fp, " PGD: %lx => %lx\n", (ulong)pgd, pgd_pte);
+ }
+
+ return pgd_pte;
+}
+
+/*
+ * Find an entry in the third-level page table..
+ * pud = pud_offset(pgd, address);
+ */
+static ulong
+x86_64_pud_offset(ulong pgd_pte, ulong vaddr, int verbose, int IS_XEN)
+{
+ ulong *pud;
+ ulong pud_paddr;
+ ulong pud_pte;
+
+ pud_paddr = pgd_pte & PHYSICAL_PAGE_MASK;
+
+ if(IS_XEN) {
+ pud_paddr = xen_m2p(pud_paddr);
+ if (verbose)
+ fprintf(fp, " PGD: %lx\n", pud_paddr);
+ }
+
+ FILL_PUD(pud_paddr, PHYSADDR, PAGESIZE());
+ pud = ((ulong *)pud_paddr) + pgd_index(vaddr);
+ pud_pte = ULONG(machdep->pud + PAGEOFFSET(pud));
+ if (verbose) {
+ if(IS_XEN)
+ fprintf(fp, " PUD: %lx => %lx [machine]\n", (ulong)pud, pud_pte);
+ else
+ fprintf(fp, " PUD: %lx => %lx\n", (ulong)pud, pud_pte);
+ }
+
+ return pud_pte;
+}
+
+/*
+ * Find an entry in the middle page table..
+ * pmd = pmd_offset(pud, address);
+ */
+static ulong
+x86_64_pmd_offset(ulong pud_pte, ulong vaddr, int verbose, int IS_XEN)
+{
+ ulong *pmd;
+ ulong pmd_paddr;
+ ulong pmd_pte;
+
+ pmd_paddr = pud_pte & PHYSICAL_PAGE_MASK;
+
+ if(IS_XEN) {
+ pmd_paddr = xen_m2p(pmd_paddr);
+ if (verbose)
+ fprintf(fp, " PUD: %lx\n", pmd_paddr);
+ }
+
+ FILL_PMD(pmd_paddr, PHYSADDR, PAGESIZE());
+ pmd = ((ulong *)pmd_paddr) + pmd_index(vaddr);
+ pmd_pte = ULONG(machdep->pmd + PAGEOFFSET(pmd));
+ if (verbose) {
+ if(IS_XEN)
+ fprintf(fp, " PMD: %lx => %lx [machine]\n", (ulong)pmd, pmd_pte);
+ else
+ fprintf(fp, " PMD: %lx => %lx\n", (ulong)pmd, pmd_pte);
+ }
+ return pmd_pte;
+}
+
+/*
+ * Find an entry in the pet page table..
+ * pmd = pmd_offset(pud, address);
+ */
+static ulong
+x86_64_pte_offset(ulong pmd_pte, ulong vaddr, int verbose, int IS_XEN)
+{
+ ulong *ptep;
+ ulong pte_paddr;
+ ulong pte;
+
+ pte_paddr = pmd_pte & PHYSICAL_PAGE_MASK;
+
+ if(IS_XEN) {
+ pte_paddr = xen_m2p(pte_paddr);
+ if (verbose)
+ fprintf(fp, " PMD: %lx\n", pte_paddr);
+ }
+
+ FILL_PTBL(pte_paddr, PHYSADDR, PAGESIZE());
+ ptep = ((ulong *)pte_paddr) + pte_index(vaddr);
+ pte = ULONG(machdep->ptbl + PAGEOFFSET(ptep));
+ if (verbose) {
+ if(IS_XEN)
+ fprintf(fp, " PTE: %lx => %lx [machine]\n", (ulong)ptep, pte);
+ else
+ fprintf(fp, " PTE: %lx => %lx\n", (ulong)ptep, pte);
+ }
+
+ return pte;
+}
/*
* Translates a user virtual address to its physical address. cmd_vtop()
@@ -1588,18 +1740,9 @@ x86_64_is_uvaddr(ulong addr, struct task_context *tc)
static int
x86_64_uvtop_level4(struct task_context *tc, ulong uvaddr, physaddr_t *paddr, int
verbose)
{
- ulong mm;
- ulong *pml;
- ulong pml_paddr;
- ulong pml_pte;
- ulong *pgd;
- ulong pgd_paddr;
ulong pgd_pte;
- ulong *pmd;
- ulong pmd_paddr;
+ ulong pud_pte;
ulong pmd_pte;
- ulong *ptep;
- ulong pte_paddr;
ulong pte;
physaddr_t physpage;
@@ -1614,43 +1757,25 @@ x86_64_uvtop_level4(struct task_context *tc, ulong uvaddr,
physaddr_t *paddr, in
if ((machdep->flags & VM_5LEVEL) && x86_64_task_uses_5level(tc))
return x86_64_uvtop_5level(tc, uvaddr, paddr, verbose);
- if ((mm = task_mm(tc->task, TRUE)))
- pml = ULONG_PTR(tt->mm_struct + OFFSET(mm_struct_pgd));
- else
- readmem(tc->mm_struct + OFFSET(mm_struct_pgd), KVADDR, &pml,
- sizeof(long), "mm_struct pgd", FAULT_ON_ERROR);
-
- pml_paddr = x86_64_VTOP((ulong)pml);
- FILL_UPML(pml_paddr, PHYSADDR, PAGESIZE());
- pml = ((ulong *)pml_paddr) + pml4_index(uvaddr);
- pml_pte = ULONG(machdep->machspec->upml + PAGEOFFSET(pml));
- if (verbose)
- fprintf(fp, " PML: %lx => %lx\n", (ulong)pml, pml_pte);
- if (!(pml_pte & _PAGE_PRESENT))
+ pgd_pte = x86_64_upgd_offset(tc, uvaddr, verbose, FALSE);
+ if (!(pgd_pte & _PAGE_PRESENT))
goto no_upage;
- pgd_paddr = pml_pte & PHYSICAL_PAGE_MASK;
- FILL_PGD(pgd_paddr, PHYSADDR, PAGESIZE());
- pgd = ((ulong *)pgd_paddr) + pgd_index(uvaddr);
- pgd_pte = ULONG(machdep->pgd + PAGEOFFSET(pgd));
- if (verbose)
- fprintf(fp, " PUD: %lx => %lx\n", (ulong)pgd, pgd_pte);
- if (!(pgd_pte & _PAGE_PRESENT))
+ /*
+ * pud = pud_offset(pgd, address);
+ */
+ pud_pte = x86_64_pud_offset(pgd_pte, uvaddr, verbose, FALSE);
+ if (!(pud_pte & _PAGE_PRESENT))
goto no_upage;
/*
- * pmd = pmd_offset(pgd, address);
+ * pmd = pmd_offset(pud, address);
*/
- pmd_paddr = pgd_pte & PHYSICAL_PAGE_MASK;
- FILL_PMD(pmd_paddr, PHYSADDR, PAGESIZE());
- pmd = ((ulong *)pmd_paddr) + pmd_index(uvaddr);
- pmd_pte = ULONG(machdep->pmd + PAGEOFFSET(pmd));
- if (verbose)
- fprintf(fp, " PMD: %lx => %lx\n", (ulong)pmd, pmd_pte);
+ pmd_pte = x86_64_pmd_offset(pud_pte, uvaddr, verbose, FALSE);
if (!(pmd_pte & (_PAGE_PRESENT | _PAGE_PROTNONE)))
goto no_upage;
if (pmd_pte & _PAGE_PSE) {
- if (verbose) {
+ if (verbose) {
fprintf(fp, " PAGE: %lx (2MB)\n\n",
PAGEBASE(pmd_pte) & PHYSICAL_PAGE_MASK);
x86_64_translate_pte(pmd_pte, 0, 0);
@@ -1666,13 +1791,7 @@ x86_64_uvtop_level4(struct task_context *tc, ulong uvaddr,
physaddr_t *paddr, in
* ptep = pte_offset_map(pmd, address);
* pte = *ptep;
*/
- pte_paddr = pmd_pte & PHYSICAL_PAGE_MASK;
- FILL_PTBL(pte_paddr, PHYSADDR, PAGESIZE());
- ptep = ((ulong *)pte_paddr) + pte_index(uvaddr);
- pte = ULONG(machdep->ptbl + PAGEOFFSET(ptep));
- if (verbose)
- fprintf(fp, " PTE: %lx => %lx\n", (ulong)ptep, pte);
-
+ pte = x86_64_pte_offset(pmd_pte, uvaddr, verbose, FALSE);
if (!(pte & (_PAGE_PRESENT | _PAGE_PROTNONE))) {
*paddr = pte;
@@ -1714,19 +1833,10 @@ x86_64_uvtop_5level(struct task_context *tc, ulong uvaddr,
physaddr_t *paddr, in
static int
x86_64_uvtop_level4_xen_wpt(struct task_context *tc, ulong uvaddr, physaddr_t *paddr, int
verbose)
{
- ulong mm;
- ulong *pml;
- ulong pml_paddr;
- ulong pml_pte;
- ulong *pgd;
- ulong pgd_paddr;
ulong pgd_pte;
- ulong *pmd;
- ulong pmd_paddr;
+ ulong pud_pte;
ulong pmd_pte;
ulong pseudo_pmd_pte;
- ulong *ptep;
- ulong pte_paddr;
ulong pte;
ulong pseudo_pte;
physaddr_t physpage;
@@ -1740,45 +1850,18 @@ x86_64_uvtop_level4_xen_wpt(struct task_context *tc, ulong uvaddr,
physaddr_t *p
if (IS_KVADDR(uvaddr))
return x86_64_kvtop(tc, uvaddr, paddr, verbose);
- if ((mm = task_mm(tc->task, TRUE)))
- pml = ULONG_PTR(tt->mm_struct + OFFSET(mm_struct_pgd));
- else
- readmem(tc->mm_struct + OFFSET(mm_struct_pgd), KVADDR, &pml,
- sizeof(long), "mm_struct pgd", FAULT_ON_ERROR);
-
- pml_paddr = x86_64_VTOP((ulong)pml);
- FILL_UPML(pml_paddr, PHYSADDR, PAGESIZE());
- pml = ((ulong *)pml_paddr) + pml4_index(uvaddr);
- pml_pte = ULONG(machdep->machspec->upml + PAGEOFFSET(pml));
- if (verbose)
- fprintf(fp, " PML: %lx => %lx [machine]\n", (ulong)pml, pml_pte);
- if (!(pml_pte & _PAGE_PRESENT))
+ pgd_pte = x86_64_upgd_offset(tc, uvaddr, verbose, TRUE);
+ if (!(pgd_pte & _PAGE_PRESENT))
goto no_upage;
- pgd_paddr = pml_pte & PHYSICAL_PAGE_MASK;
- pgd_paddr = xen_m2p(pgd_paddr);
- if (verbose)
- fprintf(fp, " PML: %lx\n", pgd_paddr);
- FILL_PGD(pgd_paddr, PHYSADDR, PAGESIZE());
- pgd = ((ulong *)pgd_paddr) + pgd_index(uvaddr);
- pgd_pte = ULONG(machdep->pgd + PAGEOFFSET(pgd));
- if (verbose)
- fprintf(fp, " PUD: %lx => %lx [machine]\n", (ulong)pgd,
pgd_pte);
- if (!(pgd_pte & _PAGE_PRESENT))
+ pud_pte = x86_64_pud_offset(pgd_pte, uvaddr, verbose, TRUE);
+ if (!(pud_pte & _PAGE_PRESENT))
goto no_upage;
/*
- * pmd = pmd_offset(pgd, address);
+ * pmd = pmd_offset(pud, address);
*/
- pmd_paddr = pgd_pte & PHYSICAL_PAGE_MASK;
- pmd_paddr = xen_m2p(pmd_paddr);
- if (verbose)
- fprintf(fp, " PUD: %lx\n", pmd_paddr);
- FILL_PMD(pmd_paddr, PHYSADDR, PAGESIZE());
- pmd = ((ulong *)pmd_paddr) + pmd_index(uvaddr);
- pmd_pte = ULONG(machdep->pmd + PAGEOFFSET(pmd));
- if (verbose)
- fprintf(fp, " PMD: %lx => %lx [machine]\n", (ulong)pmd,
pmd_pte);
+ pmd_pte = x86_64_pmd_offset(pud_pte, uvaddr, verbose, TRUE);
if (!(pmd_pte & _PAGE_PRESENT))
goto no_upage;
if (pmd_pte & _PAGE_PSE) {
@@ -1817,15 +1900,7 @@ x86_64_uvtop_level4_xen_wpt(struct task_context *tc, ulong uvaddr,
physaddr_t *p
* ptep = pte_offset_map(pmd, address);
* pte = *ptep;
*/
- pte_paddr = pmd_pte & PHYSICAL_PAGE_MASK;
- pte_paddr = xen_m2p(pte_paddr);
- if (verbose)
- fprintf(fp, " PMD: %lx\n", pte_paddr);
- FILL_PTBL(pte_paddr, PHYSADDR, PAGESIZE());
- ptep = ((ulong *)pte_paddr) + pte_index(uvaddr);
- pte = ULONG(machdep->ptbl + PAGEOFFSET(ptep));
- if (verbose)
- fprintf(fp, " PTE: %lx => %lx [machine]\n", (ulong)ptep, pte);
+ pte = x86_64_pte_offset(pmd_pte, uvaddr, verbose, TRUE);
if (!(pte & (_PAGE_PRESENT))) {
*paddr = pte;
@@ -1835,7 +1910,7 @@ x86_64_uvtop_level4_xen_wpt(struct task_context *tc, ulong uvaddr,
physaddr_t *p
}
goto no_upage;
}
-
+
pseudo_pte = xen_m2p(pte & PHYSICAL_PAGE_MASK);
if (verbose)
fprintf(fp, " PTE: %lx\n", pseudo_pte + PAGEOFFSET(pte));
@@ -1864,12 +1939,8 @@ x86_64_uvtop_level4_rhel4_xen_wpt(struct task_context *tc, ulong
uvaddr, physadd
ulong *pgd;
ulong pgd_paddr;
ulong pgd_pte;
- ulong *pmd;
- ulong pmd_paddr;
ulong pmd_pte;
ulong pseudo_pmd_pte;
- ulong *ptep;
- ulong pte_paddr;
ulong pte;
ulong pseudo_pte;
physaddr_t physpage;
@@ -1901,15 +1972,7 @@ x86_64_uvtop_level4_rhel4_xen_wpt(struct task_context *tc, ulong
uvaddr, physadd
/*
* pmd = pmd_offset(pgd, address);
*/
- pmd_paddr = pgd_pte & PHYSICAL_PAGE_MASK;
- pmd_paddr = xen_m2p(pmd_paddr);
- if (verbose)
- fprintf(fp, " PGD: %lx\n", pmd_paddr);
- FILL_PMD(pmd_paddr, PHYSADDR, PAGESIZE());
- pmd = ((ulong *)pmd_paddr) + pmd_index(uvaddr);
- pmd_pte = ULONG(machdep->pmd + PAGEOFFSET(pmd));
- if (verbose)
- fprintf(fp, " PMD: %lx => %lx [machine]\n", (ulong)pmd,
pmd_pte);
+ pmd_pte = x86_64_pmd_offset(pgd_pte, uvaddr, verbose, TRUE);
if (!(pmd_pte & _PAGE_PRESENT))
goto no_upage;
if (pmd_pte & _PAGE_PSE) {
@@ -1948,15 +2011,7 @@ x86_64_uvtop_level4_rhel4_xen_wpt(struct task_context *tc, ulong
uvaddr, physadd
* ptep = pte_offset_map(pmd, address);
* pte = *ptep;
*/
- pte_paddr = pmd_pte & PHYSICAL_PAGE_MASK;
- pte_paddr = xen_m2p(pte_paddr);
- if (verbose)
- fprintf(fp, " PMD: %lx\n", pte_paddr);
- FILL_PTBL(pte_paddr, PHYSADDR, PAGESIZE());
- ptep = ((ulong *)pte_paddr) + pte_index(uvaddr);
- pte = ULONG(machdep->ptbl + PAGEOFFSET(ptep));
- if (verbose)
- fprintf(fp, " PTE: %lx => %lx [machine]\n", (ulong)ptep, pte);
+ pte = x86_64_pte_offset(pmd_pte, uvaddr, verbose, TRUE);
if (!(pte & (_PAGE_PRESENT))) {
*paddr = pte;
@@ -1995,11 +2050,7 @@ x86_64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t
*paddr, int verbo
ulong *pgd;
ulong pgd_paddr;
ulong pgd_pte;
- ulong *pmd;
- ulong pmd_paddr;
ulong pmd_pte;
- ulong *ptep;
- ulong pte_paddr;
ulong pte;
physaddr_t physpage;
@@ -2032,12 +2083,7 @@ x86_64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t
*paddr, int verbo
/*
* pmd = pmd_offset(pgd, address);
*/
- pmd_paddr = pgd_pte & PHYSICAL_PAGE_MASK;
- FILL_PMD(pmd_paddr, PHYSADDR, PAGESIZE());
- pmd = ((ulong *)pmd_paddr) + pmd_index(uvaddr);
- pmd_pte = ULONG(machdep->pmd + PAGEOFFSET(pmd));
- if (verbose)
- fprintf(fp, " PMD: %lx => %lx\n", (ulong)pmd, pmd_pte);
+ pmd_pte = x86_64_pmd_offset(pgd_pte, uvaddr, verbose, FALSE);
if (!(pmd_pte & _PAGE_PRESENT))
goto no_upage;
if (pmd_pte & _PAGE_PSE) {
@@ -2057,12 +2103,7 @@ x86_64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t
*paddr, int verbo
* ptep = pte_offset_map(pmd, address);
* pte = *ptep;
*/
- pte_paddr = pmd_pte & PHYSICAL_PAGE_MASK;
- FILL_PTBL(pte_paddr, PHYSADDR, PAGESIZE());
- ptep = ((ulong *)pte_paddr) + pte_index(uvaddr);
- pte = ULONG(machdep->ptbl + PAGEOFFSET(ptep));
- if (verbose)
- fprintf(fp, " PTE: %lx => %lx\n", (ulong)ptep, pte);
+ pte = x86_64_pte_offset(pmd_pte, uvaddr, verbose, FALSE);
if (!(pte & (_PAGE_PRESENT))) {
*paddr = pte;
@@ -2088,7 +2129,6 @@ no_upage:
return FALSE;
}
-
/*
* Translates a kernel virtual address to its physical address. cmd_vtop()
* sets the verbose flag so that the pte translation gets displayed; all
@@ -2098,14 +2138,8 @@ static int
x86_64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t *paddr, int verbose)
{
ulong *pml4;
- ulong *pgd;
- ulong pgd_paddr;
- ulong pgd_pte;
- ulong *pmd;
- ulong pmd_paddr;
+ ulong pud_pte;
ulong pmd_pte;
- ulong *ptep;
- ulong pte_paddr;
ulong pte;
physaddr_t physpage;
@@ -2116,12 +2150,7 @@ x86_64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t
*paddr, int verbo
* stage phys_base is not initialized yet and x86_64_VTOP()
* does not work. Jump to the code of pagetable translation.
*/
- FILL_PML4();
- pml4 = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
- if (verbose) {
- fprintf(fp, "PML4 DIRECTORY: %lx\n", vt->kernel_pgd[0]);
- fprintf(fp, "PAGE DIRECTORY: %lx\n", *pml4);
- }
+ pgd = x86_64_kpgd_offset(kvaddr, verbose, FALSE);
goto start_vtop_with_pagetable;
}
@@ -2164,35 +2193,21 @@ x86_64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t
*paddr, int verbo
/*
* pgd = pgd_offset_k(addr);
*/
- FILL_PML4();
- pml4 = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
- if (verbose) {
- fprintf(fp, "PML4 DIRECTORY: %lx\n", vt->kernel_pgd[0]);
- fprintf(fp, "PAGE DIRECTORY: %lx\n", *pml4);
- }
+ pgd = x86_64_kpgd_offset(kvaddr, verbose, FALSE);
}
start_vtop_with_pagetable:
if (!(*pml4 & _PAGE_PRESENT))
goto no_kpage;
- pgd_paddr = (*pml4) & PHYSICAL_PAGE_MASK;
- FILL_PGD(pgd_paddr, PHYSADDR, PAGESIZE());
- pgd = ((ulong *)pgd_paddr) + pgd_index(kvaddr);
- pgd_pte = ULONG(machdep->pgd + PAGEOFFSET(pgd));
- if (verbose)
- fprintf(fp, " PUD: %lx => %lx\n", (ulong)pgd, pgd_pte);
- if (!(pgd_pte & _PAGE_PRESENT))
+
+ pud_pte = x86_64_pud_offset(*pml4, kvaddr, verbose, FALSE);
+ if (!(pud_pte & _PAGE_PRESENT))
goto no_kpage;
/*
- * pmd = pmd_offset(pgd, addr);
+ * pmd = pmd_offset(pud, address);
*/
- pmd_paddr = pgd_pte & PHYSICAL_PAGE_MASK;
- FILL_PMD(pmd_paddr, PHYSADDR, PAGESIZE());
- pmd = ((ulong *)pmd_paddr) + pmd_index(kvaddr);
- pmd_pte = ULONG(machdep->pmd + PAGEOFFSET(pmd));
- if (verbose)
- fprintf(fp, " PMD: %lx => %lx\n", (ulong)pmd, pmd_pte);
+ pmd_pte = x86_64_pmd_offset(pud_pte, kvaddr, verbose, FALSE);
if (!(pmd_pte & _PAGE_PRESENT))
goto no_kpage;
if (pmd_pte & _PAGE_PSE) {
@@ -2212,12 +2227,7 @@ start_vtop_with_pagetable:
* ptep = pte_offset_map(pmd, addr);
* pte = *ptep;
*/
- pte_paddr = pmd_pte & PHYSICAL_PAGE_MASK;
- FILL_PTBL(pte_paddr, PHYSADDR, PAGESIZE());
- ptep = ((ulong *)pte_paddr) + pte_index(kvaddr);
- pte = ULONG(machdep->ptbl + PAGEOFFSET(ptep));
- if (verbose)
- fprintf(fp, " PTE: %lx => %lx\n", (ulong)ptep, pte);
+ pte = x86_64_pte_offset(pmd_pte, kvaddr, verbose, FALSE);
if (!(pte & (_PAGE_PRESENT))) {
if (pte && verbose) {
fprintf(fp, "\n");
@@ -2251,15 +2261,9 @@ static int
x86_64_kvtop_xen_wpt(struct task_context *tc, ulong kvaddr, physaddr_t *paddr, int
verbose)
{
ulong *pml4;
- ulong *pgd;
- ulong pgd_paddr;
- ulong pgd_pte;
- ulong *pmd;
- ulong pmd_paddr;
+ ulong pud_pte;
ulong pmd_pte;
ulong pseudo_pmd_pte;
- ulong *ptep;
- ulong pte_paddr;
ulong pte;
ulong pseudo_pte;
physaddr_t physpage;
@@ -2268,38 +2272,18 @@ x86_64_kvtop_xen_wpt(struct task_context *tc, ulong kvaddr,
physaddr_t *paddr, i
/*
* pgd = pgd_offset_k(addr);
*/
- FILL_PML4();
- pml4 = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
- if (verbose) {
- fprintf(fp, "PML4 DIRECTORY: %lx\n", vt->kernel_pgd[0]);
- fprintf(fp, "PAGE DIRECTORY: %lx [machine]\n", *pml4);
- }
+ pml4 = x86_64_kpgd_offset(kvaddr, verbose, TRUE);
if (!(*pml4 & _PAGE_PRESENT))
goto no_kpage;
- pgd_paddr = (*pml4) & PHYSICAL_PAGE_MASK;
- pgd_paddr = xen_m2p(pgd_paddr);
- if (verbose)
- fprintf(fp, "PAGE DIRECTORY: %lx\n", pgd_paddr);
- FILL_PGD(pgd_paddr, PHYSADDR, PAGESIZE());
- pgd = ((ulong *)pgd_paddr) + pgd_index(kvaddr);
- pgd_pte = ULONG(machdep->pgd + PAGEOFFSET(pgd));
- if (verbose)
- fprintf(fp, " PUD: %lx => %lx [machine]\n", (ulong)pgd,
pgd_pte);
- if (!(pgd_pte & _PAGE_PRESENT))
+
+ pud_pte = x86_64_pud_offset(*pml4, kvaddr, verbose, TRUE);
+ if (!(pud_pte & _PAGE_PRESENT))
goto no_kpage;
/*
* pmd = pmd_offset(pgd, addr);
*/
- pmd_paddr = pgd_pte & PHYSICAL_PAGE_MASK;
- pmd_paddr = xen_m2p(pmd_paddr);
- if (verbose)
- fprintf(fp, " PUD: %lx\n", pmd_paddr);
- FILL_PMD(pmd_paddr, PHYSADDR, PAGESIZE());
- pmd = ((ulong *)pmd_paddr) + pmd_index(kvaddr);
- pmd_pte = ULONG(machdep->pmd + PAGEOFFSET(pmd));
- if (verbose)
- fprintf(fp, " PMD: %lx => %lx [machine]\n", (ulong)pmd,
pmd_pte);
+ pmd_pte = x86_64_pmd_offset(pud_pte, kvaddr, verbose, TRUE);
if (!(pmd_pte & _PAGE_PRESENT))
goto no_kpage;
if (pmd_pte & _PAGE_PSE) {
@@ -2338,15 +2322,7 @@ x86_64_kvtop_xen_wpt(struct task_context *tc, ulong kvaddr,
physaddr_t *paddr, i
* ptep = pte_offset_map(pmd, addr);
* pte = *ptep;
*/
- pte_paddr = pmd_pte & PHYSICAL_PAGE_MASK;
- pte_paddr = xen_m2p(pte_paddr);
- if (verbose)
- fprintf(fp, " PMD: %lx\n", pte_paddr);
- FILL_PTBL(pte_paddr, PHYSADDR, PAGESIZE());
- ptep = ((ulong *)pte_paddr) + pte_index(kvaddr);
- pte = ULONG(machdep->ptbl + PAGEOFFSET(ptep));
- if (verbose)
- fprintf(fp, " PTE: %lx => %lx [machine]\n", (ulong)ptep,
pte);
+ pte = x86_64_pte_offset(pmd_pte, kvaddr, verbose, TRUE);
if (!(pte & (_PAGE_PRESENT))) {
if (pte && verbose) {
fprintf(fp, "\n");
--
2.14.3
4:51 a.m.
New subject: [PATCH v3 3/6] x86_64: Unify the page table parsing for 4-level mode
Repace the PML4 and UPML with the common PGD and Use PUD macro in the
4-level paging table in x86_64. make the x86_64_vtop_xxx() uniform for
the different situation.
Signed-off-by: Dou Liyang <douly.fnst(a)cn.fujitsu.com>
---
defs.h | 53 ++++++-------
sadump.c | 9 ++-
x86_64.c | 260 ++++++++++++++++++++++++++++++---------------------------------
3 files changed, 148 insertions(+), 174 deletions(-)
diff --git a/defs.h b/defs.h
index e0b77ce..5804206 100644
--- a/defs.h
+++ b/defs.h
@@ -3332,56 +3332,44 @@ struct arm64_stackframe {
#define VTOP(X) x86_64_VTOP((ulong)(X))
#define IS_VMALLOC_ADDR(X) x86_64_IS_VMALLOC_ADDR((ulong)(X))
-#define PML4_SHIFT 39
-#define PTRS_PER_PML4 512
+/* origin level page */
#define PGDIR_SHIFT 30
#define PTRS_PER_PGD 512
#define PMD_SHIFT 21
#define PTRS_PER_PMD 512
#define PTRS_PER_PTE 512
+/* 4 level page */
+#define PGDIR_SHIFT_4LEVEL 39
+#define PTRS_PER_PGD_4LEVEL 512
+#define PUD_SHIFT 30
+#define PTRS_PER_PUD 512
+
+/* 5 level page */
#define PGDIR_SHIFT_5LEVEL 48
#define PTRS_PER_PGD_5LEVEL 512
#define P4D_SHIFT 39
#define PTRS_PER_P4D 512
#define __PGDIR_SHIFT (machdep->machspec->pgdir_shift)
+#define __PTRS_PER_PGD (machdep->machspec->ptrs_per_pgd)
-#define pml4_index(address) (((address) >> PML4_SHIFT) & (PTRS_PER_PML4-1))
+#define pgd_index(address) (((address) >> __PGDIR_SHIFT) &
(__PTRS_PER_PGD-1))
#define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
-#define pgd_index(address) (((address) >> __PGDIR_SHIFT) & (PTRS_PER_PGD-1))
+#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
-#define IS_LAST_PML4_READ(pml4) ((ulong)(pml4) ==
machdep->machspec->last_pml4_read)
-
-#define FILL_PML4() \
- if (!(pc->flags & RUNTIME) || ACTIVE()) { \
- if (!IS_LAST_PML4_READ(vt->kernel_pgd[0])) { \
- readmem(vt->kernel_pgd[0], KVADDR, machdep->machspec->pml4, \
- PAGESIZE(), "init_level4_pgt", FAULT_ON_ERROR); \
- machdep->machspec->last_pml4_read = (ulong)(vt->kernel_pgd[0]); \
- } \
+#define FILL_TOP_PGD() \
+ if (!(pc->flags & RUNTIME) || ACTIVE()) { \
+ FILL_PGD(vt->kernel_pgd[0], KVADDR, PAGESIZE()); \
}
-#define FILL_PML4_HYPER() \
- if (!machdep->machspec->last_pml4_read) { \
- unsigned long idle_pg_table = symbol_exists("idle_pg_table_4") ? \
- symbol_value("idle_pg_table_4") : \
- symbol_value("idle_pg_table"); \
- readmem(idle_pg_table, KVADDR, machdep->machspec->pml4, PAGESIZE(), \
- "idle_pg_table", FAULT_ON_ERROR); \
- machdep->machspec->last_pml4_read = idle_pg_table; \
- }
-
-#define IS_LAST_UPML_READ(pml) ((ulong)(pml) == machdep->machspec->last_upml_read)
-
-#define FILL_UPML(PML, TYPE, SIZE) \
- if (!IS_LAST_UPML_READ(PML)) { \
- readmem((ulonglong)((ulong)(PML)), TYPE, machdep->machspec->upml, \
- SIZE, "pml page", FAULT_ON_ERROR);
\
- machdep->machspec->last_upml_read = (ulong)(PML); \
- }
+#define FILL_TOP_PGD_HYPER() \
+ unsigned long idle_pg_table = symbol_exists("idle_pg_table_4") ? \
+ symbol_value("idle_pg_table_4") : \
+ symbol_value("idle_pg_table"); \
+ FILL_PGD(idle_pg_table, KVADDR, PAGESIZE());
#define IS_LAST_P4D_READ(p4d) ((ulong)(p4d) == machdep->machspec->last_p4d_read)
@@ -5749,7 +5737,7 @@ struct machine_specific {
ulong modules_vaddr;
ulong modules_end;
ulong phys_base;
- char *pml4;
+ char *pml4;
char *upml;
ulong last_upml_read;
ulong last_pml4_read;
@@ -5773,6 +5761,7 @@ struct machine_specific {
ulong irq_stack_gap;
ulong kpti_entry_stack;
ulong kpti_entry_stack_size;
+ ulong ptrs_per_pgd;
};
#define KSYMS_START (0x1)
diff --git a/sadump.c b/sadump.c
index 25cefe9..17cfd93 100644
--- a/sadump.c
+++ b/sadump.c
@@ -2057,10 +2057,11 @@ sadump_calc_kaslr_offset(ulong *kaslr_offset)
* TODO: XEN and 5-level is not supported
*/
vt->kernel_pgd[0] = pgd;
- machdep->machspec->last_pml4_read = vt->kernel_pgd[0];
+ machdep->last_pgd_read = vt->kernel_pgd[0];
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_2_6;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT;
- if (!readmem(pgd, PHYSADDR, machdep->machspec->pml4, PAGESIZE(),
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT_4LEVEL;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_4LEVEL;
+ if (!readmem(pgd, PHYSADDR, machdep->pgd, PAGESIZE(),
"pgd", RETURN_ON_ERROR))
goto quit;
@@ -2108,7 +2109,7 @@ sadump_calc_kaslr_offset(ulong *kaslr_offset)
ret = TRUE;
quit:
vt->kernel_pgd[0] = 0;
- machdep->machspec->last_pml4_read = 0;
+ machdep->last_pgd_read = 0;
return ret;
}
#else
diff --git a/x86_64.c b/x86_64.c
index e3a8db7..f5e70b5 100644
--- a/x86_64.c
+++ b/x86_64.c
@@ -168,20 +168,17 @@ x86_64_init(int when)
machdep->pageoffset = machdep->pagesize - 1;
machdep->pagemask = ~((ulonglong)machdep->pageoffset);
machdep->stacksize = machdep->pagesize * 2;
- if ((machdep->machspec->upml = (char *)malloc(PAGESIZE())) ==
NULL)
- error(FATAL, "cannot malloc upml space.");
- if ((machdep->pgd = (char *)malloc(PAGESIZE())) == NULL)
+ if ((machdep->pgd = (char *)malloc(PAGESIZE())) == NULL)
error(FATAL, "cannot malloc pgd space.");
+ if ((machdep->pud = (char *)malloc(PAGESIZE())) == NULL)
+ error(FATAL, "cannot malloc pud space.");
if ((machdep->pmd = (char *)malloc(PAGESIZE())) == NULL)
error(FATAL, "cannot malloc pmd space.");
if ((machdep->ptbl = (char *)malloc(PAGESIZE())) == NULL)
error(FATAL, "cannot malloc ptbl space.");
- if ((machdep->machspec->pml4 =
- (char *)malloc(PAGESIZE()*2)) == NULL)
- error(FATAL, "cannot malloc pml4 space.");
- machdep->machspec->last_upml_read = 0;
- machdep->machspec->last_pml4_read = 0;
+
machdep->last_pgd_read = 0;
+ machdep->last_pud_read = 0;
machdep->last_pmd_read = 0;
machdep->last_ptbl_read = 0;
machdep->verify_paddr = x86_64_verify_paddr;
@@ -235,12 +232,10 @@ x86_64_init(int when)
machdep->machspec->modules_vaddr = MODULES_VADDR_ORIG;
machdep->machspec->modules_end = MODULES_END_ORIG;
- free(machdep->machspec->upml);
- machdep->machspec->upml = NULL;
-
machdep->uvtop = x86_64_uvtop;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_2_6;
machdep->machspec->pgdir_shift = PGDIR_SHIFT;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD;
break;
case VM_2_6_11:
@@ -266,7 +261,8 @@ x86_64_init(int when)
machdep->uvtop = x86_64_uvtop_level4;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_2_6;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT;
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT_4LEVEL;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_4LEVEL;
break;
case VM_XEN:
@@ -278,7 +274,6 @@ x86_64_init(int when)
machdep->machspec->modules_vaddr = MODULES_VADDR_XEN;
machdep->machspec->modules_end = MODULES_END_XEN;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_XEN;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT;
break;
case VM_XEN_RHEL4:
@@ -290,7 +285,6 @@ x86_64_init(int when)
machdep->machspec->modules_vaddr =
MODULES_VADDR_XEN_RHEL4;
machdep->machspec->modules_end = MODULES_END_XEN_RHEL4;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_XEN;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT;
break;
case VM_5LEVEL:
@@ -304,6 +298,7 @@ x86_64_init(int when)
machdep->machspec->vmemmap_end = VMEMMAP_END_5LEVEL;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_5LEVEL;
machdep->machspec->pgdir_shift = PGDIR_SHIFT_5LEVEL;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_5LEVEL;
if ((machdep->machspec->p4d = (char *)malloc(PAGESIZE())) == NULL)
error(FATAL, "cannot malloc p4d space.");
machdep->machspec->last_p4d_read = 0;
@@ -631,14 +626,21 @@ x86_64_init(int when)
case VM_XEN:
case VM_2_6_11:
machdep->uvtop = x86_64_uvtop_level4_xen_wpt;
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT_4LEVEL;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_4LEVEL;
break;
case VM_XEN_RHEL4:
machdep->uvtop = x86_64_uvtop_level4_rhel4_xen_wpt;
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD;
break;
}
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_XEN;
- } else
- machdep->uvtop = x86_64_uvtop_level4;
+ } else {
+ machdep->uvtop = x86_64_uvtop_level4;
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT_4LEVEL;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_4LEVEL;
+ }
MEMBER_OFFSET_INIT(vcpu_guest_context_user_regs,
"vcpu_guest_context", "user_regs");
ASSIGN_OFFSET(cpu_user_regs_rsp) =
@@ -816,9 +818,11 @@ x86_64_dump_machdep_table(ulong arg)
fprintf(fp, " value_to_symbol: x86_64_value_to_symbol()\n");
fprintf(fp, " in_alternate_stack: x86_64_in_alternate_stack()\n");
fprintf(fp, " last_pgd_read: %lx\n", machdep->last_pgd_read);
+ fprintf(fp, " last_pud_read: %lx\n", machdep->last_pud_read);
fprintf(fp, " last_pmd_read: %lx\n", machdep->last_pmd_read);
fprintf(fp, " last_ptbl_read: %lx\n", machdep->last_ptbl_read);
fprintf(fp, " pgd: %lx\n", (ulong)machdep->pgd);
+ fprintf(fp, " pud: %lx\n", (ulong)machdep->pud);
fprintf(fp, " pmd: %lx\n", (ulong)machdep->pmd);
fprintf(fp, " ptbl: %lx\n", (ulong)machdep->ptbl);
fprintf(fp, " ptrs_per_pgd: %d\n", machdep->ptrs_per_pgd);
@@ -853,15 +857,25 @@ x86_64_dump_machdep_table(ulong arg)
fprintf(fp, " pgdir_shift: %ld\n", ms->pgdir_shift);
fprintf(fp, " GART_start: %lx\n", ms->GART_start);
fprintf(fp, " GART_end: %lx\n", ms->GART_end);
- fprintf(fp, " pml4: %lx\n", (ulong)ms->pml4);
- fprintf(fp, " last_pml4_read: %lx\n",
(ulong)ms->last_pml4_read);
+
+ /* pml4 and upml is legacy for extension modules */
+ if (ms->pml4) {
+ fprintf(fp, " pml4: %lx\n", (ulong)ms->pml4);
+ fprintf(fp, " last_pml4_read: %lx\n", (ulong)ms->last_pml4_read);
+
+ } else {
+ fprintf(fp, " pml4: (unused)\n");
+ fprintf(fp, " last_pml4_read: (unused)\n");
+ }
+
if (ms->upml) {
- fprintf(fp, " upml: %lx\n", (ulong)ms->upml);
- fprintf(fp, " last_upml_read: %lx\n",
(ulong)ms->last_upml_read);
+ fprintf(fp, " upml: %lx\n", (ulong)ms->upml);
+ fprintf(fp, " last_upml_read: %lx\n", (ulong)ms->last_upml_read);
} else {
- fprintf(fp, " upml: (unused)\n");
- fprintf(fp, " last_upml_read: (unused)\n");
+ fprintf(fp, " upml: (unused)\n");
+ fprintf(fp, " last_upml_read: (unused)\n");
}
+
if (ms->p4d) {
fprintf(fp, " p4d: %lx\n", (ulong)ms->p4d);
fprintf(fp, " last_p4d_read: %lx\n",
(ulong)ms->last_p4d_read);
@@ -1529,7 +1543,7 @@ x86_64_init_kernel_pgd(void)
for (i = 0; i < NR_CPUS; i++)
vt->kernel_pgd[i] = kernel_pgt;
- FILL_PML4();
+ FILL_TOP_PGD();
}
/*
@@ -1589,8 +1603,8 @@ x86_64_kpgd_offset(ulong kvaddr, int verbose, int IS_XEN)
{
ulong *pgd;
- FILL_PML4();
- pgd = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
+ FILL_TOP_PGD();
+ pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
if (verbose) {
fprintf(fp, "PGD DIRECTORY: %lx\n", vt->kernel_pgd[0]);
if(IS_XEN)
@@ -1613,16 +1627,16 @@ x86_64_upgd_offset(struct task_context *tc, ulong uvaddr, int
verbose, int IS_XE
ulong pgd_paddr;
ulong pgd_pte;
- if (task_mm(tc->task, TRUE))
- pgd = ULONG_PTR(tt->mm_struct + OFFSET(mm_struct_pgd));
- else
- readmem(tc->mm_struct + OFFSET(mm_struct_pgd), KVADDR, &pgd,
- sizeof(long), "mm_struct pgd", FAULT_ON_ERROR);
+ if (task_mm(tc->task, TRUE))
+ pgd = ULONG_PTR(tt->mm_struct + OFFSET(mm_struct_pgd));
+ else
+ readmem(tc->mm_struct + OFFSET(mm_struct_pgd), KVADDR, &pgd,
+ sizeof(long), "mm_struct pgd", FAULT_ON_ERROR);
- pgd_paddr = x86_64_VTOP((ulong)pgd);
- FILL_UPML(pgd_paddr, PHYSADDR, PAGESIZE());
- pgd = ((ulong *)pgd_paddr) + pml4_index(uvaddr);
- pgd_pte = ULONG(machdep->machspec->pml4 + PAGEOFFSET(pgd));
+ pgd_paddr = x86_64_VTOP((ulong)pgd);
+ FILL_PGD(pgd_paddr, PHYSADDR, PAGESIZE());
+ pgd = ((ulong *)pgd_paddr) + pgd_index(uvaddr);
+ pgd_pte = ULONG(machdep->pgd + PAGEOFFSET(pgd));
if (verbose) {
if(IS_XEN)
fprintf(fp, " PGD: %lx => %lx [machine]\n", (ulong)pgd, pgd_pte);
@@ -1653,7 +1667,7 @@ x86_64_pud_offset(ulong pgd_pte, ulong vaddr, int verbose, int
IS_XEN)
}
FILL_PUD(pud_paddr, PHYSADDR, PAGESIZE());
- pud = ((ulong *)pud_paddr) + pgd_index(vaddr);
+ pud = ((ulong *)pud_paddr) + pud_index(vaddr);
pud_pte = ULONG(machdep->pud + PAGEOFFSET(pud));
if (verbose) {
if(IS_XEN)
@@ -1935,9 +1949,6 @@ no_upage:
static int
x86_64_uvtop_level4_rhel4_xen_wpt(struct task_context *tc, ulong uvaddr, physaddr_t
*paddr, int verbose)
{
- ulong mm;
- ulong *pgd;
- ulong pgd_paddr;
ulong pgd_pte;
ulong pmd_pte;
ulong pseudo_pmd_pte;
@@ -1954,18 +1965,7 @@ x86_64_uvtop_level4_rhel4_xen_wpt(struct task_context *tc, ulong
uvaddr, physadd
if (IS_KVADDR(uvaddr))
return x86_64_kvtop(tc, uvaddr, paddr, verbose);
- if ((mm = task_mm(tc->task, TRUE)))
- pgd = ULONG_PTR(tt->mm_struct + OFFSET(mm_struct_pgd));
- else
- readmem(tc->mm_struct + OFFSET(mm_struct_pgd), KVADDR, &pgd,
- sizeof(long), "mm_struct pgd", FAULT_ON_ERROR);
-
- pgd_paddr = x86_64_VTOP((ulong)pgd);
- FILL_PGD(pgd_paddr, PHYSADDR, PAGESIZE());
- pgd = ((ulong *)pgd_paddr) + pgd_index(uvaddr);
- pgd_pte = ULONG(machdep->pgd + PAGEOFFSET(pgd));
- if (verbose)
- fprintf(fp, " PGD: %lx => %lx [machine]\n", (ulong)pgd,
pgd_pte);
+ pgd_pte = x86_64_upgd_offset(tc, uvaddr, verbose, TRUE);
if (!(pgd_pte & _PAGE_PRESENT))
goto no_upage;
@@ -2046,9 +2046,6 @@ no_upage:
static int
x86_64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t *paddr, int verbose)
{
- ulong mm;
- ulong *pgd;
- ulong pgd_paddr;
ulong pgd_pte;
ulong pmd_pte;
ulong pte;
@@ -2062,21 +2059,10 @@ x86_64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t
*paddr, int verbo
if (IS_KVADDR(uvaddr))
return x86_64_kvtop(tc, uvaddr, paddr, verbose);
- /*
- * pgd = pgd_offset(mm, address);
- */
- if ((mm = task_mm(tc->task, TRUE)))
- pgd = ULONG_PTR(tt->mm_struct + OFFSET(mm_struct_pgd));
- else
- readmem(tc->mm_struct + OFFSET(mm_struct_pgd), KVADDR, &pgd,
- sizeof(long), "mm_struct pgd", FAULT_ON_ERROR);
-
- pgd_paddr = x86_64_VTOP((ulong)pgd);
- FILL_PGD(pgd_paddr, PHYSADDR, PAGESIZE());
- pgd = ((ulong *)pgd_paddr) + pgd_index(uvaddr);
- pgd_pte = ULONG(machdep->pgd + PAGEOFFSET(pgd));
- if (verbose)
- fprintf(fp, " PGD: %lx => %lx\n", (ulong)pgd, pgd_pte);
+ /*
+ * pgd = pgd_offset(mm, address);
+ */
+ pgd_pte = x86_64_upgd_offset(tc, uvaddr, verbose, FALSE);
if (!(pgd_pte & _PAGE_PRESENT))
goto no_upage;
@@ -2137,7 +2123,7 @@ no_upage:
static int
x86_64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t *paddr, int verbose)
{
- ulong *pml4;
+ ulong *pgd;
ulong pud_pte;
ulong pmd_pte;
ulong pte;
@@ -2166,11 +2152,11 @@ x86_64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t
*paddr, int verbo
*paddr = kvaddr - DIRECTMAP_VIRT_START;
return TRUE;
}
- FILL_PML4_HYPER();
- pml4 = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
+ FILL_TOP_PGD_HYPER();
+ pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
if (verbose) {
- fprintf(fp, "PML4 DIRECTORY: %lx\n", vt->kernel_pgd[0]);
- fprintf(fp, "PAGE DIRECTORY: %lx\n", *pml4);
+ fprintf(fp, "PGD DIRECTORY: %lx\n", vt->kernel_pgd[0]);
+ fprintf(fp, "PAGE DIRECTORY: %lx\n", *pgd);
}
} else {
if (!vt->vmalloc_start) {
@@ -2197,10 +2183,10 @@ x86_64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t
*paddr, int verbo
}
start_vtop_with_pagetable:
- if (!(*pml4 & _PAGE_PRESENT))
+ if (!(*pgd & _PAGE_PRESENT))
goto no_kpage;
- pud_pte = x86_64_pud_offset(*pml4, kvaddr, verbose, FALSE);
+ pud_pte = x86_64_pud_offset(*pgd, kvaddr, verbose, FALSE);
if (!(pud_pte & _PAGE_PRESENT))
goto no_kpage;
@@ -2260,7 +2246,7 @@ x86_64_kvtop_5level(struct task_context *tc, ulong kvaddr,
physaddr_t *paddr, in
static int
x86_64_kvtop_xen_wpt(struct task_context *tc, ulong kvaddr, physaddr_t *paddr, int
verbose)
{
- ulong *pml4;
+ ulong *pgd;
ulong pud_pte;
ulong pmd_pte;
ulong pseudo_pmd_pte;
@@ -2272,11 +2258,11 @@ x86_64_kvtop_xen_wpt(struct task_context *tc, ulong kvaddr,
physaddr_t *paddr, i
/*
* pgd = pgd_offset_k(addr);
*/
- pml4 = x86_64_kpgd_offset(kvaddr, verbose, TRUE);
- if (!(*pml4 & _PAGE_PRESENT))
+ pgd = x86_64_kpgd_offset(kvaddr, verbose, TRUE);
+ if (!(*pgd & _PAGE_PRESENT))
goto no_kpage;
- pud_pte = x86_64_pud_offset(*pml4, kvaddr, verbose, TRUE);
+ pud_pte = x86_64_pud_offset(*pgd, kvaddr, verbose, TRUE);
if (!(pud_pte & _PAGE_PRESENT))
goto no_kpage;
@@ -5865,7 +5851,8 @@ parse_cmdline_args(void)
void
x86_64_clear_machdep_cache(void)
{
- machdep->machspec->last_upml_read = 0;
+ if (machdep->last_pgd_read != vt->kernel_pgd[0])
+ machdep->last_pgd_read = 0;
}
#define PUSH_RBP_MOV_RSP_RBP 0xe5894855
@@ -6244,12 +6231,12 @@ use_cr3:
if (CRASHDEBUG(1))
fprintf(fp, "x86_64_xen_kdump_p2m_create: cr3: %lx\n",
xkd->cr3);
- if (!readmem(PTOB(xkd->cr3), PHYSADDR, machdep->machspec->pml4,
+ if (!readmem(PTOB(xkd->cr3), PHYSADDR, machdep->pgd,
PAGESIZE(), "xen kdump cr3 page", RETURN_ON_ERROR))
error(FATAL, "cannot read xen kdump cr3 page\n");
if (CRASHDEBUG(7))
- x86_64_debug_dump_page(fp, machdep->machspec->pml4,
+ x86_64_debug_dump_page(fp, machdep->pgd,
"contents of PML4 page:");
/*
@@ -6284,7 +6271,7 @@ use_cr3:
if (CRASHDEBUG(1))
fprintf(fp, "phys_to_machine_mapping: %lx\n", kvaddr);
- machdep->last_pgd_read = BADADDR;
+ machdep->last_pud_read = BADADDR;
machdep->last_pmd_read = BADADDR;
machdep->last_ptbl_read = BADADDR;
@@ -6299,7 +6286,7 @@ use_cr3:
fprintf(fp, "\n");
}
- machdep->last_pgd_read = 0;
+ machdep->last_pud_read = 0;
machdep->last_ptbl_read = 0;
machdep->last_pmd_read = 0;
pc->curcmd_flags &= ~XEN_MACHINE_ADDR;
@@ -6313,33 +6300,33 @@ static char *
x86_64_xen_kdump_load_page(ulong kvaddr, char *pgbuf)
{
ulong mfn;
- ulong *pml4, *pgd, *pmd, *ptep;
+ ulong *pgd, *pud, *pmd, *ptep;
- pml4 = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
- mfn = ((*pml4) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
+ pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
+ mfn = ((*pgd) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
if (CRASHDEBUG(3))
fprintf(fp,
- "[%lx] pml4: %lx mfn: %lx pml4_index: %lx\n",
- kvaddr, *pml4, mfn, pml4_index(kvaddr));
+ "[%lx] pgd: %lx mfn: %lx pgd_index: %lx\n",
+ kvaddr, *pgd, mfn, pgd_index(kvaddr));
- if (!readmem(PTOB(mfn), PHYSADDR, machdep->pgd, PAGESIZE(),
+ if (!readmem(PTOB(mfn), PHYSADDR, machdep->pud, PAGESIZE(),
"xen kdump pud page", RETURN_ON_ERROR))
error(FATAL, "cannot read/find pud page\n");
- machdep->last_pgd_read = mfn;
+ machdep->last_pud_read = mfn;
if (CRASHDEBUG(7))
- x86_64_debug_dump_page(fp, machdep->pgd,
+ x86_64_debug_dump_page(fp, machdep->pud,
"contents of page upper directory page:");
- pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
- mfn = ((*pgd) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
+ pud = ((ulong *)machdep->pud) + pud_index(kvaddr);
+ mfn = ((*pud) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
if (CRASHDEBUG(3))
fprintf(fp,
- "[%lx] pgd: %lx mfn: %lx pgd_index: %lx\n",
- kvaddr, *pgd, mfn, pgd_index(kvaddr));
+ "[%lx] pud: %lx mfn: %lx pud_index: %lx\n",
+ kvaddr, *pgd, mfn, pud_index(kvaddr));
if (!readmem(PTOB(mfn), PHYSADDR, machdep->pmd, PAGESIZE(),
"xen kdump pmd page", RETURN_ON_ERROR))
@@ -6392,21 +6379,21 @@ static ulong
x86_64_xen_kdump_page_mfn(ulong kvaddr)
{
ulong mfn;
- ulong *pml4, *pgd, *pmd, *ptep;
+ ulong *pgd, *pud, *pmd, *ptep;
- pml4 = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
- mfn = ((*pml4) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
+ pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
+ mfn = ((*pgd) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
- if ((mfn != machdep->last_pgd_read) &&
- !readmem(PTOB(mfn), PHYSADDR, machdep->pgd, PAGESIZE(),
+ if ((mfn != machdep->last_pud_read) &&
+ !readmem(PTOB(mfn), PHYSADDR, machdep->pud, PAGESIZE(),
"xen kdump pud entry", RETURN_ON_ERROR))
error(FATAL, "cannot read/find pud page\n");
- machdep->last_pgd_read = mfn;
+ machdep->last_pud_read = mfn;
- pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
- mfn = ((*pgd) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
+ pud = ((ulong *)machdep->pud) + pud_index(kvaddr);
+ mfn = ((*pud) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
- if ((mfn != machdep->last_pmd_read) &&
+ if ((mfn != machdep->last_pmd_read) &&
!readmem(PTOB(mfn), PHYSADDR, machdep->pmd, PAGESIZE(),
"xen kdump pmd entry", RETURN_ON_ERROR))
error(FATAL, "cannot read/find pmd page\n");
@@ -6761,12 +6748,12 @@ x86_64_xendump_p2m_create(struct xendump_data *xd)
mfn = ctrlreg[3] >> PAGESHIFT();
- if (!xc_core_mfn_to_page(mfn, machdep->machspec->pml4))
+ if (!xc_core_mfn_to_page(mfn, machdep->pgd))
error(FATAL, "cannot read/find cr3 page\n");
if (CRASHDEBUG(7))
- x86_64_debug_dump_page(xd->ofp, machdep->machspec->pml4,
- "contents of PML4 page:");
+ x86_64_debug_dump_page(xd->ofp, machdep->pgd,
+ "contents of PGD page:");
/*
* kernel version < 2.6.27 => end_pfn
@@ -6847,12 +6834,12 @@ x86_64_pvops_xendump_p2m_create(struct xendump_data *xd)
mfn = ctrlreg[3] >> PAGESHIFT();
- if (!xc_core_mfn_to_page(mfn, machdep->machspec->pml4))
+ if (!xc_core_mfn_to_page(mfn, machdep->pgd))
error(FATAL, "cannot read/find cr3 page\n");
if (CRASHDEBUG(7))
- x86_64_debug_dump_page(xd->ofp, machdep->machspec->pml4,
- "contents of PML4 page:");
+ x86_64_debug_dump_page(xd->ofp, machdep->pgd,
+ "contents of PGD page:");
/*
* kernel version < 2.6.27 => end_pfn
@@ -7028,32 +7015,32 @@ static char *
x86_64_xendump_load_page(ulong kvaddr, struct xendump_data *xd)
{
ulong mfn;
- ulong *pml4, *pgd, *pmd, *ptep;
+ ulong *pgd, *pud, *pmd, *ptep;
- pml4 = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
- mfn = ((*pml4) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
+ pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
+ mfn = ((*pgd) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
if (CRASHDEBUG(3))
fprintf(xd->ofp,
- "[%lx] pml4: %lx mfn: %lx pml4_index: %lx\n",
- kvaddr, *pml4, mfn, pml4_index(kvaddr));
+ "[%lx] pgd: %lx mfn: %lx pgd_index: %lx\n",
+ kvaddr, *pgd, mfn, pgd_index(kvaddr));
- if (!xc_core_mfn_to_page(mfn, machdep->pgd))
+ if (!xc_core_mfn_to_page(mfn, machdep->pud))
error(FATAL, "cannot read/find pud page\n");
- machdep->last_pgd_read = mfn;
+ machdep->last_pud_read = mfn;
if (CRASHDEBUG(7))
- x86_64_debug_dump_page(xd->ofp, machdep->pgd,
+ x86_64_debug_dump_page(xd->ofp, machdep->pud,
"contents of page upper directory page:");
- pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
- mfn = ((*pgd) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
+ pud = ((ulong *)machdep->pud) + pud_index(kvaddr);
+ mfn = ((*pud) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
if (CRASHDEBUG(3))
fprintf(xd->ofp,
- "[%lx] pgd: %lx mfn: %lx pgd_index: %lx\n",
- kvaddr, *pgd, mfn, pgd_index(kvaddr));
+ "[%lx] pud: %lx mfn: %lx pud_index: %lx\n",
+ kvaddr, *pud, mfn, pud_index(kvaddr));
if (!xc_core_mfn_to_page(mfn, machdep->pmd))
error(FATAL, "cannot read/find pmd page\n");
@@ -7107,18 +7094,18 @@ x86_64_xendump_page_index(ulong kvaddr, struct xendump_data *xd)
{
int idx;
ulong mfn;
- ulong *pml4, *pgd, *pmd, *ptep;
+ ulong *pgd, *pud, *pmd, *ptep;
- pml4 = ((ulong *)machdep->machspec->pml4) + pml4_index(kvaddr);
- mfn = ((*pml4) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
+ pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
+ mfn = ((*pgd) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
- if ((mfn != machdep->last_pgd_read) &&
- !xc_core_mfn_to_page(mfn, machdep->pgd))
+ if ((mfn != machdep->last_pud_read) &&
+ !xc_core_mfn_to_page(mfn, machdep->pud))
error(FATAL, "cannot read/find pud page\n");
- machdep->last_pgd_read = mfn;
+ machdep->last_pud_read = mfn;
- pgd = ((ulong *)machdep->pgd) + pgd_index(kvaddr);
- mfn = ((*pgd) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
+ pud = ((ulong *)machdep->pud) + pud_index(kvaddr);
+ mfn = ((*pud) & PHYSICAL_PAGE_MASK) >> PAGESHIFT();
if ((mfn != machdep->last_pmd_read) &&
!xc_core_mfn_to_page(mfn, machdep->pmd))
@@ -7609,20 +7596,17 @@ x86_64_init_hyper(int when)
machdep->pageoffset = machdep->pagesize - 1;
machdep->pagemask = ~((ulonglong)machdep->pageoffset);
machdep->stacksize = machdep->pagesize * 2;
- if ((machdep->machspec->upml = (char *)malloc(PAGESIZE())) ==
NULL)
- error(FATAL, "cannot malloc upml space.");
if ((machdep->pgd = (char *)malloc(PAGESIZE())) == NULL)
error(FATAL, "cannot malloc pgd space.");
+ if ((machdep->pud = (char *)malloc(PAGESIZE())) == NULL)
+ error(FATAL, "cannot malloc pud space.");
if ((machdep->pmd = (char *)malloc(PAGESIZE())) == NULL)
error(FATAL, "cannot malloc pmd space.");
if ((machdep->ptbl = (char *)malloc(PAGESIZE())) == NULL)
error(FATAL, "cannot malloc ptbl space.");
- if ((machdep->machspec->pml4 =
- (char *)malloc(PAGESIZE()*2)) == NULL)
- error(FATAL, "cannot malloc pml4 space.");
- machdep->machspec->last_upml_read = 0;
- machdep->machspec->last_pml4_read = 0;
+
machdep->last_pgd_read = 0;
+ machdep->last_pud_read = 0;
machdep->last_pmd_read = 0;
machdep->last_ptbl_read = 0;
machdep->verify_paddr = generic_verify_paddr;
--
2.14.3
4:51 a.m.
New subject: [PATCH v3 4/6] x86_64.c: Add the VMEMMAP support for 5 level page table
Append the "VMEMMAP" flag to machdep's flags too.
Signed-off-by: Dou Liyang <douly.fnst(a)cn.fujitsu.com>
---
x86_64.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/x86_64.c b/x86_64.c
index f5e70b5..c8c6819 100644
--- a/x86_64.c
+++ b/x86_64.c
@@ -296,6 +296,8 @@ x86_64_init(int when)
machdep->machspec->modules_end = MODULES_END_5LEVEL;
machdep->machspec->vmemmap_vaddr = VMEMMAP_VADDR_5LEVEL;
machdep->machspec->vmemmap_end = VMEMMAP_END_5LEVEL;
+ if (symbol_exists("vmemmap_populate"))
+ machdep->flags |= VMEMMAP;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_5LEVEL;
machdep->machspec->pgdir_shift = PGDIR_SHIFT_5LEVEL;
machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_5LEVEL;
--
2.14.3
4:51 a.m.
New subject: [PATCH v3 5/6] x86_64: Add 5-level paging support for x86_64_k/uvtop()
Add the support for x86_64 5-level page tables. Squeeze the 5-level support
into the currently existing x86_64_kvtop() and x86_64_uvtop() functions instead
of using the new xxx_5level() function placeholders.
Signed-off-by: Dou Liyang <douly.fnst(a)cn.fujitsu.com>
---
x86_64.c | 90 +++++++++++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 63 insertions(+), 27 deletions(-)
diff --git a/x86_64.c b/x86_64.c
index c8c6819..d3d5ab9 100644
--- a/x86_64.c
+++ b/x86_64.c
@@ -19,13 +19,11 @@
#ifdef X86_64
static int x86_64_kvtop(struct task_context *, ulong, physaddr_t *, int);
-static int x86_64_kvtop_5level(struct task_context *, ulong, physaddr_t *, int);
static int x86_64_kvtop_xen_wpt(struct task_context *, ulong, physaddr_t *, int);
static int x86_64_uvtop(struct task_context *, ulong, physaddr_t *, int);
static int x86_64_uvtop_level4(struct task_context *, ulong, physaddr_t *, int);
static int x86_64_uvtop_level4_xen_wpt(struct task_context *, ulong, physaddr_t *, int);
static int x86_64_uvtop_level4_rhel4_xen_wpt(struct task_context *, ulong, physaddr_t *,
int);
-static int x86_64_uvtop_5level(struct task_context *, ulong, physaddr_t *, int);
static int x86_64_task_uses_5level(struct task_context *);
static ulong x86_64_vmalloc_start(void);
static int x86_64_is_task_addr(ulong);
@@ -81,6 +79,7 @@ static int x86_64_is_kvaddr(ulong);
static int x86_64_is_uvaddr(ulong, struct task_context *);
static ulong *x86_64_kpgd_offset(ulong, int, int);
static ulong x86_64_upgd_offset(struct task_context *, ulong, int, int);
+static ulong x86_64_p4d_offset(ulong, ulong, int, int);
static ulong x86_64_pud_offset(ulong, ulong, int, int);
static ulong x86_64_pmd_offset(ulong, ulong, int, int);
static ulong x86_64_pte_offset(ulong, ulong, int, int);
@@ -1649,6 +1648,31 @@ x86_64_upgd_offset(struct task_context *tc, ulong uvaddr, int
verbose, int IS_XE
return pgd_pte;
}
+/*
+ * Find an entry in the fourth-level page table..
+ * p4d = p4d_offset(pgd, address);
+ */
+static ulong
+x86_64_p4d_offset(ulong pgd_pte, ulong vaddr, int verbose, int IS_XEN)
+{
+ ulong *p4d;
+ ulong p4d_paddr;
+ ulong p4d_pte;
+
+ p4d_paddr = pgd_pte & PHYSICAL_PAGE_MASK;
+ FILL_P4D(p4d_paddr, PHYSADDR, PAGESIZE());
+ p4d = ((ulong *)p4d_paddr) + p4d_index(vaddr);
+ p4d_pte = ULONG(machdep->machspec->p4d + PAGEOFFSET(p4d));
+ if (verbose) {
+ if(IS_XEN)
+ fprintf(fp, " P4D: %lx => %lx [machine]\n", (ulong)p4d, p4d_pte);
+ else
+ fprintf(fp, " P4D: %lx => %lx\n", (ulong)p4d, p4d_pte);
+ }
+
+ return p4d_pte;
+}
+
/*
* Find an entry in the third-level page table..
* pud = pud_offset(pgd, address);
@@ -1770,17 +1794,30 @@ x86_64_uvtop_level4(struct task_context *tc, ulong uvaddr,
physaddr_t *paddr, in
if (IS_KVADDR(uvaddr))
return x86_64_kvtop(tc, uvaddr, paddr, verbose);
- if ((machdep->flags & VM_5LEVEL) && x86_64_task_uses_5level(tc))
- return x86_64_uvtop_5level(tc, uvaddr, paddr, verbose);
-
pgd_pte = x86_64_upgd_offset(tc, uvaddr, verbose, FALSE);
if (!(pgd_pte & _PAGE_PRESENT))
goto no_upage;
- /*
- * pud = pud_offset(pgd, address);
- */
- pud_pte = x86_64_pud_offset(pgd_pte, uvaddr, verbose, FALSE);
+ /* If the VM is in 5-level page table*/
+ if (machdep->flags & VM_5LEVEL && x86_64_task_uses_5level(tc)) {
+ ulong p4d_pte;
+ /*
+ * p4d = p4d_offset(pgd, address);
+ */
+ p4d_pte = x86_64_p4d_offset(pgd_pte, uvaddr, verbose, FALSE);
+ if (!(p4d_pte & _PAGE_PRESENT))
+ goto no_upage;
+ /*
+ * pud = pud_offset(p4d, address);
+ */
+ pud_pte = x86_64_pud_offset(p4d_pte, uvaddr, verbose, FALSE);
+ } else {
+ /*
+ * pud = pud_offset(pgd, address);
+ */
+ pud_pte = x86_64_pud_offset(pgd_pte, uvaddr, verbose, FALSE);
+ }
+
if (!(pud_pte & _PAGE_PRESENT))
goto no_upage;
@@ -1839,13 +1876,6 @@ x86_64_task_uses_5level(struct task_context *tc)
return FALSE;
}
-static int
-x86_64_uvtop_5level(struct task_context *tc, ulong uvaddr, physaddr_t *paddr, int
verbose)
-{
- error(INFO, "support for 5-level page tables TBD\n");
- return FALSE;
-}
-
static int
x86_64_uvtop_level4_xen_wpt(struct task_context *tc, ulong uvaddr, physaddr_t *paddr, int
verbose)
{
@@ -2175,9 +2205,6 @@ x86_64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t
*paddr, int verbo
if (XEN() && (kt->xen_flags & WRITABLE_PAGE_TABLES))
return (x86_64_kvtop_xen_wpt(tc, kvaddr, paddr, verbose));
- if (machdep->flags & VM_5LEVEL)
- return (x86_64_kvtop_5level(tc, kvaddr, paddr, verbose));
-
/*
* pgd = pgd_offset_k(addr);
*/
@@ -2188,7 +2215,23 @@ start_vtop_with_pagetable:
if (!(*pgd & _PAGE_PRESENT))
goto no_kpage;
- pud_pte = x86_64_pud_offset(*pgd, kvaddr, verbose, FALSE);
+ /* If the VM is in 5-level page table*/
+ if (machdep->flags & VM_5LEVEL) {
+ ulong p4d_pte;
+ /*
+ * p4d = p4d_offset(pgd, address);
+ */
+ p4d_pte = x86_64_p4d_offset(*pgd, kvaddr, verbose, FALSE);
+ if (!(p4d_pte & _PAGE_PRESENT))
+ goto no_kpage;
+ /*
+ * pud = pud_offset(p4d, address);
+ */
+ pud_pte = x86_64_pud_offset(p4d_pte, kvaddr, verbose, FALSE);
+ } else {
+ pud_pte = x86_64_pud_offset(*pgd, kvaddr, verbose, FALSE);
+ }
+
if (!(pud_pte & _PAGE_PRESENT))
goto no_kpage;
@@ -2238,13 +2281,6 @@ no_kpage:
return FALSE;
}
-static int
-x86_64_kvtop_5level(struct task_context *tc, ulong kvaddr, physaddr_t *paddr, int
verbose)
-{
- error(INFO, "support for 5-level page tables TBD\n");
- return FALSE;
-}
-
static int
x86_64_kvtop_xen_wpt(struct task_context *tc, ulong kvaddr, physaddr_t *paddr, int
verbose)
{
--
2.14.3
4:51 a.m.
New subject: [PATCH v3 6/6] x86_64: Fix the misusage of PGDIR_SHIFT
Due to the backwards compatibility, we should setup machdep->machspec->pgdir_shift
correctly in x86_64_init() for all versions of linux. this patch makes the x86_64
default page level as 4-level paging table and sets the 4-level paging table for
each Linux versions, and add a new function named x86_64_upgd_offset_legacy()
for old linuxes to get the user page global directory entry.
Signed-off-by: Dou Liyang <douly.fnst(a)cn.fujitsu.com>
---
defs.h | 15 +++++++--------
sadump.c | 4 ++--
x86_64.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++----------
3 files changed, 55 insertions(+), 20 deletions(-)
diff --git a/defs.h b/defs.h
index 5804206..ff5494c 100644
--- a/defs.h
+++ b/defs.h
@@ -3332,19 +3332,18 @@ struct arm64_stackframe {
#define VTOP(X) x86_64_VTOP((ulong)(X))
#define IS_VMALLOC_ADDR(X) x86_64_IS_VMALLOC_ADDR((ulong)(X))
-/* origin level page */
-#define PGDIR_SHIFT 30
+/*
+ * the default page table level for x86_64:
+ * 4 level page tables
+ */
+#define PGDIR_SHIFT 39
#define PTRS_PER_PGD 512
+#define PUD_SHIFT 30
+#define PTRS_PER_PUD 512
#define PMD_SHIFT 21
#define PTRS_PER_PMD 512
#define PTRS_PER_PTE 512
-/* 4 level page */
-#define PGDIR_SHIFT_4LEVEL 39
-#define PTRS_PER_PGD_4LEVEL 512
-#define PUD_SHIFT 30
-#define PTRS_PER_PUD 512
-
/* 5 level page */
#define PGDIR_SHIFT_5LEVEL 48
#define PTRS_PER_PGD_5LEVEL 512
diff --git a/sadump.c b/sadump.c
index 17cfd93..d19b40a 100644
--- a/sadump.c
+++ b/sadump.c
@@ -2059,8 +2059,8 @@ sadump_calc_kaslr_offset(ulong *kaslr_offset)
vt->kernel_pgd[0] = pgd;
machdep->last_pgd_read = vt->kernel_pgd[0];
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_2_6;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT_4LEVEL;
- machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_4LEVEL;
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD;
if (!readmem(pgd, PHYSADDR, machdep->pgd, PAGESIZE(),
"pgd", RETURN_ON_ERROR))
goto quit;
diff --git a/x86_64.c b/x86_64.c
index d3d5ab9..a6b1f50 100644
--- a/x86_64.c
+++ b/x86_64.c
@@ -79,6 +79,7 @@ static int x86_64_is_kvaddr(ulong);
static int x86_64_is_uvaddr(ulong, struct task_context *);
static ulong *x86_64_kpgd_offset(ulong, int, int);
static ulong x86_64_upgd_offset(struct task_context *, ulong, int, int);
+static ulong x86_64_upgd_offset_legacy(struct task_context *, ulong, int, int);
static ulong x86_64_p4d_offset(ulong, ulong, int, int);
static ulong x86_64_pud_offset(ulong, ulong, int, int);
static ulong x86_64_pmd_offset(ulong, ulong, int, int);
@@ -260,8 +261,8 @@ x86_64_init(int when)
machdep->uvtop = x86_64_uvtop_level4;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_2_6;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT_4LEVEL;
- machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_4LEVEL;
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD;
break;
case VM_XEN:
@@ -273,6 +274,8 @@ x86_64_init(int when)
machdep->machspec->modules_vaddr = MODULES_VADDR_XEN;
machdep->machspec->modules_end = MODULES_END_XEN;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_XEN;
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD;
break;
case VM_XEN_RHEL4:
@@ -284,6 +287,8 @@ x86_64_init(int when)
machdep->machspec->modules_vaddr =
MODULES_VADDR_XEN_RHEL4;
machdep->machspec->modules_end = MODULES_END_XEN_RHEL4;
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_XEN;
+ machdep->machspec->pgdir_shift = PGDIR_SHIFT;
+ machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD;
break;
case VM_5LEVEL:
@@ -627,20 +632,14 @@ x86_64_init(int when)
case VM_XEN:
case VM_2_6_11:
machdep->uvtop = x86_64_uvtop_level4_xen_wpt;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT_4LEVEL;
- machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_4LEVEL;
break;
case VM_XEN_RHEL4:
machdep->uvtop = x86_64_uvtop_level4_rhel4_xen_wpt;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT;
- machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD;
break;
}
machdep->machspec->physical_mask_shift = __PHYSICAL_MASK_SHIFT_XEN;
} else {
machdep->uvtop = x86_64_uvtop_level4;
- machdep->machspec->pgdir_shift = PGDIR_SHIFT_4LEVEL;
- machdep->machspec->ptrs_per_pgd = PTRS_PER_PGD_4LEVEL;
}
MEMBER_OFFSET_INIT(vcpu_guest_context_user_regs,
"vcpu_guest_context", "user_regs");
@@ -1617,6 +1616,43 @@ x86_64_kpgd_offset(ulong kvaddr, int verbose, int IS_XEN)
return pgd;
}
+/*
+ * In x86 64 bit system, Linux uses the 4-level page table as the default both
+ * in Kernel page tables and user page tables.
+ *
+ * But in some old versions(pre-2.6.11), the 3-level page table is used for
+ * user page tables.
+ *
+ * So reuse the PUD and find the user pgd entry for this older version Linux..
+ * pgd = pgd_offset(mm, address);
+ */
+static ulong
+x86_64_upgd_offset_legacy(struct task_context *tc, ulong uvaddr, int verbose, int
IS_XEN)
+{
+ ulong *pud;
+ ulong pud_paddr;
+ ulong pud_pte;
+
+ if (task_mm(tc->task, TRUE))
+ pud = ULONG_PTR(tt->mm_struct + OFFSET(mm_struct_pgd));
+ else
+ readmem(tc->mm_struct + OFFSET(mm_struct_pgd), KVADDR, &pud,
+ sizeof(long), "mm_struct pgd", FAULT_ON_ERROR);
+
+ pud_paddr = x86_64_VTOP((ulong)pud);
+ FILL_PUD(pud_paddr, PHYSADDR, PAGESIZE());
+ pud = ((ulong *)pud_paddr) + pud_index(uvaddr);
+ pud_pte = ULONG(machdep->pud + PAGEOFFSET(pud));
+ if (verbose) {
+ if(IS_XEN)
+ fprintf(fp, " PGD: %lx => %lx [machine]\n", (ulong)pud, pud_pte);
+ else
+ fprintf(fp, " PGD: %lx => %lx\n", (ulong)pud, pud_pte);
+ }
+
+ return pud_pte;
+}
+
/*
* Find the user pgd entry..
* pgd = pgd_offset(mm, address);
@@ -1997,7 +2033,7 @@ x86_64_uvtop_level4_rhel4_xen_wpt(struct task_context *tc, ulong
uvaddr, physadd
if (IS_KVADDR(uvaddr))
return x86_64_kvtop(tc, uvaddr, paddr, verbose);
- pgd_pte = x86_64_upgd_offset(tc, uvaddr, verbose, TRUE);
+ pgd_pte = x86_64_upgd_offset_legacy(tc, uvaddr, verbose, TRUE);
if (!(pgd_pte & _PAGE_PRESENT))
goto no_upage;
@@ -2094,7 +2130,7 @@ x86_64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t
*paddr, int verbo
/*
* pgd = pgd_offset(mm, address);
*/
- pgd_pte = x86_64_upgd_offset(tc, uvaddr, verbose, FALSE);
+ pgd_pte = x86_64_upgd_offset_legacy(tc, uvaddr, verbose, FALSE);
if (!(pgd_pte & _PAGE_PRESENT))
goto no_upage;
--
2.14.3
1:49 p.m.
Hello Dou,
I have reviewed this second phase patchset, and have successfully tested it on
existing kernels. I must say that it is *very* well done -- I truly appreciate
the effort that you have undertaken.
The only thing I added was a reference to the "--machdep vm=5level"
command line option in the crash.8 man page and for "crash --help".
I've queued it for crash-7.2.1:
https://github.com/crash-utility/crash/commit/94d01ce01d7bdd7e3bbc5dccb25...
I look forward to phase 3!
Thanks,
Dave
----- Original Message -----
Changelog:
V1-->V2:
-Fix the backwards compatibility issues suggested by Dave
-Test with "mod","kmem -f","vm -p" command again.
V2-->V3:
-keep the pml4, upml, last_upml_read and last_pml4_read members in struct
machine_specific like before for extension modules suggested by Dave
-move ptrs_per_pgd to the end of the machine_specific data structure
-rewirte the changelog
-rebase the patchset
-retest it
I found Dave had alread done the first phase of future support for x86_64
5-level page tables(commit 307e7f35f510). when I asked him about the
state of this work, he gave me a more detailed answer and suggestion.
I follow his advice, and do the following job.
1. Refine the original logical:
1) Create some new common function for getting the offset of page table
2) Repace the PML4 and UPML with the common PGD:
machdep->machspec->pml4/upml ==> machdep->pdg
3) Using the PUD in x86_64
2. Add 5-level page tables support for x86_64_k/uvtop()
This patchset is the second phase of the work, As Dave said, we need to be
a manner of determining very early on whether the kernel page tables are
using 5-level and whether each user-space task is using 4- or 5-level page
tables. These will be done after this phase.
About test work:
I have tested this patchset with 4-level and 5-level paging table.
sadump are not be tested.
Dou Liyang (6):
defs.h: Fix the PHYSICAL_PAGE_MASK macro
x86_64: Extract public page table mapping code
x86_64: Unify the page table parsing for 4-level mode
x86_64.c: Add the VMEMMAP support for 5 level page table
x86_64: Add 5-level paging support for x86_64_k/uvtop()
x86_64: Fix the misusage of PGDIR_SHIFT
defs.h | 56 ++---
sadump.c | 7 +-
x86_64.c | 750
+++++++++++++++++++++++++++++++++------------------------------
3 files changed, 418 insertions(+), 395 deletions(-)
--
2.14.3
8:23 p.m.
Hi Dave,
At 01/31/2018 03:49 AM, Dave Anderson wrote:
Hello Dou,
I have reviewed this second phase patchset, and have successfully tested it on
existing kernels. I must say that it is *very* well done -- I truly appreciate
the effort that you have undertaken.
The only thing I added was a reference to the "--machdep vm=5level"
command line option in the crash.8 man page and for "crash --help".
I've queued it for crash-7.2.1:
https://github.com/crash-utility/crash/commit/94d01ce01d7bdd7e3bbc5dccb25...
Thank you so much for reviewing and merging this patchset.
I look forward to phase 3!
Ok, I will try my best to do phase 3:
As you said in the changelog, the third phase contains two tasks:
1) automatically detect whether the kernel proper for 5 level page
tables.
2) detect whether an individual user task, is utilizing 5-level page
tables
For task 1, I saw Kirill had sent the v8 patchset. IMO, it's better
to do this task after kirill's patches are merged. is it right?
For task 2, I am sorry I have no idea about it now, I will learn it
first, any suggestions do you have?
Thanks,
dou
Thanks,
Dave
----- Original Message -----
> Changelog:
> V1-->V2:
> -Fix the backwards compatibility issues suggested by Dave
> -Test with "mod","kmem -f","vm -p" command again.
> V2-->V3:
> -keep the pml4, upml, last_upml_read and last_pml4_read members in struct
> machine_specific like before for extension modules suggested by Dave
> -move ptrs_per_pgd to the end of the machine_specific data structure
> -rewirte the changelog
> -rebase the patchset
> -retest it
>
> I found Dave had alread done the first phase of future support for x86_64
> 5-level page tables(commit 307e7f35f510). when I asked him about the
> state of this work, he gave me a more detailed answer and suggestion.
> I follow his advice, and do the following job.
>
> 1. Refine the original logical:
> 1) Create some new common function for getting the offset of page table
> 2) Repace the PML4 and UPML with the common PGD:
> machdep->machspec->pml4/upml ==> machdep->pdg
> 3) Using the PUD in x86_64
>
> 2. Add 5-level page tables support for x86_64_k/uvtop()
>
> This patchset is the second phase of the work, As Dave said, we need to be
> a manner of determining very early on whether the kernel page tables are
> using 5-level and whether each user-space task is using 4- or 5-level page
> tables. These will be done after this phase.
>
> About test work:
>
> I have tested this patchset with 4-level and 5-level paging table.
>
> sadump are not be tested.
>
> Dou Liyang (6):
> defs.h: Fix the PHYSICAL_PAGE_MASK macro
> x86_64: Extract public page table mapping code
> x86_64: Unify the page table parsing for 4-level mode
> x86_64.c: Add the VMEMMAP support for 5 level page table
> x86_64: Add 5-level paging support for x86_64_k/uvtop()
> x86_64: Fix the misusage of PGDIR_SHIFT
>
> defs.h | 56 ++---
> sadump.c | 7 +-
> x86_64.c | 750
> +++++++++++++++++++++++++++++++++------------------------------
> 3 files changed, 418 insertions(+), 395 deletions(-)
>
> --
> 2.14.3
>
>
>
>
9:02 a.m.
----- Original Message -----
Hi Dave,
At 01/31/2018 03:49 AM, Dave Anderson wrote:
>
> Hello Dou,
>
> I have reviewed this second phase patchset, and have successfully tested it
> on
> existing kernels. I must say that it is *very* well done -- I truly
> appreciate
> the effort that you have undertaken.
>
> The only thing I added was a reference to the "--machdep vm=5level"
> command line option in the crash.8 man page and for "crash --help".
>
> I've queued it for crash-7.2.1:
>
>
https://github.com/crash-utility/crash/commit/94d01ce01d7bdd7e3bbc5dccb25...
>
Thank you so much for reviewing and merging this patchset.
> I look forward to phase 3!
>
Ok, I will try my best to do phase 3:
As you said in the changelog, the third phase contains two tasks:
1) automatically detect whether the kernel proper for 5 level page
tables.
2) detect whether an individual user task, is utilizing 5-level page
tables
For task 1, I saw Kirill had sent the v8 patchset. IMO, it's better
to do this task after kirill's patches are merged. is it right?
Sure, that's fine with me. At that point there should just be a kernel
variable that, if it exists, and is set appropriately, would be the
best clue.
For task 2, I am sorry I have no idea about it now, I will learn it
first, any suggestions do you have?
No, I don't understand the specifics. The user-space reference in the
Documentation/x86/x86_64/5level-paging.txt file seems to imply that
5-level is set up on the fly if an mmap MAP_FIXED request goes beyond
the 47-bit limit (?). Maybe it would be possible to look at the contents
of the page directory or p4d (or whichever), and check whether it "folds"
back into itself or points to the extra page of PTEs?
Thanks,
Dave
Thanks,
dou
> Thanks,
> Dave
>
> ----- Original Message -----
>> Changelog:
>> V1-->V2:
>> -Fix the backwards compatibility issues suggested by Dave
>> -Test with "mod","kmem -f","vm -p" command
again.
>> V2-->V3:
>> -keep the pml4, upml, last_upml_read and last_pml4_read members in
>> struct
>> machine_specific like before for extension modules suggested by Dave
>> -move ptrs_per_pgd to the end of the machine_specific data structure
>> -rewirte the changelog
>> -rebase the patchset
>> -retest it
>>
>> I found Dave had alread done the first phase of future support for x86_64
>> 5-level page tables(commit 307e7f35f510). when I asked him about the
>> state of this work, he gave me a more detailed answer and suggestion.
>> I follow his advice, and do the following job.
>>
>> 1. Refine the original logical:
>> 1) Create some new common function for getting the offset of page table
>> 2) Repace the PML4 and UPML with the common PGD:
>> machdep->machspec->pml4/upml ==> machdep->pdg
>> 3) Using the PUD in x86_64
>>
>> 2. Add 5-level page tables support for x86_64_k/uvtop()
>>
>> This patchset is the second phase of the work, As Dave said, we need to be
>> a manner of determining very early on whether the kernel page tables are
>> using 5-level and whether each user-space task is using 4- or 5-level
>> page
>> tables. These will be done after this phase.
>>
>> About test work:
>>
>> I have tested this patchset with 4-level and 5-level paging table.
>>
>> sadump are not be tested.
>>
>> Dou Liyang (6):
>> defs.h: Fix the PHYSICAL_PAGE_MASK macro
>> x86_64: Extract public page table mapping code
>> x86_64: Unify the page table parsing for 4-level mode
>> x86_64.c: Add the VMEMMAP support for 5 level page table
>> x86_64: Add 5-level paging support for x86_64_k/uvtop()
>> x86_64: Fix the misusage of PGDIR_SHIFT
>>
>> defs.h | 56 ++---
>> sadump.c | 7 +-
>> x86_64.c | 750
>> +++++++++++++++++++++++++++++++++------------------------------
>> 3 files changed, 418 insertions(+), 395 deletions(-)
>>
>> --
>> 2.14.3
>>
>>
>>
>>
>
>
>
2345
days inactive
2347
days old
devel@lists.crash-utility.osci.io
9 comments
2 participants
participants (2)
-
Dave Anderson -
Dou Liyang