[Crash-utility] [PATCH 4/4] arm64: Support 16K page, 48 VA bits and 4 level page table

Add 16K page, 48 VA bits and 4 level page table support. Signed-off-by: Kuan-Ying Lee <kuan-ying.lee(a)canonical.com&gt; --- arm64.c | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- defs.h | 20 +++++++++ 2 files changed, 142 insertions(+), 6 deletions(-) diff --git a/arm64.c b/arm64.c index d8b030d4b0b0..75f145725f2d 100644 --- a/arm64.c +++ b/arm64.c @@ -44,6 +44,7 @@ static int arm64_vtop_2level_64k(ulong, ulong, physaddr_t *, int); static int arm64_vtop_3level_64k(ulong, ulong, physaddr_t *, int); static int arm64_vtop_2level_16k(ulong, ulong, physaddr_t *, int); static int arm64_vtop_3level_16k(ulong, ulong, physaddr_t *, int); +static int arm64_vtop_4level_16k(ulong, ulong, physaddr_t *, int); static int arm64_vtop_3level_4k(ulong, ulong, physaddr_t *, int); static int arm64_vtop_4level_4k(ulong, ulong, physaddr_t *, int); static ulong arm64_get_task_pgd(ulong); @@ -414,7 +415,24 @@ arm64_init(int when) break; case 16384: - if (machdep->machspec->VA_BITS == 47) { + if (machdep->machspec->VA_BITS == 48) { + machdep->flags |= VM_L4_16K; + if (!machdep->ptrs_per_pgd) + machdep->ptrs_per_pgd = PTRS_PER_PGD_L4_16K; + if ((machdep->pgd = + (char *)malloc(machdep->ptrs_per_pgd * 8)) == NULL) + error(FATAL, "cannot malloc pgd space."); + if ((machdep->pud = + (char *)malloc(PTRS_PER_PUD_L4_16K * 8)) == NULL) + error(FATAL, "cannot malloc pud space."); + if ((machdep->pmd = + (char *)malloc(PTRS_PER_PMD_L4_16K * 8)) == NULL) + error(FATAL, "cannot malloc pmd space."); + if ((machdep->ptbl = + (char *)malloc(PTRS_PER_PTE_L4_16K * 8)) == NULL) + error(FATAL, "cannot malloc ptbl space."); + } + else if (machdep->machspec->VA_BITS == 47) { machdep->flags |= VM_L3_16K; if (!machdep->ptrs_per_pgd) machdep->ptrs_per_pgd = PTRS_PER_PGD_L3_16K; @@ -427,6 +445,7 @@ arm64_init(int when) if ((machdep->ptbl = (char *)malloc(PTRS_PER_PTE_L3_16K * 8)) == NULL) error(FATAL, "cannot malloc ptbl space."); + machdep->pud = NULL; /* not used */ } else if (machdep->machspec->VA_BITS == 36) { machdep->flags |= VM_L2_16K; if (!machdep->ptrs_per_pgd) @@ -438,10 +457,10 @@ arm64_init(int when) (char *)malloc(PTRS_PER_PTE_L2_16K * 8)) == NULL) error(FATAL, "cannot malloc ptbl space."); machdep->pmd = NULL; /* not used */ + machdep->pud = NULL; /* not used */ } else { - error(FATAL, "Do not support 48 bits or 52 bits, 4-level for 16K page now."); + error(FATAL, "Do not support 52 bits, 4-level for 16K page now."); } - machdep->pud = NULL; /* not used */ break; case 65536: @@ -1091,6 +1110,8 @@ arm64_dump_machdep_table(ulong arg) fprintf(fp, "%sVM_L2_16K", others++ ? "|" : ""); if (machdep->flags & VM_L3_16K) fprintf(fp, "%sVM_L3_16K", others++ ? "|" : ""); + if (machdep->flags & VM_L4_16K) + fprintf(fp, "%sVM_L4_16K", others++ ? "|" : ""); if (machdep->flags & VM_L3_4K) fprintf(fp, "%sVM_L3_4K", others++ ? "|" : ""); if (machdep->flags & VM_L4_4K) @@ -1142,6 +1163,8 @@ arm64_dump_machdep_table(ulong arg) "arm64_vtop_2level_16k" : machdep->flags & VM_L3_16K ? "arm64_vtop_3level_16k" : + machdep->flags & VM_L4_16K ? + "arm64_vtop_4level_16k" : machdep->flags & VM_L3_64K ? "arm64_vtop_3level_64k" : "arm64_vtop_2level_64k"); fprintf(fp, " kvtop: arm64_kvtop()->%s()\n", @@ -1153,6 +1176,8 @@ arm64_dump_machdep_table(ulong arg) "arm64_vtop_2level_16k" : machdep->flags & VM_L3_16K ? "arm64_vtop_3level_16k" : + machdep->flags & VM_L4_16K ? + "arm64_vtop_4level_16k" : machdep->flags & VM_L3_64K ? "arm64_vtop_3level_64k" : "arm64_vtop_2level_64k"); fprintf(fp, " get_task_pgd: arm64_get_task_pgd()\n"); @@ -1187,7 +1212,7 @@ arm64_dump_machdep_table(ulong arg) fprintf(fp, " line_number_hooks: (not used)\n"); fprintf(fp, " last_pgd_read: %lx\n", machdep->last_pgd_read); fprintf(fp, " last_pud_read: "); - if (!(machdep->flags & VM_L4_4K)) + if ((!(machdep->flags & VM_L4_4K)) && (!(machdep->flags & VM_L4_16K))) fprintf(fp, "(not used)\n"); else fprintf(fp, "%lx\n", machdep->last_pud_read); @@ -1841,7 +1866,7 @@ arm64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t *paddr, int verbos kernel_pgd = vt->kernel_pgd[0]; *paddr = 0; - switch (machdep->flags & (VM_L2_64K|VM_L3_64K|VM_L3_4K|VM_L4_4K|VM_L2_16K|VM_L3_16K)) + switch (machdep->flags & (VM_L2_64K|VM_L3_64K|VM_L3_4K|VM_L4_4K|VM_L2_16K|VM_L3_16K|VM_L4_16K)) { case VM_L2_64K: return arm64_vtop_2level_64k(kernel_pgd, kvaddr, paddr, verbose); @@ -1855,6 +1880,8 @@ arm64_kvtop(struct task_context *tc, ulong kvaddr, physaddr_t *paddr, int verbos return arm64_vtop_2level_16k(kernel_pgd, kvaddr, paddr, verbose); case VM_L3_16K: return arm64_vtop_3level_16k(kernel_pgd, kvaddr, paddr, verbose); + case VM_L4_16K: + return arm64_vtop_4level_16k(kernel_pgd, kvaddr, paddr, verbose); default: return FALSE; } @@ -1870,7 +1897,7 @@ arm64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t *paddr, int verbos *paddr = 0; - switch (machdep->flags & (VM_L2_64K|VM_L3_64K|VM_L3_4K|VM_L4_4K|VM_L2_16K|VM_L3_16K)) + switch (machdep->flags & (VM_L2_64K|VM_L3_64K|VM_L3_4K|VM_L4_4K|VM_L2_16K|VM_L3_16K|VM_L4_16K)) { case VM_L2_64K: return arm64_vtop_2level_64k(user_pgd, uvaddr, paddr, verbose); @@ -1884,6 +1911,8 @@ arm64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t *paddr, int verbos return arm64_vtop_2level_16k(user_pgd, uvaddr, paddr, verbose); case VM_L3_16K: return arm64_vtop_3level_16k(user_pgd, uvaddr, paddr, verbose); + case VM_L4_16K: + return arm64_vtop_4level_16k(user_pgd, uvaddr, paddr, verbose); default: return FALSE; } @@ -1906,6 +1935,7 @@ arm64_uvtop(struct task_context *tc, ulong uvaddr, physaddr_t *paddr, int verbos #define SECTION_PAGE_MASK_32MB ((long)(~((MEGABYTES(32))-1))) #define SECTION_PAGE_MASK_512MB ((long)(~((MEGABYTES(512))-1))) #define SECTION_PAGE_MASK_1GB ((long)(~((GIGABYTES(1))-1))) +#define SECTION_PAGE_MASK_64GB ((long)(~((GIGABYTES(64))-1))) static int arm64_vtop_2level_64k(ulong pgd, ulong vaddr, physaddr_t *paddr, int verbose) @@ -2188,6 +2218,92 @@ no_page: return FALSE; } +static int +arm64_vtop_4level_16k(ulong pgd, ulong vaddr, physaddr_t *paddr, int verbose) +{ + ulong *pgd_base, *pgd_ptr, pgd_val; + ulong *pud_base, *pud_ptr, pud_val; + ulong *pmd_base, *pmd_ptr, pmd_val; + ulong *pte_base, *pte_ptr, pte_val; + + if (verbose) + fprintf(fp, "PAGE DIRECTORY: %lx\n", pgd); + + pgd_base = (ulong *)pgd; + FILL_PGD(pgd_base, KVADDR, PTRS_PER_PGD_L4_16K * sizeof(ulong)); + pgd_ptr = pgd_base + (((vaddr) >> PGDIR_SHIFT_L4_16K) & (PTRS_PER_PGD_L4_16K - 1)); + pgd_val = ULONG(machdep->pgd + PGDIR_OFFSET_L4_16K(pgd_ptr)); + if (verbose) + fprintf(fp, " PGD: %lx => %lx\n", (ulong)pgd_ptr, pgd_val); + if (!pgd_val) + goto no_page; + + pud_base = (ulong *)PTOV(PTE_TO_PHYS(pgd_val)); + FILL_PUD(pud_base, KVADDR, PTRS_PER_PUD_L4_16K * sizeof(ulong)); + pud_ptr = pud_base + (((vaddr) >> PUD_SHIFT_L4_16K) & (PTRS_PER_PUD_L4_16K - 1)); + pud_val = ULONG(machdep->pud + PAGEOFFSET(pud_ptr)); + if (verbose) + fprintf(fp, " PUD: %lx => %lx\n", (ulong)pud_ptr, pud_val); + if (!pud_val) + goto no_page; + + /* + * TODO: + * PUD Section mapping is only supported for LPA2 is enabled. + * LPA2 depends on CONFIG_ARM64_16K_PAGES and CONFIG_ARM64_PA_BITS_52. + */ + + pmd_base = (ulong *)PTOV(PTE_TO_PHYS(pud_val)); + FILL_PMD(pmd_base, KVADDR, PTRS_PER_PMD_L4_16K * sizeof(ulong)); + pmd_ptr = pmd_base + (((vaddr) >> PMD_SHIFT_L4_16K) & (PTRS_PER_PMD_L4_16K - 1)); + pmd_val = ULONG(machdep->pmd + PAGEOFFSET(pmd_ptr)); + if (verbose) + fprintf(fp, " PMD: %lx => %lx\n", (ulong)pmd_ptr, pmd_val); + if (!pmd_val) + goto no_page; + + if ((pmd_val & PMD_TYPE_MASK) == PMD_TYPE_SECT) { + ulong sectionbase = PTE_TO_PHYS(pmd_val) & SECTION_PAGE_MASK_32MB; + if (verbose) { + fprintf(fp, " PAGE: %lx (32MB%s)\n\n", sectionbase, + IS_ZEROPAGE(sectionbase) ? ", ZERO PAGE" : ""); + arm64_translate_pte(pmd_val, 0, 0); + } + *paddr = sectionbase + (vaddr & ~SECTION_PAGE_MASK_32MB); + return TRUE; + } + + pte_base = (ulong *)PTOV(PTE_TO_PHYS(pmd_val)); + FILL_PTBL(pte_base, KVADDR, PTRS_PER_PTE_L4_16K * sizeof(ulong)); + pte_ptr = pte_base + (((vaddr) >> machdep->pageshift) & (PTRS_PER_PTE_L4_16K - 1)); + pte_val = ULONG(machdep->ptbl + PAGEOFFSET(pte_ptr)); + if (verbose) + fprintf(fp, " PTE: %lx => %lx\n", (ulong)pte_ptr, pte_val); + if (!pte_val) + goto no_page; + + if (pte_val & PTE_VALID) { + *paddr = PTE_TO_PHYS(pte_val) + PAGEOFFSET(vaddr); + if (verbose) { + fprintf(fp, " PAGE: %lx %s\n\n", PAGEBASE(*paddr), + IS_ZEROPAGE(PAGEBASE(*paddr)) ? "(ZERO PAGE)" : ""); + arm64_translate_pte(pte_val, 0, 0); + } + } else { + if (IS_UVADDR(vaddr, NULL)) + *paddr = pte_val; + if (verbose) { + fprintf(fp, "\n"); + arm64_translate_pte(pte_val, 0, 0); + } + goto no_page; + } + + return TRUE; +no_page: + return FALSE; +} + static int arm64_vtop_3level_4k(ulong pgd, ulong vaddr, physaddr_t *paddr, int verbose) { diff --git a/defs.h b/defs.h index 9f4d676e25f9..7299952b7240 100644 --- a/defs.h +++ b/defs.h @@ -3329,6 +3329,25 @@ typedef signed int s32; #define PMD_MASK_L3_16K (~(PMD_SIZE_L3_16K-1)) #define PGDIR_OFFSET_L3_16K(X) (((ulong)(X)) & ((machdep->ptrs_per_pgd * 8) - 1)) +/* + * 4-levels / 16K pages + * 48-bit VA + */ +#define PTRS_PER_PGD_L4_16K ((1UL) << (48 - 47)) +#define PTRS_PER_PUD_L4_16K (2048) +#define PTRS_PER_PMD_L4_16K (2048) +#define PTRS_PER_PTE_L4_16K (2048) +#define PGDIR_SHIFT_L4_16K (47) +#define PGDIR_SIZE_L4_16K ((1UL) << PGDIR_SHIFT_L4_16K) +#define PGDIR_MASK_L4_16K (~(PGDIR_SIZE_L4_16K-1)) +#define PUD_SHIFT_L4_16K (36) +#define PUD_SIZE_L4_16K ((1UL) << PUD_SHIFT_L4_16K) +#define PUD_MASK_L4_16K (~(PUD_SIZE_L4_16K-1)) +#define PMD_SHIFT_L4_16K (25) +#define PMD_SIZE_L4_16K (1UL << PMD_SHIFT_L4_16K) +#define PMD_MASK_L4_16K (~(PMD_SIZE_L4_16K-1)) +#define PGDIR_OFFSET_L4_16K(X) (((ulong)(X)) & ((machdep->ptrs_per_pgd * 8) - 1)) + /* * 3-levels / 64K pages * 48-bit, 52-bit VA @@ -3399,6 +3418,7 @@ typedef signed int s32; #define ARM64_MTE (0x2000) #define VM_L3_16K (0x4000) #define VM_L2_16K (0x8000) +#define VM_L4_16K (0x10000) /* * Get kimage_voffset from /dev/crash -- 2.43.0