Skip to content

Commit 4b91428

Browse files
chleroyScott Wood
chleroy
authored and
Scott Wood
committed
powerpc/8xx: Implement support of hugepages
8xx uses a two level page table with two different linux page size support (4k and 16k). 8xx also support two different hugepage sizes 512k and 8M. In order to support them on linux we define two different page table layout. The size of pages is in the PGD entry, using PS field (bits 28-29): 00 : Small pages (4k or 16k) 01 : 512k pages 10 : reserved 11 : 8M pages For 512K hugepage size a pgd entry have the below format [<hugepte address >0101] . The hugepte table allocated will contain 8 entries pointing to 512K huge pte in 4k pages mode and 64 entries in 16k pages mode. For 8M in 16k mode, a pgd entry have the below format [<hugepte address >1101] . The hugepte table allocated will contain 8 entries pointing to 8M huge pte. For 8M in 4k mode, multiple pgd entries point to the same hugepte address and pgd entry will have the below format [<hugepte address>1101]. The hugepte table allocated will only have one entry. For the time being, we do not support CPU15 ERRATA when HUGETLB is selected Signed-off-by: Christophe Leroy <[email protected]> Reviewed-by: Aneesh Kumar K.V <[email protected]> (v3, for the generic bits) Signed-off-by: Scott Wood <[email protected]>
1 parent 03bb2d6 commit 4b91428

File tree

11 files changed

+225
-30
lines changed

11 files changed

+225
-30
lines changed

‎arch/powerpc/include/asm/hugetlb.h

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,20 @@ static inline void __local_flush_hugetlb_page(struct vm_area_struct *vma,
5151
static inline pte_t *hugepd_page(hugepd_t hpd)
5252
{
5353
BUG_ON(!hugepd_ok(hpd));
54+
#ifdef CONFIG_PPC_8xx
55+
return (pte_t *)__va(hpd.pd & ~(_PMD_PAGE_MASK | _PMD_PRESENT_MASK));
56+
#else
5457
return (pte_t *)((hpd.pd & ~HUGEPD_SHIFT_MASK) | PD_HUGE);
58+
#endif
5559
}
5660

5761
static inline unsigned int hugepd_shift(hugepd_t hpd)
5862
{
63+
#ifdef CONFIG_PPC_8xx
64+
return ((hpd.pd & _PMD_PAGE_MASK) >> 1) + 17;
65+
#else
5966
return hpd.pd & HUGEPD_SHIFT_MASK;
67+
#endif
6068
}
6169

6270
#endif /* CONFIG_PPC_BOOK3S_64 */
@@ -99,7 +107,15 @@ static inline int is_hugepage_only_range(struct mm_struct *mm,
99107

100108
void book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea,
101109
pte_t pte);
110+
#ifdef CONFIG_PPC_8xx
111+
static inline void flush_hugetlb_page(struct vm_area_struct *vma,
112+
unsigned long vmaddr)
113+
{
114+
flush_tlb_page(vma, vmaddr);
115+
}
116+
#else
102117
void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
118+
#endif
103119

104120
void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
105121
unsigned long end, unsigned long floor,
@@ -205,7 +221,8 @@ static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr,
205221
* are reserved early in the boot process by memblock instead of via
206222
* the .dts as on IBM platforms.
207223
*/
208-
#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_FSL_BOOK3E)
224+
#if defined(CONFIG_HUGETLB_PAGE) && (defined(CONFIG_PPC_FSL_BOOK3E) || \
225+
defined(CONFIG_PPC_8xx))
209226
extern void __init reserve_hugetlb_gpages(void);
210227
#else
211228
static inline void reserve_hugetlb_gpages(void)

‎arch/powerpc/include/asm/mmu-8xx.h

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,41 @@ typedef struct {
172172

173173
#define PHYS_IMMR_BASE (mfspr(SPRN_IMMR) & 0xfff80000)
174174
#define VIRT_IMMR_BASE (__fix_to_virt(FIX_IMMR_BASE))
175+
176+
/* Page size definitions, common between 32 and 64-bit
177+
*
178+
* shift : is the "PAGE_SHIFT" value for that page size
179+
* penc : is the pte encoding mask
180+
*
181+
*/
182+
struct mmu_psize_def {
183+
unsigned int shift; /* number of bits */
184+
unsigned int enc; /* PTE encoding */
185+
unsigned int ind; /* Corresponding indirect page size shift */
186+
unsigned int flags;
187+
#define MMU_PAGE_SIZE_DIRECT 0x1 /* Supported as a direct size */
188+
#define MMU_PAGE_SIZE_INDIRECT 0x2 /* Supported as an indirect size */
189+
};
190+
191+
extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
192+
193+
static inline int shift_to_mmu_psize(unsigned int shift)
194+
{
195+
int psize;
196+
197+
for (psize = 0; psize < MMU_PAGE_COUNT; ++psize)
198+
if (mmu_psize_defs[psize].shift == shift)
199+
return psize;
200+
return -1;
201+
}
202+
203+
static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
204+
{
205+
if (mmu_psize_defs[mmu_psize].shift)
206+
return mmu_psize_defs[mmu_psize].shift;
207+
BUG();
208+
}
209+
175210
#endif /* !__ASSEMBLY__ */
176211

177212
#if defined(CONFIG_PPC_4K_PAGES)

‎arch/powerpc/include/asm/mmu.h

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -264,19 +264,20 @@ static inline bool early_radix_enabled(void)
264264
#define MMU_PAGE_64K 2
265265
#define MMU_PAGE_64K_AP 3 /* "Admixed pages" (hash64 only) */
266266
#define MMU_PAGE_256K 4
267-
#define MMU_PAGE_1M 5
268-
#define MMU_PAGE_2M 6
269-
#define MMU_PAGE_4M 7
270-
#define MMU_PAGE_8M 8
271-
#define MMU_PAGE_16M 9
272-
#define MMU_PAGE_64M 10
273-
#define MMU_PAGE_256M 11
274-
#define MMU_PAGE_1G 12
275-
#define MMU_PAGE_16G 13
276-
#define MMU_PAGE_64G 14
267+
#define MMU_PAGE_512K 5
268+
#define MMU_PAGE_1M 6
269+
#define MMU_PAGE_2M 7
270+
#define MMU_PAGE_4M 8
271+
#define MMU_PAGE_8M 9
272+
#define MMU_PAGE_16M 10
273+
#define MMU_PAGE_64M 11
274+
#define MMU_PAGE_256M 12
275+
#define MMU_PAGE_1G 13
276+
#define MMU_PAGE_16G 14
277+
#define MMU_PAGE_64G 15
277278

278279
/* N.B. we need to change the type of hpte_page_sizes if this gets to be > 16 */
279-
#define MMU_PAGE_COUNT 15
280+
#define MMU_PAGE_COUNT 16
280281

281282
#ifdef CONFIG_PPC_BOOK3S_64
282283
#include <asm/book3s/64/mmu.h>

‎arch/powerpc/include/asm/nohash/32/pte-8xx.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
#define _PMD_BAD 0x0ff0
5050
#define _PMD_PAGE_MASK 0x000c
5151
#define _PMD_PAGE_8M 0x000c
52+
#define _PMD_PAGE_512K 0x0004
5253

5354
/* Until my rework is finished, 8xx still needs atomic PTE updates */
5455
#define PTE_ATOMIC_UPDATES 1

‎arch/powerpc/include/asm/nohash/pgtable.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,11 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
226226
#ifdef CONFIG_HUGETLB_PAGE
227227
static inline int hugepd_ok(hugepd_t hpd)
228228
{
229+
#ifdef CONFIG_PPC_8xx
230+
return ((hpd.pd & 0x4) != 0);
231+
#else
229232
return (hpd.pd > 0);
233+
#endif
230234
}
231235

232236
static inline int pmd_huge(pmd_t pmd)

‎arch/powerpc/include/asm/reg_8xx.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#ifndef _ASM_POWERPC_REG_8xx_H
55
#define _ASM_POWERPC_REG_8xx_H
66

7-
#include <asm/mmu-8xx.h>
7+
#include <asm/mmu.h>
88

99
/* Cache control on the MPC8xx is provided through some additional
1010
* special purpose registers.

‎arch/powerpc/kernel/head_8xx.S

Lines changed: 113 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,9 @@
7373
#define RPN_PATTERN 0x00f0
7474
#endif
7575

76+
#define PAGE_SHIFT_512K 19
77+
#define PAGE_SHIFT_8M 23
78+
7679
__HEAD
7780
_ENTRY(_stext);
7881
_ENTRY(_start);
@@ -322,7 +325,7 @@ SystemCall:
322325
#endif
323326

324327
InstructionTLBMiss:
325-
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
328+
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
326329
mtspr SPRN_SPRG_SCRATCH2, r3
327330
#endif
328331
EXCEPTION_PROLOG_0
@@ -332,33 +335,45 @@ InstructionTLBMiss:
332335
*/
333336
mfspr r10, SPRN_SRR0 /* Get effective address of fault */
334337
INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
335-
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
336338
/* Only modules will cause ITLB Misses as we always
337339
* pin the first 8MB of kernel memory */
340+
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
338341
mfcr r3
342+
#endif
343+
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
339344
IS_KERNEL(r11, r10)
340345
#endif
341346
mfspr r11, SPRN_M_TW /* Get level 1 table */
342347
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
343348
BRANCH_UNLESS_KERNEL(3f)
344349
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
345350
3:
346-
mtcr r3
347351
#endif
348352
/* Insert level 1 index */
349353
rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
350354
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
351355

352356
/* Extract level 2 index */
353357
rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
358+
#ifdef CONFIG_HUGETLB_PAGE
359+
mtcr r11
360+
bt- 28, 10f /* bit 28 = Large page (8M) */
361+
bt- 29, 20f /* bit 29 = Large page (8M or 512k) */
362+
#endif
354363
rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
355364
lwz r10, 0(r10) /* Get the pte */
356-
365+
4:
366+
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
367+
mtcr r3
368+
#endif
357369
/* Insert the APG into the TWC from the Linux PTE. */
358370
rlwimi r11, r10, 0, 25, 26
359371
/* Load the MI_TWC with the attributes for this "segment." */
360372
MTSPR_CPU6(SPRN_MI_TWC, r11, r3) /* Set segment attributes */
361373

374+
#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
375+
rlwimi r10, r11, 1, MI_SPS16K
376+
#endif
362377
#ifdef CONFIG_SWAP
363378
rlwinm r11, r10, 32-5, _PAGE_PRESENT
364379
and r11, r11, r10
@@ -371,16 +386,45 @@ InstructionTLBMiss:
371386
* set. All other Linux PTE bits control the behavior
372387
* of the MMU.
373388
*/
389+
#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
390+
rlwimi r10, r11, 0, 0x0ff0 /* Set 24-27, clear 20-23 */
391+
#else
374392
rlwimi r10, r11, 0, 0x0ff8 /* Set 24-27, clear 20-23,28 */
393+
#endif
375394
MTSPR_CPU6(SPRN_MI_RPN, r10, r3) /* Update TLB entry */
376395

377396
/* Restore registers */
378-
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
397+
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
379398
mfspr r3, SPRN_SPRG_SCRATCH2
380399
#endif
381400
EXCEPTION_EPILOG_0
382401
rfi
383402

403+
#ifdef CONFIG_HUGETLB_PAGE
404+
10: /* 8M pages */
405+
#ifdef CONFIG_PPC_16K_PAGES
406+
/* Extract level 2 index */
407+
rlwinm r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
408+
/* Add level 2 base */
409+
rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
410+
#else
411+
/* Level 2 base */
412+
rlwinm r10, r11, 0, ~HUGEPD_SHIFT_MASK
413+
#endif
414+
lwz r10, 0(r10) /* Get the pte */
415+
rlwinm r11, r11, 0, 0xf
416+
b 4b
417+
418+
20: /* 512k pages */
419+
/* Extract level 2 index */
420+
rlwinm r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
421+
/* Add level 2 base */
422+
rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
423+
lwz r10, 0(r10) /* Get the pte */
424+
rlwinm r11, r11, 0, 0xf
425+
b 4b
426+
#endif
427+
384428
. = 0x1200
385429
DataStoreTLBMiss:
386430
mtspr SPRN_SPRG_SCRATCH2, r3
@@ -407,7 +451,6 @@ _ENTRY(DTLBMiss_jmp)
407451
#endif
408452
blt cr7, DTLBMissLinear
409453
3:
410-
mtcr r3
411454
mfspr r10, SPRN_MD_EPN
412455

413456
/* Insert level 1 index */
@@ -418,8 +461,15 @@ _ENTRY(DTLBMiss_jmp)
418461
*/
419462
/* Extract level 2 index */
420463
rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
464+
#ifdef CONFIG_HUGETLB_PAGE
465+
mtcr r11
466+
bt- 28, 10f /* bit 28 = Large page (8M) */
467+
bt- 29, 20f /* bit 29 = Large page (8M or 512k) */
468+
#endif
421469
rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
422470
lwz r10, 0(r10) /* Get the pte */
471+
4:
472+
mtcr r3
423473

424474
/* Insert the Guarded flag and APG into the TWC from the Linux PTE.
425475
* It is bit 26-27 of both the Linux PTE and the TWC (at least
@@ -434,6 +484,11 @@ _ENTRY(DTLBMiss_jmp)
434484
rlwimi r11, r10, 32-5, 30, 30
435485
MTSPR_CPU6(SPRN_MD_TWC, r11, r3)
436486

487+
/* In 4k pages mode, SPS (bit 28) in RPN must match PS[1] (bit 29)
488+
* In 16k pages mode, SPS is always 1 */
489+
#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
490+
rlwimi r10, r11, 1, MD_SPS16K
491+
#endif
437492
/* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
438493
* We also need to know if the insn is a load/store, so:
439494
* Clear _PAGE_PRESENT and load that which will
@@ -455,7 +510,11 @@ _ENTRY(DTLBMiss_jmp)
455510
* of the MMU.
456511
*/
457512
li r11, RPN_PATTERN
513+
#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
514+
rlwimi r10, r11, 0, 24, 27 /* Set 24-27 */
515+
#else
458516
rlwimi r10, r11, 0, 24, 28 /* Set 24-27, clear 28 */
517+
#endif
459518
rlwimi r10, r11, 0, 20, 20 /* clear 20 */
460519
MTSPR_CPU6(SPRN_MD_RPN, r10, r3) /* Update TLB entry */
461520

@@ -465,6 +524,30 @@ _ENTRY(DTLBMiss_jmp)
465524
EXCEPTION_EPILOG_0
466525
rfi
467526

527+
#ifdef CONFIG_HUGETLB_PAGE
528+
10: /* 8M pages */
529+
/* Extract level 2 index */
530+
#ifdef CONFIG_PPC_16K_PAGES
531+
rlwinm r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
532+
/* Add level 2 base */
533+
rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
534+
#else
535+
/* Level 2 base */
536+
rlwinm r10, r11, 0, ~HUGEPD_SHIFT_MASK
537+
#endif
538+
lwz r10, 0(r10) /* Get the pte */
539+
rlwinm r11, r11, 0, 0xf
540+
b 4b
541+
542+
20: /* 512k pages */
543+
/* Extract level 2 index */
544+
rlwinm r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
545+
/* Add level 2 base */
546+
rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
547+
lwz r10, 0(r10) /* Get the pte */
548+
rlwinm r11, r11, 0, 0xf
549+
b 4b
550+
#endif
468551

469552
/* This is an instruction TLB error on the MPC8xx. This could be due
470553
* to many reasons, such as executing guarded memory or illegal instruction
@@ -586,6 +669,9 @@ _ENTRY(FixupDAR_cmp)
586669
/* Insert level 1 index */
587670
3: rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
588671
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
672+
mtcr r11
673+
bt 28,200f /* bit 28 = Large page (8M) */
674+
bt 29,202f /* bit 29 = Large page (8M or 512K) */
589675
rlwinm r11, r11,0,0,19 /* Extract page descriptor page address */
590676
/* Insert level 2 index */
591677
rlwimi r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
@@ -611,6 +697,27 @@ _ENTRY(FixupDAR_cmp)
611697
141: mfspr r10,SPRN_SPRG_SCRATCH2
612698
b DARFixed /* Nope, go back to normal TLB processing */
613699

700+
/* concat physical page address(r11) and page offset(r10) */
701+
200:
702+
#ifdef CONFIG_PPC_16K_PAGES
703+
rlwinm r11, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
704+
rlwimi r11, r10, 32 - (PAGE_SHIFT_8M - 2), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
705+
#else
706+
rlwinm r11, r10, 0, ~HUGEPD_SHIFT_MASK
707+
#endif
708+
lwz r11, 0(r11) /* Get the pte */
709+
/* concat physical page address(r11) and page offset(r10) */
710+
rlwimi r11, r10, 0, 32 - PAGE_SHIFT_8M, 31
711+
b 201b
712+
713+
202:
714+
rlwinm r11, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
715+
rlwimi r11, r10, 32 - (PAGE_SHIFT_512K - 2), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
716+
lwz r11, 0(r11) /* Get the pte */
717+
/* concat physical page address(r11) and page offset(r10) */
718+
rlwimi r11, r10, 0, 32 - PAGE_SHIFT_512K, 31
719+
b 201b
720+
614721
144: mfspr r10, SPRN_DSISR
615722
rlwinm r10, r10,0,7,5 /* Clear store bit for buggy dcbst insn */
616723
mtspr SPRN_DSISR, r10

0 commit comments

Comments
 (0)