^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-or-later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2008 Mark Nelson, IBM Corp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <asm/ppc_asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <asm/asm-offsets.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/feature-fixups.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) .section ".toc","aw"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) PPC64_CACHES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) .tc ppc64_caches[TC],ppc64_caches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) .section ".text"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) _GLOBAL_TOC(copy_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) lis r5,PAGE_SIZE@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #ifdef CONFIG_PPC_BOOK3S_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) b copypage_power7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) ori r5,r5,PAGE_SIZE@l
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) ld r10,PPC64_CACHES@toc(r2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) lwz r11,DCACHEL1LOGBLOCKSIZE(r10) /* log2 of cache block size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) lwz r12,DCACHEL1BLOCKSIZE(r10) /* get cache block size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) li r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) srd r8,r5,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) mtctr r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) .Lsetup:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) dcbt r9,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) dcbz r9,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) add r9,r9,r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) bdnz .Lsetup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) END_FTR_SECTION_IFSET(CPU_FTR_CP_USE_DCBTZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) addi r3,r3,-8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) srdi r8,r5,7 /* page is copied in 128 byte strides */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) addi r8,r8,-1 /* one stride copied outside loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) mtctr r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) ld r5,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) ld r6,8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) ld r7,16(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) ldu r8,24(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) 1: std r5,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) std r6,16(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) ld r9,8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) ld r10,16(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) std r7,24(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) std r8,32(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) ld r11,24(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) ld r12,32(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) std r9,40(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) std r10,48(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) ld r5,40(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) ld r6,48(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) std r11,56(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) std r12,64(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) ld r7,56(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) ld r8,64(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) std r5,72(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) std r6,80(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) ld r9,72(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) ld r10,80(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) std r7,88(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) std r8,96(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) ld r11,88(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) ld r12,96(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) std r9,104(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) std r10,112(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) ld r5,104(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) ld r6,112(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) std r11,120(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) stdu r12,128(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) ld r7,120(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) ldu r8,128(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) bdnz 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) std r5,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) std r6,16(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) ld r9,8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) ld r10,16(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) std r7,24(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) std r8,32(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) ld r11,24(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) ld r12,32(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) std r9,40(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) std r10,48(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) ld r5,40(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) ld r6,48(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) std r11,56(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) std r12,64(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) ld r7,56(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) ld r8,64(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) std r5,72(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) std r6,80(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) ld r9,72(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) ld r10,80(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) std r7,88(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) std r8,96(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) ld r11,88(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) ld r12,96(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) std r9,104(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) std r10,112(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) std r11,120(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) std r12,128(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) EXPORT_SYMBOL(copy_page)