^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * This file is subject to the terms and conditions of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * License. See the file "COPYING" in the main directory of this archive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2007 Maciej W. Rozycki
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2008 Thiemo Seufer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (C) 2012 MIPS Technologies, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/proc_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/bugs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/cacheops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/cpu-type.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/inst.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/prefetch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/bootinfo.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/mipsregs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <asm/war.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #ifdef CONFIG_SIBYTE_DMA_PAGEOPS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <asm/sibyte/sb1250.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/sibyte/sb1250_regs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/sibyte/sb1250_dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/uasm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /* Registers used in the assembled routines. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define ZERO 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define AT 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define A0 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define A1 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define A2 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define T0 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define T1 9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define T2 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define T3 11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define T9 25
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define RA 31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /* Handle labels (which must be positive integers). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) enum label_id {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) label_clear_nopref = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) label_clear_pref,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) label_copy_nopref,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) label_copy_pref_both,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) label_copy_pref_store,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) UASM_L_LA(_clear_nopref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) UASM_L_LA(_clear_pref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) UASM_L_LA(_copy_nopref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) UASM_L_LA(_copy_pref_both)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) UASM_L_LA(_copy_pref_store)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* We need one branch and therefore one relocation per target label. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static struct uasm_label labels[5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static struct uasm_reloc relocs[5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * R6 has a limited offset of the pref instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * Skip it if the offset is more than 9 bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define _uasm_i_pref(a, b, c, d) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (cpu_has_mips_r6) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (c <= 0xff && c >= -0x100) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) uasm_i_pref(a, b, c, d);\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) } else { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) uasm_i_pref(a, b, c, d); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) } while(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static int pref_bias_clear_store;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static int pref_bias_copy_load;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static int pref_bias_copy_store;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static u32 pref_src_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static u32 pref_dst_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static int clear_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static int copy_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) static int half_clear_loop_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static int half_copy_loop_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static int cache_line_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define cache_line_mask() (cache_line_size - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (off > 0x7fff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) uasm_i_lui(buf, T9, uasm_rel_hi(off));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) uasm_i_addiu(buf, T9, ZERO, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) uasm_i_daddu(buf, reg1, reg2, T9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (off > 0x7fff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) uasm_i_lui(buf, T9, uasm_rel_hi(off));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) UASM_i_ADDU(buf, reg1, reg2, T9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) UASM_i_ADDIU(buf, reg1, reg2, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) static void set_prefetch_parameters(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) clear_word_size = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) clear_word_size = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (cpu_has_64bit_gp_regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) copy_word_size = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) copy_word_size = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * The pref's used here are using "streaming" hints, which cause the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * copied data to be kicked out of the cache sooner. A page copy often
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * ends up copying a lot more data than is commonly used, so this seems
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * to make sense in terms of reducing cache pollution, but I've no real
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * performance data to back this up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (cpu_has_prefetch) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * XXX: Most prefetch bias values in here are based on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * guesswork.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) cache_line_size = cpu_dcache_line_size();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) switch (current_cpu_type()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) case CPU_R5500:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) case CPU_TX49XX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /* These processors only support the Pref_Load. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) pref_bias_copy_load = 256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) case CPU_R10000:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) case CPU_R12000:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) case CPU_R14000:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) case CPU_R16000:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * Those values have been experimentally tuned for an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * Origin 200.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) pref_bias_clear_store = 512;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) pref_bias_copy_load = 256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) pref_bias_copy_store = 256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) pref_src_mode = Pref_LoadStreamed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) pref_dst_mode = Pref_StoreStreamed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) case CPU_SB1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) case CPU_SB1A:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) pref_bias_clear_store = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) pref_bias_copy_load = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) pref_bias_copy_store = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * SB1 pass1 Pref_LoadStreamed/Pref_StoreStreamed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * hints are broken.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (current_cpu_type() == CPU_SB1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) (current_cpu_data.processor_id & 0xff) < 0x02) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) pref_src_mode = Pref_Load;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) pref_dst_mode = Pref_Store;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) pref_src_mode = Pref_LoadStreamed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) pref_dst_mode = Pref_StoreStreamed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) case CPU_LOONGSON64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /* Loongson-3 only support the Pref_Load/Pref_Store. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) pref_bias_clear_store = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) pref_bias_copy_load = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) pref_bias_copy_store = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) pref_src_mode = Pref_Load;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) pref_dst_mode = Pref_Store;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) pref_bias_clear_store = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) pref_bias_copy_load = 256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) pref_bias_copy_store = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) pref_src_mode = Pref_LoadStreamed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (cpu_has_mips_r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * Bit 30 (Pref_PrepareForStore) has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * removed from MIPS R6. Use bit 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * (Pref_StoreStreamed).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) pref_dst_mode = Pref_StoreStreamed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) pref_dst_mode = Pref_PrepareForStore;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (cpu_has_cache_cdex_s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) cache_line_size = cpu_scache_line_size();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) else if (cpu_has_cache_cdex_p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) cache_line_size = cpu_dcache_line_size();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * Too much unrolling will overflow the available space in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * clear_space_array / copy_page_array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) half_clear_loop_size = min(16 * clear_word_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) max(cache_line_size >> 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 4 * clear_word_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) half_copy_loop_size = min(16 * copy_word_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) max(cache_line_size >> 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 4 * copy_word_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static void build_clear_store(u32 **buf, int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) uasm_i_sd(buf, ZERO, off, A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) uasm_i_sw(buf, ZERO, off, A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static inline void build_clear_pref(u32 **buf, int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (off & cache_line_mask())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (pref_bias_clear_store) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) _uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) } else if (cache_line_size == (half_clear_loop_size << 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (cpu_has_cache_cdex_s) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) } else if (cpu_has_cache_cdex_p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) cpu_is_r4600_v1_x()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) uasm_i_nop(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) uasm_i_nop(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) uasm_i_nop(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) uasm_i_nop(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) cpu_is_r4600_v2_x())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) uasm_i_lw(buf, ZERO, ZERO, AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) extern u32 __clear_page_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) extern u32 __clear_page_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) extern u32 __copy_page_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) extern u32 __copy_page_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) void build_clear_page(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) int off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) u32 *buf = &__clear_page_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct uasm_label *l = labels;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct uasm_reloc *r = relocs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static atomic_t run_once = ATOMIC_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (atomic_xchg(&run_once, 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) memset(labels, 0, sizeof(labels));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) memset(relocs, 0, sizeof(relocs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) set_prefetch_parameters();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * This algorithm makes the following assumptions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * - The prefetch bias is a multiple of 2 words.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * - The prefetch bias is less than one page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) BUG_ON(pref_bias_clear_store % (2 * clear_word_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) BUG_ON(PAGE_SIZE < pref_bias_clear_store);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) off = PAGE_SIZE - pref_bias_clear_store;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (off > 0xffff || !pref_bias_clear_store)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) pg_addiu(&buf, A2, A0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) uasm_i_ori(&buf, A2, A0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) && cpu_is_r4600_v2_x())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * cache_line_size : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) while (off) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) build_clear_pref(&buf, -off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) off -= cache_line_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) uasm_l_clear_pref(&l, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) build_clear_pref(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) build_clear_store(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) off += clear_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) } while (off < half_clear_loop_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) pg_addiu(&buf, A0, A0, 2 * off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) off = -off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) build_clear_pref(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (off == -clear_word_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) uasm_il_bne(&buf, &r, A0, A2, label_clear_pref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) build_clear_store(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) off += clear_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) } while (off < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (pref_bias_clear_store) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) pg_addiu(&buf, A2, A0, pref_bias_clear_store);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) uasm_l_clear_nopref(&l, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) build_clear_store(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) off += clear_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) } while (off < half_clear_loop_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) pg_addiu(&buf, A0, A0, 2 * off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) off = -off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (off == -clear_word_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) uasm_il_bne(&buf, &r, A0, A2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) label_clear_nopref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) build_clear_store(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) off += clear_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) } while (off < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) uasm_i_jr(&buf, RA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) uasm_i_nop(&buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) BUG_ON(buf > &__clear_page_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) uasm_resolve_relocs(relocs, labels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) pr_debug("Synthesized clear page handler (%u instructions).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) (u32)(buf - &__clear_page_start));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) pr_debug("\t.set push\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) pr_debug("\t.set noreorder\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) for (i = 0; i < (buf - &__clear_page_start); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) pr_debug("\t.word 0x%08x\n", (&__clear_page_start)[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) pr_debug("\t.set pop\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static void build_copy_load(u32 **buf, int reg, int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (cpu_has_64bit_gp_regs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) uasm_i_ld(buf, reg, off, A1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) uasm_i_lw(buf, reg, off, A1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) static void build_copy_store(u32 **buf, int reg, int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (cpu_has_64bit_gp_regs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) uasm_i_sd(buf, reg, off, A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) uasm_i_sw(buf, reg, off, A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) static inline void build_copy_load_pref(u32 **buf, int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (off & cache_line_mask())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (pref_bias_copy_load)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) _uasm_i_pref(buf, pref_src_mode, pref_bias_copy_load + off, A1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) static inline void build_copy_store_pref(u32 **buf, int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (off & cache_line_mask())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (pref_bias_copy_store) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) _uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) } else if (cache_line_size == (half_copy_loop_size << 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (cpu_has_cache_cdex_s) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) } else if (cpu_has_cache_cdex_p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) cpu_is_r4600_v1_x()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) uasm_i_nop(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) uasm_i_nop(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) uasm_i_nop(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) uasm_i_nop(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) cpu_is_r4600_v2_x())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) uasm_i_lw(buf, ZERO, ZERO, AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) void build_copy_page(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) int off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) u32 *buf = &__copy_page_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct uasm_label *l = labels;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) struct uasm_reloc *r = relocs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) static atomic_t run_once = ATOMIC_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) if (atomic_xchg(&run_once, 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) memset(labels, 0, sizeof(labels));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) memset(relocs, 0, sizeof(relocs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) set_prefetch_parameters();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * This algorithm makes the following assumptions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * - All prefetch biases are multiples of 8 words.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * - The prefetch biases are less than one page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * - The store prefetch bias isn't greater than the load
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * prefetch bias.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) BUG_ON(pref_bias_copy_load % (8 * copy_word_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) BUG_ON(pref_bias_copy_store % (8 * copy_word_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) BUG_ON(PAGE_SIZE < pref_bias_copy_load);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) BUG_ON(pref_bias_copy_store > pref_bias_copy_load);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) off = PAGE_SIZE - pref_bias_copy_load;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) if (off > 0xffff || !pref_bias_copy_load)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) pg_addiu(&buf, A2, A0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) uasm_i_ori(&buf, A2, A0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) && cpu_is_r4600_v2_x())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) cache_line_size : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) while (off) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) build_copy_load_pref(&buf, -off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) off -= cache_line_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) off = cache_line_size ? min(8, pref_bias_copy_store / cache_line_size) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) cache_line_size : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) while (off) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) build_copy_store_pref(&buf, -off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) off -= cache_line_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) uasm_l_copy_pref_both(&l, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) build_copy_load_pref(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) build_copy_load(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) build_copy_load_pref(&buf, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) build_copy_load(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) build_copy_load_pref(&buf, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) build_copy_load(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) build_copy_load_pref(&buf, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) build_copy_load(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) build_copy_store_pref(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) build_copy_store(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) build_copy_store_pref(&buf, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) build_copy_store(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) build_copy_store_pref(&buf, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) build_copy_store(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) build_copy_store_pref(&buf, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) build_copy_store(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) off += 4 * copy_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) } while (off < half_copy_loop_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) pg_addiu(&buf, A1, A1, 2 * off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) pg_addiu(&buf, A0, A0, 2 * off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) off = -off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) build_copy_load_pref(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) build_copy_load(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) build_copy_load_pref(&buf, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) build_copy_load(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) build_copy_load_pref(&buf, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) build_copy_load(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) build_copy_load_pref(&buf, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) build_copy_load(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) build_copy_store_pref(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) build_copy_store(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) build_copy_store_pref(&buf, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) build_copy_store(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) build_copy_store_pref(&buf, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) build_copy_store(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) build_copy_store_pref(&buf, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (off == -(4 * copy_word_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_both);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) build_copy_store(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) off += 4 * copy_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) } while (off < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) if (pref_bias_copy_load - pref_bias_copy_store) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) pg_addiu(&buf, A2, A0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) pref_bias_copy_load - pref_bias_copy_store);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) uasm_l_copy_pref_store(&l, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) build_copy_load(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) build_copy_load(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) build_copy_load(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) build_copy_load(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) build_copy_store_pref(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) build_copy_store(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) build_copy_store_pref(&buf, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) build_copy_store(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) build_copy_store_pref(&buf, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) build_copy_store(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) build_copy_store_pref(&buf, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) build_copy_store(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) off += 4 * copy_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) } while (off < half_copy_loop_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) pg_addiu(&buf, A1, A1, 2 * off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) pg_addiu(&buf, A0, A0, 2 * off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) off = -off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) build_copy_load(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) build_copy_load(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) build_copy_load(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) build_copy_load(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) build_copy_store_pref(&buf, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) build_copy_store(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) build_copy_store_pref(&buf, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) build_copy_store(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) build_copy_store_pref(&buf, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) build_copy_store(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) build_copy_store_pref(&buf, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (off == -(4 * copy_word_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) uasm_il_bne(&buf, &r, A2, A0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) label_copy_pref_store);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) build_copy_store(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) off += 4 * copy_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) } while (off < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) if (pref_bias_copy_store) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) pg_addiu(&buf, A2, A0, pref_bias_copy_store);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) uasm_l_copy_nopref(&l, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) build_copy_load(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) build_copy_load(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) build_copy_load(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) build_copy_load(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) build_copy_store(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) build_copy_store(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) build_copy_store(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) build_copy_store(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) off += 4 * copy_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) } while (off < half_copy_loop_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) pg_addiu(&buf, A1, A1, 2 * off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) pg_addiu(&buf, A0, A0, 2 * off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) off = -off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) build_copy_load(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) build_copy_load(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) build_copy_load(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) build_copy_load(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) build_copy_store(&buf, T0, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) build_copy_store(&buf, T1, off + copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) build_copy_store(&buf, T2, off + 2 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (off == -(4 * copy_word_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) uasm_il_bne(&buf, &r, A2, A0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) label_copy_nopref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) build_copy_store(&buf, T3, off + 3 * copy_word_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) off += 4 * copy_word_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) } while (off < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) uasm_i_jr(&buf, RA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) uasm_i_nop(&buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) BUG_ON(buf > &__copy_page_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) uasm_resolve_relocs(relocs, labels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) pr_debug("Synthesized copy page handler (%u instructions).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) (u32)(buf - &__copy_page_start));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) pr_debug("\t.set push\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) pr_debug("\t.set noreorder\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) for (i = 0; i < (buf - &__copy_page_start); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) pr_debug("\t.word 0x%08x\n", (&__copy_page_start)[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) pr_debug("\t.set pop\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) #ifdef CONFIG_SIBYTE_DMA_PAGEOPS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) extern void clear_page_cpu(void *page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) extern void copy_page_cpu(void *to, void *from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * Pad descriptors to cacheline, since each is exclusively owned by a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * particular CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct dmadscr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) u64 dscr_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) u64 dscr_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) u64 pad_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) u64 pad_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) } ____cacheline_aligned_in_smp page_descr[DM_NUM_CHANNELS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) void clear_page(void *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) u64 to_phys = CPHYSADDR((unsigned long)page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) unsigned int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) /* if the page is not in KSEG0, use old way */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) if ((long)KSEGX((unsigned long)page) != (long)CKSEG0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) return clear_page_cpu(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_ZERO_MEM |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) page_descr[cpu].dscr_b = V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * Don't really want to do it this way, but there's no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * reliable way to delay completion detection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) & M_DM_DSCR_BASE_INTERRUPT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) EXPORT_SYMBOL(clear_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) void copy_page(void *to, void *from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) u64 from_phys = CPHYSADDR((unsigned long)from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) u64 to_phys = CPHYSADDR((unsigned long)to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) unsigned int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) /* if any page is not in KSEG0, use old way */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) if ((long)KSEGX((unsigned long)to) != (long)CKSEG0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) || (long)KSEGX((unsigned long)from) != (long)CKSEG0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return copy_page_cpu(to, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_L2C_DEST |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) M_DM_DSCRA_INTERRUPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) page_descr[cpu].dscr_b = from_phys | V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * Don't really want to do it this way, but there's no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * reliable way to delay completion detection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) & M_DM_DSCR_BASE_INTERRUPT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) EXPORT_SYMBOL(copy_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) #endif /* CONFIG_SIBYTE_DMA_PAGEOPS */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags