1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2003, 2004 Ralf Baechle (ralf@linux-mips.org) 7 */ 8#include <linux/config.h> 9#include <linux/init.h> 10#include <linux/kernel.h> 11#include <linux/sched.h> 12#include <linux/mm.h> 13#include <linux/module.h> 14#include <linux/proc_fs.h> 15 16#include <asm/cacheops.h> 17#include <asm/inst.h> 18#include <asm/io.h> 19#include <asm/page.h> 20#include <asm/pgtable.h> 21#include <asm/prefetch.h> 22#include <asm/system.h> 23#include <asm/bootinfo.h> 24#include <asm/mipsregs.h> 25#include <asm/mmu_context.h> 26#include <asm/cpu.h> 27#include <asm/war.h> 28 29#define half_scache_line_size() (cpu_scache_line_size() >> 1) 30 31/* 32 * Maximum sizes: 33 * 34 * R4000 128 bytes S-cache: 0x58 bytes 35 * R4600 v1.7: 0x5c bytes 36 * R4600 v2.0: 0x60 bytes 37 * With prefetching, 16 byte strides 0xa0 bytes 38 */ 39 40static unsigned int clear_page_array[0x130 / 4]; 41 42void clear_page(void * page) __attribute__((alias("clear_page_array"))); 43 44/* 45 * Maximum sizes: 46 * 47 * R4000 128 bytes S-cache: 0x11c bytes 48 * R4600 v1.7: 0x080 bytes 49 * R4600 v2.0: 0x07c bytes 50 * With prefetching, 16 byte strides 0x0b8 bytes 51 */ 52static unsigned int copy_page_array[0x148 / 4]; 53 54void copy_page(void *to, void *from) __attribute__((alias("copy_page_array"))); 55 56/* 57 * An address fits into a single register so it's safe to use 64-bit registers 58 * if we have 64-bit adresses. 59 */ 60#define cpu_has_64bit_registers cpu_has_64bit_addresses 61 62/* 63 * This is suboptimal for 32-bit kernels; we assume that R10000 is only used 64 * with 64-bit kernels. The prefetch offsets have been experimentally tuned 65 * an Origin 200. 66 */ 67static int pref_offset_clear __initdata = 512; 68static int pref_offset_copy __initdata = 256; 69 70static unsigned int pref_src_mode __initdata; 71static unsigned int pref_dst_mode __initdata; 72 73static int load_offset __initdata; 74static int store_offset __initdata; 75 76static unsigned int __initdata *dest, *epc; 77 78static unsigned int instruction_pending; 79static union mips_instruction delayed_mi; 80 81static void __init emit_instruction(union mips_instruction mi) 82{ 83 if (instruction_pending) 84 *epc++ = delayed_mi.word; 85 86 instruction_pending = 1; 87 delayed_mi = mi; 88} 89 90static inline void flush_delay_slot_or_nop(void) 91{ 92 if (instruction_pending) { 93 *epc++ = delayed_mi.word; 94 instruction_pending = 0; 95 return; 96 } 97 98 *epc++ = 0; 99} 100 101static inline unsigned int *label(void) 102{ 103 if (instruction_pending) { 104 *epc++ = delayed_mi.word; 105 instruction_pending = 0; 106 } 107 108 return epc; 109} 110 111static inline void build_insn_word(unsigned int word) 112{ 113 union mips_instruction mi; 114 115 mi.word = word; 116 117 emit_instruction(mi); 118} 119 120static inline void build_nop(void) 121{ 122 build_insn_word(0); /* nop */ 123} 124 125static inline void build_src_pref(int advance) 126{ 127 if (!(load_offset & (cpu_dcache_line_size() - 1))) { 128 union mips_instruction mi; 129 130 mi.i_format.opcode = pref_op; 131 mi.i_format.rs = 5; /* $a1 */ 132 mi.i_format.rt = pref_src_mode; 133 mi.i_format.simmediate = load_offset + advance; 134 135 emit_instruction(mi); 136 } 137} 138 139static inline void __build_load_reg(int reg) 140{ 141 union mips_instruction mi; 142 unsigned int width; 143 144 if (cpu_has_64bit_registers) { 145 mi.i_format.opcode = ld_op; 146 width = 8; 147 } else { 148 mi.i_format.opcode = lw_op; 149 width = 4; 150 } 151 mi.i_format.rs = 5; /* $a1 */ 152 mi.i_format.rt = reg; /* $reg */ 153 mi.i_format.simmediate = load_offset; 154 155 load_offset += width; 156 emit_instruction(mi); 157} 158 159static inline void build_load_reg(int reg) 160{ 161 if (cpu_has_prefetch) 162 build_src_pref(pref_offset_copy); 163 164 __build_load_reg(reg); 165} 166 167static inline void build_dst_pref(int advance) 168{ 169 if (!(store_offset & (cpu_dcache_line_size() - 1))) { 170 union mips_instruction mi; 171 172 mi.i_format.opcode = pref_op; 173 mi.i_format.rs = 4; /* $a0 */ 174 mi.i_format.rt = pref_dst_mode; 175 mi.i_format.simmediate = store_offset + advance; 176 177 emit_instruction(mi); 178 } 179} 180 181static inline void build_cdex_s(void) 182{ 183 union mips_instruction mi; 184 185 if ((store_offset & (cpu_scache_line_size() - 1))) 186 return; 187 188 mi.c_format.opcode = cache_op; 189 mi.c_format.rs = 4; /* $a0 */ 190 mi.c_format.c_op = 3; /* Create Dirty Exclusive */ 191 mi.c_format.cache = 3; /* Secondary Data Cache */ 192 mi.c_format.simmediate = store_offset; 193 194 emit_instruction(mi); 195} 196 197static inline void build_cdex_p(void) 198{ 199 union mips_instruction mi; 200 201 if (store_offset & (cpu_dcache_line_size() - 1)) 202 return; 203 204 if (R4600_V1_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2010)) { 205 build_nop(); 206 build_nop(); 207 build_nop(); 208 build_nop(); 209 } 210 211 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020)) 212 build_insn_word(0x8c200000); /* lw $zero, ($at) */ 213 214 mi.c_format.opcode = cache_op; 215 mi.c_format.rs = 4; /* $a0 */ 216 mi.c_format.c_op = 3; /* Create Dirty Exclusive */ 217 mi.c_format.cache = 1; /* Data Cache */ 218 mi.c_format.simmediate = store_offset; 219 220 emit_instruction(mi); 221} 222 223static void __build_store_reg(int reg) 224{ 225 union mips_instruction mi; 226 unsigned int width; 227 228 if (cpu_has_64bit_gp_regs || 229 (cpu_has_64bit_zero_reg && reg == 0)) { 230 mi.i_format.opcode = sd_op; 231 width = 8; 232 } else { 233 mi.i_format.opcode = sw_op; 234 width = 4; 235 } 236 mi.i_format.rs = 4; /* $a0 */ 237 mi.i_format.rt = reg; /* $reg */ 238 mi.i_format.simmediate = store_offset; 239 240 store_offset += width; 241 emit_instruction(mi); 242} 243 244static inline void build_store_reg(int reg) 245{ 246 if (cpu_has_prefetch) 247 if (reg) 248 build_dst_pref(pref_offset_copy); 249 else 250 build_dst_pref(pref_offset_clear); 251 else if (cpu_has_cache_cdex_s) 252 build_cdex_s(); 253 else if (cpu_has_cache_cdex_p) 254 build_cdex_p(); 255 256 __build_store_reg(reg); 257} 258 259static inline void build_addiu_a2_a0(unsigned long offset) 260{ 261 union mips_instruction mi; 262 263 BUG_ON(offset > 0x7fff); 264 265 mi.i_format.opcode = cpu_has_64bit_addresses ? daddiu_op : addiu_op; 266 mi.i_format.rs = 4; /* $a0 */ 267 mi.i_format.rt = 6; /* $a2 */ 268 mi.i_format.simmediate = offset; 269 270 emit_instruction(mi); 271} 272 273static inline void build_addiu_a1(unsigned long offset) 274{ 275 union mips_instruction mi; 276 277 BUG_ON(offset > 0x7fff); 278 279 mi.i_format.opcode = cpu_has_64bit_addresses ? daddiu_op : addiu_op; 280 mi.i_format.rs = 5; /* $a1 */ 281 mi.i_format.rt = 5; /* $a1 */ 282 mi.i_format.simmediate = offset; 283 284 load_offset -= offset; 285 286 emit_instruction(mi); 287} 288 289static inline void build_addiu_a0(unsigned long offset) 290{ 291 union mips_instruction mi; 292 293 BUG_ON(offset > 0x7fff); 294 295 mi.i_format.opcode = cpu_has_64bit_addresses ? daddiu_op : addiu_op; 296 mi.i_format.rs = 4; /* $a0 */ 297 mi.i_format.rt = 4; /* $a0 */ 298 mi.i_format.simmediate = offset; 299 300 store_offset -= offset; 301 302 emit_instruction(mi); 303} 304 305static inline void build_bne(unsigned int *dest) 306{ 307 union mips_instruction mi; 308 309 mi.i_format.opcode = bne_op; 310 mi.i_format.rs = 6; /* $a2 */ 311 mi.i_format.rt = 4; /* $a0 */ 312 mi.i_format.simmediate = dest - epc - 1; 313 314 *epc++ = mi.word; 315 flush_delay_slot_or_nop(); 316} 317 318static inline void build_jr_ra(void) 319{ 320 union mips_instruction mi; 321 322 mi.r_format.opcode = spec_op; 323 mi.r_format.rs = 31; 324 mi.r_format.rt = 0; 325 mi.r_format.rd = 0; 326 mi.r_format.re = 0; 327 mi.r_format.func = jr_op; 328 329 *epc++ = mi.word; 330 flush_delay_slot_or_nop(); 331} 332 333void __init build_clear_page(void) 334{ 335 unsigned int loop_start; 336 337 epc = (unsigned int *) &clear_page_array; 338 instruction_pending = 0; 339 store_offset = 0; 340 341 if (cpu_has_prefetch) { 342 switch (current_cpu_data.cputype) { 343 case CPU_RM9000: 344 /* 345 * As a workaround for erratum G105 which make the 346 * PrepareForStore hint unusable we fall back to 347 * StoreRetained on the RM9000. Once it is known which 348 * versions of the RM9000 we'll be able to condition- 349 * alize this. 350 */ 351 352 case CPU_R10000: 353 case CPU_R12000: 354 pref_src_mode = Pref_LoadStreamed; 355 pref_dst_mode = Pref_StoreRetained; 356 break; 357 358 default: 359 pref_src_mode = Pref_LoadStreamed; 360 pref_dst_mode = Pref_PrepareForStore; 361 break; 362 } 363 } 364 365 build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0)); 366 367 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020)) 368 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */ 369 370dest = label(); 371 do { 372 build_store_reg(0); 373 build_store_reg(0); 374 build_store_reg(0); 375 build_store_reg(0); 376 } while (store_offset < half_scache_line_size()); 377 build_addiu_a0(2 * store_offset); 378 loop_start = store_offset; 379 do { 380 build_store_reg(0); 381 build_store_reg(0); 382 build_store_reg(0); 383 build_store_reg(0); 384 } while ((store_offset - loop_start) < half_scache_line_size()); 385 build_bne(dest); 386 387 if (cpu_has_prefetch && pref_offset_clear) { 388 build_addiu_a2_a0(pref_offset_clear); 389 dest = label(); 390 loop_start = store_offset; 391 do { 392 __build_store_reg(0); 393 __build_store_reg(0); 394 __build_store_reg(0); 395 __build_store_reg(0); 396 } while ((store_offset - loop_start) < half_scache_line_size()); 397 build_addiu_a0(2 * store_offset); 398 loop_start = store_offset; 399 do { 400 __build_store_reg(0); 401 __build_store_reg(0); 402 __build_store_reg(0); 403 __build_store_reg(0); 404 } while ((store_offset - loop_start) < half_scache_line_size()); 405 build_bne(dest); 406 } 407 408 build_jr_ra(); 409 410 flush_icache_range((unsigned long)&clear_page_array, 411 (unsigned long) epc); 412 413 BUG_ON(epc > clear_page_array + ARRAY_SIZE(clear_page_array)); 414} 415 416void __init build_copy_page(void) 417{ 418 unsigned int loop_start; 419 420 epc = (unsigned int *) ©_page_array; 421 store_offset = load_offset = 0; 422 instruction_pending = 0; 423 424 build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0)); 425 426 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020)) 427 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */ 428 429dest = label(); 430 loop_start = store_offset; 431 do { 432 build_load_reg( 8); 433 build_load_reg( 9); 434 build_load_reg(10); 435 build_load_reg(11); 436 build_store_reg( 8); 437 build_store_reg( 9); 438 build_store_reg(10); 439 build_store_reg(11); 440 } while ((store_offset - loop_start) < half_scache_line_size()); 441 build_addiu_a0(2 * store_offset); 442 build_addiu_a1(2 * load_offset); 443 loop_start = store_offset; 444 do { 445 build_load_reg( 8); 446 build_load_reg( 9); 447 build_load_reg(10); 448 build_load_reg(11); 449 build_store_reg( 8); 450 build_store_reg( 9); 451 build_store_reg(10); 452 build_store_reg(11); 453 } while ((store_offset - loop_start) < half_scache_line_size()); 454 build_bne(dest); 455 456 if (cpu_has_prefetch && pref_offset_copy) { 457 build_addiu_a2_a0(pref_offset_copy); 458 dest = label(); 459 loop_start = store_offset; 460 do { 461 __build_load_reg( 8); 462 __build_load_reg( 9); 463 __build_load_reg(10); 464 __build_load_reg(11); 465 __build_store_reg( 8); 466 __build_store_reg( 9); 467 __build_store_reg(10); 468 __build_store_reg(11); 469 } while ((store_offset - loop_start) < half_scache_line_size()); 470 build_addiu_a0(2 * store_offset); 471 build_addiu_a1(2 * load_offset); 472 loop_start = store_offset; 473 do { 474 __build_load_reg( 8); 475 __build_load_reg( 9); 476 __build_load_reg(10); 477 __build_load_reg(11); 478 __build_store_reg( 8); 479 __build_store_reg( 9); 480 __build_store_reg(10); 481 __build_store_reg(11); 482 } while ((store_offset - loop_start) < half_scache_line_size()); 483 build_bne(dest); 484 } 485 486 build_jr_ra(); 487 488 flush_icache_range((unsigned long)©_page_array, 489 (unsigned long) epc); 490 491 BUG_ON(epc > copy_page_array + ARRAY_SIZE(copy_page_array)); 492} 493