Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239/* SPDX-License-Identifier: GPL-2.0-only *//* * Copyright (C) 1994-2002 Russell King * Copyright (c) 2003, 2020 ARM Limited * All Rights Reserved */
#include <linux/init.h>#include <linux/linkage.h>#include <asm/assembler.h>#include <asm/page.h>
#ifdef __ARMEB__#define LOW_OFFSET 0x4#define HIGH_OFFSET 0x0#else#define LOW_OFFSET 0x0#define HIGH_OFFSET 0x4#endif
/* * __fixup_pv_table - patch the stub instructions with the delta between * PHYS_OFFSET and PAGE_OFFSET, which is assumed to be * 2 MiB aligned. * * Called from head.S, which expects the following registers to be preserved: * r1 = machine no, r2 = atags or dtb, * r8 = phys_offset, r9 = cpuid, r10 = procinfo */ __HEADENTRY(__fixup_pv_table) mov r0, r8, lsr #PAGE_SHIFT @ convert to PFN str_l r0, __pv_phys_pfn_offset, r3
adr_l r0, __pv_offset subs r3, r8, #PAGE_OFFSET @ PHYS_OFFSET - PAGE_OFFSET mvn ip, #0 strcc ip, [r0, #HIGH_OFFSET] @ save to __pv_offset high bits str r3, [r0, #LOW_OFFSET] @ save to __pv_offset low bits
mov r0, r3, lsr #21 @ constant for add/sub instructions teq r3, r0, lsl #21 @ must be 2 MiB aligned bne 0f
adr_l r4, __pv_table_begin adr_l r5, __pv_table_end b __fixup_a_pv_table
0: mov r0, r0 @ deadloop on error b 0bENDPROC(__fixup_pv_table)
.text__fixup_a_pv_table: adr_l r6, __pv_offset ldr r0, [r6, #HIGH_OFFSET] @ pv_offset high word ldr r6, [r6, #LOW_OFFSET] @ pv_offset low word cmn r0, #1#ifdef CONFIG_THUMB2_KERNEL @ @ The Thumb-2 versions of the patchable sequences are @ @ phys-to-virt: movw <reg>, #offset<31:21> @ lsl <reg>, #21 @ sub <VA>, <PA>, <reg> @ @ virt-to-phys (non-LPAE): movw <reg>, #offset<31:21> @ lsl <reg>, #21 @ add <PA>, <VA>, <reg> @ @ virt-to-phys (LPAE): movw <reg>, #offset<31:21> @ lsl <reg>, #21 @ adds <PAlo>, <VA>, <reg> @ mov <PAhi>, #offset<39:32> @ adc <PAhi>, <PAhi>, #0 @ @ In the non-LPAE case, all patchable instructions are MOVW @ instructions, where we need to patch in the offset into the @ second halfword of the opcode (the 16-bit immediate is encoded @ as imm4:i:imm3:imm8) @ @ 15 11 10 9 4 3 0 15 14 12 11 8 7 0 @ +-----------+---+-------------+------++---+------+----+------+ @ MOVW | 1 1 1 1 0 | i | 1 0 0 1 0 0 | imm4 || 0 | imm3 | Rd | imm8 | @ +-----------+---+-------------+------++---+------+----+------+ @ @ In the LPAE case, we also need to patch in the high word of the @ offset into the immediate field of the MOV instruction, or patch it @ to a MVN instruction if the offset is negative. In this case, we @ need to inspect the first halfword of the opcode, to check whether @ it is MOVW or MOV/MVN, and to perform the MOV to MVN patching if @ needed. The encoding of the immediate is rather complex for values @ of i:imm3 != 0b0000, but fortunately, we never need more than 8 lower @ order bits, which can be patched into imm8 directly (and i:imm3 @ cleared) @ @ 15 11 10 9 5 0 15 14 12 11 8 7 0 @ +-----------+---+---------------------++---+------+----+------+ @ MOV | 1 1 1 1 0 | i | 0 0 0 1 0 0 1 1 1 1 || 0 | imm3 | Rd | imm8 | @ MVN | 1 1 1 1 0 | i | 0 0 0 1 1 0 1 1 1 1 || 0 | imm3 | Rd | imm8 | @ +-----------+---+---------------------++---+------+----+------+ @ moveq r0, #0x200000 @ set bit 21, mov to mvn instruction lsrs r3, r6, #29 @ isolate top 3 bits of displacement ubfx r6, r6, #21, #8 @ put bits 28:21 into the MOVW imm8 field bfi r6, r3, #12, #3 @ put bits 31:29 into the MOVW imm3 field b .Lnext.Lloop: add r7, r4 adds r4, #4 @ clears Z flag#ifdef CONFIG_ARM_LPAE ldrh ip, [r7]ARM_BE8(rev16 ip, ip) tst ip, #0x200 @ MOVW has bit 9 set, MVN has it clear bne 0f @ skip to MOVW handling (Z flag is clear) bic ip, #0x20 @ clear bit 5 (MVN -> MOV) orr ip, ip, r0, lsr #16 @ MOV -> MVN if offset < 0ARM_BE8(rev16 ip, ip) strh ip, [r7] @ Z flag is set0:#endif ldrh ip, [r7, #2]ARM_BE8(rev16 ip, ip) and ip, #0xf00 @ clear everything except Rd field orreq ip, r0 @ Z flag set -> MOV/MVN -> patch in high bits orrne ip, r6 @ Z flag clear -> MOVW -> patch in low bitsARM_BE8(rev16 ip, ip) strh ip, [r7, #2]#else#ifdef CONFIG_CPU_ENDIAN_BE8@ in BE8, we load data in BE, but instructions still in LE#define PV_BIT24 0x00000001#define PV_IMM8_MASK 0xff000000#define PV_IMMR_MSB 0x00080000#else#define PV_BIT24 0x01000000#define PV_IMM8_MASK 0x000000ff#define PV_IMMR_MSB 0x00000800#endif
@ @ The ARM versions of the patchable sequences are @ @ phys-to-virt: sub <VA>, <PA>, #offset<31:24>, lsl #24 @ sub <VA>, <PA>, #offset<23:16>, lsl #16 @ @ virt-to-phys (non-LPAE): add <PA>, <VA>, #offset<31:24>, lsl #24 @ add <PA>, <VA>, #offset<23:16>, lsl #16 @ @ virt-to-phys (LPAE): movw <reg>, #offset<31:20> @ adds <PAlo>, <VA>, <reg>, lsl #20 @ mov <PAhi>, #offset<39:32> @ adc <PAhi>, <PAhi>, #0 @ @ In the non-LPAE case, all patchable instructions are ADD or SUB @ instructions, where we need to patch in the offset into the @ immediate field of the opcode, which is emitted with the correct @ rotation value. (The effective value of the immediate is imm12<7:0> @ rotated right by [2 * imm12<11:8>] bits) @ @ 31 28 27 23 22 20 19 16 15 12 11 0 @ +------+-----------------+------+------+-------+ @ ADD | cond | 0 0 1 0 1 0 0 0 | Rn | Rd | imm12 | @ SUB | cond | 0 0 1 0 0 1 0 0 | Rn | Rd | imm12 | @ MOV | cond | 0 0 1 1 1 0 1 0 | Rn | Rd | imm12 | @ MVN | cond | 0 0 1 1 1 1 1 0 | Rn | Rd | imm12 | @ +------+-----------------+------+------+-------+ @ @ In the LPAE case, we use a MOVW instruction to carry the low offset @ word, and patch in the high word of the offset into the immediate @ field of the subsequent MOV instruction, or patch it to a MVN @ instruction if the offset is negative. We can distinguish MOVW @ instructions based on bits 23:22 of the opcode, and ADD/SUB can be @ distinguished from MOV/MVN (all using the encodings above) using @ bit 24. @ @ 31 28 27 23 22 20 19 16 15 12 11 0 @ +------+-----------------+------+------+-------+ @ MOVW | cond | 0 0 1 1 0 0 0 0 | imm4 | Rd | imm12 | @ +------+-----------------+------+------+-------+ @ moveq r0, #0x400000 @ set bit 22, mov to mvn instruction mov r3, r6, lsr #16 @ put offset bits 31-16 into r3 mov r6, r6, lsr #24 @ put offset bits 31-24 into r6 and r3, r3, #0xf0 @ only keep offset bits 23-20 in r3 b .Lnext.Lloop: ldr ip, [r7, r4]#ifdef CONFIG_ARM_LPAE tst ip, #PV_BIT24 @ ADD/SUB have bit 24 clear beq 1fARM_BE8(rev ip, ip) tst ip, #0xc00000 @ MOVW has bits 23:22 clear bic ip, ip, #0x400000 @ clear bit 22 bfc ip, #0, #12 @ clear imm12 field of MOV[W] instruction orreq ip, ip, r6, lsl #4 @ MOVW -> mask in offset bits 31-24 orreq ip, ip, r3, lsr #4 @ MOVW -> mask in offset bits 23-20 orrne ip, ip, r0 @ MOV -> mask in offset bits 7-0 (or bit 22)ARM_BE8(rev ip, ip) b 2f1:#endif tst ip, #PV_IMMR_MSB @ rotation value >= 16 ? bic ip, ip, #PV_IMM8_MASK orreq ip, ip, r6 ARM_BE8(, lsl #24) @ mask in offset bits 31-24 orrne ip, ip, r3 ARM_BE8(, lsl #24) @ mask in offset bits 23-202: str ip, [r7, r4] add r4, r4, #4#endif
.Lnext: cmp r4, r5 ldrcc r7, [r4] @ use branch for delay slot bcc .Lloop ret lrENDPROC(__fixup_a_pv_table)
ENTRY(fixup_pv_table) stmfd sp!, {r4 - r7, lr} mov r4, r0 @ r0 = table start add r5, r0, r1 @ r1 = table size bl __fixup_a_pv_table ldmfd sp!, {r4 - r7, pc}ENDPROC(fixup_pv_table)
.data .align 2 .globl __pv_phys_pfn_offset .type __pv_phys_pfn_offset, %object__pv_phys_pfn_offset: .word 0 .size __pv_phys_pfn_offset, . -__pv_phys_pfn_offset
.globl __pv_offset .type __pv_offset, %object__pv_offset: .quad 0 .size __pv_offset, . -__pv_offset