select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
select BUILDTIME_TABLE_SORT
select CLONE_BACKWARDS2
+ select DCACHE_WORD_ACCESS if !KMSAN
select DMA_OPS if PCI
select DYNAMIC_FTRACE if FUNCTION_TRACER
select FUNCTION_ALIGNMENT_8B if CC_IS_GCC
#define EX_TYPE_UA_LOAD_MEM 4
#define EX_TYPE_UA_LOAD_REG 5
#define EX_TYPE_UA_LOAD_REGPAIR 6
+#define EX_TYPE_ZEROPAD 7
#define EX_DATA_REG_ERR_SHIFT 0
#define EX_DATA_REG_ERR GENMASK(3, 0)
#define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \
__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0)
+#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \
+ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0)
+
#endif /* __ASM_EXTABLE_H */
#define _ASM_WORD_AT_A_TIME_H
#include <linux/kernel.h>
+#include <asm/asm-extable.h>
#include <asm/bitsperlong.h>
struct word_at_a_time {
return ~1UL << data;
}
+/*
+ * Load an unaligned word from kernel space.
+ *
+ * In the (very unlikely) case of the word being a page-crosser
+ * and the next page not being mapped, take the exception and
+ * return zeroes in the non-existing part.
+ */
+static inline unsigned long load_unaligned_zeropad(const void *addr)
+{
+ unsigned long data;
+
+ asm volatile(
+ "0: lg %[data],0(%[addr])\n"
+ "1: nopr %%r7\n"
+ EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])
+ EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr])
+ : [data] "=d" (data)
+ : [addr] "a" (addr), "m" (*(unsigned long *)addr));
+ return data;
+}
+
#endif /* _ASM_WORD_AT_A_TIME_H */
return true;
}
+static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs)
+{
+ unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data);
+ unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data);
+ unsigned long data, addr, offset;
+
+ addr = regs->gprs[reg_addr];
+ offset = addr & (sizeof(unsigned long) - 1);
+ addr &= ~(sizeof(unsigned long) - 1);
+ data = *(unsigned long *)addr;
+ data <<= BITS_PER_BYTE * offset;
+ regs->gprs[reg_data] = data;
+ regs->psw.addr = extable_fixup(ex);
+ return true;
+}
+
bool fixup_exception(struct pt_regs *regs)
{
const struct exception_table_entry *ex;
return ex_handler_ua_load_reg(ex, false, regs);
case EX_TYPE_UA_LOAD_REGPAIR:
return ex_handler_ua_load_reg(ex, true, regs);
+ case EX_TYPE_ZEROPAD:
+ return ex_handler_zeropad(ex, regs);
}
panic("invalid exception table entry");
}