return fls64(reg->umax_value);
}
-/* See comment for mark_nocsr_pattern_for_call() */
-static void check_nocsr_stack_contract(struct bpf_verifier_env *env, struct bpf_func_state *state,
- int insn_idx, int off)
+/* See comment for mark_fastcall_pattern_for_call() */
+static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
+ struct bpf_func_state *state, int insn_idx, int off)
{
struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
struct bpf_insn_aux_data *aux = env->insn_aux_data;
int i;
- if (subprog->nocsr_stack_off <= off || aux[insn_idx].nocsr_pattern)
+ if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)
return;
- /* access to the region [max_stack_depth .. nocsr_stack_off)
- * from something that is not a part of the nocsr pattern,
- * disable nocsr rewrites for current subprogram by setting
- * nocsr_stack_off to a value smaller than any possible offset.
+ /* access to the region [max_stack_depth .. fastcall_stack_off)
+ * from something that is not a part of the fastcall pattern,
+ * disable fastcall rewrites for current subprogram by setting
+ * fastcall_stack_off to a value smaller than any possible offset.
*/
- subprog->nocsr_stack_off = S16_MIN;
- /* reset nocsr aux flags within subprogram,
+ subprog->fastcall_stack_off = S16_MIN;
+ /* reset fastcall aux flags within subprogram,
* happens at most once per subprogram
*/
for (i = subprog->start; i < (subprog + 1)->start; ++i) {
- aux[i].nocsr_spills_num = 0;
- aux[i].nocsr_pattern = 0;
+ aux[i].fastcall_spills_num = 0;
+ aux[i].fastcall_pattern = 0;
}
}
if (err)
return err;
- check_nocsr_stack_contract(env, state, insn_idx, off);
+ check_fastcall_stack_contract(env, state, insn_idx, off);
mark_stack_slot_scratched(env, spi);
if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {
bool reg_value_fits;
return err;
}
- check_nocsr_stack_contract(env, state, insn_idx, min_off);
+ check_fastcall_stack_contract(env, state, insn_idx, min_off);
/* Variable offset writes destroy any spilled pointers in range. */
for (i = min_off; i < max_off; i++) {
u8 new_type, *stype;
reg = ®_state->stack[spi].spilled_ptr;
mark_stack_slot_scratched(env, spi);
- check_nocsr_stack_contract(env, state, env->insn_idx, off);
+ check_fastcall_stack_contract(env, state, env->insn_idx, off);
if (is_spilled_reg(®_state->stack[spi])) {
u8 spill_size = 1;
min_off = reg->smin_value + off;
max_off = reg->smax_value + off;
mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
- check_nocsr_stack_contract(env, ptr_state, env->insn_idx, min_off);
+ check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
return 0;
}
struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx];
int min_valid_off, max_bpf_stack;
- /* If accessing instruction is a spill/fill from nocsr pattern,
+ /* If accessing instruction is a spill/fill from bpf_fastcall pattern,
* add room for all caller saved registers below MAX_BPF_STACK.
- * In case if nocsr rewrite won't happen maximal stack depth
+ * In case if bpf_fastcall rewrite won't happen maximal stack depth
* would be checked by check_max_stack_depth_subprog().
*/
max_bpf_stack = MAX_BPF_STACK;
- if (aux->nocsr_pattern)
+ if (aux->fastcall_pattern)
max_bpf_stack += CALLER_SAVED_REGS * BPF_REG_SIZE;
if (t == BPF_WRITE || env->allow_uninit_stack)
/* Return a bitmask specifying which caller saved registers are
* clobbered by a call to a helper *as if* this helper follows
- * no_caller_saved_registers contract:
+ * bpf_fastcall contract:
* - includes R0 if function is non-void;
* - includes R1-R5 if corresponding parameter has is described
* in the function prototype.
*/
-static u32 helper_nocsr_clobber_mask(const struct bpf_func_proto *fn)
+static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)
{
u8 mask;
int i;
}
/* True if do_misc_fixups() replaces calls to helper number 'imm',
- * replacement patch is presumed to follow no_caller_saved_registers contract
- * (see mark_nocsr_pattern_for_call() below).
+ * replacement patch is presumed to follow bpf_fastcall contract
+ * (see mark_fastcall_pattern_for_call() below).
*/
static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
{
}
}
-/* GCC and LLVM define a no_caller_saved_registers function attribute.
+/* LLVM define a bpf_fastcall function attribute.
* This attribute means that function scratches only some of
* the caller saved registers defined by ABI.
* For BPF the set of such registers could be defined as follows:
*
* The contract between kernel and clang allows to simultaneously use
* such functions and maintain backwards compatibility with old
- * kernels that don't understand no_caller_saved_registers calls
- * (nocsr for short):
+ * kernels that don't understand bpf_fastcall calls:
*
- * - for nocsr calls clang allocates registers as-if relevant r0-r5
+ * - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5
* registers are not scratched by the call;
*
- * - as a post-processing step, clang visits each nocsr call and adds
+ * - as a post-processing step, clang visits each bpf_fastcall call and adds
* spill/fill for every live r0-r5;
*
* - stack offsets used for the spill/fill are allocated as lowest
* purposes;
*
* - when kernel loads a program, it looks for such patterns
- * (nocsr function surrounded by spills/fills) and checks if
- * spill/fill stack offsets are used exclusively in nocsr patterns;
+ * (bpf_fastcall function surrounded by spills/fills) and checks if
+ * spill/fill stack offsets are used exclusively in fastcall patterns;
*
* - if so, and if verifier or current JIT inlines the call to the
- * nocsr function (e.g. a helper call), kernel removes unnecessary
+ * bpf_fastcall function (e.g. a helper call), kernel removes unnecessary
* spill/fill pairs;
*
* - when old kernel loads a program, presence of spill/fill pairs
* r0 += r2;
* exit;
*
- * The purpose of mark_nocsr_pattern_for_call is to:
+ * The purpose of mark_fastcall_pattern_for_call is to:
* - look for such patterns;
- * - mark spill and fill instructions in env->insn_aux_data[*].nocsr_pattern;
- * - mark set env->insn_aux_data[*].nocsr_spills_num for call instruction;
- * - update env->subprog_info[*]->nocsr_stack_off to find an offset
- * at which nocsr spill/fill stack slots start;
- * - update env->subprog_info[*]->keep_nocsr_stack.
+ * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern;
+ * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction;
+ * - update env->subprog_info[*]->fastcall_stack_off to find an offset
+ * at which bpf_fastcall spill/fill stack slots start;
+ * - update env->subprog_info[*]->keep_fastcall_stack.
*
- * The .nocsr_pattern and .nocsr_stack_off are used by
- * check_nocsr_stack_contract() to check if every stack access to
- * nocsr spill/fill stack slot originates from spill/fill
- * instructions, members of nocsr patterns.
+ * The .fastcall_pattern and .fastcall_stack_off are used by
+ * check_fastcall_stack_contract() to check if every stack access to
+ * fastcall spill/fill stack slot originates from spill/fill
+ * instructions, members of fastcall patterns.
*
- * If such condition holds true for a subprogram, nocsr patterns could
- * be rewritten by remove_nocsr_spills_fills().
- * Otherwise nocsr patterns are not changed in the subprogram
+ * If such condition holds true for a subprogram, fastcall patterns could
+ * be rewritten by remove_fastcall_spills_fills().
+ * Otherwise bpf_fastcall patterns are not changed in the subprogram
* (code, presumably, generated by an older clang version).
*
* For example, it is *not* safe to remove spill/fill below:
* r0 += r1; exit;
* exit;
*/
-static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,
- struct bpf_subprog_info *subprog,
- int insn_idx, s16 lowest_off)
+static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
+ struct bpf_subprog_info *subprog,
+ int insn_idx, s16 lowest_off)
{
struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;
struct bpf_insn *call = &env->prog->insnsi[insn_idx];
if (get_helper_proto(env, call->imm, &fn) < 0)
/* error would be reported later */
return;
- clobbered_regs_mask = helper_nocsr_clobber_mask(fn);
- can_be_inlined = fn->allow_nocsr &&
+ clobbered_regs_mask = helper_fastcall_clobber_mask(fn);
+ can_be_inlined = fn->allow_fastcall &&
(verifier_inlines_helper_call(env, call->imm) ||
bpf_jit_inlines_helper_call(call->imm));
}
if (stx->off != off || ldx->off != off)
break;
expected_regs_mask &= ~BIT(stx->src_reg);
- env->insn_aux_data[insn_idx - i].nocsr_pattern = 1;
- env->insn_aux_data[insn_idx + i].nocsr_pattern = 1;
+ env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;
+ env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;
}
if (i == 1)
return;
- /* Conditionally set 'nocsr_spills_num' to allow forward
+ /* Conditionally set 'fastcall_spills_num' to allow forward
* compatibility when more helper functions are marked as
- * nocsr at compile time than current kernel supports, e.g:
+ * bpf_fastcall at compile time than current kernel supports, e.g:
*
* 1: *(u64 *)(r10 - 8) = r1
- * 2: call A ;; assume A is nocsr for current kernel
+ * 2: call A ;; assume A is bpf_fastcall for current kernel
* 3: r1 = *(u64 *)(r10 - 8)
* 4: *(u64 *)(r10 - 8) = r1
- * 5: call B ;; assume B is not nocsr for current kernel
+ * 5: call B ;; assume B is not bpf_fastcall for current kernel
* 6: r1 = *(u64 *)(r10 - 8)
*
- * There is no need to block nocsr rewrite for such program.
- * Set 'nocsr_pattern' for both calls to keep check_nocsr_stack_contract() happy,
- * don't set 'nocsr_spills_num' for call B so that remove_nocsr_spills_fills()
+ * There is no need to block bpf_fastcall rewrite for such program.
+ * Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy,
+ * don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()
* does not remove spill/fill pair {4,6}.
*/
if (can_be_inlined)
- env->insn_aux_data[insn_idx].nocsr_spills_num = i - 1;
+ env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;
else
- subprog->keep_nocsr_stack = 1;
- subprog->nocsr_stack_off = min(subprog->nocsr_stack_off, off);
+ subprog->keep_fastcall_stack = 1;
+ subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);
}
-static int mark_nocsr_patterns(struct bpf_verifier_env *env)
+static int mark_fastcall_patterns(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprog = env->subprog_info;
struct bpf_insn *insn;
continue;
lowest_off = min(lowest_off, insn->off);
}
- /* use this offset to find nocsr patterns */
+ /* use this offset to find fastcall patterns */
for (i = subprog->start; i < (subprog + 1)->start; ++i) {
insn = env->prog->insnsi + i;
if (insn->code != (BPF_JMP | BPF_CALL))
continue;
- mark_nocsr_pattern_for_call(env, subprog, i, lowest_off);
+ mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);
}
}
return 0;
return 0;
}
-/* Remove unnecessary spill/fill pairs, members of nocsr pattern,
+/* Remove unnecessary spill/fill pairs, members of fastcall pattern,
* adjust subprograms stack depth when possible.
*/
-static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)
+static int remove_fastcall_spills_fills(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprog = env->subprog_info;
struct bpf_insn_aux_data *aux = env->insn_aux_data;
int i, j;
for (i = 0; i < insn_cnt; i++, insn++) {
- if (aux[i].nocsr_spills_num > 0) {
- spills_num = aux[i].nocsr_spills_num;
+ if (aux[i].fastcall_spills_num > 0) {
+ spills_num = aux[i].fastcall_spills_num;
/* NOPs would be removed by opt_remove_nops() */
for (j = 1; j <= spills_num; ++j) {
*(insn - j) = NOP;
modified = true;
}
if ((subprog + 1)->start == i + 1) {
- if (modified && !subprog->keep_nocsr_stack)
- subprog->stack_depth = -subprog->nocsr_stack_off;
+ if (modified && !subprog->keep_fastcall_stack)
+ subprog->stack_depth = -subprog->fastcall_stack_off;
subprog++;
modified = false;
}
if (ret < 0)
goto skip_full_check;
- ret = mark_nocsr_patterns(env);
+ ret = mark_fastcall_patterns(env);
if (ret < 0)
goto skip_full_check;
* allocate additional slots.
*/
if (ret == 0)
- ret = remove_nocsr_spills_fills(env);
+ ret = remove_fastcall_spills_fills(env);
if (ret == 0)
ret = check_max_stack_depth(env);
projects / linux-2.6-microblaze.git / commitdiff