In cases where we know the task is not sleeping, try to optimise
away the indirect call to task->tk_action() by replacing it with
a direct call.
Only change tail calls, to allow gcc to perform tail call
elimination.
Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>
}
#endif /* CONFIG_SUNRPC_SWAP */
}
#endif /* CONFIG_SUNRPC_SWAP */
+static inline bool
+rpc_task_need_resched(const struct rpc_task *task)
+{
+ if (RPC_IS_QUEUED(task) || task->tk_callback)
+ return true;
+ return false;
+}
+
#endif /* _LINUX_SUNRPC_SCHED_H_ */
#endif /* _LINUX_SUNRPC_SCHED_H_ */
clnt->cl_stats->rpccnt++;
task->tk_action = call_reserve;
rpc_task_set_transport(task, clnt);
clnt->cl_stats->rpccnt++;
task->tk_action = call_reserve;
rpc_task_set_transport(task, clnt);
task->tk_status = 0;
task->tk_action = call_reserveresult;
xprt_reserve(task);
task->tk_status = 0;
task->tk_action = call_reserveresult;
xprt_reserve(task);
+ if (rpc_task_need_resched(task))
+ return;
+ call_reserveresult(task);
}
static void call_retry_reserve(struct rpc_task *task);
}
static void call_retry_reserve(struct rpc_task *task);
if (status >= 0) {
if (task->tk_rqstp) {
task->tk_action = call_refresh;
if (status >= 0) {
if (task->tk_rqstp) {
task->tk_action = call_refresh;
/* fall through */
case -EAGAIN: /* woken up; retry */
task->tk_action = call_retry_reserve;
/* fall through */
case -EAGAIN: /* woken up; retry */
task->tk_action = call_retry_reserve;
+ call_retry_reserve(task);
return;
case -EIO: /* probably a shutdown */
break;
return;
case -EIO: /* probably a shutdown */
break;
task->tk_status = 0;
task->tk_action = call_reserveresult;
xprt_retry_reserve(task);
task->tk_status = 0;
task->tk_action = call_reserveresult;
xprt_retry_reserve(task);
+ if (rpc_task_need_resched(task))
+ return;
+ call_reserveresult(task);
task->tk_status = 0;
task->tk_client->cl_stats->rpcauthrefresh++;
rpcauth_refreshcred(task);
task->tk_status = 0;
task->tk_client->cl_stats->rpcauthrefresh++;
rpcauth_refreshcred(task);
+ if (rpc_task_need_resched(task))
+ return;
+ call_refreshresult(task);
case 0:
if (rpcauth_uptodatecred(task)) {
task->tk_action = call_allocate;
case 0:
if (rpcauth_uptodatecred(task)) {
task->tk_action = call_allocate;
return;
}
/* Use rate-limiting and a max number of retries if refresh
return;
}
/* Use rate-limiting and a max number of retries if refresh
task->tk_cred_retry--;
dprintk("RPC: %5u %s: retry refresh creds\n",
task->tk_pid, __func__);
task->tk_cred_retry--;
dprintk("RPC: %5u %s: retry refresh creds\n",
task->tk_pid, __func__);
return;
}
dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
return;
}
dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
task->tk_status = 0;
task->tk_action = call_encode;
task->tk_status = 0;
task->tk_action = call_encode;
+ if (req->rq_buffer) {
+ call_encode(task);
if (proc->p_proc != 0) {
BUG_ON(proc->p_arglen == 0);
if (proc->p_proc != 0) {
BUG_ON(proc->p_arglen == 0);
status = xprt->ops->buf_alloc(task);
xprt_inject_disconnect(xprt);
status = xprt->ops->buf_alloc(task);
xprt_inject_disconnect(xprt);
+ if (status == 0) {
+ if (rpc_task_need_resched(task))
+ return;
+ call_encode(task);
if (status != -ENOMEM) {
rpc_exit(task, status);
return;
if (status != -ENOMEM) {
rpc_exit(task, status);
return;
xprt_request_enqueue_receive(task);
xprt_request_enqueue_transmit(task);
out:
xprt_request_enqueue_receive(task);
xprt_request_enqueue_transmit(task);
out:
- task->tk_action = call_transmit;
- /* Check that the connection is OK */
- if (!xprt_bound(task->tk_xprt))
- task->tk_action = call_bind;
- else if (!xprt_connected(task->tk_xprt))
- task->tk_action = call_connect;
+ task->tk_action = call_bind;
+ call_bind(task);
+ if (xprt_bound(xprt)) {
+ task->tk_action = call_connect;
+ call_connect(task);
+ return;
+ }
+
- task->tk_action = call_connect;
- if (!xprt_bound(xprt)) {
- task->tk_action = call_bind_status;
- task->tk_timeout = xprt->bind_timeout;
- xprt->ops->rpcbind(task);
- }
+ task->tk_action = call_bind_status;
+ task->tk_timeout = xprt->bind_timeout;
+ xprt->ops->rpcbind(task);
dprint_status(task);
task->tk_status = 0;
task->tk_action = call_connect;
dprint_status(task);
task->tk_status = 0;
task->tk_action = call_connect;
+ if (xprt_connected(xprt)) {
+ task->tk_action = call_transmit;
+ call_transmit(task);
+ return;
+ }
+
dprintk("RPC: %5u call_connect xprt %p %s connected\n",
task->tk_pid, xprt,
(xprt_connected(xprt) ? "is" : "is not"));
dprintk("RPC: %5u call_connect xprt %p %s connected\n",
task->tk_pid, xprt,
(xprt_connected(xprt) ? "is" : "is not"));
- task->tk_action = call_transmit;
- if (!xprt_connected(xprt)) {
- task->tk_action = call_connect_status;
- if (task->tk_status < 0)
- return;
- if (task->tk_flags & RPC_TASK_NOCONNECT) {
- rpc_exit(task, -ENOTCONN);
- return;
- }
- xprt_connect(task);
+ task->tk_action = call_connect_status;
+ if (task->tk_status < 0)
+ return;
+ if (task->tk_flags & RPC_TASK_NOCONNECT) {
+ rpc_exit(task, -ENOTCONN);
+ return;
case 0:
clnt->cl_stats->netreconn++;
task->tk_action = call_transmit;
case 0:
clnt->cl_stats->netreconn++;
task->tk_action = call_transmit;
return;
}
rpc_exit(task, status);
return;
}
rpc_exit(task, status);
dprint_status(task);
task->tk_action = call_transmit_status;
dprint_status(task);
task->tk_action = call_transmit_status;
+ if (!xprt_prepare_transmit(task))
+ return;
+ task->tk_status = 0;
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
- if (!xprt_prepare_transmit(task))
+ if (!xprt_connected(task->tk_xprt)) {
+ task->tk_status = -ENOTCONN;
- task->tk_status = 0;
- if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
- if (!xprt_connected(task->tk_xprt)) {
- task->tk_status = -ENOTCONN;
- return;
- }
- xprt_transmit(task);
}
xprt_end_transmit(task);
}
xprt_end_transmit(task);
+ if (rpc_task_need_resched(task))
+ return;
+ call_transmit_status(task);
* Common case: success. Force the compiler to put this
* test first.
*/
* Common case: success. Force the compiler to put this
* test first.
*/
- if (task->tk_status == 0) {
- xprt_request_wait_receive(task);
+ if (rpc_task_transmitted(task)) {
+ if (task->tk_status == 0)
+ xprt_request_wait_receive(task);
+ if (rpc_task_need_resched(task))
+ return;
+ call_status(task);
{
xprt_request_enqueue_transmit(task);
task->tk_action = call_bc_transmit;
{
xprt_request_enqueue_transmit(task);
task->tk_action = call_bc_transmit;
+ call_bc_transmit(task);
status = task->tk_status;
if (status >= 0) {
task->tk_action = call_decode;
status = task->tk_status;
if (status >= 0) {
task->tk_action = call_decode;