#include <linux/module.h>
#include <linux/delay.h>
#include <linux/mlx5/driver.h>
-#include <linux/mlx5/cmd.h>
#include "mlx5_core.h"
#include "lib/eq.h"
u8 ec_function;
s32 npages;
struct work_struct work;
+ u8 release_all;
};
struct fw_page {
static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
s32 *npages, int boot)
{
- u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0};
- u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {0};
+ u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {};
+ u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {};
int err;
MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES);
MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES);
MLX5_SET(query_pages_in, in, embedded_cpu_function, mlx5_core_is_ecpf(dev));
- err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+ err = mlx5_cmd_exec_inout(dev, query_pages, in, out);
if (err)
return err;
return err;
}
-static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr)
+static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id)
{
- struct fw_page *fp;
+ struct fw_page *fp = NULL;
+ struct fw_page *iter;
unsigned n;
- if (list_empty(&dev->priv.free_list))
+ list_for_each_entry(iter, &dev->priv.free_list, list) {
+ if (iter->func_id != func_id)
+ continue;
+ fp = iter;
+ }
+
+ if (list_empty(&dev->priv.free_list) || !fp)
return -ENOMEM;
- fp = list_entry(dev->priv.free_list.next, struct fw_page, list);
n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask));
if (n >= MLX5_NUM_4K_IN_PAGE) {
mlx5_core_warn(dev, "alloc 4k bug\n");
#define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT)
+static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp,
+ bool in_free_list)
+{
+ rb_erase(&fwp->rb_node, &dev->priv.page_root);
+ if (in_free_list)
+ list_del(&fwp->list);
+ dma_unmap_page(dev->device, fwp->addr & MLX5_U64_4K_PAGE_MASK,
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ __free_page(fwp->page);
+ kfree(fwp);
+}
+
static void free_4k(struct mlx5_core_dev *dev, u64 addr)
{
struct fw_page *fwp;
fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK);
if (!fwp) {
- mlx5_core_warn(dev, "page not found\n");
+ mlx5_core_warn_rl(dev, "page not found\n");
return;
}
-
n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT;
fwp->free_count++;
set_bit(n, &fwp->bitmask);
- if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) {
- rb_erase(&fwp->rb_node, &dev->priv.page_root);
- if (fwp->free_count != 1)
- list_del(&fwp->list);
- dma_unmap_page(dev->device, addr & MLX5_U64_4K_PAGE_MASK,
- PAGE_SIZE, DMA_BIDIRECTIONAL);
- __free_page(fwp->page);
- kfree(fwp);
- } else if (fwp->free_count == 1) {
+ if (fwp->free_count == MLX5_NUM_4K_IN_PAGE)
+ free_fwp(dev, fwp, fwp->free_count != 1);
+ else if (fwp->free_count == 1)
list_add(&fwp->list, &dev->priv.free_list);
- }
}
static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id)
static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id,
bool ec_function)
{
- u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0};
- u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0};
+ u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {};
int err;
MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
MLX5_SET(manage_pages_in, in, function_id, func_id);
MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function);
- err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+ err = mlx5_cmd_exec_in(dev, manage_pages, in);
if (err)
mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n",
func_id, err);
for (i = 0; i < npages; i++) {
retry:
- err = alloc_4k(dev, &addr);
+ err = alloc_4k(dev, &addr, func_id);
if (err) {
if (err == -ENOMEM)
err = alloc_system_page(dev, func_id);
return err;
}
+static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id,
+ bool ec_function)
+{
+ struct rb_node *p;
+ int npages = 0;
+
+ p = rb_first(&dev->priv.page_root);
+ while (p) {
+ struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node);
+
+ p = rb_next(p);
+ if (fwp->func_id != func_id)
+ continue;
+ npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count);
+ free_fwp(dev, fwp, fwp->free_count);
+ }
+
+ dev->priv.fw_pages -= npages;
+ if (func_id)
+ dev->priv.vfs_pages -= npages;
+ else if (mlx5_core_is_ecpf(dev) && !ec_function)
+ dev->priv.peer_pf_pages -= npages;
+
+ mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x\n",
+ npages, ec_function, func_id);
+}
+
static int reclaim_pages_cmd(struct mlx5_core_dev *dev,
u32 *in, int in_size, u32 *out, int out_size)
{
int *nclaimed, bool ec_function)
{
int outlen = MLX5_ST_SZ_BYTES(manage_pages_out);
- u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0};
+ u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {};
int num_claimed;
u32 *out;
int err;
struct mlx5_core_dev *dev = req->dev;
int err = 0;
- if (req->npages < 0)
+ if (req->release_all)
+ release_all_pages(dev, req->func_id, req->ec_function);
+ else if (req->npages < 0)
err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL,
req->ec_function);
else if (req->npages > 0)
enum {
EC_FUNCTION_MASK = 0x8000,
+ RELEASE_ALL_PAGES_MASK = 0x4000,
};
static int req_pages_handler(struct notifier_block *nb,
struct mlx5_priv *priv;
struct mlx5_eqe *eqe;
bool ec_function;
+ bool release_all;
u16 func_id;
s32 npages;
func_id = be16_to_cpu(eqe->data.req_pages.func_id);
npages = be32_to_cpu(eqe->data.req_pages.num_pages);
ec_function = be16_to_cpu(eqe->data.req_pages.ec_function) & EC_FUNCTION_MASK;
- mlx5_core_dbg(dev, "page request for func 0x%x, npages %d\n",
- func_id, npages);
+ release_all = be16_to_cpu(eqe->data.req_pages.ec_function) &
+ RELEASE_ALL_PAGES_MASK;
+ mlx5_core_dbg(dev, "page request for func 0x%x, npages %d, release_all %d\n",
+ func_id, npages, release_all);
req = kzalloc(sizeof(*req), GFP_ATOMIC);
if (!req) {
mlx5_core_warn(dev, "failed to allocate pages request\n");
req->func_id = func_id;
req->npages = npages;
req->ec_function = ec_function;
+ req->release_all = release_all;
INIT_WORK(&req->work, pages_work_handler);
queue_work(dev->priv.pg_wq, &req->work);
return NOTIFY_OK;
projects / linux-2.6-microblaze.git / blobdiff