#define DM_BUFIO_MEMORY_PERCENT 2
#define DM_BUFIO_VMALLOC_PERCENT 25
#define DM_BUFIO_WRITEBACK_RATIO 3
+#define DM_BUFIO_LOW_WATERMARK_RATIO 16
/*
* Check buffer ages in this interval (seconds)
unsigned char list_mode; /* LIST_* */
blk_status_t read_error;
blk_status_t write_error;
+ unsigned accessed;
unsigned hold_count;
unsigned long state;
unsigned long last_accessed;
static LIST_HEAD(global_queue);
+static unsigned long global_num = 0;
+
/*
* Buffers are freed after this timeout
*/
*/
static DEFINE_MUTEX(dm_bufio_clients_lock);
+static struct workqueue_struct *dm_bufio_wq;
+static struct delayed_work dm_bufio_cleanup_old_work;
+static struct work_struct dm_bufio_replacement_work;
+
+
#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
static void buffer_record_stack(struct dm_buffer *b)
{
if (dm_bufio_current_allocated > dm_bufio_peak_allocated)
dm_bufio_peak_allocated = dm_bufio_current_allocated;
+ b->accessed = 1;
+
if (!unlink) {
list_add(&b->global_list, &global_queue);
+ global_num++;
+ if (dm_bufio_current_allocated > dm_bufio_cache_size)
+ queue_work(dm_bufio_wq, &dm_bufio_replacement_work);
} else {
list_del(&b->global_list);
+ global_num--;
}
spin_unlock(&global_spinlock);
{
struct dm_bufio_client *c = b->c;
+ b->accessed = 1;
+
BUG_ON(!c->n_buffers[b->list_mode]);
c->n_buffers[b->list_mode]--;
dm_bufio_unlock(c);
}
+static void do_global_cleanup(struct work_struct *w)
+{
+ struct dm_bufio_client *locked_client = NULL;
+ struct dm_bufio_client *current_client;
+ struct dm_buffer *b;
+ unsigned spinlock_hold_count;
+ unsigned long threshold = dm_bufio_cache_size -
+ dm_bufio_cache_size / DM_BUFIO_LOW_WATERMARK_RATIO;
+ unsigned long loops = global_num * 2;
+
+ mutex_lock(&dm_bufio_clients_lock);
+
+ while (1) {
+ cond_resched();
+
+ spin_lock(&global_spinlock);
+ if (unlikely(dm_bufio_current_allocated <= threshold))
+ break;
+
+ spinlock_hold_count = 0;
+get_next:
+ if (!loops--)
+ break;
+ if (unlikely(list_empty(&global_queue)))
+ break;
+ b = list_entry(global_queue.prev, struct dm_buffer, global_list);
+
+ if (b->accessed) {
+ b->accessed = 0;
+ list_move(&b->global_list, &global_queue);
+ if (likely(++spinlock_hold_count < 16))
+ goto get_next;
+ spin_unlock(&global_spinlock);
+ continue;
+ }
+
+ current_client = b->c;
+ if (unlikely(current_client != locked_client)) {
+ if (locked_client)
+ dm_bufio_unlock(locked_client);
+
+ if (!dm_bufio_trylock(current_client)) {
+ spin_unlock(&global_spinlock);
+ dm_bufio_lock(current_client);
+ locked_client = current_client;
+ continue;
+ }
+
+ locked_client = current_client;
+ }
+
+ spin_unlock(&global_spinlock);
+
+ if (unlikely(!__try_evict_buffer(b, GFP_KERNEL))) {
+ spin_lock(&global_spinlock);
+ list_move(&b->global_list, &global_queue);
+ spin_unlock(&global_spinlock);
+ }
+ }
+
+ spin_unlock(&global_spinlock);
+
+ if (locked_client)
+ dm_bufio_unlock(locked_client);
+
+ mutex_unlock(&dm_bufio_clients_lock);
+}
+
static void cleanup_old_buffers(void)
{
unsigned long max_age_hz = get_max_age_hz();
mutex_unlock(&dm_bufio_clients_lock);
}
-static struct workqueue_struct *dm_bufio_wq;
-static struct delayed_work dm_bufio_work;
-
static void work_fn(struct work_struct *w)
{
cleanup_old_buffers();
- queue_delayed_work(dm_bufio_wq, &dm_bufio_work,
+ queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
DM_BUFIO_WORK_TIMER_SECS * HZ);
}
if (!dm_bufio_wq)
return -ENOMEM;
- INIT_DELAYED_WORK(&dm_bufio_work, work_fn);
- queue_delayed_work(dm_bufio_wq, &dm_bufio_work,
+ INIT_DELAYED_WORK(&dm_bufio_cleanup_old_work, work_fn);
+ INIT_WORK(&dm_bufio_replacement_work, do_global_cleanup);
+ queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
DM_BUFIO_WORK_TIMER_SECS * HZ);
return 0;
{
int bug = 0;
- cancel_delayed_work_sync(&dm_bufio_work);
+ cancel_delayed_work_sync(&dm_bufio_cleanup_old_work);
+ flush_workqueue(dm_bufio_wq);
destroy_workqueue(dm_bufio_wq);
if (dm_bufio_client_count) {