/* page count of a compressed cluster */
#define erofs_clusterpages(sbi) ((1 << (sbi)->clusterbits) / PAGE_SIZE)
+#define Z_EROFS_NR_INLINE_PAGEVECS 3
+
+#if (Z_EROFS_CLUSTER_MAX_PAGES > Z_EROFS_NR_INLINE_PAGEVECS)
+#define EROFS_PCPUBUF_NR_PAGES Z_EROFS_CLUSTER_MAX_PAGES
+#else
+#define EROFS_PCPUBUF_NR_PAGES Z_EROFS_NR_INLINE_PAGEVECS
+#endif
+
+#else
+#define EROFS_PCPUBUF_NR_PAGES 0
#endif
typedef u64 erofs_off_t;
extern struct shrinker erofs_shrinker_info;
struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp);
+
+#if (EROFS_PCPUBUF_NR_PAGES > 0)
+void *erofs_get_pcpubuf(unsigned int pagenr);
+#define erofs_put_pcpubuf(buf) do { \
+ (void)&(buf); \
+ preempt_enable(); \
+} while (0)
+#else
+static inline void *erofs_get_pcpubuf(unsigned int pagenr)
+{
+ return ERR_PTR(-ENOTSUPP);
+}
+
+#define erofs_put_pcpubuf(buf) do {} while (0)
+#endif
+
void erofs_register_super(struct super_block *sb);
void erofs_unregister_super(struct super_block *sb);
if (IS_ERR(work))
return PTR_ERR(work);
got_it:
- z_erofs_pagevec_ctor_init(&builder->vector,
- Z_EROFS_VLE_INLINE_PAGEVECS,
+ z_erofs_pagevec_ctor_init(&builder->vector, Z_EROFS_NR_INLINE_PAGEVECS,
work->pagevec, work->vcnt);
if (builder->role >= Z_EROFS_VLE_WORK_PRIMARY) {
for (i = 0; i < nr_pages; ++i)
pages[i] = NULL;
- z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_VLE_INLINE_PAGEVECS,
+ z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_NR_INLINE_PAGEVECS,
work->pagevec, 0);
for (i = 0; i < work->vcnt; ++i) {
*
*/
-#define Z_EROFS_VLE_INLINE_PAGEVECS 3
-
struct z_erofs_vle_work {
struct mutex lock;
union {
/* L: pagevec */
- erofs_vtptr_t pagevec[Z_EROFS_VLE_INLINE_PAGEVECS];
+ erofs_vtptr_t pagevec[Z_EROFS_NR_INLINE_PAGEVECS];
struct rcu_head rcu;
};
};
return -EIO;
}
-#if Z_EROFS_CLUSTER_MAX_PAGES > Z_EROFS_VLE_INLINE_PAGEVECS
-#define EROFS_PERCPU_NR_PAGES Z_EROFS_CLUSTER_MAX_PAGES
-#else
-#define EROFS_PERCPU_NR_PAGES Z_EROFS_VLE_INLINE_PAGEVECS
-#endif
-
-static struct {
- char data[PAGE_SIZE * EROFS_PERCPU_NR_PAGES];
-} erofs_pcpubuf[NR_CPUS];
-
int z_erofs_vle_plain_copy(struct page **compressed_pages,
unsigned int clusterpages,
struct page **pages,
char *percpu_data;
bool mirrored[Z_EROFS_CLUSTER_MAX_PAGES] = { 0 };
- preempt_disable();
- percpu_data = erofs_pcpubuf[smp_processor_id()].data;
+ percpu_data = erofs_get_pcpubuf(0);
+ if (IS_ERR(percpu_data))
+ return PTR_ERR(percpu_data);
j = 0;
for (i = 0; i < nr_pages; j = i++) {
if (src && !mirrored[j])
kunmap_atomic(src);
- preempt_enable();
+ erofs_put_pcpubuf(percpu_data);
return 0;
}
unsigned int nr_pages, i, j;
int ret;
- if (outlen + pageofs > EROFS_PERCPU_NR_PAGES * PAGE_SIZE)
+ if (outlen + pageofs > EROFS_PCPUBUF_NR_PAGES * PAGE_SIZE)
return -ENOTSUPP;
nr_pages = DIV_ROUND_UP(outlen + pageofs, PAGE_SIZE);
return -ENOMEM;
}
- preempt_disable();
- vout = erofs_pcpubuf[smp_processor_id()].data;
+ vout = erofs_get_pcpubuf(0);
+ if (IS_ERR(vout))
+ return PTR_ERR(vout);
ret = z_erofs_unzip_lz4(vin, vout + pageofs,
clusterpages * PAGE_SIZE, outlen);
}
out:
- preempt_enable();
+ erofs_put_pcpubuf(vout);
if (clusterpages == 1)
kunmap_atomic(vin);
int ret;
if (overlapped) {
- preempt_disable();
- vin = erofs_pcpubuf[smp_processor_id()].data;
+ vin = erofs_get_pcpubuf(0);
+ if (IS_ERR(vin))
+ return PTR_ERR(vin);
for (i = 0; i < clusterpages; ++i) {
void *t = kmap_atomic(compressed_pages[i]);
if (ret > 0)
ret = 0;
- if (!overlapped) {
+ if (overlapped) {
+ erofs_put_pcpubuf(vin);
+ } else {
if (clusterpages == 1)
kunmap_atomic(vin);
else
erofs_vunmap(vin, clusterpages);
- } else {
- preempt_enable();
}
return ret;
}
return page;
}
+#if (EROFS_PCPUBUF_NR_PAGES > 0)
+static struct {
+ u8 data[PAGE_SIZE * EROFS_PCPUBUF_NR_PAGES];
+} ____cacheline_aligned_in_smp erofs_pcpubuf[NR_CPUS];
+
+void *erofs_get_pcpubuf(unsigned int pagenr)
+{
+ preempt_disable();
+ return &erofs_pcpubuf[smp_processor_id()].data[pagenr * PAGE_SIZE];
+}
+#endif
+
/* global shrink count (for all mounted EROFS instances) */
static atomic_long_t erofs_global_shrink_cnt;