Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiederm...

[linux-2.6-microblaze.git] / lib / raid6 / neon.uc

diff --git a/lib/raid6/neon.uc b/lib/raid6/neon.uc
index 4fa51b7..d5242f5 100644 (file)
--- a/lib/raid6/neon.uc
+++ b/lib/raid6/neon.uc
@@ -46,8 +46,12 @@ static inline unative_t SHLBYTE(unative_t v)
  */
 static inline unative_t MASK(unative_t v)
 {
-       const uint8x16_t temp = NBYTES(0);
-       return (unative_t)vcltq_s8((int8x16_t)v, (int8x16_t)temp);
+       return (unative_t)vshrq_n_s8((int8x16_t)v, 7);
+}
+
+static inline unative_t PMUL(unative_t v, unative_t u)
+{
+       return (unative_t)vmulq_p8((poly8x16_t)v, (poly8x16_t)u);
 }
 
 void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
@@ -110,7 +114,30 @@ void raid6_neon$#_xor_syndrome_real(int disks, int start, int stop,
                        wq$$ = veorq_u8(w1$$, wd$$);
                }
                /* P/Q left side optimization */
-               for ( z = start-1 ; z >= 0 ; z-- ) {
+               for ( z = start-1 ; z >= 3 ; z -= 4 ) {
+                       w2$$ = vshrq_n_u8(wq$$, 4);
+                       w1$$ = vshlq_n_u8(wq$$, 4);
+
+                       w2$$ = PMUL(w2$$, x1d);
+                       wq$$ = veorq_u8(w1$$, w2$$);
+               }
+
+               switch (z) {
+               case 2:
+                       w2$$ = vshrq_n_u8(wq$$, 5);
+                       w1$$ = vshlq_n_u8(wq$$, 3);
+
+                       w2$$ = PMUL(w2$$, x1d);
+                       wq$$ = veorq_u8(w1$$, w2$$);
+                       break;
+               case 1:
+                       w2$$ = vshrq_n_u8(wq$$, 6);
+                       w1$$ = vshlq_n_u8(wq$$, 2);
+
+                       w2$$ = PMUL(w2$$, x1d);
+                       wq$$ = veorq_u8(w1$$, w2$$);
+                       break;
+               case 0:
                        w2$$ = MASK(wq$$);
                        w1$$ = SHLBYTE(wq$$);