1041 lines
33 KiB
C
Executable file
1041 lines
33 KiB
C
Executable file
#include <stdio.h>
|
|
#include <stdint.h>
|
|
#include <string.h>
|
|
#include "scaler_neon.h"
|
|
|
|
//
|
|
// arm NEON / C integer scalers for miyoomini
|
|
// args/ src : src offset address of top left corner
|
|
// dst : dst offset address of top left corner
|
|
// sw : src width pixels
|
|
// sh : src height pixels
|
|
// sp : src pitch (stride) bytes if 0, (src width * [2|4]) is used
|
|
// dp : dst pitch (stride) bytes if 0, (src width * [2|4] * multiplier) is used
|
|
//
|
|
// ** NOTE **
|
|
// since 32bit aligned addresses need to be processed for NEON scalers,
|
|
// x-offset and stride pixels must be even# in the case of 16bpp,
|
|
// if odd#, then handled by the C scaler
|
|
//
|
|
|
|
// memcpy_neon (dst/src must be aligned 4, size must be aligned 2)
|
|
static inline void memcpy_neon(void* dst, void* src, uint32_t size) {
|
|
asm volatile (
|
|
" bic r4, %2, #127 ;"
|
|
" add r3, %0, %2 ;" // r3 = endofs
|
|
" add r4, %0, r4 ;" // r4 = s128ofs
|
|
" cmp %[s], r4 ;"
|
|
" beq 2f ;"
|
|
"1: vldmia %[s]!, {q8-q15} ;" // 128 bytes
|
|
" vstmia %[d]!, {q8-q15} ;"
|
|
" cmp %[s], r4 ;"
|
|
" bne 1b ;"
|
|
"2: cmp %[s], r3 ;"
|
|
" beq 7f ;"
|
|
" tst %[sz], #64 ;"
|
|
" beq 3f ;"
|
|
" vldmia %[s]!, {q8-q11} ;" // 64 bytes
|
|
" vstmia %[d]!, {q8-q11} ;"
|
|
" cmp %[s], r3 ;"
|
|
" beq 7f ;"
|
|
"3: tst %[sz], #32 ;"
|
|
" beq 4f ;"
|
|
" vldmia %[s]!, {q12-q13} ;" // 32 bytes
|
|
" vstmia %[d]!, {q12-q13} ;"
|
|
" cmp %[s], r3 ;"
|
|
" beq 7f ;"
|
|
"4: tst %[sz], #16 ;"
|
|
" beq 5f ;"
|
|
" vldmia %[s]!, {q14} ;" // 16 bytes
|
|
" vstmia %[d]!, {q14} ;"
|
|
" cmp %[s], r3 ;"
|
|
" beq 7f ;"
|
|
"5: tst %[sz], #8 ;"
|
|
" beq 6f ;"
|
|
" vldmia %[s]!, {d30} ;" // 8 bytes
|
|
" vstmia %[d]!, {d30} ;"
|
|
" cmp %[s], r3 ;"
|
|
" beq 7f ;"
|
|
"6: ldrh r4, [%[s]],#2 ;" // rest
|
|
" strh r4, [%[d]],#2 ;"
|
|
" cmp %[s], r3 ;"
|
|
" bne 6b ;"
|
|
"7: "
|
|
: [s]"+r"(src), [d]"+r"(dst)
|
|
: [sz]"r"(size)
|
|
: "r3","r4","q8","q9","q10","q11","q12","q13","q14","q15","memory","cc"
|
|
);
|
|
}
|
|
|
|
//
|
|
// NEON scalers
|
|
//
|
|
|
|
void scale1x_n16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw*sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*1; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale1x_c16(src,dst,sw,sh,sp,dp); return; }
|
|
if ((swl == sp)&&(sp == dp)) memcpy_neon(dst, src, sp*sh);
|
|
else for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp) memcpy_neon(dst, src, swl);
|
|
}
|
|
|
|
void scale1x_n32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw*sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*1; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale1x_c32(src,dst,sw,sh,sp,dp); return; }
|
|
if ((swl == sp)&&(sp == dp)) memcpy_neon(dst, src, sp*sh);
|
|
else for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp) memcpy_neon(dst, src, swl);
|
|
}
|
|
|
|
void scale2x_n16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*2; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale2x_c16(src,dst,sw,sh,sp,dp); return; }
|
|
uint32_t swl64 = swl & ~63;
|
|
uint32_t swrest = swl & 63;
|
|
uint32_t sadd = sp - swl;
|
|
uint32_t dadd = dp*2 - swl*2;
|
|
uint8_t* finofs = (uint8_t*)src + (sp*sh);
|
|
asm volatile (
|
|
"1: add lr, %0, %2 ;" // lr = x64bytes offset
|
|
" add r9, %0, %3 ;" // r9 = lineend offset
|
|
" add r10, %1, %7 ;" // r10 = 2x line offset
|
|
" cmp %0, lr ;"
|
|
" beq 3f ;"
|
|
"2: vldmia %0!, {q8-q11} ;" // 32 pixels 64 bytes
|
|
" vdup.16 d0, d23[3] ;"
|
|
" vdup.16 d1, d23[2] ;"
|
|
" vext.16 d31, d1,d0,#2 ;"
|
|
" vdup.16 d0, d23[1] ;"
|
|
" vdup.16 d1, d23[0] ;"
|
|
" vext.16 d30, d1,d0,#2 ;"
|
|
" vdup.16 d0, d22[3] ;"
|
|
" vdup.16 d1, d22[2] ;"
|
|
" vext.16 d29, d1,d0,#2 ;"
|
|
" vdup.16 d0, d22[1] ;"
|
|
" vdup.16 d1, d22[0] ;"
|
|
" vext.16 d28, d1,d0,#2 ;"
|
|
" vdup.16 d0, d21[3] ;"
|
|
" vdup.16 d1, d21[2] ;"
|
|
" vext.16 d27, d1,d0,#2 ;"
|
|
" vdup.16 d0, d21[1] ;"
|
|
" vdup.16 d1, d21[0] ;"
|
|
" vext.16 d26, d1,d0,#2 ;"
|
|
" vdup.16 d0, d20[3] ;"
|
|
" vdup.16 d1, d20[2] ;"
|
|
" vext.16 d25, d1,d0,#2 ;"
|
|
" vdup.16 d0, d20[1] ;"
|
|
" vdup.16 d1, d20[0] ;"
|
|
" vext.16 d24, d1,d0,#2 ;"
|
|
" vdup.16 d0, d19[3] ;"
|
|
" vdup.16 d1, d19[2] ;"
|
|
" vext.16 d23, d1,d0,#2 ;"
|
|
" vdup.16 d0, d19[1] ;"
|
|
" vdup.16 d1, d19[0] ;"
|
|
" vext.16 d22, d1,d0,#2 ;"
|
|
" vdup.16 d0, d18[3] ;"
|
|
" vdup.16 d1, d18[2] ;"
|
|
" vext.16 d21, d1,d0,#2 ;"
|
|
" vdup.16 d0, d18[1] ;"
|
|
" vdup.16 d1, d18[0] ;"
|
|
" vext.16 d20, d1,d0,#2 ;"
|
|
" vdup.16 d0, d17[3] ;"
|
|
" vdup.16 d1, d17[2] ;"
|
|
" vext.16 d19, d1,d0,#2 ;"
|
|
" vdup.16 d0, d17[1] ;"
|
|
" vdup.16 d1, d17[0] ;"
|
|
" vext.16 d18, d1,d0,#2 ;"
|
|
" vdup.16 d0, d16[3] ;"
|
|
" vdup.16 d1, d16[2] ;"
|
|
" vext.16 d17, d1,d0,#2 ;"
|
|
" vdup.16 d0, d16[1] ;"
|
|
" vdup.16 d1, d16[0] ;"
|
|
" vext.16 d16, d1,d0,#2 ;"
|
|
" cmp %0, lr ;"
|
|
" vstmia %1!, {q8-q15} ;"
|
|
" vstmia r10!, {q8-q15} ;"
|
|
" bne 2b ;"
|
|
"3: cmp %0, r9 ;"
|
|
" beq 5f ;"
|
|
" tst %8, #32 ;"
|
|
" beq 4f ;"
|
|
" vldmia %0!,{q8-q9} ;" // 16 pixels
|
|
" vdup.16 d0, d19[3] ;"
|
|
" vdup.16 d1, d19[2] ;"
|
|
" vext.16 d23, d1,d0,#2 ;"
|
|
" vdup.16 d0, d19[1] ;"
|
|
" vdup.16 d1, d19[0] ;"
|
|
" vext.16 d22, d1,d0,#2 ;"
|
|
" vdup.16 d0, d18[3] ;"
|
|
" vdup.16 d1, d18[2] ;"
|
|
" vext.16 d21, d1,d0,#2 ;"
|
|
" vdup.16 d0, d18[1] ;"
|
|
" vdup.16 d1, d18[0] ;"
|
|
" vext.16 d20, d1,d0,#2 ;"
|
|
" vdup.16 d0, d17[3] ;"
|
|
" vdup.16 d1, d17[2] ;"
|
|
" vext.16 d19, d1,d0,#2 ;"
|
|
" vdup.16 d0, d17[1] ;"
|
|
" vdup.16 d1, d17[0] ;"
|
|
" vext.16 d18, d1,d0,#2 ;"
|
|
" vdup.16 d0, d16[3] ;"
|
|
" vdup.16 d1, d16[2] ;"
|
|
" vext.16 d17, d1,d0,#2 ;"
|
|
" vdup.16 d0, d16[1] ;"
|
|
" vdup.16 d1, d16[0] ;"
|
|
" vext.16 d16, d1,d0,#2 ;"
|
|
" cmp %0, r9 ;"
|
|
" vstmia %1!, {q8-q11} ;"
|
|
" vstmia r10!, {q8-q11} ;"
|
|
" beq 5f ;"
|
|
"4: ldrh lr, [%0],#2 ;" // rest
|
|
" orr lr, lr, lsl #16 ;"
|
|
" cmp %0, r9 ;"
|
|
" str lr, [%1],#4 ;"
|
|
" str lr, [r10],#4 ;"
|
|
" bne 4b ;"
|
|
"5: add %0, %0, %4 ;"
|
|
" add %1, %1, %5 ;"
|
|
" cmp %0, %6 ;"
|
|
" bne 1b "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl64), "r"(swl), "r"(sadd), "r"(dadd), "r"(finofs), "r"(dp), "r"(swrest)
|
|
: "r9","r10","lr","q0","q8","q9","q10","q11","q12","q13","q14","q15","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale2x_n32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*2; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale2x_c32(src,dst,sw,sh,sp,dp); return; }
|
|
uint32_t swl64 = swl & ~63;
|
|
uint32_t sadd = sp - swl;
|
|
uint32_t dadd = dp*2 - swl*2;
|
|
uint8_t* finofs = (uint8_t*)src + (sp*sh);
|
|
asm volatile (
|
|
"1: add lr, %0, %2 ;" // lr = x64bytes offset
|
|
" add r8, %0, %3 ;" // r8 = lineend offset
|
|
" add r9, %1, %7 ;" // r9 = 2x line offset
|
|
" cmp %0, lr ;"
|
|
" beq 3f ;"
|
|
"2: vldmia %0!, {q8-q11} ;" // 16 pixels 64 bytes
|
|
" vdup.32 d31, d23[1] ;"
|
|
" vdup.32 d30, d23[0] ;"
|
|
" vdup.32 d29, d22[1] ;"
|
|
" vdup.32 d28, d22[0] ;"
|
|
" vdup.32 d27, d21[1] ;"
|
|
" vdup.32 d26, d21[0] ;"
|
|
" vdup.32 d25, d20[1] ;"
|
|
" vdup.32 d24, d20[0] ;"
|
|
" vdup.32 d23, d19[1] ;"
|
|
" vdup.32 d22, d19[0] ;"
|
|
" vdup.32 d21, d18[1] ;"
|
|
" vdup.32 d20, d18[0] ;"
|
|
" vdup.32 d19, d17[1] ;"
|
|
" vdup.32 d18, d17[0] ;"
|
|
" vdup.32 d17, d16[1] ;"
|
|
" vdup.32 d16, d16[0] ;"
|
|
" cmp %0, lr ;"
|
|
" vstmia %1!, {q8-q15} ;"
|
|
" vstmia r9!, {q8-q15} ;"
|
|
" bne 2b ;"
|
|
"3: cmp %0, r8 ;"
|
|
" beq 5f ;"
|
|
"4: ldr lr, [%0],#4 ;" // rest
|
|
" vdup.32 d16, lr ;"
|
|
" cmp %0, r8 ;"
|
|
" vstmia %1!, {d16} ;"
|
|
" vstmia r9!, {d16} ;"
|
|
" bne 4b ;"
|
|
"5: add %0, %0, %4 ;"
|
|
" add %1, %1, %5 ;"
|
|
" cmp %0, %6 ;"
|
|
" bne 1b "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl64), "r"(swl), "r"(sadd), "r"(dadd), "r"(finofs), "r"(dp)
|
|
: "r8","r9","lr","q8","q9","q10","q11","q12","q13","q14","q15","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale3x_n16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*3; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale3x_c16(src,dst,sw,sh,sp,dp); return; }
|
|
uint32_t swl32 = swl & ~31;
|
|
uint32_t sadd = sp - swl;
|
|
uint32_t dadd = dp - swl*3;
|
|
uint32_t dwl = swl*3;
|
|
uint32_t dwl128 = dwl & ~127;
|
|
uint32_t dwrest = dwl & 127;
|
|
uint8_t* finofs = (uint8_t*)src + (sp*sh);
|
|
asm volatile (
|
|
"1: mov r11,%1 ;" // dst push
|
|
" add lr, %0, %2 ;" // lr = x32bytes offset
|
|
" add r10, %0, %3 ;" // r10 = lineend offset
|
|
" cmp %0, lr ;"
|
|
" beq 3f ;"
|
|
"2: vldmia %0!, {q8-q9} ;" // 16 pixels 32 bytes
|
|
" vdup.16 d31, d19[3] ;" // FFFF
|
|
" vdup.16 d30, d19[2] ;" // EEEE
|
|
" vdup.16 d29, d19[1] ;" // DDDD
|
|
" vdup.16 d28, d19[0] ;" // CCCC
|
|
" vext.16 d27, d30,d31,#3 ;" // EFFF
|
|
" vext.16 d26, d29,d30,#2 ;" // DDEE
|
|
" vext.16 d25, d28,d29,#1 ;" // CCCD
|
|
" vdup.16 d31, d18[3] ;" // BBBB
|
|
" vdup.16 d30, d18[2] ;" // AAAA
|
|
" vdup.16 d29, d18[1] ;" // 9999
|
|
" vdup.16 d28, d18[0] ;" // 8888
|
|
" vext.16 d24, d30,d31,#3 ;" // ABBB
|
|
" vext.16 d23, d29,d30,#2 ;" // 99AA
|
|
" vext.16 d22, d28,d29,#1 ;" // 8889
|
|
" vdup.16 d31, d17[3] ;" // 7777
|
|
" vdup.16 d30, d17[2] ;" // 6666
|
|
" vdup.16 d29, d17[1] ;" // 5555
|
|
" vdup.16 d28, d17[0] ;" // 4444
|
|
" vext.16 d21, d30,d31,#3 ;" // 6777
|
|
" vext.16 d20, d29,d30,#2 ;" // 5566
|
|
" vext.16 d19, d28,d29,#1 ;" // 4445
|
|
" vdup.16 d31, d16[3] ;" // 3333
|
|
" vdup.16 d30, d16[2] ;" // 2222
|
|
" vdup.16 d29, d16[1] ;" // 1111
|
|
" vdup.16 d28, d16[0] ;" // 0000
|
|
" vext.16 d18, d30,d31,#3 ;" // 2333
|
|
" vext.16 d17, d29,d30,#2 ;" // 1122
|
|
" vext.16 d16, d28,d29,#1 ;" // 0001
|
|
" cmp %0, lr ;"
|
|
" vstmia %1!, {q8-q13} ;"
|
|
" bne 2b ;"
|
|
"3: cmp %0, r10 ;"
|
|
" beq 5f ;"
|
|
"4: ldrh lr, [%0],#2 ;" // rest
|
|
" orr lr, lr, lsl #16 ;"
|
|
" cmp %0, r10 ;"
|
|
" str lr, [%1],#4 ;"
|
|
" strh lr, [%1],#2 ;"
|
|
" bne 4b ;"
|
|
"5: add %0, %4 ;"
|
|
" add %1, %5 ;"
|
|
" mov r12, %1 ;" // r12 = 2x line offset
|
|
" add %1, %8 ;" //
|
|
" add %1, %5 ;" // %1 = 3x line offset
|
|
" add lr, r11, %7 ;" // lr = x128bytes offset
|
|
" add r10, r11, %8 ;" // r10 = lineend offset
|
|
" cmp r11, lr ;"
|
|
" beq 7f ;"
|
|
"6: vldmia r11!, {q8-q15} ;" // 64 pixels 128 bytes
|
|
" vstmia r12!, {q8-q15} ;"
|
|
" vstmia %1!, {q8-q15} ;"
|
|
" cmp r11, lr ;"
|
|
" bne 6b ;"
|
|
"7: cmp r11, r10 ;"
|
|
" beq 10f ;"
|
|
" tst %9, #64 ;"
|
|
" beq 8f ;"
|
|
" vldmia r11!, {q8-q11} ;" // 32 pixels
|
|
" vstmia r12!, {q8-q11} ;"
|
|
" vstmia %1!, {q8-q11} ;"
|
|
" cmp r11, r10 ;"
|
|
" beq 10f ;"
|
|
"8: tst %9, #32 ;"
|
|
" beq 9f ;"
|
|
" vldmia r11!, {q8-q9} ;" // 16 pixels
|
|
" vstmia r12!, {q8-q9} ;"
|
|
" vstmia %1!, {q8-q9} ;"
|
|
" cmp r11, r10 ;"
|
|
" beq 10f ;"
|
|
"9: ldrh lr, [r11],#2 ;" // rest
|
|
" strh lr, [r12],#2 ;"
|
|
" strh lr, [%1],#2 ;"
|
|
" cmp r11, r10 ;"
|
|
" bne 9b ;"
|
|
"10: add %1, %5 ;"
|
|
" cmp %0, %6 ;"
|
|
" bne 1b "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl32), "r"(swl), "r"(sadd), "r"(dadd), "r"(finofs), "r"(dwl128), "r"(dwl), "r"(dwrest)
|
|
: "r10","r11","r12","lr","q8","q9","q10","q11","q12","q13","q14","q15","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale3x_n32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*3; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale3x_c32(src,dst,sw,sh,sp,dp); return; }
|
|
uint32_t swl32 = swl & ~31;
|
|
uint32_t sadd = sp - swl;
|
|
uint32_t dadd = dp - swl*3;
|
|
uint32_t dwl = swl*3;
|
|
uint32_t dwl128 = dwl & ~127;
|
|
uint32_t dwrest = dwl & 127;
|
|
uint8_t* finofs = (uint8_t*)src + (sp*sh);
|
|
asm volatile (
|
|
"1: mov r11,%1 ;" // dst push
|
|
" add lr, %0, %2 ;" // lr = x32bytes offset
|
|
" add r10, %0, %3 ;" // r10 = lineend offset
|
|
" cmp %0, lr ;"
|
|
" beq 3f ;"
|
|
"2: vldmia %0!,{q8-q9} ;" // 8 pixels 32 bytes
|
|
" vdup.32 q15, d19[1] ;" // 7777
|
|
" vdup.32 q14, d19[0] ;" // 6666
|
|
" vdup.32 q1, d18[1] ;" // 5555
|
|
" vdup.32 q0, d18[0] ;" // 4444
|
|
" vext.32 q13, q14,q15,#3 ;" // 6777
|
|
" vext.32 q12, q1,q14,#2 ;" // 5566
|
|
" vext.32 q11, q0,q1,#1 ;" // 4445
|
|
" vdup.32 q15, d17[1] ;" // 3333
|
|
" vdup.32 q14, d17[0] ;" // 2222
|
|
" vdup.32 q1, d16[1] ;" // 1111
|
|
" vdup.32 q0, d16[0] ;" // 0000
|
|
" vext.32 q10, q14,q15,#3 ;" // 2333
|
|
" vext.32 q9, q1,q14,#2 ;" // 1122
|
|
" vext.32 q8, q0,q1,#1 ;" // 0001
|
|
" cmp %0, lr ;"
|
|
" vstmia %1!,{q8-q13} ;"
|
|
" bne 2b ;"
|
|
"3: cmp %0, r10 ;"
|
|
" beq 5f ;"
|
|
"4: ldr lr, [%0],#4 ;" // rest
|
|
" vdup.32 d16, lr ;"
|
|
" cmp %0, r10 ;"
|
|
" vstmia %1!, {d16} ;"
|
|
" str lr, [%1],#4 ;"
|
|
" bne 4b ;"
|
|
"5: add %0, %4 ;"
|
|
" add %1, %5 ;"
|
|
" mov r12, %1 ;" // r12 = 2x line offset
|
|
" add %1, %8 ;" //
|
|
" add %1, %5 ;" // %1 = 3x line offset
|
|
" add lr, r11, %7 ;" // lr = x128bytes offset
|
|
" add r10, r11, %8 ;" // r10 = lineend offset
|
|
" cmp r11, lr ;"
|
|
" beq 7f ;"
|
|
"6: vldmia r11!, {q8-q15} ;" // 32 pixels 128 bytes
|
|
" vstmia r12!, {q8-q15} ;"
|
|
" vstmia %1!, {q8-q15} ;"
|
|
" cmp r11, lr ;"
|
|
" bne 6b ;"
|
|
"7: cmp r11, r10 ;"
|
|
" beq 10f ;"
|
|
" tst %9, #64 ;"
|
|
" beq 8f ;"
|
|
" vldmia r11!, {q8-q11} ;" // 16 pixels
|
|
" vstmia r12!, {q8-q11} ;"
|
|
" vstmia %1!, {q8-q11} ;"
|
|
" cmp r11, r10 ;"
|
|
" beq 10f ;"
|
|
"8: tst %9, #32 ;"
|
|
" beq 9f ;"
|
|
" vldmia r11!, {q8-q9} ;" // 8 pixels
|
|
" vstmia r12!, {q8-q9} ;"
|
|
" vstmia %1!, {q8-q9} ;"
|
|
" cmp r11, r10 ;"
|
|
" beq 10f ;"
|
|
"9: ldr lr, [r11],#4 ;" // rest
|
|
" str lr, [r12],#4 ;"
|
|
" str lr, [%1],#4 ;"
|
|
" cmp r11, r10 ;"
|
|
" bne 9b ;"
|
|
"10: add %1, %5 ;"
|
|
" cmp %0, %6 ;"
|
|
" bne 1b "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl32), "r"(swl), "r"(sadd), "r"(dadd), "r"(finofs), "r"(dwl128), "r"(dwl), "r"(dwrest)
|
|
: "r10","r11","r12","lr","q0","q1","q8","q9","q10","q11","q12","q13","q14","q15","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale4x_n16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*4; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale4x_c16(src,dst,sw,sh,sp,dp); return; }
|
|
uint32_t swl32 = swl & ~31;
|
|
uint32_t sadd = sp - swl;
|
|
uint32_t dadd = dp*4 - swl*4;
|
|
uint8_t* finofs = (uint8_t*)src + (sp*sh);
|
|
asm volatile (
|
|
"1: add lr, %0, %2 ;" // lr = x32bytes offset
|
|
" add r8, %0, %3 ;" // r8 = lineend offset
|
|
" add r9, %1, %7 ;" // r9 = 2x line offset
|
|
" add r10, r9, %7 ;" // r10 = 3x line offset
|
|
" add r11, r10, %7 ;" // r11 = 4x line offset
|
|
" cmp %0, lr ;"
|
|
" beq 3f ;"
|
|
"2: vldmia %0!,{q8-q9} ;" // 16 pixels 32 bytes
|
|
" vdup.16 d31,d19[3] ;"
|
|
" vdup.16 d30,d19[2] ;"
|
|
" vdup.16 d29,d19[1] ;"
|
|
" vdup.16 d28,d19[0] ;"
|
|
" vdup.16 d27,d18[3] ;"
|
|
" vdup.16 d26,d18[2] ;"
|
|
" vdup.16 d25,d18[1] ;"
|
|
" vdup.16 d24,d18[0] ;"
|
|
" vdup.16 d23,d17[3] ;"
|
|
" vdup.16 d22,d17[2] ;"
|
|
" vdup.16 d21,d17[1] ;"
|
|
" vdup.16 d20,d17[0] ;"
|
|
" vdup.16 d19,d16[3] ;"
|
|
" vdup.16 d18,d16[2] ;"
|
|
" vdup.16 d17,d16[1] ;"
|
|
" vdup.16 d16,d16[0] ;"
|
|
" cmp %0, lr ;"
|
|
" vstmia %1!,{q8-q15} ;"
|
|
" vstmia r9!,{q8-q15} ;"
|
|
" vstmia r10!,{q8-q15} ;"
|
|
" vstmia r11!,{q8-q15} ;"
|
|
" bne 2b ;"
|
|
"3: cmp %0, r8 ;"
|
|
" beq 5f ;"
|
|
"4: ldrh lr, [%0],#2 ;" // rest
|
|
" vdup.16 d16, lr ;"
|
|
" cmp %0, r8 ;"
|
|
" vstmia %1!, {d16} ;"
|
|
" vstmia r9!, {d16} ;"
|
|
" vstmia r10!, {d16} ;"
|
|
" vstmia r11!, {d16} ;"
|
|
" bne 4b ;"
|
|
"5: add %0, %0, %4 ;"
|
|
" add %1, %1, %5 ;"
|
|
" cmp %0, %6 ;"
|
|
" bne 1b "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl32), "r"(swl), "r"(sadd), "r"(dadd), "r"(finofs), "r"(dp)
|
|
: "r8","r9","r10","r11","lr","q8","q9","q10","q11","q12","q13","q14","q15","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale4x_n32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*4; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale4x_c32(src,dst,sw,sh,sp,dp); return; }
|
|
uint32_t swl32 = swl & ~31;
|
|
uint32_t sadd = sp - swl;
|
|
uint32_t dadd = dp*4 - swl*4;
|
|
uint8_t* finofs = (uint8_t*)src + (sp*sh);
|
|
asm volatile (
|
|
"1: add lr, %0, %2 ;" // lr = x32bytes offset
|
|
" add r8, %0, %3 ;" // r8 = lineend offset
|
|
" add r9, %1, %7 ;" // r9 = 2x line offset
|
|
" add r10, r9, %7 ;" // r10 = 3x line offset
|
|
" add r11, r10, %7 ;" // r11 = 4x line offset
|
|
" cmp %0, lr ;"
|
|
" beq 3f ;"
|
|
"2: vldmia %0!,{q8-q9} ;" // 8 pixels 32 bytes
|
|
" vdup.32 q15,d19[1] ;"
|
|
" vdup.32 q14,d19[0] ;"
|
|
" vdup.32 q13,d18[1] ;"
|
|
" vdup.32 q12,d18[0] ;"
|
|
" vdup.32 q11,d17[1] ;"
|
|
" vdup.32 q10,d17[0] ;"
|
|
" vdup.32 q9,d16[1] ;"
|
|
" vdup.32 q8,d16[0] ;"
|
|
" cmp %0, lr ;"
|
|
" vstmia %1!,{q8-q15} ;"
|
|
" vstmia r9!,{q8-q15} ;"
|
|
" vstmia r10!,{q8-q15} ;"
|
|
" vstmia r11!,{q8-q15} ;"
|
|
" bne 2b ;"
|
|
"3: cmp %0, r8 ;"
|
|
" beq 5f ;"
|
|
"4: ldr lr, [%0],#4 ;" // rest
|
|
" vdup.32 q8, lr ;"
|
|
" cmp %0, r8 ;"
|
|
" vstmia %1!, {q8} ;"
|
|
" vstmia r9!, {q8} ;"
|
|
" vstmia r10!, {q8} ;"
|
|
" vstmia r11!, {q8} ;"
|
|
" bne 4b ;"
|
|
"5: add %0, %0, %4 ;"
|
|
" add %1, %1, %5 ;"
|
|
" cmp %0, %6 ;"
|
|
" bne 1b "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl32), "r"(swl), "r"(sadd), "r"(dadd), "r"(finofs), "r"(dp)
|
|
: "r8","r9","r10","r11","lr","q8","q9","q10","q11","q12","q13","q14","q15","memory","cc"
|
|
);
|
|
}
|
|
|
|
static inline void scale5x_n16line(void* src, void* dst, uint32_t swl) {
|
|
asm volatile (
|
|
" bic r4, %2, #15 ;" // r4 = swl16
|
|
" add r3, %0, %2 ;" // r3 = lineend offset
|
|
" add r4, %0, r4 ;" // r4 = x16bytes offset
|
|
" cmp %0, r4 ;"
|
|
" beq 2f ;"
|
|
"1: vldmia %0!, {q8} ;" // 8 pixels 16 bytes
|
|
" vdup.16 d25, d17[3] ;" // 7777
|
|
" vdup.16 d27, d17[2] ;" // 6666
|
|
" vdup.16 d26, d17[1] ;" // 5555
|
|
" vdup.16 d21, d17[0] ;" // 4444
|
|
" vext.16 d24, d27,d25,#1 ;" // 6667
|
|
" vext.16 d23, d26,d27,#2 ;" // 5566
|
|
" vext.16 d22, d21,d26,#3 ;" // 4555
|
|
" vdup.16 d20, d16[3] ;" // 3333
|
|
" vdup.16 d27, d16[2] ;" // 2222
|
|
" vdup.16 d26, d16[1] ;" // 1111
|
|
" vdup.16 d16, d16[0] ;" // 0000
|
|
" vext.16 d19, d27,d20,#1 ;" // 2223
|
|
" vext.16 d18, d26,d27,#2 ;" // 1122
|
|
" vext.16 d17, d16,d26,#3 ;" // 0111
|
|
" cmp %0, r4 ;"
|
|
" vstmia %1!, {q8-q12} ;"
|
|
" bne 1b ;"
|
|
"2: cmp %0, r3 ;"
|
|
" beq 4f ;"
|
|
"3: ldrh r4, [%0],#2 ;" // rest
|
|
" orr r4, r4, lsl #16 ;"
|
|
" cmp %0, r3 ;"
|
|
" str r4, [%1],#4 ;"
|
|
" str r4, [%1],#4 ;"
|
|
" strh r4, [%1],#2 ;"
|
|
" bne 3b ;"
|
|
"4: "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl)
|
|
: "r3","r4","q8","q9","q10","q11","q12","q13","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale5x_n16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint16_t);
|
|
uint32_t dwl = swl*5;
|
|
if (!sp) { sp = swl; } if (!dp) { dp = dwl; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale5x_c16(src,dst,sw,sh,sp,dp); return; }
|
|
void* __restrict dstsrc;
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp) {
|
|
scale5x_n16line(src, dst, swl);
|
|
dstsrc = dst; dst = (uint8_t*)dst+dp;
|
|
for (uint32_t i=4; i>0; i--, dst=(uint8_t*)dst+dp) memcpy_neon(dst, dstsrc, dwl);
|
|
}
|
|
}
|
|
|
|
static inline void scale5x_n32line(void* src, void* dst, uint32_t swl) {
|
|
asm volatile (
|
|
" bic r4, %2, #15 ;" // r4 = swl16
|
|
" add r3, %0, %2 ;" // r3 = lineend offset
|
|
" add r4, %0, r4 ;" // r4 = x16bytes offset
|
|
" cmp %0, r4 ;"
|
|
" beq 2f ;"
|
|
"1: vldmia %0!,{q8} ;" // 4 pixels 16 bytes
|
|
" vdup.32 q12, d17[1] ;" // 3333
|
|
" vdup.32 q14, d17[0] ;" // 2222
|
|
" vdup.32 q13, d16[1] ;" // 1111
|
|
" vdup.32 q8, d16[0] ;" // 0000
|
|
" vext.32 q11, q14,q12,#1 ;" // 2223
|
|
" vext.32 q10, q13,q14,#2 ;" // 1122
|
|
" vext.32 q9, q8,q13,#3 ;" // 0111
|
|
" cmp %0, r4 ;"
|
|
" vstmia %1!,{q8-q12} ;"
|
|
" bne 1b ;"
|
|
"2: cmp %0, r3 ;"
|
|
" beq 4f ;"
|
|
"3: ldr r4, [%0],#4 ;" // rest
|
|
" vdup.32 q8, r4 ;"
|
|
" cmp %0, r3 ;"
|
|
" vstmia %1!, {q8} ;"
|
|
" str r4, [%1],#4 ;"
|
|
" bne 3b ;"
|
|
"4: "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl)
|
|
: "r3","r4","q8","q9","q10","q11","q12","q13","q14","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale5x_n32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint32_t);
|
|
uint32_t dwl = swl*5;
|
|
if (!sp) { sp = swl; } if (!dp) { dp = dwl; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale5x_c32(src,dst,sw,sh,sp,dp); return; }
|
|
void* __restrict dstsrc;
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp) {
|
|
scale5x_n32line(src, dst, swl);
|
|
dstsrc = dst; dst = (uint8_t*)dst+dp;
|
|
for (uint32_t i=4; i>0; i--, dst=(uint8_t*)dst+dp) memcpy_neon(dst, dstsrc, dwl);
|
|
}
|
|
}
|
|
|
|
static inline void scale6x_n16line(void* src, void* dst, uint32_t swl) {
|
|
asm volatile (
|
|
" bic r4, %2, #15 ;" // r4 = swl16
|
|
" add r3, %0, %2 ;" // r3 = lineend offset
|
|
" add r4, %0, r4 ;" // r4 = x16bytes offset
|
|
" cmp %0, r4 ;"
|
|
" beq 2f ;"
|
|
"1: vldmia %0!, {q8} ;" // 8 pixels 16 bytes
|
|
" vdup.16 d27, d17[3] ;" // 7777
|
|
" vdup.16 d25, d17[2] ;" // 6666
|
|
" vdup.16 d24, d17[1] ;" // 5555
|
|
" vdup.16 d22, d17[0] ;" // 4444
|
|
" vext.16 d26, d25,d27,#2 ;" // 6677
|
|
" vext.16 d23, d22,d24,#2 ;" // 4455
|
|
" vdup.16 d21, d16[3] ;" // 3333
|
|
" vdup.16 d19, d16[2] ;" // 2222
|
|
" vdup.16 d18, d16[1] ;" // 1111
|
|
" vdup.16 d16, d16[0] ;" // 0000
|
|
" vext.16 d20, d19,d21,#2 ;" // 2233
|
|
" vext.16 d17, d16,d18,#2 ;" // 0011
|
|
" cmp %0, r4 ;"
|
|
" vstmia %1!, {q8-q13} ;"
|
|
" bne 1b ;"
|
|
"2: cmp %0, r3 ;"
|
|
" beq 4f ;"
|
|
"3: ldrh r4, [%0],#2 ;" // rest
|
|
" orr r4, r4, lsl #16 ;"
|
|
" vdup.32 d16, r4 ;"
|
|
" cmp %0, r3 ;"
|
|
" vstmia %1!, {d16} ;"
|
|
" str r4, [%1],#4 ;"
|
|
" bne 3b ;"
|
|
"4: "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl)
|
|
: "r3","r4","q8","q9","q10","q11","q12","q13","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale6x_n16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint16_t);
|
|
uint32_t dwl = swl*6;
|
|
if (!sp) { sp = swl; } if (!dp) { dp = dwl; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale6x_c16(src,dst,sw,sh,sp,dp); return; }
|
|
void* __restrict dstsrc;
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp) {
|
|
scale6x_n16line(src, dst, swl);
|
|
dstsrc = dst; dst = (uint8_t*)dst+dp;
|
|
for (uint32_t i=5; i>0; i--, dst=(uint8_t*)dst+dp) memcpy_neon(dst, dstsrc, dwl);
|
|
}
|
|
}
|
|
|
|
static inline void scale6x_n32line(void* src, void* dst, uint32_t swl) {
|
|
asm volatile (
|
|
" bic r4, %2, #15 ;" // r4 = swl16
|
|
" add r3, %0, %2 ;" // r3 = lineend offset
|
|
" add r4, %0, r4 ;" // r4 = x16bytes offset
|
|
" cmp %0, r4 ;"
|
|
" beq 2f ;"
|
|
"1: vldmia %0!,{q8} ;" // 4 pixels 16 bytes
|
|
" vdup.32 q13, d17[1] ;" // 3333
|
|
" vdup.32 q11, d17[0] ;" // 2222
|
|
" vdup.32 q10, d16[1] ;" // 1111
|
|
" vdup.32 q8, d16[0] ;" // 0000
|
|
" vext.32 q12, q11,q13,#2 ;" // 2233
|
|
" vext.32 q9, q8,q10,#2 ;" // 0011
|
|
" cmp %0, r4 ;"
|
|
" vstmia %1!,{q8-q13} ;"
|
|
" bne 1b ;"
|
|
"2: cmp %0, r3 ;"
|
|
" beq 4f ;"
|
|
"3: ldr r4, [%0],#4 ;" // rest
|
|
" vdup.32 q8, r4 ;"
|
|
" vmov d18, d16 ;"
|
|
" cmp %0, r3 ;"
|
|
" vstmia %1!, {d16-d18} ;"
|
|
" bne 3b ;"
|
|
"4: "
|
|
: "+r"(src), "+r"(dst)
|
|
: "r"(swl)
|
|
: "r3","r4","q8","q9","q10","q11","q12","q13","memory","cc"
|
|
);
|
|
}
|
|
|
|
void scale6x_n32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw * sizeof(uint32_t);
|
|
uint32_t dwl = swl*6;
|
|
if (!sp) { sp = swl; } if (!dp) { dp = dwl; }
|
|
if ( ((uintptr_t)src&3)||((uintptr_t)dst&3)||(sp&3)||(dp&3) ) { scale6x_c32(src,dst,sw,sh,sp,dp); return; }
|
|
void* __restrict dstsrc;
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp) {
|
|
scale6x_n32line(src, dst, swl);
|
|
dstsrc = dst; dst = (uint8_t*)dst+dp;
|
|
for (uint32_t i=5; i>0; i--, dst=(uint8_t*)dst+dp) memcpy_neon(dst, dstsrc, dwl);
|
|
}
|
|
}
|
|
|
|
//
|
|
// C scalers
|
|
//
|
|
|
|
void scale1x_c16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw*sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*1; }
|
|
if ((swl == sp)&&(sp == dp)) memcpy(dst, src, sp*sh);
|
|
else for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp) memcpy(dst, src, swl);
|
|
}
|
|
|
|
void scale1x_c32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t swl = sw*sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } if (!dp) { dp = swl*1; }
|
|
if ((swl == sp)&&(sp == dp)) memcpy(dst, src, sp*sh);
|
|
else for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp) memcpy(dst, src, swl);
|
|
}
|
|
|
|
void scale2x_c16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, dpix1, dpix2, swl = sw*sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } swl*=2; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*2) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<(sw/2); x++, dx+=2) {
|
|
pix = s[x];
|
|
dpix1=(pix & 0x0000FFFF)|(pix<<16);
|
|
dpix2=(pix & 0xFFFF0000)|(pix>>16);
|
|
d[dx] = dpix1; d[dx+1] = dpix2;
|
|
}
|
|
if (sw&1) {
|
|
uint16_t *s16 = (uint16_t*)s;
|
|
uint16_t pix16 = s16[x*2];
|
|
d[dx] = pix16|(pix16<<16);
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
}
|
|
}
|
|
|
|
void scale2x_c32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, swl = sw*sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } swl*=2; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*2) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<sw; x++, dx+=2) {
|
|
pix = s[x];
|
|
d[dx] = pix; d[dx+1] = pix;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
}
|
|
}
|
|
|
|
void scale3x_c16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, dpix1, dpix2, swl = sw*sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } swl*=3; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*3) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<(sw/2); x++, dx+=3) {
|
|
pix = s[x];
|
|
dpix1=(pix & 0x0000FFFF)|(pix<<16);
|
|
dpix2=(pix & 0xFFFF0000)|(pix>>16);
|
|
d[dx] = dpix1; d[dx+1] = pix; d[dx+2] = dpix2;
|
|
}
|
|
if (sw&1) {
|
|
uint16_t *s16 = (uint16_t*)s;
|
|
uint16_t *d16 = (uint16_t*)d;
|
|
uint16_t pix16 = s16[x*2];
|
|
dpix1 = pix16|(pix16<<16);
|
|
d[dx] = dpix1; d16[(dx+1)*2] = pix16;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*2, dst, swl);
|
|
}
|
|
}
|
|
|
|
void scale3x_c32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, swl = sw*sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } swl*=3; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*3) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<sw; x++, dx+=3) {
|
|
pix = s[x];
|
|
d[dx] = pix; d[dx+1] = pix; d[dx+2] = pix;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*2, dst, swl);
|
|
}
|
|
}
|
|
|
|
void scale4x_c16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, dpix1, dpix2, swl = sw*sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } swl*=4; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*4) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<(sw/2); x++, dx+=4) {
|
|
pix = s[x];
|
|
dpix1=(pix & 0x0000FFFF)|(pix<<16);
|
|
dpix2=(pix & 0xFFFF0000)|(pix>>16);
|
|
d[dx] = dpix1; d[dx+1] = dpix1; d[dx+2] = dpix2; d[dx+3] = dpix2;
|
|
}
|
|
if (sw&1) {
|
|
uint16_t *s16 = (uint16_t*)s;
|
|
uint16_t pix16 = s16[x*2];
|
|
dpix1 = pix16|(pix16<<16);
|
|
d[dx] = dpix1; d[dx+1] = dpix1;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*2, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*3, dst, swl);
|
|
}
|
|
}
|
|
|
|
// faster than 4x_c16 when -Ofast/-O3 and aligned width, however dp must be 4xN
|
|
void scale4x_c16b(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; } if (!sp) { sp = sw*sizeof(uint16_t); } if (!dp) { dp = sw*sizeof(uint16_t)*4; }
|
|
uint32_t x, dx, pix, dpix1, dpix2, dp32 = dp / sizeof(uint32_t);
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*4) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<(sw/2); x++, dx+=4) {
|
|
pix = s[x];
|
|
dpix1=(pix & 0x0000FFFF)|(pix<<16);
|
|
dpix2=(pix & 0xFFFF0000)|(pix>>16);
|
|
d[dx] = dpix1; d[dx+1] = dpix1; d[dx+2] = dpix2; d[dx+3] = dpix2;
|
|
d[dp32+dx] = dpix1; d[dp32+dx+1]= dpix1; d[dp32+dx+2]= dpix2; d[dp32+dx+3]= dpix2;
|
|
d[dp32*2+dx] = dpix1; d[dp32*2+dx+1]= dpix1; d[dp32*2+dx+2]= dpix2; d[dp32*2+dx+3]= dpix2;
|
|
d[dp32*3+dx] = dpix1; d[dp32*3+dx+1]= dpix1; d[dp32*3+dx+2]= dpix2; d[dp32*3+dx+3]= dpix2;
|
|
}
|
|
if (sw&1) {
|
|
uint16_t *s16 = (uint16_t*)s;
|
|
uint16_t pix16 = s16[x*2];
|
|
dpix1 = pix16|(pix16<<16);
|
|
d[dx] = dpix1; d[dx+1] = dpix1;
|
|
d[dp32+dx] = dpix1; d[dp32+dx+1] = dpix1;
|
|
d[dp32*2+dx] = dpix1; d[dp32*2+dx+1] = dpix1;
|
|
d[dp32*3+dx] = dpix1; d[dp32*3+dx+1] = dpix1;
|
|
}
|
|
}
|
|
}
|
|
|
|
void scale4x_c32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, swl = sw*sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } swl*=4; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*4) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<sw; x++, dx+=4) {
|
|
pix = s[x];
|
|
d[dx] = pix; d[dx+1] = pix; d[dx+2] = pix; d[dx+3] = pix;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*2, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*3, dst, swl);
|
|
}
|
|
}
|
|
|
|
// faster than 4x_c32 when -Ofast/-O3 and aligned width
|
|
void scale4x_c32b(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; } if (!sp) { sp = sw*sizeof(uint32_t); } if (!dp) { dp = sw*sizeof(uint32_t)*4; }
|
|
uint32_t x, dx, pix, dp32 = dp / sizeof(uint32_t);
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*4) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<sw; x++, dx+=4) {
|
|
pix = s[x];
|
|
d[dx] = pix; d[dx+1] = pix; d[dx+2] = pix; d[dx+3] = pix;
|
|
d[dp32+dx] = pix; d[dp32+dx+1]= pix; d[dp32+dx+2]= pix; d[dp32+dx+3]= pix;
|
|
d[dp32*2+dx] = pix; d[dp32*2+dx+1]= pix; d[dp32*2+dx+2]= pix; d[dp32*2+dx+3]= pix;
|
|
d[dp32*3+dx] = pix; d[dp32*3+dx+1]= pix; d[dp32*3+dx+2]= pix; d[dp32*3+dx+3]= pix;
|
|
}
|
|
}
|
|
}
|
|
|
|
void scale5x_c16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, dpix1, dpix2, swl = sw*sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } swl*=5; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*5) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<(sw/2); x++, dx+=5) {
|
|
pix = s[x];
|
|
dpix1=(pix & 0x0000FFFF)|(pix<<16);
|
|
dpix2=(pix & 0xFFFF0000)|(pix>>16);
|
|
d[dx] = dpix1; d[dx+1] = dpix1; d[dx+2] = pix; d[dx+3] = dpix2; d[dx+4] = dpix2;
|
|
}
|
|
if (sw&1) {
|
|
uint16_t *s16 = (uint16_t*)s;
|
|
uint16_t *d16 = (uint16_t*)d;
|
|
uint16_t pix16 = s16[x*2];
|
|
dpix1 = pix16|(pix16<<16);
|
|
d[dx] = dpix1; d[dx+1] = dpix1; d16[(dx+2)*2] = pix16;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*2, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*3, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*4, dst, swl);
|
|
}
|
|
}
|
|
|
|
void scale5x_c32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, swl = sw*sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } swl*=5; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*5) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<sw; x++, dx+=5) {
|
|
pix = s[x];
|
|
d[dx] = pix; d[dx+1] = pix; d[dx+2] = pix; d[dx+3] = pix; d[dx+4] = pix;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*2, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*3, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*4, dst, swl);
|
|
}
|
|
}
|
|
|
|
void scale6x_c16(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, dpix1, dpix2, swl = sw*sizeof(uint16_t);
|
|
if (!sp) { sp = swl; } swl*=6; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*6) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<(sw/2); x++, dx+=6) {
|
|
pix = s[x];
|
|
dpix1=(pix & 0x0000FFFF)|(pix<<16);
|
|
dpix2=(pix & 0xFFFF0000)|(pix>>16);
|
|
d[dx] = dpix1; d[dx+1] = dpix1; d[dx+2] = dpix1; d[dx+3] = dpix2; d[dx+4] = dpix2; d[dx+5] = dpix2;
|
|
}
|
|
if (sw&1) {
|
|
uint16_t *s16 = (uint16_t*)s;
|
|
uint16_t pix16 = s16[x*2];
|
|
dpix1 = pix16|(pix16<<16);
|
|
d[dx] = dpix1; d[dx+1] = dpix1; d[dx+2] = dpix1;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*2, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*3, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*4, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*5, dst, swl);
|
|
}
|
|
}
|
|
|
|
void scale6x_c32(void* __restrict src, void* __restrict dst, uint32_t sw, uint32_t sh, uint32_t sp, uint32_t dp) {
|
|
if (!sw||!sh) { return; }
|
|
uint32_t x, dx, pix, swl = sw*sizeof(uint32_t);
|
|
if (!sp) { sp = swl; } swl*=6; if (!dp) { dp = swl; }
|
|
for (; sh>0; sh--, src=(uint8_t*)src+sp, dst=(uint8_t*)dst+dp*6) {
|
|
uint32_t *s = (uint32_t* __restrict)src;
|
|
uint32_t *d = (uint32_t* __restrict)dst;
|
|
for (x=dx=0; x<sw; x++, dx+=6) {
|
|
pix = s[x];
|
|
d[dx] = pix; d[dx+1] = pix; d[dx+2] = pix; d[dx+3] = pix; d[dx+4] = pix; d[dx+5] = pix;
|
|
}
|
|
memcpy((uint8_t*)dst+dp*1, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*2, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*3, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*4, dst, swl);
|
|
memcpy((uint8_t*)dst+dp*5, dst, swl);
|
|
}
|
|
}
|