dengzhihao
/
CMSIS-DSP
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CMSIS-DSP: FFT bit reversal unrolling

Browse Source 
Issue index update earlier, adding more distance with
           subsequent gather loads
           Added Out-of-place variants
pull/19/head
FabKlein 5 years ago
parent 0a5a96d904
commit a509fa39d4
5 changed files with 539 additions and 453 deletions
Show Stats Download Patch File Download Diff File

267
PrivateInclude/arm_vec_fft.h
Unescape Escape View File

@ -47,6 +47,273 @@ extern "C"
#define MVE_CMPLX_SUB_FX_A_ixB(A,B) vhcaddq_rot270(A,B)
/**
@brief In-place 32 bit reversal function for helium
@param[in,out] pSrc points to in-place buffer of unknown 32-bit data type
@param[in] bitRevLen bit reversal table length
@param[in] pBitRevTab points to bit reversal table
@return none
*/
__STATIC_INLINE void arm_bitreversal_32_inpl_mve(
uint32_t *pSrc,
const uint16_t bitRevLen,
const uint16_t *pBitRevTab)
{
uint64_t *src = (uint64_t *) pSrc;
int32_t blkCnt; /* loop counters */
uint32x4_t bitRevTabOff;
uint32x4_t one = vdupq_n_u32(1);
uint64x2_t inLow, inHigh;
uint64x2_t bitRevOff1Low, bitRevOff0Low;
uint64x2_t bitRevOff1High, bitRevOff0High;
/* load scheduling to increase gather load idx update / gather load distance */
bitRevTabOff = vldrhq_u32(pBitRevTab);
pBitRevTab += 4;
bitRevOff0Low = vmullbq_int_u32(bitRevTabOff, one);
bitRevOff0High = vmulltq_int_u32(bitRevTabOff, one);
blkCnt = bitRevLen / 8;
while (blkCnt > 0) {
bitRevTabOff = vldrhq_u32(pBitRevTab);
pBitRevTab += 4;
/* 64-bit index expansion */
bitRevOff1Low = vmullbq_int_u32(bitRevTabOff, one);
bitRevOff1High = vmulltq_int_u32(bitRevTabOff, one);
inLow = vldrdq_gather_offset_u64(src, bitRevOff0Low);
inHigh = vldrdq_gather_offset_u64(src, bitRevOff0High);
vstrdq_scatter_offset_u64(src, bitRevOff0Low, inHigh);
vstrdq_scatter_offset_u64(src, bitRevOff0High, inLow);
/* unrolled */
bitRevTabOff = vldrhq_u32(pBitRevTab);
pBitRevTab += 4;
bitRevOff0Low = vmullbq_int_u32(bitRevTabOff, one);
bitRevOff0High = vmulltq_int_u32(bitRevTabOff, one);
inLow = vldrdq_gather_offset_u64(src, bitRevOff1Low);
inHigh = vldrdq_gather_offset_u64(src, bitRevOff1High);
vstrdq_scatter_offset_u64(src, bitRevOff1Low, inHigh);
vstrdq_scatter_offset_u64(src, bitRevOff1High, inLow);
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
if (bitRevLen & 7) {
/* FFT size = 16 */
inLow = vldrdq_gather_offset_u64(src, bitRevOff0Low);
inHigh = vldrdq_gather_offset_u64(src, bitRevOff0High);
vstrdq_scatter_offset_u64(src, bitRevOff0Low, inHigh);
vstrdq_scatter_offset_u64(src, bitRevOff0High, inLow);
}
}
/**
@brief In-place 16 bit reversal function for helium
@param[in,out] pSrc points to in-place buffer of unknown 16-bit data type
@param[in] bitRevLen bit reversal table length
@param[in] pBitRevTab points to bit reversal table
@return none
*/
__STATIC_INLINE void arm_bitreversal_16_inpl_mve(
uint16_t *pSrc,
const uint16_t bitRevLen,
const uint16_t *pBitRevTab)
{
uint32_t *src = (uint32_t *) pSrc;
int32_t blkCnt; /* loop counters */
uint32x4_t bitRevTabOff;
uint16x8_t one = vdupq_n_u16(1);
uint32x4_t bitRevOff1Low, bitRevOff0Low;
uint32x4_t bitRevOff1High, bitRevOff0High;
uint32x4_t inLow, inHigh;
/* load scheduling to increase gather load idx update / gather load distance */
bitRevTabOff = vldrhq_u16(pBitRevTab);
pBitRevTab += 8;
bitRevOff0Low = vmullbq_int_u16(bitRevTabOff, one);
bitRevOff0High = vmulltq_int_u16(bitRevTabOff, one);
bitRevOff0Low = vshrq_n_u16(bitRevOff0Low, 3);
bitRevOff0High = vshrq_n_u16(bitRevOff0High, 3);
blkCnt = (bitRevLen / 16);
while (blkCnt > 0U) {
bitRevTabOff = vldrhq_u16(pBitRevTab);
pBitRevTab += 8;
bitRevOff1Low = vmullbq_int_u16(bitRevTabOff, one);
bitRevOff1High = vmulltq_int_u16(bitRevTabOff, one);
bitRevOff1Low = vshrq_n_u16(bitRevOff1Low, 3);
bitRevOff1High = vshrq_n_u16(bitRevOff1High, 3);
inLow = vldrwq_gather_shifted_offset_u32(src, bitRevOff0Low);
inHigh = vldrwq_gather_shifted_offset_u32(src, bitRevOff0High);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff0Low, inHigh);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff0High, inLow);
/* loop unrolling */
bitRevTabOff = vldrhq_u16(pBitRevTab);
pBitRevTab += 8;
bitRevOff0Low = vmullbq_int_u16(bitRevTabOff, one);
bitRevOff0High = vmulltq_int_u16(bitRevTabOff, one);
bitRevOff0Low = vshrq_n_u16(bitRevOff0Low, 3);
bitRevOff0High = vshrq_n_u16(bitRevOff0High, 3);
inLow = vldrwq_gather_shifted_offset_u32(src, bitRevOff1Low);
inHigh = vldrwq_gather_shifted_offset_u32(src, bitRevOff1High);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff1Low, inHigh);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff1High, inLow);
blkCnt--;
}
/* tail handling */
blkCnt = bitRevLen & 0xf;
if (blkCnt == 8) {
inLow = vldrwq_gather_shifted_offset_u32(src, bitRevOff0Low);
inHigh = vldrwq_gather_shifted_offset_u32(src, bitRevOff0High);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff0Low, inHigh);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff0High, inLow);
} else if (blkCnt == 12) {
/* FFT 16 special case */
mve_pred16_t p = vctp16q(4);
bitRevTabOff = vldrhq_z_u16(pBitRevTab, p);
inLow = vldrwq_gather_shifted_offset_u32(src, bitRevOff0Low);
inHigh = vldrwq_gather_shifted_offset_u32(src, bitRevOff0High);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff0Low, inHigh);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff0High, inLow);
bitRevOff0Low = vmullbq_int_u16(bitRevTabOff, one);
bitRevOff0High = vmulltq_int_u16(bitRevTabOff, one);
bitRevOff0Low = vshrq_n_u16(bitRevOff0Low, 3);
bitRevOff0High = vshrq_n_u16(bitRevOff0High, 3);
inLow = vldrwq_gather_shifted_offset_z_u32(src, bitRevOff0Low, p);
inHigh = vldrwq_gather_shifted_offset_z_u32(src, bitRevOff0High, p);
vstrwq_scatter_shifted_offset_p_u32(src, bitRevOff0Low, inHigh, p);
vstrwq_scatter_shifted_offset_p_u32(src, bitRevOff0High, inLow, p);
}
}
/**
@brief Out-of-place 32 bit reversal function for helium
@param[out] pDst points to destination buffer of unknown 32-bit data type
@param[in] pSrc points to input buffer of unknown 32-bit data type
@param[in] fftLen FFT length
@return none
*/
__STATIC_INLINE void arm_bitreversal_32_outpl_mve(void *pDst, void *pSrc, uint32_t fftLen)
{
uint32x4_t idxOffs0, idxOffs1, bitRevOffs0, bitRevOffs1;
uint32_t bitRevPos, blkCnt;
uint32_t *pDst32 = (uint32_t *) pDst;
/* fwd indexes */
idxOffs0 = vdupq_n_u32(0);
idxOffs1 = vdupq_n_u32(0);
idxOffs0[0] = 0; idxOffs0[2] = 4;
idxOffs1[0] = 8; idxOffs1[2] = 12;
bitRevPos = (31 - __CLZ(fftLen)) + 5;
blkCnt = fftLen >> 2;
/* issued earlier to increase gather load idx update / gather load distance */
/* bit-reverse fwd indexes */
bitRevOffs0 = vbrsrq(idxOffs0, bitRevPos);
bitRevOffs1 = vbrsrq(idxOffs1, bitRevPos);
while (blkCnt > 0U) {
uint64x2_t vecIn;
vecIn = vldrdq_gather_offset_u64(pSrc, (int64x2_t) bitRevOffs0);
idxOffs0 = idxOffs0 + 16;
vst1q(pDst32, (uint32x4_t) vecIn);
pDst32 += 4;
bitRevOffs0 = vbrsrq(idxOffs0, bitRevPos);
vecIn = vldrdq_gather_offset_u64(pSrc, (int64x2_t) bitRevOffs1);
idxOffs1 = idxOffs1 + 16;
vst1q(pDst32, (uint32x4_t) vecIn);
pDst32 += 4;
bitRevOffs1 = vbrsrq(idxOffs1, bitRevPos);
blkCnt--;
}
}
/**
@brief Out-of-place 16 bit reversal function for helium
@param[out] pDst points to destination buffer of unknown 16-bit data type
@param[in] pSrc points to input buffer of unknown 16-bit data type
@param[in] fftLen FFT length
@return none
*/
__STATIC_INLINE void arm_bitreversal_16_outpl_mve(void *pDst, void *pSrc, uint32_t fftLen)
{
uint32x4_t idxOffs0, idxOffs1, bitRevOffs0, bitRevOffs1;
uint32_t bitRevPos, blkCnt;
uint16_t *pDst16 = (uint16_t *) pDst;
uint32_t incrIdx = 0;
/* fwd indexes */
idxOffs0 = vidupq_wb_u32(&incrIdx, 4); // {0, 4, 8, 12}
idxOffs1 = vidupq_wb_u32(&incrIdx, 4); // {16, 20, 24, 28}
bitRevPos = (31 - __CLZ(fftLen)) + 4;
blkCnt = fftLen >> 3;
/* issued earlier to increase gather load idx update / gather load distance */
/* bit-reverse fwd indexes */
bitRevOffs0 = vbrsrq(idxOffs0, bitRevPos);
bitRevOffs1 = vbrsrq(idxOffs1, bitRevPos);
while (blkCnt > 0U) {
uint32x4_t vecIn;
vecIn = vldrwq_gather_offset_s32(pSrc, bitRevOffs0);
idxOffs0 = idxOffs0 + 32;
vst1q(pDst16, (uint16x8_t) vecIn);
pDst16 += 8;
bitRevOffs0 = vbrsrq(idxOffs0, bitRevPos);
vecIn = vldrwq_gather_offset_s32(pSrc, bitRevOffs1);
idxOffs1 = idxOffs1 + 32;
vst1q(pDst16, (uint16x8_t) vecIn);
pDst16 += 8;
bitRevOffs1 = vbrsrq(idxOffs1, bitRevPos);
blkCnt--;
}
}
#endif /* (defined(ARM_MATH_MVEF) || defined(ARM_MATH_HELIUM)) && !defined(ARM_MATH_AUTOVECTORIZE)*/

62
Source/TransformFunctions/arm_cfft_f16.c
Unescape Escape View File

@ -88,66 +88,6 @@ static float16_t arm_inverse_fft_length_f16(uint16_t fftLen)
}
static void arm_bitreversal_f16_inpl_mve(
uint16_t *pSrc,
const uint16_t bitRevLen,
const uint16_t *pBitRevTab)
{
uint32_t *src = (uint32_t *)pSrc;
uint32_t blkCnt; /* loop counters */
uint32x4_t bitRevTabOff;
uint16x8_t one = vdupq_n_u16(1);
blkCnt = (bitRevLen / 2) / 4;
while (blkCnt > 0U) {
bitRevTabOff = vldrhq_u16(pBitRevTab);
pBitRevTab += 8;
uint32x4_t bitRevOff1 = vmullbq_int_u16(bitRevTabOff, one);
uint32x4_t bitRevOff2 = vmulltq_int_u16(bitRevTabOff, one);
bitRevOff1 = bitRevOff1 >> 3;
bitRevOff2 = bitRevOff2 >> 3;
uint32x4_t in1 = vldrwq_gather_shifted_offset_u32(src, bitRevOff1);
uint32x4_t in2 = vldrwq_gather_shifted_offset_u32(src, bitRevOff2);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff1, in2);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff2, in1);
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
/*
* tail
* (will be merged thru tail predication)
*/
blkCnt = bitRevLen & 7;
if (blkCnt > 0U) {
mve_pred16_t p0 = vctp16q(blkCnt);
bitRevTabOff = vldrhq_z_u16(pBitRevTab, p0);
uint32x4_t bitRevOff1 = vmullbq_int_u16(bitRevTabOff, one);
uint32x4_t bitRevOff2 = vmulltq_int_u16(bitRevTabOff, one);
bitRevOff1 = bitRevOff1 >> 3;
bitRevOff2 = bitRevOff2 >> 3;
uint32x4_t in1 = vldrwq_gather_shifted_offset_z_u32(src, bitRevOff1, p0);
uint32x4_t in2 = vldrwq_gather_shifted_offset_z_u32(src, bitRevOff2, p0);
vstrwq_scatter_shifted_offset_p_u32(src, bitRevOff1, in2, p0);
vstrwq_scatter_shifted_offset_p_u32(src, bitRevOff2, in1, p0);
}
}
static void _arm_radix4_butterfly_f16_mve(const arm_cfft_instance_f16 * S,float16_t * pSrc, uint32_t fftLen)
{
f16x8_t vecTmp0, vecTmp1;
@ -634,7 +574,7 @@ void arm_cfft_f16(
if (bitReverseFlag)
{
arm_bitreversal_f16_inpl_mve((uint16_t*)pSrc, S->bitRevLength, S->pBitRevTable);
arm_bitreversal_16_inpl_mve((uint16_t*)pSrc, S->bitRevLength, S->pBitRevTable);
}
}

33
Source/TransformFunctions/arm_cfft_f32.c
Unescape Escape View File

@ -87,37 +87,6 @@ static float32_t arm_inverse_fft_length_f32(uint16_t fftLen)
}
static void arm_bitreversal_f32_inpl_mve(
uint32_t *pSrc,
const uint16_t bitRevLen,
const uint16_t *pBitRevTab)
{
uint64_t *src = (uint64_t *) pSrc;
uint32_t blkCnt; /* loop counters */
uint32x4_t bitRevTabOff;
uint32x4_t one = vdupq_n_u32(1);
blkCnt = (bitRevLen / 2) / 2;
while (blkCnt > 0U) {
bitRevTabOff = vldrhq_u32(pBitRevTab);
pBitRevTab += 4;
uint64x2_t bitRevOff1 = vmullbq_int_u32(bitRevTabOff, one);
uint64x2_t bitRevOff2 = vmulltq_int_u32(bitRevTabOff, one);
uint64x2_t in1 = vldrdq_gather_offset_u64(src, bitRevOff1);
uint64x2_t in2 = vldrdq_gather_offset_u64(src, bitRevOff2);
vstrdq_scatter_offset_u64(src, bitRevOff1, in2);
vstrdq_scatter_offset_u64(src, bitRevOff2, in1);
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
}
static void _arm_radix4_butterfly_f32_mve(const arm_cfft_instance_f32 * S,float32_t * pSrc, uint32_t fftLen)
@ -607,7 +576,7 @@ void arm_cfft_f32(
if (bitReverseFlag)
{
arm_bitreversal_f32_inpl_mve((uint32_t*)pSrc, S->bitRevLength, S->pBitRevTable);
arm_bitreversal_32_inpl_mve((uint32_t*)pSrc, S->bitRevLength, S->pBitRevTable);
}
}

59
Source/TransformFunctions/arm_cfft_q15.c
Unescape Escape View File

@ -33,65 +33,6 @@
#include "arm_vec_fft.h"
static void arm_bitreversal_16_inpl_mve(
uint16_t *pSrc,
const uint16_t bitRevLen,
const uint16_t *pBitRevTab)
{
uint32_t *src = (uint32_t *)pSrc;
uint32_t blkCnt; /* loop counters */
uint32x4_t bitRevTabOff;
uint16x8_t one = vdupq_n_u16(1);
blkCnt = (bitRevLen / 2) / 4;
while (blkCnt > 0U) {
bitRevTabOff = vldrhq_u16(pBitRevTab);
pBitRevTab += 8;
uint32x4_t bitRevOff1 = vmullbq_int_u16(bitRevTabOff, one);
uint32x4_t bitRevOff2 = vmulltq_int_u16(bitRevTabOff, one);
bitRevOff1 = bitRevOff1 >> 3;
bitRevOff2 = bitRevOff2 >> 3;
uint32x4_t in1 = vldrwq_gather_shifted_offset_u32(src, bitRevOff1);
uint32x4_t in2 = vldrwq_gather_shifted_offset_u32(src, bitRevOff2);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff1, in2);
vstrwq_scatter_shifted_offset_u32(src, bitRevOff2, in1);
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
/*
* tail
* (will be merged thru tail predication)
*/
blkCnt = bitRevLen & 7;
if (blkCnt > 0U) {
mve_pred16_t p0 = vctp16q(blkCnt);
bitRevTabOff = vldrhq_z_u16(pBitRevTab, p0);
uint32x4_t bitRevOff1 = vmullbq_int_u16(bitRevTabOff, one);
uint32x4_t bitRevOff2 = vmulltq_int_u16(bitRevTabOff, one);
bitRevOff1 = bitRevOff1 >> 3;
bitRevOff2 = bitRevOff2 >> 3;
uint32x4_t in1 = vldrwq_gather_shifted_offset_z_u32(src, bitRevOff1, p0);
uint32x4_t in2 = vldrwq_gather_shifted_offset_z_u32(src, bitRevOff2, p0);
vstrwq_scatter_shifted_offset_p_u32(src, bitRevOff1, in2, p0);
vstrwq_scatter_shifted_offset_p_u32(src, bitRevOff2, in1, p0);
}
}
static void _arm_radix4_butterfly_q15_mve(
const arm_cfft_instance_q15 * S,
q15_t *pSrc,

31
Source/TransformFunctions/arm_cfft_q31.c
Unescape Escape View File

@ -34,37 +34,6 @@
#include "arm_vec_fft.h"
static void arm_bitreversal_32_inpl_mve(
uint32_t *pSrc,
const uint16_t bitRevLen,
const uint16_t *pBitRevTab)
{
uint64_t *src = (uint64_t *) pSrc;
uint32_t blkCnt; /* loop counters */
uint32x4_t bitRevTabOff;
uint32x4_t one = vdupq_n_u32(1);
blkCnt = (bitRevLen / 2) / 2;
while (blkCnt > 0U) {
bitRevTabOff = vldrhq_u32(pBitRevTab);
pBitRevTab += 4;
uint64x2_t bitRevOff1 = vmullbq_int_u32(bitRevTabOff, one);
uint64x2_t bitRevOff2 = vmulltq_int_u32(bitRevTabOff, one);
uint64x2_t in1 = vldrdq_gather_offset_u64(src, bitRevOff1);
uint64x2_t in2 = vldrdq_gather_offset_u64(src, bitRevOff2);
vstrdq_scatter_offset_u64(src, bitRevOff1, in2);
vstrdq_scatter_offset_u64(src, bitRevOff2, in1);
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
}
static void _arm_radix4_butterfly_q31_mve(
const arm_cfft_instance_q31 * S,

Write Preview
Loading…
Cancel
Save