dengzhihao
/
CMSIS-DSP
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CMSIS-DSP: New MVE implementations of the complex dot product and vector product.

Browse Source
pull/19/head
Christophe Favergeon 5 years ago
parent c4283d209f
commit 8fff9ebe29
14 changed files with 738 additions and 364 deletions
Show Stats Download Patch File Download Diff File

88
Source/ComplexMathFunctions/arm_cmplx_dot_prod_f16.c
Unescape Escape View File

@ -86,42 +86,92 @@ void arm_cmplx_dot_prod_f16(
float16_t * realResult,
float16_t * imagResult)
{
uint32_t blockSize = numSamples * CMPLX_DIM; /* loop counters */
uint32_t blkCnt;
int32_t blkCnt;
float16_t real_sum, imag_sum;
f16x8_t vecSrcA, vecSrcB;
f16x8_t vec_acc = vdupq_n_f16(0.0f);
f16x8_t vec_acc = vdupq_n_f16(0.0f16);
f16x8_t vecSrcC, vecSrcD;
blkCnt = (numSamples >> 3);
blkCnt -= 1;
if (blkCnt > 0) {
/* should give more freedom to generate stall free code */
vecSrcA = vld1q( pSrcA);
vecSrcB = vld1q( pSrcB);
pSrcA += 8;
pSrcB += 8;
/* Compute 2 complex samples at a time */
blkCnt = blockSize >> 3U;
while (blkCnt > 0) {
vec_acc = vcmlaq(vec_acc, vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
pSrcA += 8;
while (blkCnt > 0U)
{
vec_acc = vcmlaq_rot90(vec_acc, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
pSrcB += 8;
vec_acc = vcmlaq(vec_acc, vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
pSrcA += 8;
vec_acc = vcmlaq_rot90(vec_acc, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
pSrcB += 8;
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
/* process last elements out of the loop avoid the armclang breaking the SW pipeline */
vec_acc = vcmlaq(vec_acc, vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
vec_acc = vcmlaq_rot90(vec_acc, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
vec_acc = vcmlaq(vec_acc, vecSrcC, vecSrcD);
vec_acc = vcmlaq_rot90(vec_acc, vecSrcC, vecSrcD);
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
* tail
*/
blkCnt = CMPLX_DIM * (numSamples & 7);
while (blkCnt > 0) {
mve_pred16_t p = vctp16q(blkCnt);
pSrcA += 8;
pSrcB += 8;
blkCnt--;
vecSrcA = vldrhq_z_f16(pSrcA, p);
vecSrcB = vldrhq_z_f16(pSrcB, p);
vec_acc = vcmlaq_m(vec_acc, vecSrcA, vecSrcB, p);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p);
blkCnt -= 8;
}
} else {
/* small vector */
blkCnt = numSamples * CMPLX_DIM;
vec_acc = vdupq_n_f16(0.0f16);
/* Tail */
blkCnt = (blockSize & 7);
do {
mve_pred16_t p = vctp16q(blkCnt);
if (blkCnt > 0U)
{
mve_pred16_t p0 = vctp16q(blkCnt);
vecSrcA = vld1q(pSrcA);
vecSrcB = vld1q(pSrcB);
vec_acc = vcmlaq_m(vec_acc, vecSrcA, vecSrcB, p0);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p0);
vecSrcA = vldrhq_z_f16(pSrcA, p);
vecSrcB = vldrhq_z_f16(pSrcB, p);
vec_acc = vcmlaq_m(vec_acc, vecSrcA, vecSrcB, p);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p);
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
*/
pSrcA += 8;
pSrcB += 8;
blkCnt -= 8;
}
while (blkCnt > 0);
}
/* Sum the partial parts */

94
Source/ComplexMathFunctions/arm_cmplx_dot_prod_f32.c
Unescape Escape View File

@ -83,56 +83,94 @@ void arm_cmplx_dot_prod_f32(
float32_t * realResult,
float32_t * imagResult)
{
uint32_t blockSize = numSamples * CMPLX_DIM; /* loop counters */
uint32_t blkCnt;
int32_t blkCnt;
float32_t real_sum, imag_sum;
f32x4_t vecSrcA, vecSrcB;
f32x4_t vec_acc = vdupq_n_f32(0.0f);
float32_t a0,b0,c0,d0;
/* Compute 2 complex samples at a time */
blkCnt = blockSize >> 2U;
f32x4_t vecSrcC, vecSrcD;
while (blkCnt > 0U)
{
blkCnt = numSamples >> 2;
blkCnt -= 1;
if (blkCnt > 0) {
/* should give more freedom to generate stall free code */
vecSrcA = vld1q(pSrcA);
vecSrcB = vld1q(pSrcB);
pSrcA += 4;
pSrcB += 4;
while (blkCnt > 0) {
vec_acc = vcmlaq(vec_acc, vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
pSrcA += 4;
vec_acc = vcmlaq_rot90(vec_acc, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
pSrcB += 4;
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
*/
vec_acc = vcmlaq(vec_acc, vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
pSrcA += 4;
vec_acc = vcmlaq_rot90(vec_acc, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
pSrcB += 4;
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
/* process last elements out of the loop avoid the armclang breaking the SW pipeline */
vec_acc = vcmlaq(vec_acc, vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
real_sum = vgetq_lane(vec_acc, 0) + vgetq_lane(vec_acc, 2);
imag_sum = vgetq_lane(vec_acc, 1) + vgetq_lane(vec_acc, 3);
vec_acc = vcmlaq_rot90(vec_acc, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
/* Tail */
blkCnt = (blockSize & 3) >> 1;
vec_acc = vcmlaq(vec_acc, vecSrcC, vecSrcD);
vec_acc = vcmlaq_rot90(vec_acc, vecSrcC, vecSrcD);
while (blkCnt > 0U)
{
a0 = *pSrcA++;
b0 = *pSrcA++;
c0 = *pSrcB++;
d0 = *pSrcB++;
/*
* tail
*/
blkCnt = CMPLX_DIM * (numSamples & 3);
while (blkCnt > 0) {
mve_pred16_t p = vctp32q(blkCnt);
pSrcA += 4;
pSrcB += 4;
vecSrcA = vldrwq_z_f32(pSrcA, p);
vecSrcB = vldrwq_z_f32(pSrcB, p);
vec_acc = vcmlaq_m(vec_acc, vecSrcA, vecSrcB, p);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p);
blkCnt -= 4;
}
} else {
/* small vector */
blkCnt = numSamples * CMPLX_DIM;
vec_acc = vdupq_n_f32(0.0f);
real_sum += a0 * c0;
imag_sum += a0 * d0;
real_sum -= b0 * d0;
imag_sum += b0 * c0;
do {
mve_pred16_t p = vctp32q(blkCnt);
/* Decrement loop counter */
blkCnt--;
vecSrcA = vldrwq_z_f32(pSrcA, p);
vecSrcB = vldrwq_z_f32(pSrcB, p);
vec_acc = vcmlaq_m(vec_acc, vecSrcA, vecSrcB, p);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p);
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
*/
pSrcA += 4;
pSrcB += 4;
blkCnt -= 4;
}
while (blkCnt > 0);
}
real_sum = vgetq_lane(vec_acc, 0) + vgetq_lane(vec_acc, 2);
imag_sum = vgetq_lane(vec_acc, 1) + vgetq_lane(vec_acc, 3);
/*
* Store the real and imaginary results in the destination buffers

90
Source/ComplexMathFunctions/arm_cmplx_dot_prod_q15.c
Unescape Escape View File

@ -62,27 +62,22 @@ void arm_cmplx_dot_prod_q15(
q31_t * realResult,
q31_t * imagResult)
{
uint32_t blockSize = numSamples * CMPLX_DIM; /* loop counters */
uint32_t blkCnt;
q15_t a0,b0,c0,d0;
q63_t accReal = 0LL; q63_t accImag = 0LL;
int32_t blkCnt;
q63_t accReal = 0LL;
q63_t accImag = 0LL;
q15x8_t vecSrcA, vecSrcB;
q15x8_t vecSrcC, vecSrcD;
blkCnt = (numSamples >> 3);
blkCnt -= 1;
if (blkCnt > 0) {
/* should give more freedom to generate stall free code */
vecSrcA = vld1q(pSrcA);
vecSrcB = vld1q(pSrcB);
pSrcA += 8;
pSrcB += 8;
/* Compute 4 complex samples at a time */
blkCnt = blockSize >> 3;
while (blkCnt > 0U)
{
q15x8_t vecSrcC, vecSrcD;
while (blkCnt > 0) {
accReal = vmlsldavaq(accReal, vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
@ -105,33 +100,60 @@ void arm_cmplx_dot_prod_q15(
blkCnt--;
}
/* Tail */
pSrcA -= 8;
pSrcB -= 8;
/* process last elements out of the loop avoid the armclang breaking the SW pipeline */
accReal = vmlsldavaq(accReal, vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
accImag = vmlaldavaxq(accImag, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
blkCnt = (blockSize & 7) >> 1;
accReal = vmlsldavaq(accReal, vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
while (blkCnt > 0U)
{
a0 = *pSrcA++;
b0 = *pSrcA++;
c0 = *pSrcB++;
d0 = *pSrcB++;
accImag = vmlaldavaxq(accImag, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
accReal += (q31_t)a0 * c0;
accImag += (q31_t)a0 * d0;
accReal -= (q31_t)b0 * d0;
accImag += (q31_t)b0 * c0;
/*
* tail
*/
blkCnt = CMPLX_DIM * (numSamples & 7);
do {
mve_pred16_t p = vctp16q(blkCnt);
/* Decrement loop counter */
blkCnt--;
pSrcA += 8;
pSrcB += 8;
vecSrcA = vldrhq_z_s16(pSrcA, p);
vecSrcB = vldrhq_z_s16(pSrcB, p);
accReal = vmlsldavaq_p(accReal, vecSrcA, vecSrcB, p);
accImag = vmlaldavaxq_p(accImag, vecSrcA, vecSrcB, p);
blkCnt -= 8;
}
while ((int32_t) blkCnt > 0);
} else {
blkCnt = numSamples * CMPLX_DIM;
while (blkCnt > 0) {
mve_pred16_t p = vctp16q(blkCnt);
/* Store real and imaginary result in 8.24 format */
/* Convert real data in 34.30 to 8.24 by 6 right shifts */
*realResult = (q31_t) (accReal >> 6);
/* Convert imaginary data in 34.30 to 8.24 by 6 right shifts */
*imagResult = (q31_t) (accImag >> 6);
vecSrcA = vldrhq_z_s16(pSrcA, p);
vecSrcB = vldrhq_z_s16(pSrcB, p);
accReal = vmlsldavaq_p(accReal, vecSrcA, vecSrcB, p);
accImag = vmlaldavaxq_p(accImag, vecSrcA, vecSrcB, p);
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
*/
pSrcA += 8;
pSrcB += 8;
blkCnt -= 8;
}
}
*realResult = asrl(accReal, (14 - 8));
*imagResult = asrl(accImag, (14 - 8));
}
#else
void arm_cmplx_dot_prod_q15(

101
Source/ComplexMathFunctions/arm_cmplx_dot_prod_q31.c
Unescape Escape View File

@ -64,60 +64,99 @@ void arm_cmplx_dot_prod_q31(
q63_t * realResult,
q63_t * imagResult)
{
uint32_t blockSize = numSamples * CMPLX_DIM; /* loop counters */
uint32_t blkCnt;
q31x4_t vecSrcA, vecSrcB;
int32_t blkCnt;
q63_t accReal = 0LL;
q63_t accImag = 0LL;
q31x4_t vecSrcA, vecSrcB;
q31x4_t vecSrcC, vecSrcD;
q31_t a0,b0,c0,d0;
blkCnt = numSamples >> 2;
blkCnt -= 1;
if (blkCnt > 0) {
/* should give more freedom to generate stall free code */
vecSrcA = vld1q(pSrcA);
vecSrcB = vld1q(pSrcB);
pSrcA += 4;
pSrcB += 4;
/* Compute 2 complex samples at a time */
blkCnt = blockSize >> 2U;
while (blkCnt > 0) {
while (blkCnt > 0U)
{
accReal = vrmlsldavhaq(accReal, vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
pSrcA += 4;
accImag = vrmlaldavhaxq(accImag, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
pSrcB += 4;
accReal = vrmlsldavhaq(accReal, vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
pSrcA += 4;
accImag = vrmlaldavhaxq(accImag, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
pSrcB += 4;
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
/* process last elements out of the loop avoid the armclang breaking the SW pipeline */
accReal = vrmlsldavhaq(accReal, vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
accImag = vrmlaldavhaxq(accImag, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
accReal = vrmlsldavhaq(accReal, vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
accImag = vrmlaldavhaxq(accImag, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
* tail
*/
blkCnt = CMPLX_DIM * (numSamples & 3);
do {
mve_pred16_t p = vctp32q(blkCnt);
pSrcA += 4;
pSrcB += 4;
blkCnt --;
}
accReal = asrl(accReal, (14 - 8));
accImag = asrl(accImag, (14 - 8));
vecSrcA = vldrwq_z_s32(pSrcA, p);
vecSrcB = vldrwq_z_s32(pSrcB, p);
/* Tail */
blkCnt = (blockSize & 3) >> 1;
accReal = vrmlsldavhaq_p(accReal, vecSrcA, vecSrcB, p);
accImag = vrmlaldavhaxq_p(accImag, vecSrcA, vecSrcB, p);
while (blkCnt > 0U)
{
a0 = *pSrcA++;
b0 = *pSrcA++;
c0 = *pSrcB++;
d0 = *pSrcB++;
blkCnt -= 4;
}
while ((int32_t) blkCnt > 0);
} else {
blkCnt = numSamples * CMPLX_DIM;
while (blkCnt > 0) {
mve_pred16_t p = vctp32q(blkCnt);
accReal += ((q63_t)a0 * c0) >> 14;
accImag += ((q63_t)a0 * d0) >> 14;
accReal -= ((q63_t)b0 * d0) >> 14;
accImag += ((q63_t)b0 * c0) >> 14;
vecSrcA = vldrwq_z_s32(pSrcA, p);
vecSrcB = vldrwq_z_s32(pSrcB, p);
/* Decrement loop counter */
blkCnt--;
accReal = vrmlsldavhaq_p(accReal, vecSrcA, vecSrcB, p);
accImag = vrmlaldavhaxq_p(accImag, vecSrcA, vecSrcB, p);
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
*/
pSrcA += 4;
pSrcB += 4;
blkCnt -= 4;
}
}
*realResult = asrl(accReal, (14 - 8));
*imagResult = asrl(accImag, (14 - 8));
/* Store real and imaginary result in destination buffer. */
*realResult = accReal;
*imagResult = accImag;
}
#else

118
Source/ComplexMathFunctions/arm_cmplx_mult_cmplx_f16.c
Unescape Escape View File

@ -78,51 +78,105 @@ void arm_cmplx_mult_cmplx_f16(
float16_t * pDst,
uint32_t numSamples)
{
int32_t blkCnt; /* loop counters */
int32_t blockSize = numSamples;
f16x8_t vecA;
f16x8_t vecB;
f16x8_t vecDst;
int32_t blkCnt;
f16x8_t vecSrcA, vecSrcB;
f16x8_t vecSrcC, vecSrcD;
f16x8_t vec_acc;
blkCnt = (numSamples >> 3);
blkCnt -= 1;
if (blkCnt > 0) {
/* should give more freedom to generate stall free code */
vecSrcA = vld1q(pSrcA);
vecSrcB = vld1q(pSrcB);
pSrcA += 8;
pSrcB += 8;
blkCnt = blockSize * CMPLX_DIM;
blkCnt = blkCnt >> 3;
while (blkCnt > 0) {
vec_acc = vcmulq(vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
pSrcA += 8;
while (blkCnt > 0)
{
vecA = vldrhq_f16(pSrcA);
vecB = vldrhq_f16(pSrcB);
/* C[2 * i] = A[2 * i] * B[2 * i] - A[2 * i + 1] * B[2 * i + 1]. */
vecDst = vcmulq(vecA, vecB);
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i]. */
vecDst = vcmlaq_rot90(vecDst, vecA, vecB);
vstrhq_f16(pDst, vecDst);
vec_acc = vcmlaq_rot90(vec_acc, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
pSrcB += 8;
vst1q(pDst, vec_acc);
pDst += 8;
vec_acc = vcmulq(vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
pSrcA += 8;
vec_acc = vcmlaq_rot90(vec_acc, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
pSrcB += 8;
vst1q(pDst, vec_acc);
pDst += 8;
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
/* process last elements out of the loop avoid the armclang breaking the SW pipeline */
vec_acc = vcmulq(vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
vec_acc = vcmlaq_rot90(vec_acc, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
vst1q(pDst, vec_acc);
pDst += 8;
vec_acc = vcmulq(vecSrcC, vecSrcD);
vec_acc = vcmlaq_rot90(vec_acc, vecSrcC, vecSrcD);
vst1q(pDst, vec_acc);
pDst += 8;
/*
* tail
*/
blkCnt = CMPLX_DIM * (numSamples & 7);
while (blkCnt > 0) {
mve_pred16_t p = vctp16q(blkCnt);
pSrcA += 8;
pSrcB += 8;
vecSrcA = vldrhq_z_f16(pSrcA, p);
vecSrcB = vldrhq_z_f16(pSrcB, p);
vec_acc = vcmulq_m(vuninitializedq_f16(),vecSrcA, vecSrcB, p);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p);
vstrhq_p_f16(pDst, vec_acc, p);
pDst += 8;
blkCnt -= 8;
}
} else {
/* small vector */
blkCnt = numSamples * CMPLX_DIM;
_Float16 a, b, c, d; /* Temporary variables to store real and imaginary values */
/* Tail */
blkCnt = (blockSize & 7) >> 1;
while (blkCnt > 0)
{
/* C[2 * i ] = A[2 * i] * B[2 * i ] - A[2 * i + 1] * B[2 * i + 1]. */
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i ]. */
do {
mve_pred16_t p = vctp16q(blkCnt);
a = *pSrcA++;
b = *pSrcA++;
c = *pSrcB++;
d = *pSrcB++;
vecSrcA = vldrhq_z_f16(pSrcA, p);
vecSrcB = vldrhq_z_f16(pSrcB, p);
/* store result in destination buffer. */
*pDst++ = (a * c) - (b * d);
*pDst++ = (a * d) + (b * c);
vec_acc = vcmulq_m(vuninitializedq_f16(),vecSrcA, vecSrcB, p);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p);
vstrhq_p_f16(pDst, vec_acc, p);
pDst += 8;
/* Decrement loop counter */
blkCnt--;
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
*/
pSrcA += 8;
pSrcB += 8;
blkCnt -= 8;
}
while (blkCnt > 0);
}
}

126
Source/ComplexMathFunctions/arm_cmplx_mult_cmplx_f32.c
Unescape Escape View File

@ -76,54 +76,104 @@ void arm_cmplx_mult_cmplx_f32(
float32_t * pDst,
uint32_t numSamples)
{
uint32_t blkCnt; /* loop counters */
uint32_t blockSize = numSamples; /* loop counters */
float32_t a, b, c, d; /* Temporary variables to store real and imaginary values */
int32_t blkCnt;
f32x4_t vecSrcA, vecSrcB;
f32x4_t vecSrcC, vecSrcD;
f32x4_t vec_acc;
blkCnt = numSamples >> 2;
blkCnt -= 1;
if (blkCnt > 0) {
/* should give more freedom to generate stall free code */
vecSrcA = vld1q(pSrcA);
vecSrcB = vld1q(pSrcB);
pSrcA += 4;
pSrcB += 4;
while (blkCnt > 0) {
vec_acc = vcmulq(vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
pSrcA += 4;
vec_acc = vcmlaq_rot90(vec_acc, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
pSrcB += 4;
vst1q(pDst, vec_acc);
pDst += 4;
vec_acc = vcmulq(vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
pSrcA += 4;
vec_acc = vcmlaq_rot90(vec_acc, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
pSrcB += 4;
vst1q(pDst, vec_acc);
pDst += 4;
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
f32x4x2_t vecA;
f32x4x2_t vecB;
f32x4x2_t vecDst;
/* process last elements out of the loop avoid the armclang breaking the SW pipeline */
vec_acc = vcmulq(vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
/* Compute 4 complex outputs at a time */
blkCnt = blockSize >> 2;
while (blkCnt > 0U)
{
vecA = vld2q(pSrcA); // load & separate real/imag pSrcA (de-interleave 2)
vecB = vld2q(pSrcB); // load & separate real/imag pSrcB
pSrcA += 8;
pSrcB += 8;
vec_acc = vcmlaq_rot90(vec_acc, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
vst1q(pDst, vec_acc);
pDst += 4;
/* C[2 * i] = A[2 * i] * B[2 * i] - A[2 * i + 1] * B[2 * i + 1]. */
vecDst.val[0] = vmulq(vecA.val[0], vecB.val[0]);
vecDst.val[0] = vfmsq(vecDst.val[0],vecA.val[1], vecB.val[1]);
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i]. */
vecDst.val[1] = vmulq(vecA.val[0], vecB.val[1]);
vecDst.val[1] = vfmaq(vecDst.val[1], vecA.val[1], vecB.val[0]);
vec_acc = vcmulq(vecSrcC, vecSrcD);
vec_acc = vcmlaq_rot90(vec_acc, vecSrcC, vecSrcD);
vst1q(pDst, vec_acc);
pDst += 4;
vst2q(pDst, vecDst);
pDst += 8;
/*
* tail
*/
blkCnt = CMPLX_DIM * (numSamples & 3);
while (blkCnt > 0) {
mve_pred16_t p = vctp32q(blkCnt);
pSrcA += 4;
pSrcB += 4;
blkCnt--;
vecSrcA = vldrwq_z_f32(pSrcA, p);
vecSrcB = vldrwq_z_f32(pSrcB, p);
vec_acc = vcmulq_m(vuninitializedq_f32(),vecSrcA, vecSrcB, p);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p);
vstrwq_p_f32(pDst, vec_acc, p);
pDst += 4;
blkCnt -= 4;
}
} else {
/* small vector */
blkCnt = numSamples * CMPLX_DIM;
vec_acc = vdupq_n_f32(0.0f);
/* Tail */
blkCnt = blockSize & 3;
while (blkCnt > 0U)
{
/* C[2 * i ] = A[2 * i] * B[2 * i ] - A[2 * i + 1] * B[2 * i + 1]. */
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i ]. */
do {
mve_pred16_t p = vctp32q(blkCnt);
a = *pSrcA++;
b = *pSrcA++;
c = *pSrcB++;
d = *pSrcB++;
vecSrcA = vldrwq_z_f32(pSrcA, p);
vecSrcB = vldrwq_z_f32(pSrcB, p);
/* store result in destination buffer. */
*pDst++ = (a * c) - (b * d);
*pDst++ = (a * d) + (b * c);
vec_acc = vcmulq_m(vuninitializedq_f32(),vecSrcA, vecSrcB, p);
vec_acc = vcmlaq_rot90_m(vec_acc, vecSrcA, vecSrcB, p);
vstrwq_p_f32(pDst, vec_acc, p);
pDst += 4;
/* Decrement loop counter */
blkCnt--;
/*
* Decrement the blkCnt loop counter
* Advance vector source and destination pointers
*/
pSrcA += 4;
pSrcB += 4;
blkCnt -= 4;
}
while (blkCnt > 0);
}
}

122
Source/ComplexMathFunctions/arm_cmplx_mult_cmplx_q15.c
Unescape Escape View File

@ -57,54 +57,116 @@ void arm_cmplx_mult_cmplx_q15(
q15_t * pDst,
uint32_t numSamples)
{
uint32_t blkCnt; /* loop counters */
uint32_t blockSize = numSamples * CMPLX_DIM; /* loop counters */
q15_t a, b, c, d;
q15x8_t vecA;
q15x8_t vecB;
int32_t blkCnt;
q15x8_t vecSrcA, vecSrcB;
q15x8_t vecSrcC, vecSrcD;
q15x8_t vecDst;
blkCnt = blockSize >> 3;
while (blkCnt > 0U)
blkCnt = (numSamples >> 3);
blkCnt -= 1;
if (blkCnt > 0)
{
vecA = vld1q(pSrcA);
vecB = vld1q(pSrcB);
/* should give more freedom to generate stall free code */
vecSrcA = vld1q(pSrcA);
vecSrcB = vld1q(pSrcB);
pSrcA += 8;
pSrcB += 8;
while (blkCnt > 0)
{
/* C[2 * i] = A[2 * i] * B[2 * i] - A[2 * i + 1] * B[2 * i + 1]. */
vecDst = vqdmlsdhq_s16(vuninitializedq_s16(), vecA, vecB);
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i]. */
vecDst = vqdmladhxq_s16(vecDst, vecA, vecB);
vecDst = vqdmlsdhq(vuninitializedq_s16(), vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
pSrcA += 8;
vecDst = vshrq(vecDst, 2);
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i]. */
vecDst = vqdmladhxq(vecDst, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
pSrcB += 8;
vst1q(pDst, vecDst);
vstrhq_s16(pDst, vshrq(vecDst, 2));
pDst += 8;
blkCnt --;
vecDst = vqdmlsdhq(vuninitializedq_s16(), vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
pSrcA += 8;
vecDst = vqdmladhxq(vecDst, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
pSrcB += 8;
vstrhq_s16(pDst, vshrq(vecDst, 2));
pDst += 8;
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
/* process last elements out of the loop avoid the armclang breaking the SW pipeline */
vecDst = vqdmlsdhq(vuninitializedq_s16(), vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
vecDst = vqdmladhxq(vecDst, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
vstrhq_s16(pDst, vshrq(vecDst, 2));
pDst += 8;
vecDst = vqdmlsdhq(vuninitializedq_s16(), vecSrcC, vecSrcD);
vecDst = vqdmladhxq(vecDst, vecSrcC, vecSrcD);
vstrhq_s16(pDst, vshrq(vecDst, 2));
pDst += 8;
};
/*
* tail
*/
blkCnt = (blockSize & 7) >> 1;
while (blkCnt > 0U)
blkCnt = CMPLX_DIM * (numSamples & 7);
do
{
/* C[2 * i ] = A[2 * i] * B[2 * i ] - A[2 * i + 1] * B[2 * i + 1]. */
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i ]. */
mve_pred16_t p = vctp16q(blkCnt);
a = *pSrcA++;
b = *pSrcA++;
c = *pSrcB++;
d = *pSrcB++;
pSrcA += 8;
pSrcB += 8;
/* store result in 3.13 format in destination buffer. */
*pDst++ = (q15_t) ( (((q31_t) a * c) >> 17) - (((q31_t) b * d) >> 17) );
*pDst++ = (q15_t) ( (((q31_t) a * d) >> 17) + (((q31_t) b * c) >> 17) );
vecSrcA = vldrhq_z_s16(pSrcA, p);
vecSrcB = vldrhq_z_s16(pSrcB, p);
/* Decrement loop counter */
blkCnt--;
vecDst = vqdmlsdhq_m(vuninitializedq_s16(), vecSrcA, vecSrcB, p);
vecDst = vqdmladhxq_m(vecDst, vecSrcA, vecSrcB, p);
vecDst = vshrq_m(vuninitializedq_s16(), vecDst, 2, p);
vstrhq_p_s16(pDst, vecDst, p);
pDst += 8;
blkCnt -= 8;
}
while ((int32_t) blkCnt > 0);
}
else
{
blkCnt = numSamples * CMPLX_DIM;
while (blkCnt > 0) {
mve_pred16_t p = vctp16q(blkCnt);
vecSrcA = vldrhq_z_s16(pSrcA, p);
vecSrcB = vldrhq_z_s16(pSrcB, p);
vecDst = vqdmlsdhq_m(vuninitializedq_s16(), vecSrcA, vecSrcB, p);
vecDst = vqdmladhxq_m(vecDst, vecSrcA, vecSrcB, p);
vecDst = vshrq_m(vuninitializedq_s16(), vecDst, 2, p);
vstrhq_p_s16(pDst, vecDst, p);
pDst += 8;
pSrcA += 8;
pSrcB += 8;
blkCnt -= 8;
}
}
}
#else

123
Source/ComplexMathFunctions/arm_cmplx_mult_cmplx_q31.c
Unescape Escape View File

@ -57,52 +57,111 @@ void arm_cmplx_mult_cmplx_q31(
q31_t * pDst,
uint32_t numSamples)
{
uint32_t blkCnt; /* loop counters */
uint32_t blockSize = numSamples * CMPLX_DIM; /* loop counters */
q31x4_t vecA;
q31x4_t vecB;
int32_t blkCnt;
q31x4_t vecSrcA, vecSrcB;
q31x4_t vecSrcC, vecSrcD;
q31x4_t vecDst;
q31_t a, b, c, d; /* Temporary variables */
/* Compute 2 complex outputs at a time */
blkCnt = blockSize >> 2;
while (blkCnt > 0U)
{
blkCnt = numSamples >> 2;
blkCnt -= 1;
if (blkCnt > 0) {
/* should give more freedom to generate stall free code */
vecSrcA = vld1q(pSrcA);
vecSrcB = vld1q(pSrcB);
pSrcA += 4;
pSrcB += 4;
while (blkCnt > 0) {
vecA = vld1q(pSrcA);
vecB = vld1q(pSrcB);
/* C[2 * i] = A[2 * i] * B[2 * i] - A[2 * i + 1] * B[2 * i + 1]. */
vecDst = vqdmlsdhq(vuninitializedq_s32(),vecA, vecB);
vecDst = vqdmlsdhq(vuninitializedq_s32(), vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
pSrcA += 4;
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i]. */
vecDst = vqdmladhxq(vecDst, vecA, vecB);
vecDst = vqdmladhxq(vecDst, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
pSrcB += 4;
vecDst = vshrq(vecDst, 2);
vst1q(pDst, vecDst);
vst1q(pDst, vshrq(vecDst, 2));
pDst += 4;
blkCnt --;
vecDst = vqdmlsdhq(vuninitializedq_s32(), vecSrcC, vecSrcD);
vecSrcA = vld1q(pSrcA);
pSrcA += 4;
vecDst = vqdmladhxq(vecDst, vecSrcC, vecSrcD);
vecSrcB = vld1q(pSrcB);
pSrcB += 4;
vst1q(pDst, vshrq(vecDst, 2));
pDst += 4;
};
blkCnt = (blockSize & 3) >> 1;
while (blkCnt > 0U)
{
/* C[2 * i ] = A[2 * i] * B[2 * i ] - A[2 * i + 1] * B[2 * i + 1]. */
/* C[2 * i + 1] = A[2 * i] * B[2 * i + 1] + A[2 * i + 1] * B[2 * i ]. */
/*
* Decrement the blockSize loop counter
*/
blkCnt--;
}
a = *pSrcA++;
b = *pSrcA++;
c = *pSrcB++;
d = *pSrcB++;
/* process last elements out of the loop avoid the armclang breaking the SW pipeline */
vecDst = vqdmlsdhq(vuninitializedq_s32(), vecSrcA, vecSrcB);
vecSrcC = vld1q(pSrcA);
/* store result in 3.29 format in destination buffer. */
*pDst++ = (q31_t) ( (((q63_t) a * c) >> 33) - (((q63_t) b * d) >> 33) );
*pDst++ = (q31_t) ( (((q63_t) a * d) >> 33) + (((q63_t) b * c) >> 33) );
vecDst = vqdmladhxq(vecDst, vecSrcA, vecSrcB);
vecSrcD = vld1q(pSrcB);
/* Decrement loop counter */
blkCnt--;
vst1q(pDst, vshrq(vecDst, 2));
pDst += 4;
vecDst = vqdmlsdhq(vuninitializedq_s32(), vecSrcC, vecSrcD);
vecDst = vqdmladhxq(vecDst, vecSrcC, vecSrcD);
vst1q(pDst, vshrq(vecDst, 2));
pDst += 4;
/*
* tail
*/
blkCnt = CMPLX_DIM * (numSamples & 3);
do {
mve_pred16_t p = vctp32q(blkCnt);
pSrcA += 4;
pSrcB += 4;
vecSrcA = vldrwq_z_s32(pSrcA, p);
vecSrcB = vldrwq_z_s32(pSrcB, p);
vecDst = vqdmlsdhq_m(vuninitializedq_s32(), vecSrcA, vecSrcB, p);
vecDst = vqdmladhxq_m(vecDst, vecSrcA, vecSrcB, p);
vecDst = vshrq_m(vuninitializedq_s32(), vecDst, 2, p);
vstrwq_p_s32(pDst, vecDst, p);
pDst += 4;
blkCnt -= 4;
}
while ((int32_t) blkCnt > 0);
} else {
blkCnt = numSamples * CMPLX_DIM;
while (blkCnt > 0) {
mve_pred16_t p = vctp32q(blkCnt);
vecSrcA = vldrwq_z_s32(pSrcA, p);
vecSrcB = vldrwq_z_s32(pSrcB, p);
vecDst = vqdmlsdhq_m(vuninitializedq_s32(), vecSrcA, vecSrcB, p);
vecDst = vqdmladhxq_m(vecDst, vecSrcA, vecSrcB, p);
vecDst = vshrq_m(vuninitializedq_s32(), vecDst, 2, p);
vstrwq_p_s32(pDst, vecDst, p);
pDst += 4;
pSrcA += 4;
pSrcB += 4;
blkCnt -= 4;
}
}
}
#else

2
Testing/Source/Tests/BIQUADQ31.cpp
Unescape Escape View File

@ -11,7 +11,7 @@
#define ABS_32x64_ERROR_Q31 ((q31_t)25)
void checkInnerTail(q31_t *b)
static void checkInnerTail(q31_t *b)
{
ASSERT_TRUE(b[0] == 0);
ASSERT_TRUE(b[1] == 0);

2
Testing/Source/Tests/FIRF16.cpp
Unescape Escape View File

@ -16,7 +16,7 @@ a double precision computation.
static __ALIGNED(8) float16_t coeffArray[32];
#endif
void checkInnerTail(float16_t *b)
static void checkInnerTail(float16_t *b)
{
ASSERT_TRUE(b[0] == 0.0f);
ASSERT_TRUE(b[1] == 0.0f);

2
Testing/Source/Tests/FIRF32.cpp
Unescape Escape View File

@ -16,7 +16,7 @@ a double precision computation.
static __ALIGNED(8) float32_t coeffArray[32];
#endif
void checkInnerTail(float32_t *b)
static void checkInnerTail(float32_t *b)
{
ASSERT_TRUE(b[0] == 0.0f);
ASSERT_TRUE(b[1] == 0.0f);

2
Testing/Source/Tests/FIRQ15.cpp
Unescape Escape View File

@ -10,7 +10,7 @@
static __ALIGNED(8) q15_t coeffArray[32];
#endif
void checkInnerTail(q15_t *b)
static void checkInnerTail(q15_t *b)
{
ASSERT_TRUE(b[0] == 0);
ASSERT_TRUE(b[1] == 0);

2
Testing/Source/Tests/FIRQ31.cpp
Unescape Escape View File

@ -10,7 +10,7 @@
static __ALIGNED(8) q31_t coeffArray[32];
#endif
void checkInnerTail(q31_t *b)
static void checkInnerTail(q31_t *b)
{
ASSERT_TRUE(b[0] == 0);
ASSERT_TRUE(b[1] == 0);

2
Testing/Source/Tests/FIRQ7.cpp
Unescape Escape View File

@ -10,7 +10,7 @@
static __ALIGNED(8) q7_t coeffArray[32];
#endif
void checkInnerTail(q7_t *b)
static void checkInnerTail(q7_t *b)
{
ASSERT_TRUE(b[0] == 0);
ASSERT_TRUE(b[1] == 0);

Write Preview
Loading…
Cancel
Save