From 02f890cef70b8cdaba091dafbb037320ea62611d Mon Sep 17 00:00:00 2001 From: Athulya Raj Raji Mohini Date: Fri, 8 May 2026 14:04:01 +0100 Subject: [PATCH 1/2] Arm: Update Neon implementation for Filter7x7Blk Simplify the `foldingRequired` calculation by deriving the four row distances from the first row in each 4 row group. Since `startHeight` and loop row offset are `STEP_Y` aligned, any CTU wrap can only occur after those four rows. This new version improves performance by approximately ~2% when compared to the existing Neon version when benchmarked on a Neoverse V2 with LLVM 22. --- .../arm/neon/AdaptiveLoopFilter_neon.cpp | 194 +++++++++--------- 1 file changed, 100 insertions(+), 94 deletions(-) diff --git a/source/Lib/CommonLib/arm/neon/AdaptiveLoopFilter_neon.cpp b/source/Lib/CommonLib/arm/neon/AdaptiveLoopFilter_neon.cpp index 7ab485e0..88e6f4a7 100644 --- a/source/Lib/CommonLib/arm/neon/AdaptiveLoopFilter_neon.cpp +++ b/source/Lib/CommonLib/arm/neon/AdaptiveLoopFilter_neon.cpp @@ -98,6 +98,57 @@ static inline void processALF_CoeffPair_neon( const Pel* ptr0, const Pel* ptr1, accB = vaddq_s16( diff2, diff3 ); } +static inline void prepareALF7x7Params_neon( const short* filterCoeff0, const short* filterCoeff1, + const short* filterClip0, const short* filterClip1, + ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2] ) +{ + // Copy all 12 coeffs and clips. + const int16x8_t cl0_04 = vld1q_s16( filterClip0 + 0 ); + const int16x8_t cl1_04 = vld1q_s16( filterClip1 + 0 ); + const int16x4_t cl0_8 = vld1_s16( filterClip0 + 8 ); + const int16x4_t cl1_8 = vld1_s16( filterClip1 + 8 ); + + const int16x4_t cl0_0 = vget_low_s16( cl0_04 ); + const int16x4_t cl0_4 = vget_high_s16( cl0_04 ); + const int16x4_t cl1_0 = vget_low_s16( cl1_04 ); + const int16x4_t cl1_4 = vget_high_s16( cl1_04 ); + + param[0].clipA = vcombine_s16( vdup_lane_s16( cl0_0, 0 ), vdup_lane_s16( cl1_0, 0 ) ); + param[0].clipB = vcombine_s16( vdup_lane_s16( cl0_0, 1 ), vdup_lane_s16( cl1_0, 1 ) ); + param[1].clipA = vcombine_s16( vdup_lane_s16( cl0_0, 2 ), vdup_lane_s16( cl1_0, 2 ) ); + param[1].clipB = vcombine_s16( vdup_lane_s16( cl0_0, 3 ), vdup_lane_s16( cl1_0, 3 ) ); + param[2].clipA = vcombine_s16( vdup_lane_s16( cl0_4, 0 ), vdup_lane_s16( cl1_4, 0 ) ); + param[2].clipB = vcombine_s16( vdup_lane_s16( cl0_4, 1 ), vdup_lane_s16( cl1_4, 1 ) ); + param[3].clipA = vcombine_s16( vdup_lane_s16( cl0_4, 2 ), vdup_lane_s16( cl1_4, 2 ) ); + param[3].clipB = vcombine_s16( vdup_lane_s16( cl0_4, 3 ), vdup_lane_s16( cl1_4, 3 ) ); + param[4].clipA = vcombine_s16( vdup_lane_s16( cl0_8, 0 ), vdup_lane_s16( cl1_8, 0 ) ); + param[4].clipB = vcombine_s16( vdup_lane_s16( cl0_8, 1 ), vdup_lane_s16( cl1_8, 1 ) ); + param[5].clipA = vcombine_s16( vdup_lane_s16( cl0_8, 2 ), vdup_lane_s16( cl1_8, 2 ) ); + param[5].clipB = vcombine_s16( vdup_lane_s16( cl0_8, 3 ), vdup_lane_s16( cl1_8, 3 ) ); + + param[0].negClipA = vnegq_s16( param[0].clipA ); + param[0].negClipB = vnegq_s16( param[0].clipB ); + param[1].negClipA = vnegq_s16( param[1].clipA ); + param[1].negClipB = vnegq_s16( param[1].clipB ); + param[2].negClipA = vnegq_s16( param[2].clipA ); + param[2].negClipB = vnegq_s16( param[2].clipB ); + param[3].negClipA = vnegq_s16( param[3].clipA ); + param[3].negClipB = vnegq_s16( param[3].clipB ); + param[4].negClipA = vnegq_s16( param[4].clipA ); + param[4].negClipB = vnegq_s16( param[4].clipB ); + param[5].negClipA = vnegq_s16( param[5].clipA ); + param[5].negClipB = vnegq_s16( param[5].clipB ); + + const int16x8_t c0_04 = vld1q_s16( filterCoeff0 + 0 ); + const int16x8_t c1_04 = vld1q_s16( filterCoeff1 + 0 ); + param[0].coeff = vget_low_s16( c0_04 ); + param[2].coeff = vget_high_s16( c0_04 ); + param[1].coeff = vget_low_s16( c1_04 ); + param[3].coeff = vget_high_s16( c1_04 ); + param[4].coeff = vld1_s16( filterCoeff0 + 8 ); + param[5].coeff = vld1_s16( filterCoeff1 + 8 ); +} + template static inline void processALF7x7Row_neon( const Pel* pImg0, Pel* pDst, const int distance, const ptrdiff_t srcStride, const ALFGroupParam param[6], const int clpRngMax ) @@ -196,10 +247,10 @@ static inline void processALF7x7Row_neon( const Pel* pImg0, Pel* pDst, const int acc = vcombine_s16( vrshrn_n_s32( accLo, SHIFT ), vrshrn_n_s32( accHi, SHIFT ) ); } - acc = vqaddq_s16( acc, curr ); - int16x8_t dst = clip3_neon( acc, vdupq_n_s16( 0 ), vdupq_n_s16( clpRngMax ) ); + uint16x8_t dst = vvdec_vsqaddq_u16( vreinterpretq_u16_s16( curr ), acc ); + dst = vminq_u16( dst, vdupq_n_u16( clpRngMax ) ); - vst1q_s16( pDst, dst ); + vst1q_s16( pDst, vreinterpretq_s16_u16( dst ) ); } void Filter7x7Blk_neon( const AlfClassifier* classifier, const PelUnitBuf& recDst, const CPelUnitBuf& recSrc, @@ -230,118 +281,73 @@ void Filter7x7Blk_neon( const AlfClassifier* classifier, const PelUnitBuf& recDs for( int i = 0; i < height; i += STEP_Y ) { - int yVbPos = ( startHeight + i ) & ( vbCTUHeight - 1 ); // Row’s position inside its CTU. - - auto calculateNextVbPosDist = [&yVbPos, vbPos, vbCTUHeight]() -> int - { - int distance = vbPos - yVbPos; - if( ++yVbPos == vbCTUHeight ) - { - yVbPos = 0; - } - return distance; - }; - - int VbDistance[STEP_Y]; - VbDistance[0] = calculateNextVbPosDist(); - VbDistance[1] = calculateNextVbPosDist(); - VbDistance[2] = calculateNextVbPosDist(); - VbDistance[3] = calculateNextVbPosDist(); + const int yVbPos = ( startHeight + i ) & ( vbCTUHeight - 1 ); // Row’s position inside its CTU. + const int firstDistance = vbPos - yVbPos; + // The four rows require folding if any distance lies in the 7x7 VB fold range. const bool foldingRequired = - ( VbDistance[0] >= ALF_7x7_VB_FOLD_MIN_DIST && VbDistance[0] <= ALF_7x7_VB_FOLD_MAX_DIST ) || - ( VbDistance[1] >= ALF_7x7_VB_FOLD_MIN_DIST && VbDistance[1] <= ALF_7x7_VB_FOLD_MAX_DIST ) || - ( VbDistance[2] >= ALF_7x7_VB_FOLD_MIN_DIST && VbDistance[2] <= ALF_7x7_VB_FOLD_MAX_DIST ) || - ( VbDistance[3] >= ALF_7x7_VB_FOLD_MIN_DIST && VbDistance[3] <= ALF_7x7_VB_FOLD_MAX_DIST ); + firstDistance >= ALF_7x7_VB_FOLD_MIN_DIST && firstDistance <= ALF_7x7_VB_FOLD_MAX_DIST + 3; int cl_index = ( i / 4 ) * ( AdaptiveLoopFilter::m_CLASSIFICATION_BLK_SIZE / 4 ); const Pel* pImg0 = src; Pel* pDst = dst; - int j = width; - do + if( foldingRequired ) { - const AlfClassifier& cl0 = classifier[cl_index]; - const int index0 = - cl0.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl0.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; - const AlfClassifier& cl1 = classifier[cl_index + 1]; - const int index1 = - cl1.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl1.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; - - const short* filterCoeff0 = filterSet + index0; - const short* filterClip0 = fClipSet + index0; - const short* filterCoeff1 = filterSet + index1; - const short* filterClip1 = fClipSet + index1; - - ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2]; - - // Copy all 12 coeffs and clips. - const int16x8_t cl0_04 = vld1q_s16( filterClip0 + 0 ); - const int16x8_t cl1_04 = vld1q_s16( filterClip1 + 0 ); - const int16x4_t cl0_8 = vld1_s16( filterClip0 + 8 ); - const int16x4_t cl1_8 = vld1_s16( filterClip1 + 8 ); - - const int16x4_t cl0_0 = vget_low_s16( cl0_04 ); - const int16x4_t cl0_4 = vget_high_s16( cl0_04 ); - const int16x4_t cl1_0 = vget_low_s16( cl1_04 ); - const int16x4_t cl1_4 = vget_high_s16( cl1_04 ); - - param[0].clipA = vcombine_s16( vdup_lane_s16( cl0_0, 0 ), vdup_lane_s16( cl1_0, 0 ) ); - param[0].clipB = vcombine_s16( vdup_lane_s16( cl0_0, 1 ), vdup_lane_s16( cl1_0, 1 ) ); - param[1].clipA = vcombine_s16( vdup_lane_s16( cl0_0, 2 ), vdup_lane_s16( cl1_0, 2 ) ); - param[1].clipB = vcombine_s16( vdup_lane_s16( cl0_0, 3 ), vdup_lane_s16( cl1_0, 3 ) ); - param[2].clipA = vcombine_s16( vdup_lane_s16( cl0_4, 0 ), vdup_lane_s16( cl1_4, 0 ) ); - param[2].clipB = vcombine_s16( vdup_lane_s16( cl0_4, 1 ), vdup_lane_s16( cl1_4, 1 ) ); - param[3].clipA = vcombine_s16( vdup_lane_s16( cl0_4, 2 ), vdup_lane_s16( cl1_4, 2 ) ); - param[3].clipB = vcombine_s16( vdup_lane_s16( cl0_4, 3 ), vdup_lane_s16( cl1_4, 3 ) ); - param[4].clipA = vcombine_s16( vdup_lane_s16( cl0_8, 0 ), vdup_lane_s16( cl1_8, 0 ) ); - param[4].clipB = vcombine_s16( vdup_lane_s16( cl0_8, 1 ), vdup_lane_s16( cl1_8, 1 ) ); - param[5].clipA = vcombine_s16( vdup_lane_s16( cl0_8, 2 ), vdup_lane_s16( cl1_8, 2 ) ); - param[5].clipB = vcombine_s16( vdup_lane_s16( cl0_8, 3 ), vdup_lane_s16( cl1_8, 3 ) ); - - param[0].negClipA = vnegq_s16( param[0].clipA ); - param[0].negClipB = vnegq_s16( param[0].clipB ); - param[1].negClipA = vnegq_s16( param[1].clipA ); - param[1].negClipB = vnegq_s16( param[1].clipB ); - param[2].negClipA = vnegq_s16( param[2].clipA ); - param[2].negClipB = vnegq_s16( param[2].clipB ); - param[3].negClipA = vnegq_s16( param[3].clipA ); - param[3].negClipB = vnegq_s16( param[3].clipB ); - param[4].negClipA = vnegq_s16( param[4].clipA ); - param[4].negClipB = vnegq_s16( param[4].clipB ); - param[5].negClipA = vnegq_s16( param[5].clipA ); - param[5].negClipB = vnegq_s16( param[5].clipB ); - - const int16x8_t c0_04 = vld1q_s16( filterCoeff0 + 0 ); - const int16x8_t c1_04 = vld1q_s16( filterCoeff1 + 0 ); - param[0].coeff = vget_low_s16( c0_04 ); - param[2].coeff = vget_high_s16( c0_04 ); - param[1].coeff = vget_low_s16( c1_04 ); - param[3].coeff = vget_high_s16( c1_04 ); - param[4].coeff = vld1_s16( filterCoeff0 + 8 ); - param[5].coeff = vld1_s16( filterCoeff1 + 8 ); + // startHeight and i are STEP_Y-aligned, so any CTU wrap can only occur after these four row distances. + const int VbDistance[STEP_Y] = { firstDistance, firstDistance - 1, firstDistance - 2, firstDistance - 3 }; - if( foldingRequired ) + int j = width; + do { + const AlfClassifier& cl0 = classifier[cl_index]; + const int index0 = + cl0.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl0.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; + const AlfClassifier& cl1 = classifier[cl_index + 1]; + const int index1 = + cl1.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl1.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; + + ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2]; + prepareALF7x7Params_neon( filterSet + index0, filterSet + index1, fClipSet + index0, fClipSet + index1, param ); + for( int k = 0; k < STEP_Y; k++ ) { processALF7x7Row_neon( pImg0 + k * srcStride, pDst + k * dstStride, VbDistance[k], srcStride, param, clpRngMax ); } - } - else + + j -= STEP_X; + cl_index += 2; + pImg0 += STEP_X; + pDst += STEP_X; + } while( j != 0 ); + } + else + { + int j = width; + do { + const AlfClassifier& cl0 = classifier[cl_index]; + const int index0 = + cl0.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl0.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; + const AlfClassifier& cl1 = classifier[cl_index + 1]; + const int index1 = + cl1.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl1.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; + + ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2]; + prepareALF7x7Params_neon( filterSet + index0, filterSet + index1, fClipSet + index0, fClipSet + index1, param ); + for( int k = 0; k < STEP_Y; k++ ) { processALF7x7Row_neon( pImg0 + k * srcStride, pDst + k * dstStride, 0, srcStride, param, clpRngMax ); } - } - j -= STEP_X; - cl_index += 2; - pImg0 += STEP_X; - pDst += STEP_X; - } while( j != 0 ); + j -= STEP_X; + cl_index += 2; + pImg0 += STEP_X; + pDst += STEP_X; + } while( j != 0 ); + } src += srcStride * STEP_Y; dst += dstStride * STEP_Y; From bbfff1080cfa1b256f8b380d3ab1fabebb6c8af7 Mon Sep 17 00:00:00 2001 From: Athulya Raj Raji Mohini Date: Thu, 14 May 2026 13:44:27 +0100 Subject: [PATCH 2/2] Arm: Update ALF Filter7x7Blk to skip clip handling Skip clipping for fixed filter sets in Filter7x7Blk_neon. Fixed ALF luma filter sets use the default clip value, so clipping does not affect the filtered differences. Add a `isFixedFilterSet` flag through the ALF filter callbacks, including 5x5, scalar, Neon, and x86 paths. Use `isFixedFilterSet` in the Neon 7x7 path to select a no-clip implementation. The fixed-filter no-clip case shows an approximate 45% reduction when benchmarked on a Neoverse V2 with LLVM 21. Non-fixed filter sets continue to use the clipped path. Extend filterBlk unit test to check both clipped and fixed-filter 7x7 modes. --- source/Lib/CommonLib/AdaptiveLoopFilter.cpp | 10 +- source/Lib/CommonLib/AdaptiveLoopFilter.h | 6 +- .../arm/neon/AdaptiveLoopFilter_neon.cpp | 375 ++++++++++++------ .../Lib/CommonLib/x86/AdaptiveLoopFilterX86.h | 18 +- tests/vvdec_unit_test/vvdec_unit_test.cpp | 29 +- 5 files changed, 287 insertions(+), 151 deletions(-) diff --git a/source/Lib/CommonLib/AdaptiveLoopFilter.cpp b/source/Lib/CommonLib/AdaptiveLoopFilter.cpp index 6b014c81..b9a03e5e 100644 --- a/source/Lib/CommonLib/AdaptiveLoopFilter.cpp +++ b/source/Lib/CommonLib/AdaptiveLoopFilter.cpp @@ -507,6 +507,8 @@ void AdaptiveLoopFilter::filterAreaLuma( const CPelUnitBuf& srcBuf, { const short* coeff = nullptr; const short* clip = nullptr; + const bool isFixedFilterSet = filterSetIndex < NUM_FIXED_FILTER_SETS; + if( filterSetIndex >= NUM_FIXED_FILTER_SETS ) { CHECK( slice->getNumAlfAps() <= ( filterSetIndex - NUM_FIXED_FILTER_SETS ), "deduemm" ); @@ -536,7 +538,7 @@ void AdaptiveLoopFilter::filterAreaLuma( const CPelUnitBuf& srcBuf, int nWidth = std::min( j + m_CLASSIFICATION_BLK_SIZE, right ) - j; m_deriveClassificationBlk( classifier[tId].data(), srcBuf.Y(), Area( j, i, nWidth, nHeight ), m_inputBitDepth + 4, m_alfVBLumaCTUHeight, m_alfVBLumaPos ); - m_filter7x7Blk ( classifier[tId].data(), dstBuf, srcBuf, Area( j, i, nWidth, nHeight ), COMPONENT_Y, coeff, clip, clpRngs, m_alfVBLumaCTUHeight, m_alfVBLumaPos ); + m_filter7x7Blk ( classifier[tId].data(), dstBuf, srcBuf, Area( j, i, nWidth, nHeight ), COMPONENT_Y, coeff, clip, clpRngs, m_alfVBLumaCTUHeight, m_alfVBLumaPos, isFixedFilterSet ); } } } @@ -569,7 +571,7 @@ void AdaptiveLoopFilter::filterAreaChroma( const CPelUnitBuf& srcBuf, alfSliceParam.chrmClippFinal + altIdx * MAX_NUM_ALF_CHROMA_COEFF, clpRngs, m_alfVBChmaCTUHeight, - m_alfVBChmaPos ); + m_alfVBChmaPos, false ); } else { @@ -1180,8 +1182,10 @@ void AdaptiveLoopFilter::filterBlk( const AlfClassifier* classifier, const short* fClipSet, const ClpRng& clpRng, int vbCTUHeight, - int vbPos ) + int vbPos, + const bool isFixedFilterSet ) { + ( void ) isFixedFilterSet; const bool bChroma = isChroma( compId ); if( bChroma ) diff --git a/source/Lib/CommonLib/AdaptiveLoopFilter.h b/source/Lib/CommonLib/AdaptiveLoopFilter.h index 28094a47..72b629e1 100644 --- a/source/Lib/CommonLib/AdaptiveLoopFilter.h +++ b/source/Lib/CommonLib/AdaptiveLoopFilter.h @@ -127,10 +127,10 @@ class AdaptiveLoopFilter void ( *m_filter5x5Blk )( const AlfClassifier* classifier, const PelUnitBuf& recDst, const CPelUnitBuf& recSrc, const Area& blk, const ComponentID compId, const short* filterSet, const short* fClipSet, - const ClpRng& clpRng, int vbCTUHeight, int vbPos ); + const ClpRng& clpRng, int vbCTUHeight, int vbPos, const bool isFixedFilterSet ); void ( *m_filter7x7Blk )( const AlfClassifier* classifier, const PelUnitBuf& recDst, const CPelUnitBuf& recSrc, const Area& blk, const ComponentID compId, const short* filterSet, const short* fClipSet, - const ClpRng& clpRng, int vbCTUHeight, int vbPos ); + const ClpRng& clpRng, int vbCTUHeight, int vbPos, const bool isFixedFilterSet ); void ( *m_deriveClassificationBlk )( AlfClassifier *classifier, const CPelBuf& srcLuma, const Area& blk, const int shift, int vbCTUHeight, int vbPos ); protected: @@ -143,7 +143,7 @@ class AdaptiveLoopFilter void filterAreaChromaBothCc ( const CPelUnitBuf& srcBuf, PelUnitBuf& dstBuf, const Area& blkLuma, const Area& blkChroma, const Slice* slice, const APS* const* aps, const CtuAlfData& ctuAlfData, const ClpRngs& clpRngs ); template - static void filterBlk ( const AlfClassifier *classifier, const PelUnitBuf &recDst, const CPelUnitBuf& recSrc, const Area& blk, const ComponentID compId, const short* filterSet, const short* fClipSet, const ClpRng& clpRng, int vbCTUHeight, int vbPos ); + static void filterBlk ( const AlfClassifier *classifier, const PelUnitBuf &recDst, const CPelUnitBuf& recSrc, const Area& blk, const ComponentID compId, const short* filterSet, const short* fClipSet, const ClpRng& clpRng, int vbCTUHeight, int vbPos, const bool isFixedFilterSet ); void ( *m_filterCcAlf ) ( const PelBuf &dstBuf, const CPelUnitBuf &recSrc, const Area &blkDst, const Area &blkSrc, const ComponentID compId, const int16_t *filterCoeff, const ClpRngs &clpRngs, int vbCTUHeight, int vbPos ); void ( *m_filterCcAlfBoth ) ( const PelBuf& dstBufCb, const PelBuf& dstBufCr, const CPelUnitBuf &recSrc, const Area &blkDst, const Area &blkSrc, const int16_t* filterCoeffCb, const int16_t* filterCoeffCr, diff --git a/source/Lib/CommonLib/arm/neon/AdaptiveLoopFilter_neon.cpp b/source/Lib/CommonLib/arm/neon/AdaptiveLoopFilter_neon.cpp index 88e6f4a7..92e7d1e3 100644 --- a/source/Lib/CommonLib/arm/neon/AdaptiveLoopFilter_neon.cpp +++ b/source/Lib/CommonLib/arm/neon/AdaptiveLoopFilter_neon.cpp @@ -66,7 +66,6 @@ typedef struct ALFGroupParam { int16x8_t clipA, negClipA; int16x8_t clipB, negClipB; - int16x4_t coeff; } ALFGroupParam; static inline int16x8_t clip3_neon( const int16x8_t val, const int16x8_t minVal, const int16x8_t maxVal ) @@ -98,11 +97,24 @@ static inline void processALF_CoeffPair_neon( const Pel* ptr0, const Pel* ptr1, accB = vaddq_s16( diff2, diff3 ); } -static inline void prepareALF7x7Params_neon( const short* filterCoeff0, const short* filterCoeff1, - const short* filterClip0, const short* filterClip1, - ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2] ) +static inline void processALF_CoeffPairNoClip_neon( const Pel* ptr0, const Pel* ptr1, const Pel* ptr2, const Pel* ptr3, + const int16x8_t currx2, int16x8_t& accA, int16x8_t& accB ) { - // Copy all 12 coeffs and clips. + const int16x8_t v0 = vld1q_s16( ptr0 ); + const int16x8_t v1 = vld1q_s16( ptr1 ); + const int16x8_t v2 = vld1q_s16( ptr2 ); + const int16x8_t v3 = vld1q_s16( ptr3 ); + + const int16x8_t v0v1 = vaddq_s16( v0, v1 ); + const int16x8_t v2v3 = vaddq_s16( v2, v3 ); + accA = vsubq_s16( v0v1, currx2 ); + accB = vsubq_s16( v2v3, currx2 ); +} + +static inline void prepareALF7x7ClipParams_neon( const short* filterClip0, const short* filterClip1, + ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2] ) +{ + // Copy all 12 clips. const int16x8_t cl0_04 = vld1q_s16( filterClip0 + 0 ); const int16x8_t cl1_04 = vld1q_s16( filterClip1 + 0 ); const int16x4_t cl0_8 = vld1_s16( filterClip0 + 8 ); @@ -138,23 +150,154 @@ static inline void prepareALF7x7Params_neon( const short* filterCoeff0, const sh param[4].negClipB = vnegq_s16( param[4].clipB ); param[5].negClipA = vnegq_s16( param[5].clipA ); param[5].negClipB = vnegq_s16( param[5].clipB ); +} +static inline void prepareALF7x7CoeffParams_neon( const short* filterCoeff0, const short* filterCoeff1, + int16x4_t coeff[MAX_NUM_ALF_LUMA_COEFF / 2] ) +{ + // Copy all 12 coeffs. const int16x8_t c0_04 = vld1q_s16( filterCoeff0 + 0 ); const int16x8_t c1_04 = vld1q_s16( filterCoeff1 + 0 ); - param[0].coeff = vget_low_s16( c0_04 ); - param[2].coeff = vget_high_s16( c0_04 ); - param[1].coeff = vget_low_s16( c1_04 ); - param[3].coeff = vget_high_s16( c1_04 ); - param[4].coeff = vld1_s16( filterCoeff0 + 8 ); - param[5].coeff = vld1_s16( filterCoeff1 + 8 ); + coeff[0] = vget_low_s16( c0_04 ); + coeff[2] = vget_high_s16( c0_04 ); + coeff[1] = vget_low_s16( c1_04 ); + coeff[3] = vget_high_s16( c1_04 ); + coeff[4] = vld1_s16( filterCoeff0 + 8 ); + coeff[5] = vld1_s16( filterCoeff1 + 8 ); } -template -static inline void processALF7x7Row_neon( const Pel* pImg0, Pel* pDst, const int distance, const ptrdiff_t srcStride, - const ALFGroupParam param[6], const int clpRngMax ) +template +static inline int16x8_t roundAndClipALF7x7Row_neon( const int16x8_t curr, const int32x4_t accLo, const int32x4_t accHi, + const int distance, const int clpRngMax ) { - constexpr int SHIFT = AdaptiveLoopFilter::m_NUM_BITS - 1; + constexpr int shift = AdaptiveLoopFilter::m_NUM_BITS - 1; + + int16x8_t acc; + if( IsFoldingRequired && distance >= 0 && distance <= 1 ) + { + // Weaker filter, closer to VB. + acc = vcombine_s16( vrshrn_n_s32( accLo, shift + 3 ), vrshrn_n_s32( accHi, shift + 3 ) ); + } + else + { + // Regular filter strength. + acc = vcombine_s16( vrshrn_n_s32( accLo, shift ), vrshrn_n_s32( accHi, shift ) ); + } + + uint16x8_t dst = vvdec_vsqaddq_u16( vreinterpretq_u16_s16( curr ), acc ); + dst = vminq_u16( dst, vdupq_n_u16( clpRngMax ) ); + return vreinterpretq_s16_u16( dst ); +} + +template +static inline int16x8_t processALF7x7Row_Clip_neon( const Pel* pImg0, const Pel* pImg1, const Pel* pImg2, + const Pel* pImg3, const Pel* pImg4, const Pel* pImg5, + const Pel* pImg6, const ALFGroupParam* param, + const int16x4_t coeff[6], const int distance, const int clpRngMax ) +{ + const int16x8_t curr = vld1q_s16( pImg0 ); + int16x8_t a, b; + + processALF_CoeffPair_neon( pImg5 + 0, pImg6 + 0, pImg3 + 1, pImg4 - 1, curr, param[0].clipA, param[0].clipB, + param[0].negClipA, param[0].negClipB, a, b ); + int32x4_t accLo = vmull_lane_s16( vget_low_s16( a ), coeff[0], 0 ); + int32x4_t accHi = vmull_lane_s16( vget_high_s16( a ), coeff[1], 0 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[0], 1 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[1], 1 ); + + processALF_CoeffPair_neon( pImg3 + 0, pImg4 + 0, pImg3 - 1, pImg4 + 1, curr, param[1].clipA, param[1].clipB, + param[1].negClipA, param[1].negClipB, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[0], 2 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[1], 2 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[0], 3 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[1], 3 ); + + processALF_CoeffPair_neon( pImg1 + 2, pImg2 - 2, pImg1 + 1, pImg2 - 1, curr, param[2].clipA, param[2].clipB, + param[2].negClipA, param[2].negClipB, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[2], 0 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[3], 0 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[2], 1 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[3], 1 ); + + processALF_CoeffPair_neon( pImg1 + 0, pImg2 + 0, pImg1 - 1, pImg2 + 1, curr, param[3].clipA, param[3].clipB, + param[3].negClipA, param[3].negClipB, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[2], 2 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[3], 2 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[2], 3 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[3], 3 ); + + processALF_CoeffPair_neon( pImg1 - 2, pImg2 + 2, pImg0 + 3, pImg0 - 3, curr, param[4].clipA, param[4].clipB, + param[4].negClipA, param[4].negClipB, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[4], 0 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[5], 0 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[4], 1 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[5], 1 ); + + processALF_CoeffPair_neon( pImg0 + 2, pImg0 - 2, pImg0 + 1, pImg0 - 1, curr, param[5].clipA, param[5].clipB, + param[5].negClipA, param[5].negClipB, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[4], 2 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[5], 2 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[4], 3 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[5], 3 ); + + return roundAndClipALF7x7Row_neon( curr, accLo, accHi, distance, clpRngMax ); +} +template +static inline int16x8_t processALF7x7Row_noClip_neon( const Pel* pImg0, const Pel* pImg1, const Pel* pImg2, + const Pel* pImg3, const Pel* pImg4, const Pel* pImg5, + const Pel* pImg6, const int16x4_t coeff[6], const int distance, + const int clpRngMax ) +{ + const int16x8_t curr = vld1q_s16( pImg0 ); + int16x8_t currx2 = vshlq_n_s16( curr, 1 ); + + int16x8_t a, b; + + processALF_CoeffPairNoClip_neon( pImg5 + 0, pImg6 + 0, pImg3 + 1, pImg4 - 1, currx2, a, b ); + int32x4_t accLo = vmull_lane_s16( vget_low_s16( a ), coeff[0], 0 ); + int32x4_t accHi = vmull_lane_s16( vget_high_s16( a ), coeff[1], 0 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[0], 1 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[1], 1 ); + + processALF_CoeffPairNoClip_neon( pImg3 + 0, pImg4 + 0, pImg3 - 1, pImg4 + 1, currx2, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[0], 2 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[1], 2 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[0], 3 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[1], 3 ); + + processALF_CoeffPairNoClip_neon( pImg1 + 2, pImg2 - 2, pImg1 + 1, pImg2 - 1, currx2, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[2], 0 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[3], 0 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[2], 1 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[3], 1 ); + + processALF_CoeffPairNoClip_neon( pImg1 + 0, pImg2 + 0, pImg1 - 1, pImg2 + 1, currx2, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[2], 2 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[3], 2 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[2], 3 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[3], 3 ); + + processALF_CoeffPairNoClip_neon( pImg1 - 2, pImg2 + 2, pImg0 + 3, pImg0 - 3, currx2, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[4], 0 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[5], 0 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[4], 1 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[5], 1 ); + + processALF_CoeffPairNoClip_neon( pImg0 + 2, pImg0 - 2, pImg0 + 1, pImg0 - 1, currx2, a, b ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), coeff[4], 2 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), coeff[5], 2 ); + accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), coeff[4], 3 ); + accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), coeff[5], 3 ); + + return roundAndClipALF7x7Row_neon( curr, accLo, accHi, distance, clpRngMax ); +} + +template +static inline int16x8_t processALF7x7Row_neon( const Pel* pImg0, const ptrdiff_t srcStride, const int16x4_t coeff[6], + const int clpRngMax, const int distance, + const ALFGroupParam* param = nullptr ) +{ const Pel* pImg1 = pImg0 + srcStride; // y+1 const Pel* pImg2 = pImg0 - srcStride; // y-1 const Pel* pImg3 = pImg1 + srcStride; // y+2 @@ -162,7 +305,7 @@ static inline void processALF7x7Row_neon( const Pel* pImg0, Pel* pDst, const int const Pel* pImg5 = pImg3 + srcStride; // y+3 const Pel* pImg6 = pImg4 - srcStride; // y-3 - if( isFoldingRequired ) + if( IsFoldingRequired ) { // When the current line is near the VB (vbPos), some of rows access could point across the CTU boundary. // Distance = 0 or 1, reuse current row (pImg0). @@ -190,72 +333,66 @@ static inline void processALF7x7Row_neon( const Pel* pImg0, Pel* pDst, const int } } - const int16x8_t curr = vld1q_s16( pImg0 ); - - int16x8_t a, b; - processALF_CoeffPair_neon( pImg5 + 0, pImg6 + 0, pImg3 + 1, pImg4 - 1, curr, param[0].clipA, param[0].clipB, - param[0].negClipA, param[0].negClipB, a, b ); - int32x4_t accLo = vmull_lane_s16( vget_low_s16( a ), param[0].coeff, 0 ); - int32x4_t accHi = vmull_lane_s16( vget_high_s16( a ), param[1].coeff, 0 ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), param[0].coeff, 1 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), param[1].coeff, 1 ); - - processALF_CoeffPair_neon( pImg3 + 0, pImg4 + 0, pImg3 - 1, pImg4 + 1, curr, param[1].clipA, param[1].clipB, - param[1].negClipA, param[1].negClipB, a, b ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), param[0].coeff, 2 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), param[1].coeff, 2 ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), param[0].coeff, 3 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), param[1].coeff, 3 ); - - processALF_CoeffPair_neon( pImg1 + 2, pImg2 - 2, pImg1 + 1, pImg2 - 1, curr, param[2].clipA, param[2].clipB, - param[2].negClipA, param[2].negClipB, a, b ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), param[2].coeff, 0 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), param[3].coeff, 0 ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), param[2].coeff, 1 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), param[3].coeff, 1 ); - - processALF_CoeffPair_neon( pImg1 + 0, pImg2 + 0, pImg1 - 1, pImg2 + 1, curr, param[3].clipA, param[3].clipB, - param[3].negClipA, param[3].negClipB, a, b ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), param[2].coeff, 2 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), param[3].coeff, 2 ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), param[2].coeff, 3 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), param[3].coeff, 3 ); - - processALF_CoeffPair_neon( pImg1 - 2, pImg2 + 2, pImg0 + 3, pImg0 - 3, curr, param[4].clipA, param[4].clipB, - param[4].negClipA, param[4].negClipB, a, b ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), param[4].coeff, 0 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), param[5].coeff, 0 ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), param[4].coeff, 1 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), param[5].coeff, 1 ); - - processALF_CoeffPair_neon( pImg0 + 2, pImg0 - 2, pImg0 + 1, pImg0 - 1, curr, param[5].clipA, param[5].clipB, - param[5].negClipA, param[5].negClipB, a, b ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( a ), param[4].coeff, 2 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( a ), param[5].coeff, 2 ); - accLo = vmlal_lane_s16( accLo, vget_low_s16( b ), param[4].coeff, 3 ); - accHi = vmlal_lane_s16( accHi, vget_high_s16( b ), param[5].coeff, 3 ); - - int16x8_t acc; - if( isFoldingRequired && distance >= 0 && distance <= 1 ) + if( IsFixedFilterSet ) { - // Weaker filter, closer to VB. - acc = vcombine_s16( vrshrn_n_s32( accLo, SHIFT + 3 ), vrshrn_n_s32( accHi, SHIFT + 3 ) ); + return processALF7x7Row_noClip_neon( pImg0, pImg1, pImg2, pImg3, pImg4, pImg5, pImg6, coeff, + distance, clpRngMax ); } else { - // Regular filter strength. - acc = vcombine_s16( vrshrn_n_s32( accLo, SHIFT ), vrshrn_n_s32( accHi, SHIFT ) ); + return processALF7x7Row_Clip_neon( pImg0, pImg1, pImg2, pImg3, pImg4, pImg5, pImg6, param, coeff, + distance, clpRngMax ); } +} - uint16x8_t dst = vvdec_vsqaddq_u16( vreinterpretq_u16_s16( curr ), acc ); - dst = vminq_u16( dst, vdupq_n_u16( clpRngMax ) ); +template +static inline void +filterALF7x7Block4Rows_neon( const AlfClassifier* classifier, int cl_index, const int width, const Pel* pImg0, + Pel* pDst, const ptrdiff_t srcStride, const ptrdiff_t dstStride, const short* filterSet, + const short* fClipSet, const int clpRngMax, const int VbDistance[4] = nullptr ) +{ + CHECKD( width % 8, "Width must be multiple of 8!" ); - vst1q_s16( pDst, vreinterpretq_s16_u16( dst ) ); + int j = width; + do + { + const AlfClassifier& cl0 = classifier[cl_index]; + const AlfClassifier& cl1 = classifier[cl_index + 1]; + const int index0 = + cl0.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl0.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; + const int index1 = + cl1.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl1.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; + + // For 7x7, 6 pairs of coeffs. Each pair has 4 coeffs (2 for each filter). + int16x4_t coeff[MAX_NUM_ALF_LUMA_COEFF / 2]; + prepareALF7x7CoeffParams_neon( filterSet + index0, filterSet + index1, coeff ); + + ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2]; + if( !IsFixedFilterSet ) + { + prepareALF7x7ClipParams_neon( fClipSet + index0, fClipSet + index1, param ); + } + + for( int k = 0; k < 4; k++ ) + { + const int distance = IsFoldingRequired ? VbDistance[k] : 0; + const int16x8_t dst = processALF7x7Row_neon( + pImg0 + k * srcStride, srcStride, coeff, clpRngMax, distance, param ); + vst1q_s16( pDst + k * dstStride, dst ); + } + + cl_index += 2; + pImg0 += 8; + pDst += 8; + j -= 8; + } while( j != 0 ); } -void Filter7x7Blk_neon( const AlfClassifier* classifier, const PelUnitBuf& recDst, const CPelUnitBuf& recSrc, - const Area& blk, const ComponentID compId, const short* filterSet, const short* fClipSet, - const ClpRng& clpRng, int vbCTUHeight, int vbPos ) +template +static inline void calculate_Filter7x7Blk_neon( const AlfClassifier* classifier, const PelUnitBuf& recDst, + const CPelUnitBuf& recSrc, const Area& blk, const ComponentID compId, + const ClpRng& clpRng, int vbCTUHeight, int vbPos, + const short* filterSet, const short* fClipSet = nullptr ) { const CPelBuf srcLuma = recSrc.get( compId ); PelBuf dstLuma = recDst.get( compId ); @@ -270,16 +407,12 @@ void Filter7x7Blk_neon( const AlfClassifier* classifier, const PelUnitBuf& recDs const Pel* src = srcLuma.buf + blk.y * srcStride + blk.x; Pel* dst = dstLuma.buf + blk.y * dstStride + blk.x; - constexpr size_t STEP_X = 8; - constexpr size_t STEP_Y = 4; - - CHECKD( startHeight % STEP_Y, "Wrong startHeight in filtering!" ); - CHECKD( width % STEP_X, "Width must be multiple of 8!" ); - CHECKD( height % STEP_Y, "Height must be multiple of 4!" ); + CHECKD( startHeight % 4, "Wrong startHeight in filtering!" ); + CHECKD( height % 4, "Height must be multiple of 4!" ); const int clpRngMax = clpRng.max(); - for( int i = 0; i < height; i += STEP_Y ) + for( int i = 0; i < height; i += 4 ) { const int yVbPos = ( startHeight + i ) & ( vbCTUHeight - 1 ); // Row’s position inside its CTU. const int firstDistance = vbPos - yVbPos; @@ -294,63 +427,40 @@ void Filter7x7Blk_neon( const AlfClassifier* classifier, const PelUnitBuf& recDs if( foldingRequired ) { - // startHeight and i are STEP_Y-aligned, so any CTU wrap can only occur after these four row distances. - const int VbDistance[STEP_Y] = { firstDistance, firstDistance - 1, firstDistance - 2, firstDistance - 3 }; + // startHeight and i are 4-aligned, so any CTU wrap can only occur after these four row distances. + const int VbDistance[] = { firstDistance, firstDistance - 1, firstDistance - 2, firstDistance - 3 }; - int j = width; - do - { - const AlfClassifier& cl0 = classifier[cl_index]; - const int index0 = - cl0.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl0.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; - const AlfClassifier& cl1 = classifier[cl_index + 1]; - const int index1 = - cl1.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl1.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; - - ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2]; - prepareALF7x7Params_neon( filterSet + index0, filterSet + index1, fClipSet + index0, fClipSet + index1, param ); - - for( int k = 0; k < STEP_Y; k++ ) - { - processALF7x7Row_neon( pImg0 + k * srcStride, pDst + k * dstStride, VbDistance[k], srcStride, param, - clpRngMax ); - } - - j -= STEP_X; - cl_index += 2; - pImg0 += STEP_X; - pDst += STEP_X; - } while( j != 0 ); + filterALF7x7Block4Rows_neon( + classifier, cl_index, width, pImg0, pDst, srcStride, dstStride, filterSet, fClipSet, clpRngMax, VbDistance ); } else { - int j = width; - do - { - const AlfClassifier& cl0 = classifier[cl_index]; - const int index0 = - cl0.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl0.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; - const AlfClassifier& cl1 = classifier[cl_index + 1]; - const int index1 = - cl1.classIdx * MAX_NUM_ALF_LUMA_COEFF + cl1.transposeIdx * MAX_NUM_ALF_LUMA_COEFF * MAX_NUM_ALF_CLASSES; - - ALFGroupParam param[MAX_NUM_ALF_LUMA_COEFF / 2]; - prepareALF7x7Params_neon( filterSet + index0, filterSet + index1, fClipSet + index0, fClipSet + index1, param ); - - for( int k = 0; k < STEP_Y; k++ ) - { - processALF7x7Row_neon( pImg0 + k * srcStride, pDst + k * dstStride, 0, srcStride, param, clpRngMax ); - } - - j -= STEP_X; - cl_index += 2; - pImg0 += STEP_X; - pDst += STEP_X; - } while( j != 0 ); + filterALF7x7Block4Rows_neon( + classifier, cl_index, width, pImg0, pDst, srcStride, dstStride, filterSet, fClipSet, clpRngMax ); } - src += srcStride * STEP_Y; - dst += dstStride * STEP_Y; + src += srcStride * 4; + dst += dstStride * 4; + } +} + +void Filter7x7Blk_neon( const AlfClassifier* classifier, const PelUnitBuf& recDst, const CPelUnitBuf& recSrc, + const Area& blk, const ComponentID compId, const short* filterSet, const short* fClipSet, + const ClpRng& clpRng, int vbCTUHeight, int vbPos, const bool isFixedFilterSet ) +{ + + if( isFixedFilterSet ) + { + for( int i = 0; i < MAX_NUM_ALF_LUMA_COEFF - 1; i++ ) + { + CHECKD( fClipSet[i] != clpRng.max() + 1, "Wrong clipset for fixed filter set!" ); + } + calculate_Filter7x7Blk_neon( classifier, recDst, recSrc, blk, compId, clpRng, vbCTUHeight, vbPos, filterSet ); + } + else + { + calculate_Filter7x7Blk_neon( classifier, recDst, recSrc, blk, compId, clpRng, vbCTUHeight, vbPos, filterSet, + fClipSet ); } } @@ -461,8 +571,9 @@ static inline int16x8_t processALF5x5Row_neon( const Pel* pImg0, const int dista void Filter5x5Blk_neon( const AlfClassifier*, const PelUnitBuf& recDst, const CPelUnitBuf& recSrc, const Area& blk, const ComponentID compId, const short* filterSet, const short* fClipSet, const ClpRng& clpRng, - int vbCTUHeight, int vbPos ) + int vbCTUHeight, int vbPos, const bool isFixedFilterSet ) { + ( void ) isFixedFilterSet; const CPelBuf srcLuma = recSrc.get( compId ); PelBuf dstLuma = recDst.get( compId ); diff --git a/source/Lib/CommonLib/x86/AdaptiveLoopFilterX86.h b/source/Lib/CommonLib/x86/AdaptiveLoopFilterX86.h index bd068dcf..5ced5340 100644 --- a/source/Lib/CommonLib/x86/AdaptiveLoopFilterX86.h +++ b/source/Lib/CommonLib/x86/AdaptiveLoopFilterX86.h @@ -584,8 +584,10 @@ static void simdFilter5x5Blk( const AlfClassifier*, const short* fClipSet, const ClpRng& clpRng, int vbCTUHeight, - int vbPos ) + int vbPos, + const bool isFixedFilterSet ) { + ( void ) isFixedFilterSet; CHECK(!isChroma(compId), "ALF 5x5 filter is for chroma only"); @@ -773,8 +775,10 @@ void simdFilter5x5Blk( const AlfClassifier*, const short* fClipSet, const ClpRng& clpRng, int vbCTUHeight, - int vbPos ) + int vbPos, + const bool isFixedFilterSet ) { + ( void ) isFixedFilterSet; CHECK( !isChroma( compId ), "ALF 5x5 filter is for chroma only" ); @@ -999,8 +1003,10 @@ static void simdFilter7x7Blk( const AlfClassifier* classifier, const short* fClipSet, const ClpRng& clpRng, int vbCTUHeight, - int vbPos ) + int vbPos, + const bool isFixedFilterSet ) { + ( void ) isFixedFilterSet; const CPelBuf srcBuffer = recSrc.get(compId); PelBuf dstBuffer = recDst.get(compId); @@ -1215,11 +1221,13 @@ void simdFilter7x7Blk( const AlfClassifier* classifier, const short* fClipSet, const ClpRng& clpRng, int vbCTUHeight, - int vbPos ) + int vbPos, + const bool isFixedFilterSet ) { + if( blk.width & 15 ) { - simdFilter7x7Blk( classifier, recDst, recSrc, blk, compId, filterSet, fClipSet, clpRng, vbCTUHeight, vbPos ); + simdFilter7x7Blk( classifier, recDst, recSrc, blk, compId, filterSet, fClipSet, clpRng, vbCTUHeight, vbPos, isFixedFilterSet ); return; } diff --git a/tests/vvdec_unit_test/vvdec_unit_test.cpp b/tests/vvdec_unit_test/vvdec_unit_test.cpp index a56698e0..6333356d 100644 --- a/tests/vvdec_unit_test/vvdec_unit_test.cpp +++ b/tests/vvdec_unit_test/vvdec_unit_test.cpp @@ -429,10 +429,24 @@ static bool check_one_filterBlk( AdaptiveLoopFilter* ref, AdaptiveLoopFilter* op } } - ref->m_filter7x7Blk( classifier.data(), dstUnitBuf_ref, srcUnitBuf, blk, compId, coeffLuma.data(), clipLuma.data(), - clpRng, vbCTUHeight, vbPos ); - opt->m_filter7x7Blk( classifier.data(), dstUnitBuf_opt, srcUnitBuf, blk, compId, coeffLuma.data(), clipLuma.data(), - clpRng, vbCTUHeight, vbPos ); + auto check7x7 = [&]( bool isFixedFilterSet, const char* mode ) + { + std::fill( dst_ref.begin(), dst_ref.end(), 0 ); + std::fill( dst_opt.begin(), dst_opt.end(), 0 ); + + ref->m_filter7x7Blk( classifier.data(), dstUnitBuf_ref, srcUnitBuf, blk, compId, coeffLuma.data(), + clipLuma.data(), clpRng, vbCTUHeight, vbPos, isFixedFilterSet ); + opt->m_filter7x7Blk( classifier.data(), dstUnitBuf_opt, srcUnitBuf, blk, compId, coeffLuma.data(), + clipLuma.data(), clpRng, vbCTUHeight, vbPos, isFixedFilterSet ); + + return compare_values_2d( sstm.str() + mode, dst_ref.data(), dst_opt.data(), h, w, ( unsigned )dstStride ); + }; + + bool passed = check7x7( false, " isFixedFilterSet=false" ); + std::fill( clipLuma.begin(), clipLuma.end(), AdaptiveLoopFilter::m_alfClippVls[bitDepth - 8][0] ); + passed = check7x7( true, " isFixedFilterSet=true" ) && passed; + + return passed; } else { @@ -476,12 +490,11 @@ static bool check_one_filterBlk( AdaptiveLoopFilter* ref, AdaptiveLoopFilter* op // The classifier is unused for 5x5 chroma filtering, so pass nullptr. ref->m_filter5x5Blk( nullptr, dstUnitBuf_ref, srcUnitBuf, blk, compId, chromaCoeff, chrmClip.data(), clpRng, - vbCTUHeight, vbPos ); + vbCTUHeight, vbPos, /*isFixedFilterSet = */ false ); opt->m_filter5x5Blk( nullptr, dstUnitBuf_opt, srcUnitBuf, blk, compId, chromaCoeff, chrmClip.data(), clpRng, - vbCTUHeight, vbPos ); + vbCTUHeight, vbPos, /*isFixedFilterSet = */ false ); + return compare_values_2d( sstm.str(), dst_ref.data(), dst_opt.data(), h, w, ( unsigned )dstStride ); } - - return compare_values_2d( sstm.str(), dst_ref.data(), dst_opt.data(), h, w, (unsigned)dstStride ); } static bool check_deriveClassificationBlk( AdaptiveLoopFilter* ref, AdaptiveLoopFilter* opt, unsigned num_cases, int w,