eeva/jamulus
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

implement the OPUS coder calls similar to what we have in the server (leading to cleaner code)

Browse source
This commit is contained in:
Volker Fischer 2020-04-12 21:36:50 +02:00
parent af32867216
commit 9b07cbeb5f
2 changed files with 83 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

258
src/client.cpp
View file

@ -47,10 +47,13 @@ CClient::CClient ( const quint16 iPortNumber,
bWindowWasShownProfile ( false ), bWindowWasShownProfile ( false ),
bWindowWasShownConnect ( false ), bWindowWasShownConnect ( false ),
Channel ( false ), /* we need a client channel -> "false" */ Channel ( false ), /* we need a client channel -> "false" */
CurOpusEncoder ( nullptr ),
CurOpusDecoder ( nullptr ),
eAudioCompressionType ( CT_OPUS ), eAudioCompressionType ( CT_OPUS ),
iCeltNumCodedBytes ( OPUS_NUM_BYTES_MONO_LOW_QUALITY ), iCeltNumCodedBytes ( OPUS_NUM_BYTES_MONO_LOW_QUALITY ),
eAudioQuality ( AQ_NORMAL ), eAudioQuality ( AQ_NORMAL ),
eAudioChannelConf ( CC_MONO ), eAudioChannelConf ( CC_MONO ),
iNumAudioChannels ( 1 ),
bIsInitializationPhase ( true ), bIsInitializationPhase ( true ),
bMuteInputAndOutput ( false ), bMuteInputAndOutput ( false ),
Socket ( &Channel, iPortNumber ), Socket ( &Channel, iPortNumber ),
@ -74,54 +77,28 @@ CClient::CClient ( const quint16 iPortNumber,
{ {
int iOpusError; int iOpusError;
// init audio encoder/decoder (mono) // init audio encoders and decoders
OpusMode = opus_custom_mode_create ( SYSTEM_SAMPLE_RATE_HZ, OpusMode = opus_custom_mode_create ( SYSTEM_SAMPLE_RATE_HZ,
SYSTEM_FRAME_SIZE_SAMPLES, SYSTEM_FRAME_SIZE_SAMPLES,
&iOpusError ); &iOpusError );
OpusEncoderMono = opus_custom_encoder_create ( OpusMode, OpusEncoderMono = opus_custom_encoder_create ( OpusMode, 1, &iOpusError ); // mono encoder legacy
1, OpusDecoderMono = opus_custom_decoder_create ( OpusMode, 1, &iOpusError ); // mono decoder legacy
&iOpusError ); OpusEncoderStereo = opus_custom_encoder_create ( OpusMode, 2, &iOpusError ); // stereo encoder legacy
OpusDecoderStereo = opus_custom_decoder_create ( OpusMode, 2, &iOpusError ); // stereo decoder legacy
OpusDecoderMono = opus_custom_decoder_create ( OpusMode,
1,
&iOpusError );
// we require a constant bit rate // we require a constant bit rate
opus_custom_encoder_ctl ( OpusEncoderMono, opus_custom_encoder_ctl ( OpusEncoderMono, OPUS_SET_VBR ( 0 ) );
OPUS_SET_VBR ( 0 ) ); opus_custom_encoder_ctl ( OpusEncoderStereo, OPUS_SET_VBR ( 0 ) );
// we want as low delay as possible // we want as low delay as possible
opus_custom_encoder_ctl ( OpusEncoderMono, opus_custom_encoder_ctl ( OpusEncoderMono, OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) );
OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) ); opus_custom_encoder_ctl ( OpusEncoderStereo, OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) );
#if ( SYSTEM_FRAME_SIZE_SAMPLES == 128 ) #if ( SYSTEM_FRAME_SIZE_SAMPLES == 128 )
// set encoder low complexity for legacy 128 samples frame size // set encoder low complexity for legacy 128 samples frame size
opus_custom_encoder_ctl ( OpusEncoderMono, opus_custom_encoder_ctl ( OpusEncoderMono, OPUS_SET_COMPLEXITY ( 1 ) );
OPUS_SET_COMPLEXITY ( 1 ) ); opus_custom_encoder_ctl ( OpusEncoderStereo, OPUS_SET_COMPLEXITY ( 1 ) );
#endif
// init audio encoder/decoder (stereo)
OpusEncoderStereo = opus_custom_encoder_create ( OpusMode,
2,
&iOpusError );
OpusDecoderStereo = opus_custom_decoder_create ( OpusMode,
2,
&iOpusError );
// we require a constant bit rate
opus_custom_encoder_ctl ( OpusEncoderStereo,
OPUS_SET_VBR ( 0 ) );
// we want as low delay as possible
opus_custom_encoder_ctl ( OpusEncoderStereo,
OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) );
#if ( SYSTEM_FRAME_SIZE_SAMPLES == 128 )
// set encoder low complexity for legacy 128 samples frame size
opus_custom_encoder_ctl ( OpusEncoderStereo,
OPUS_SET_COMPLEXITY ( 1 ) );
#endif #endif
@ -752,8 +729,6 @@ void CClient::Init()
// calculate stereo (two channels) buffer size // calculate stereo (two channels) buffer size
iStereoBlockSizeSam = 2 * iMonoBlockSizeSam; iStereoBlockSizeSam = 2 * iMonoBlockSizeSam;
vecsAudioSndCrdMono.Init ( iMonoBlockSizeSam );
// init reverberation // init reverberation
AudioReverbL.Init ( SYSTEM_SAMPLE_RATE_HZ ); AudioReverbL.Init ( SYSTEM_SAMPLE_RATE_HZ );
AudioReverbR.Init ( SYSTEM_SAMPLE_RATE_HZ ); AudioReverbR.Init ( SYSTEM_SAMPLE_RATE_HZ );
@ -761,75 +736,46 @@ void CClient::Init()
// inits for audio coding // inits for audio coding
if ( eAudioChannelConf == CC_MONO ) if ( eAudioChannelConf == CC_MONO )
{ {
CurOpusEncoder = OpusEncoderMono;
CurOpusDecoder = OpusDecoderMono;
iNumAudioChannels = 1;
switch ( eAudioQuality ) switch ( eAudioQuality )
{ {
case AQ_LOW: case AQ_LOW: iCeltNumCodedBytes = OPUS_NUM_BYTES_MONO_LOW_QUALITY; break;
iCeltNumCodedBytes = OPUS_NUM_BYTES_MONO_LOW_QUALITY; case AQ_NORMAL: iCeltNumCodedBytes = OPUS_NUM_BYTES_MONO_NORMAL_QUALITY; break;
break; case AQ_HIGH: iCeltNumCodedBytes = OPUS_NUM_BYTES_MONO_HIGH_QUALITY; break;
case AQ_NORMAL:
iCeltNumCodedBytes = OPUS_NUM_BYTES_MONO_NORMAL_QUALITY;
break;
case AQ_HIGH:
iCeltNumCodedBytes = OPUS_NUM_BYTES_MONO_HIGH_QUALITY;
break;
} }
} }
else else
{ {
CurOpusEncoder = OpusEncoderStereo;
CurOpusDecoder = OpusDecoderStereo;
iNumAudioChannels = 2;
switch ( eAudioQuality ) switch ( eAudioQuality )
{ {
case AQ_LOW: case AQ_LOW: iCeltNumCodedBytes = OPUS_NUM_BYTES_STEREO_LOW_QUALITY; break;
iCeltNumCodedBytes = OPUS_NUM_BYTES_STEREO_LOW_QUALITY; case AQ_NORMAL: iCeltNumCodedBytes = OPUS_NUM_BYTES_STEREO_NORMAL_QUALITY; break;
break; case AQ_HIGH: iCeltNumCodedBytes = OPUS_NUM_BYTES_STEREO_HIGH_QUALITY; break;
case AQ_NORMAL:
iCeltNumCodedBytes = OPUS_NUM_BYTES_STEREO_NORMAL_QUALITY;
break;
case AQ_HIGH:
iCeltNumCodedBytes = OPUS_NUM_BYTES_STEREO_HIGH_QUALITY;
break;
} }
} }
vecCeltData.Init ( iCeltNumCodedBytes ); vecCeltData.Init ( iCeltNumCodedBytes );
if ( eAudioChannelConf == CC_MONO ) opus_custom_encoder_ctl ( CurOpusEncoder,
{ OPUS_SET_BITRATE (
opus_custom_encoder_ctl ( OpusEncoderMono, CalcBitRateBitsPerSecFromCodedBytes (
OPUS_SET_BITRATE ( iCeltNumCodedBytes, SYSTEM_FRAME_SIZE_SAMPLES ) ) );
CalcBitRateBitsPerSecFromCodedBytes (
iCeltNumCodedBytes, SYSTEM_FRAME_SIZE_SAMPLES ) ) );
}
else
{
opus_custom_encoder_ctl ( OpusEncoderStereo,
OPUS_SET_BITRATE (
CalcBitRateBitsPerSecFromCodedBytes (
iCeltNumCodedBytes, SYSTEM_FRAME_SIZE_SAMPLES ) ) );
}
// inits for network and channel // inits for network and channel
vecbyNetwData.Init ( iCeltNumCodedBytes ); vecbyNetwData.Init ( iCeltNumCodedBytes );
if ( eAudioChannelConf == CC_MONO ) // set the channel network properties
{ Channel.SetAudioStreamProperties ( eAudioCompressionType,
// set the channel network properties iCeltNumCodedBytes,
Channel.SetAudioStreamProperties ( eAudioCompressionType, iSndCrdFrameSizeFactor,
iCeltNumCodedBytes, iNumAudioChannels );
iSndCrdFrameSizeFactor,
1 );
}
else
{
// set the channel network properties
Channel.SetAudioStreamProperties ( eAudioCompressionType,
iCeltNumCodedBytes,
iSndCrdFrameSizeFactor,
2 );
}
// reset initialization phase flag and mute flag // reset initialization phase flag and mute flag
bIsInitializationPhase = true; bIsInitializationPhase = true;
@ -897,7 +843,9 @@ JitterMeas.Measure();
void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd ) void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd )
{ {
int i, j; int i, j;
unsigned char* pCurCodedData;
// Transmit signal --------------------------------------------------------- // Transmit signal ---------------------------------------------------------
// update stereo signal level meter // update stereo signal level meter
@ -907,8 +855,7 @@ void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd )
if ( iReverbLevel != 0 ) if ( iReverbLevel != 0 )
{ {
// calculate attenuation amplification factor // calculate attenuation amplification factor
const double dRevLev = const double dRevLev = static_cast<double> ( iReverbLevel ) / AUD_REVERB_MAX / 2;
static_cast<double> ( iReverbLevel ) / AUD_REVERB_MAX / 2;
if ( eAudioChannelConf == CC_STEREO ) if ( eAudioChannelConf == CC_STEREO )
{ {
@ -957,8 +904,7 @@ void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd )
// int32), after the normalization by 2, the result will fit // int32), after the normalization by 2, the result will fit
// into the old size so that cast to int16 is safe // into the old size so that cast to int16 is safe
vecsStereoSndCrd[i] = static_cast<int16_t> ( vecsStereoSndCrd[i] = static_cast<int16_t> (
( static_cast<int32_t> ( vecsStereoSndCrd[j] ) + ( static_cast<int32_t> ( vecsStereoSndCrd[j] ) + vecsStereoSndCrd[j + 1] ) / 2 );
vecsStereoSndCrd[j + 1] ) / 2 );
} }
} }
} }
@ -968,16 +914,14 @@ void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd )
{ {
// stereo // stereo
const double dAttFactStereo = static_cast<double> ( const double dAttFactStereo = static_cast<double> (
AUD_FADER_IN_MIDDLE - abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / AUD_FADER_IN_MIDDLE - abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / AUD_FADER_IN_MIDDLE;
AUD_FADER_IN_MIDDLE;
if ( iAudioInFader > AUD_FADER_IN_MIDDLE ) if ( iAudioInFader > AUD_FADER_IN_MIDDLE )
{ {
for ( i = 0, j = 0; i < iMonoBlockSizeSam; i++, j += 2 ) for ( i = 0, j = 0; i < iMonoBlockSizeSam; i++, j += 2 )
{ {
// attenuation on right channel // attenuation on right channel
vecsStereoSndCrd[j + 1] = Double2Short ( vecsStereoSndCrd[j + 1] = Double2Short ( dAttFactStereo * vecsStereoSndCrd[j + 1] );
dAttFactStereo * vecsStereoSndCrd[j + 1] );
} }
} }
else else
@ -985,8 +929,7 @@ void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd )
for ( i = 0, j = 0; i < iMonoBlockSizeSam; i++, j += 2 ) for ( i = 0, j = 0; i < iMonoBlockSizeSam; i++, j += 2 )
{ {
// attenuation on left channel // attenuation on left channel
vecsStereoSndCrd[j] = Double2Short ( vecsStereoSndCrd[j] = Double2Short ( dAttFactStereo * vecsStereoSndCrd[j] );
dAttFactStereo * vecsStereoSndCrd[j] );
} }
} }
} }
@ -997,12 +940,10 @@ void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd )
// amplified by 1/2, if the pan is set to one channel, this // amplified by 1/2, if the pan is set to one channel, this
// channel should have an amplification of 1 // channel should have an amplification of 1
const double dAttFactMono = static_cast<double> ( const double dAttFactMono = static_cast<double> (
AUD_FADER_IN_MIDDLE - abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / AUD_FADER_IN_MIDDLE - abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / AUD_FADER_IN_MIDDLE / 2;
AUD_FADER_IN_MIDDLE / 2;
const double dAmplFactMono = 0.5 + static_cast<double> ( const double dAmplFactMono = 0.5 + static_cast<double> (
abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / AUD_FADER_IN_MIDDLE / 2;
AUD_FADER_IN_MIDDLE / 2;
if ( iAudioInFader > AUD_FADER_IN_MIDDLE ) if ( iAudioInFader > AUD_FADER_IN_MIDDLE )
{ {
@ -1040,29 +981,20 @@ void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd )
// overwrite input values) // overwrite input values)
for ( i = iMonoBlockSizeSam - 1, j = iStereoBlockSizeSam - 2; i >= 0; i--, j -= 2 ) for ( i = iMonoBlockSizeSam - 1, j = iStereoBlockSizeSam - 2; i >= 0; i--, j -= 2 )
{ {
vecsStereoSndCrd[j] = vecsStereoSndCrd[j + 1] = vecsStereoSndCrd[j] = vecsStereoSndCrd[j + 1] = vecsStereoSndCrd[i];
vecsStereoSndCrd[i];
} }
} }
for ( i = 0; i < iSndCrdFrameSizeFactor; i++ ) for ( i = 0; i < iSndCrdFrameSizeFactor; i++ )
{ {
// encode current audio frame // OPUS encoding
if ( ( eAudioCompressionType == CT_OPUS ) || if ( ( eAudioCompressionType == CT_OPUS ) ||
( eAudioCompressionType == CT_OPUS64 ) ) ( eAudioCompressionType == CT_OPUS64 ) )
{ {
if ( eAudioChannelConf == CC_MONO ) if ( CurOpusEncoder != nullptr )
{ {
opus_custom_encode ( OpusEncoderMono, opus_custom_encode ( CurOpusEncoder,
&vecsStereoSndCrd[i * SYSTEM_FRAME_SIZE_SAMPLES], &vecsStereoSndCrd[i * iNumAudioChannels * SYSTEM_FRAME_SIZE_SAMPLES],
SYSTEM_FRAME_SIZE_SAMPLES,
&vecCeltData[0],
iCeltNumCodedBytes );
}
else
{
opus_custom_encode ( OpusEncoderStereo,
&vecsStereoSndCrd[i * 2 * SYSTEM_FRAME_SIZE_SAMPLES],
SYSTEM_FRAME_SIZE_SAMPLES, SYSTEM_FRAME_SIZE_SAMPLES,
&vecCeltData[0], &vecCeltData[0],
iCeltNumCodedBytes ); iCeltNumCodedBytes );
@ -1083,67 +1015,38 @@ void CClient::ProcessAudioDataIntern ( CVector<int16_t>& vecsStereoSndCrd )
const bool bReceiveDataOk = const bool bReceiveDataOk =
( Channel.GetData ( vecbyNetwData, iCeltNumCodedBytes ) == GS_BUFFER_OK ); ( Channel.GetData ( vecbyNetwData, iCeltNumCodedBytes ) == GS_BUFFER_OK );
// invalidate the buffer OK status flag if necessary // get pointer to coded data and manage the flags
if ( !bReceiveDataOk )
{
bJitterBufferOK = false;
}
// CELT decoding
if ( bReceiveDataOk ) if ( bReceiveDataOk )
{ {
// on any valid received packet, we clear the initialization phase pCurCodedData = &vecbyNetwData[0];
// flag
bIsInitializationPhase = false;
if ( ( eAudioCompressionType == CT_OPUS ) || // on any valid received packet, we clear the initialization phase flag
( eAudioCompressionType == CT_OPUS64 ) ) bIsInitializationPhase = false;
{
if ( eAudioChannelConf == CC_MONO )
{
opus_custom_decode ( OpusDecoderMono,
&vecbyNetwData[0],
iCeltNumCodedBytes,
&vecsAudioSndCrdMono[i * SYSTEM_FRAME_SIZE_SAMPLES],
SYSTEM_FRAME_SIZE_SAMPLES );
}
else
{
opus_custom_decode ( OpusDecoderStereo,
&vecbyNetwData[0],
iCeltNumCodedBytes,
&vecsStereoSndCrd[i * 2 * SYSTEM_FRAME_SIZE_SAMPLES],
SYSTEM_FRAME_SIZE_SAMPLES );
}
}
} }
else else
{ {
// lost packet // for lost packets use null pointer as coded input data
if ( ( eAudioCompressionType == CT_OPUS ) || pCurCodedData = nullptr;
( eAudioCompressionType == CT_OPUS64 ) )
// invalidate the buffer OK status flag
bJitterBufferOK = false;
}
// OPUS decoding
if ( ( eAudioCompressionType == CT_OPUS ) ||
( eAudioCompressionType == CT_OPUS64 ) )
{
if ( CurOpusDecoder != nullptr )
{ {
if ( eAudioChannelConf == CC_MONO ) opus_custom_decode ( CurOpusDecoder,
{ pCurCodedData,
opus_custom_decode ( OpusDecoderMono, iCeltNumCodedBytes,
nullptr, &vecsStereoSndCrd[i * iNumAudioChannels * SYSTEM_FRAME_SIZE_SAMPLES],
iCeltNumCodedBytes, SYSTEM_FRAME_SIZE_SAMPLES );
&vecsAudioSndCrdMono[i * SYSTEM_FRAME_SIZE_SAMPLES],
SYSTEM_FRAME_SIZE_SAMPLES );
}
else
{
opus_custom_decode ( OpusDecoderStereo,
nullptr,
iCeltNumCodedBytes,
&vecsStereoSndCrd[i * 2 * SYSTEM_FRAME_SIZE_SAMPLES],
SYSTEM_FRAME_SIZE_SAMPLES );
}
} }
} }
} }
/* /*
// TEST // TEST
// fid=fopen('c:\\temp\test2.dat','r');x=fread(fid,'int16');fclose(fid); // fid=fopen('c:\\temp\test2.dat','r');x=fread(fid,'int16');fclose(fid);
@ -1151,24 +1054,23 @@ static FILE* pFileDelay = fopen("c:\\temp\\test2.dat", "wb");
short sData[2]; short sData[2];
for (i = 0; i < iMonoBlockSizeSam; i++) for (i = 0; i < iMonoBlockSizeSam; i++)
{ {
sData[0] = (short) vecsAudioSndCrdMono[i]; sData[0] = (short) vecsStereoSndCrd[i];
fwrite(&sData, size_t(2), size_t(1), pFileDelay); fwrite(&sData, size_t(2), size_t(1), pFileDelay);
} }
fflush(pFileDelay); fflush(pFileDelay);
*/ */
// check if channel is connected and if we do not have the initialization phase
// check if channel is connected and if we do not have the initialization
// phase
if ( Channel.IsConnected() && ( !bIsInitializationPhase ) ) if ( Channel.IsConnected() && ( !bIsInitializationPhase ) )
{ {
if ( eAudioChannelConf == CC_MONO ) if ( eAudioChannelConf == CC_MONO )
{ {
// copy mono data in stereo sound card buffer // copy mono data in stereo sound card buffer (note that since the input
for ( i = 0, j = 0; i < iMonoBlockSizeSam; i++, j += 2 ) // and output is the same buffer, we have to start from the end not to
// overwrite input values)
for ( i = iMonoBlockSizeSam - 1, j = iStereoBlockSizeSam - 2; i >= 0; i--, j -= 2 )
{ {
vecsStereoSndCrd[j] = vecsStereoSndCrd[j + 1] = vecsStereoSndCrd[j] = vecsStereoSndCrd[j + 1] = vecsStereoSndCrd[i];
vecsAudioSndCrdMono[i];
} }
} }
} }

5
src/client.h
View file

@ -321,10 +321,13 @@ protected:
OpusCustomDecoder* OpusDecoderMono; OpusCustomDecoder* OpusDecoderMono;
OpusCustomEncoder* OpusEncoderStereo; OpusCustomEncoder* OpusEncoderStereo;
OpusCustomDecoder* OpusDecoderStereo; OpusCustomDecoder* OpusDecoderStereo;
OpusCustomEncoder* CurOpusEncoder;
OpusCustomDecoder* CurOpusDecoder;
EAudComprType eAudioCompressionType; EAudComprType eAudioCompressionType;
int iCeltNumCodedBytes; int iCeltNumCodedBytes;
EAudioQuality eAudioQuality; EAudioQuality eAudioQuality;
EAudChanConf eAudioChannelConf; EAudChanConf eAudioChannelConf;
int iNumAudioChannels;
bool bIsInitializationPhase; bool bIsInitializationPhase;
bool bMuteInputAndOutput; bool bMuteInputAndOutput;
CVector<unsigned char> vecCeltData; CVector<unsigned char> vecCeltData;
@ -365,8 +368,6 @@ protected:
QString strCentralServerAddress; QString strCentralServerAddress;
ECSAddType eCentralServerAddressType; ECSAddType eCentralServerAddressType;
CVector<int16_t> vecsAudioSndCrdMono;
// server settings // server settings
int iServerSockBufNumFrames; int iServerSockBufNumFrames;