diff --git a/src/channel.cpp b/src/channel.cpp index b78ab38b..a19df5d8 100755 --- a/src/channel.cpp +++ b/src/channel.cpp @@ -31,7 +31,8 @@ CChannel::CChannel ( const bool bNIsServer ) : vecdGains ( USED_NUM_CHANNELS, (double) 1.0 ), bIsEnabled ( false ), iNetwFrameSizeFact ( FRAME_SIZE_FACTOR_DEFAULT ), - iNetwFrameSize ( 20 ) // must be > 0 and should be close to a valid size + iNetwFrameSize ( 20 ), // must be > 0 and should be close to a valid size + iNumAudioChannels ( 1 ) // mono { // initial value for connection time out counter, we calculate the total // number of samples here and subtract the number of samples of the block @@ -132,13 +133,15 @@ void CChannel::SetEnable ( const bool bNEnStat ) } } -void CChannel::SetNetwFrameSizeAndFact ( const int iNewNetwFrameSize, - const int iNewNetwFrameSizeFact ) +void CChannel::SetAudioStreamProperties ( const int iNewNetwFrameSize, + const int iNewNetwFrameSizeFact, + const int iNewNumAudioChannels ) { // this function is intended for the server (not the client) QMutexLocker locker ( &Mutex ); // store new values + iNumAudioChannels = iNewNumAudioChannels; iNetwFrameSize = iNewNetwFrameSize; iNetwFrameSizeFact = iNewNetwFrameSizeFact; @@ -291,6 +294,7 @@ void CChannel::OnNetTranspPropsReceived ( CNetworkTransportProps NetworkTranspor QMutexLocker locker ( &Mutex ); // store received parameters + iNumAudioChannels = NetworkTransportProps.iNumAudioChannels; iNetwFrameSizeFact = NetworkTransportProps.iBlockSizeFact; iNetwFrameSize = NetworkTransportProps.iBaseNetworkPacketSize; @@ -314,10 +318,10 @@ void CChannel::CreateNetTranspPropsMessFromCurrentSettings() CNetworkTransportProps NetworkTransportProps ( iNetwFrameSize, iNetwFrameSizeFact, - 1, // right now we only use mono + iNumAudioChannels, SYSTEM_SAMPLE_RATE, CT_CELT, // always CELT coding - 0, + 0, // version of the codec 0 ); // send current network transport properties diff --git a/src/channel.h b/src/channel.h index 15182786..8d03b87c 100755 --- a/src/channel.h +++ b/src/channel.h @@ -99,10 +99,12 @@ public: double GetTimingStdDev() { return CycleTimeVariance.GetStdDev(); } // set/get network out buffer size and size factor - void SetNetwFrameSizeAndFact ( const int iNewNetwFrameSize, - const int iNewNetwFrameSizeFact ); + void SetAudioStreamProperties ( const int iNewNetwFrameSize, + const int iNewNetwFrameSizeFact, + const int iNewNumAudioChannels ); int GetNetwFrameSizeFact() const { return iNetwFrameSizeFact; } int GetNetwFrameSize() const { return iNetwFrameSize; } + int GetNumAudioChannels() const { return iNumAudioChannels; } double GetJitterBufferErrorRate() { return SockBuf.GetErrorRate(); } @@ -161,6 +163,8 @@ protected: int iNetwFrameSizeFact; int iNetwFrameSize; + int iNumAudioChannels; + QMutex Mutex; public slots: diff --git a/src/client.cpp b/src/client.cpp index 84f2375f..01a3e5d3 100755 --- a/src/client.cpp +++ b/src/client.cpp @@ -55,15 +55,28 @@ CClient::CClient ( const quint16 iPortNumber ) : iSndCardMonoBlockSizeSamConvBuff ( 0 ) { // init audio endocder/decoder (mono) - CeltMode = celt_mode_create ( + CeltModeMono = celt_mode_create ( SYSTEM_SAMPLE_RATE, 1, SYSTEM_FRAME_SIZE_SAMPLES, NULL ); - CeltEncoder = celt_encoder_create ( CeltMode ); - CeltDecoder = celt_decoder_create ( CeltMode ); + CeltEncoderMono = celt_encoder_create ( CeltModeMono ); + CeltDecoderMono = celt_decoder_create ( CeltModeMono ); #ifdef USE_LOW_COMPLEXITY_CELT_ENC // set encoder low complexity - celt_encoder_ctl(CeltEncoder, + celt_encoder_ctl ( CeltEncoderMono, + CELT_SET_COMPLEXITY_REQUEST, celt_int32_t ( 1 ) ); +#endif + + // init audio endocder/decoder (stereo) + CeltModeStereo = celt_mode_create ( + SYSTEM_SAMPLE_RATE, 2, SYSTEM_FRAME_SIZE_SAMPLES, NULL ); + + CeltEncoderStereo = celt_encoder_create ( CeltModeStereo ); + CeltDecoderStereo = celt_decoder_create ( CeltModeStereo ); + +#ifdef USE_LOW_COMPLEXITY_CELT_ENC + // set encoder low complexity + celt_encoder_ctl ( CeltEncoderStereo, CELT_SET_COMPLEXITY_REQUEST, celt_int32_t ( 1 ) ); #endif @@ -511,8 +524,9 @@ void CClient::Init() vecbyNetwData.Init ( iCeltNumCodedBytes ); // set the channel network properties - Channel.SetNetwFrameSizeAndFact ( iCeltNumCodedBytes, - iSndCrdFrameSizeFactor ); + Channel.SetAudioStreamProperties ( iCeltNumCodedBytes, + iSndCrdFrameSizeFactor, + 1 /* only mono right now */ ); } void CClient::AudioCallback ( CVector& psData, void* arg ) @@ -610,18 +624,25 @@ void CClient::ProcessAudioDataIntern ( CVector& vecsStereoSndCrd ) } else { - const double dAttFact = - static_cast ( AUD_FADER_IN_MIDDLE - abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / - AUD_FADER_IN_MIDDLE; + // make sure that in the middle position the two channels are + // amplified by 1/2, if the pan is set to one channel, this + // channel should have an amplification of 1 + const double dAttFact = static_cast ( + AUD_FADER_IN_MIDDLE - abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / + AUD_FADER_IN_MIDDLE / 2; + + const double dAmplFact = 0.5 + static_cast ( + abs ( AUD_FADER_IN_MIDDLE - iAudioInFader ) ) / + AUD_FADER_IN_MIDDLE / 2; if ( iAudioInFader > AUD_FADER_IN_MIDDLE ) { for ( i = 0, j = 0; i < iMonoBlockSizeSam; i++, j += 2 ) { // attenuation on right channel - vecsNetwork[i] = - Double2Short ( ( vecdAudioStereo[j] + - dAttFact * vecdAudioStereo[j + 1] ) / 2 ); + vecsNetwork[i] = Double2Short ( + dAmplFact * vecdAudioStereo[j] + + dAttFact * vecdAudioStereo[j + 1] ); } } else @@ -629,9 +650,9 @@ void CClient::ProcessAudioDataIntern ( CVector& vecsStereoSndCrd ) for ( i = 0, j = 0; i < iMonoBlockSizeSam; i++, j += 2 ) { // attenuation on left channel - vecsNetwork[i] = - Double2Short ( ( vecdAudioStereo[j + 1] + - dAttFact * vecdAudioStereo[j] ) / 2 ); + vecsNetwork[i] = Double2Short ( + dAmplFact * vecdAudioStereo[j + 1] + + dAttFact * vecdAudioStereo[j] ); } } } @@ -639,7 +660,7 @@ void CClient::ProcessAudioDataIntern ( CVector& vecsStereoSndCrd ) for ( i = 0; i < iSndCrdFrameSizeFactor; i++ ) { // encode current audio frame with CELT encoder - celt_encode ( CeltEncoder, + celt_encode ( CeltEncoderMono, &vecsNetwork[i * SYSTEM_FRAME_SIZE_SAMPLES], NULL, &vecCeltData[0], @@ -670,7 +691,7 @@ void CClient::ProcessAudioDataIntern ( CVector& vecsStereoSndCrd ) // CELT decoding if ( bReceiveDataOk ) { - celt_decode ( CeltDecoder, + celt_decode ( CeltDecoderMono, &vecbyNetwData[0], iCeltNumCodedBytes, &vecsAudioSndCrdMono[i * SYSTEM_FRAME_SIZE_SAMPLES] ); @@ -678,7 +699,7 @@ void CClient::ProcessAudioDataIntern ( CVector& vecsStereoSndCrd ) else { // lost packet - celt_decode ( CeltDecoder, + celt_decode ( CeltDecoderMono, NULL, 0, &vecsAudioSndCrdMono[i * SYSTEM_FRAME_SIZE_SAMPLES] ); diff --git a/src/client.h b/src/client.h index ebf89f7b..03dd4fa8 100755 --- a/src/client.h +++ b/src/client.h @@ -223,9 +223,12 @@ protected: bool bDoAutoSockBufSize; // audio encoder/decoder - CELTMode* CeltMode; - CELTEncoder* CeltEncoder; - CELTDecoder* CeltDecoder; + CELTMode* CeltModeMono; + CELTEncoder* CeltEncoderMono; + CELTDecoder* CeltDecoderMono; + CELTMode* CeltModeStereo; + CELTEncoder* CeltEncoderStereo; + CELTDecoder* CeltDecoderStereo; int iCeltNumCodedBytes; bool bCeltDoHighQuality; CVector vecCeltData; diff --git a/src/protocol.cpp b/src/protocol.cpp index dfc1c407..ae11634b 100755 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -897,7 +897,8 @@ bool CProtocol::EvaluateNetwTranspPropsMes ( const CVector& vecData ) return true; } - // number of channels of the audio signal, e.g. "2" is stereo (1 byte) + // number of channels of the audio signal, only mono (1 channel) or + // stereo (2 channels) allowed (1 byte) ReceivedNetwTranspProps.iNumAudioChannels = static_cast ( GetValFromStream ( vecData, iPos, 1 ) ); diff --git a/src/server.cpp b/src/server.cpp index fdf49a56..2335a930 100755 --- a/src/server.cpp +++ b/src/server.cpp @@ -175,19 +175,33 @@ CServer::CServer ( const QString& strLoggingFileName, int i; // create CELT encoder/decoder for each channel (must be done before - // enabling the channels) + // enabling the channels), create a mono and stereo encoder/decoder + // for each channel for ( i = 0; i < USED_NUM_CHANNELS; i++ ) { // init audio endocder/decoder (mono) - CeltMode[i] = celt_mode_create ( + CeltModeMono[i] = celt_mode_create ( SYSTEM_SAMPLE_RATE, 1, SYSTEM_FRAME_SIZE_SAMPLES, NULL ); - CeltEncoder[i] = celt_encoder_create ( CeltMode[i] ); - CeltDecoder[i] = celt_decoder_create ( CeltMode[i] ); + CeltEncoderMono[i] = celt_encoder_create ( CeltModeMono[i] ); + CeltDecoderMono[i] = celt_decoder_create ( CeltModeMono[i] ); #ifdef USE_LOW_COMPLEXITY_CELT_ENC // set encoder low complexity - celt_encoder_ctl(CeltEncoder[i], + celt_encoder_ctl ( CeltEncoderMono[i], + CELT_SET_COMPLEXITY_REQUEST, celt_int32_t ( 1 ) ); +#endif + + // init audio endocder/decoder (stereo) + CeltModeStereo[i] = celt_mode_create ( + SYSTEM_SAMPLE_RATE, 2, SYSTEM_FRAME_SIZE_SAMPLES, NULL ); + + CeltEncoderStereo[i] = celt_encoder_create ( CeltModeStereo[i] ); + CeltDecoderStereo[i] = celt_decoder_create ( CeltModeStereo[i] ); + +#ifdef USE_LOW_COMPLEXITY_CELT_ENC + // set encoder low complexity + celt_encoder_ctl ( CeltEncoderStereo[i], CELT_SET_COMPLEXITY_REQUEST, celt_int32_t ( 1 ) ); #endif } @@ -376,6 +390,7 @@ void CServer::OnTimer() CVector vecChanID; CVector > vecvecdGains; CVector > vecvecsData; + CVector vecNumAudioChannels; // Get data from all connected clients ------------------------------------- bool bChannelIsNowDisconnected = false; @@ -399,17 +414,24 @@ void CServer::OnTimer() const int iNumCurConnChan = vecChanID.Size(); // init temporary vectors - vecvecdGains.Init ( iNumCurConnChan ); - vecvecsData.Init ( iNumCurConnChan ); + vecvecdGains.Init ( iNumCurConnChan ); + vecvecsData.Init ( iNumCurConnChan ); + vecNumAudioChannels.Init ( iNumCurConnChan ); for ( i = 0; i < iNumCurConnChan; i++ ) { // get actual ID of current channel const int iCurChanID = vecChanID[i]; + // get and store number of audio channels + const int iCurNumAudChan = + vecChannels[iCurChanID].GetNumAudioChannels(); + + vecNumAudioChannels[i] = iCurNumAudChan; + // init vectors storing information of all channels vecvecdGains[i].Init ( iNumCurConnChan ); - vecvecsData[i].Init ( SYSTEM_FRAME_SIZE_SAMPLES ); + vecvecsData[i].Init ( iCurNumAudChan * SYSTEM_FRAME_SIZE_SAMPLES ); // get gains of all connected channels for ( j = 0; j < iNumCurConnChan; j++ ) @@ -442,18 +464,42 @@ void CServer::OnTimer() // CELT decode received data stream if ( eGetStat == GS_BUFFER_OK ) { - celt_decode ( CeltDecoder[iCurChanID], - &vecbyData[0], - iCeltNumCodedBytes, - &vecvecsData[i][0] ); + if ( iCurNumAudChan == 1 ) + { + // mono + celt_decode ( CeltDecoderMono[iCurChanID], + &vecbyData[0], + iCeltNumCodedBytes, + &vecvecsData[i][0] ); + } + else + { + // stereo + celt_decode ( CeltDecoderStereo[iCurChanID], + &vecbyData[0], + iCeltNumCodedBytes, + &vecvecsData[i][0] ); + } } else { // lost packet - celt_decode ( CeltDecoder[iCurChanID], - NULL, - 0, - &vecvecsData[i][0] ); + if ( iCurNumAudChan == 1 ) + { + // mono + celt_decode ( CeltDecoderMono[iCurChanID], + NULL, + 0, + &vecvecsData[i][0] ); + } + else + { + // stereo + celt_decode ( CeltDecoderStereo[iCurChanID], + NULL, + 0, + &vecvecsData[i][0] ); + } } // send message for get status (for GUI) @@ -491,7 +537,10 @@ void CServer::OnTimer() // generate a sparate mix for each channel // actual processing of audio data -> mix - vecsSendData = ProcessData ( vecvecsData, vecvecdGains[i] ); + vecsSendData = ProcessData ( i, + vecvecsData, + vecvecdGains[i], + vecNumAudioChannels ); // get current number of CELT coded bytes const int iCeltNumCodedBytes = @@ -500,11 +549,24 @@ void CServer::OnTimer() // CELT encoding CVector vecCeltData ( iCeltNumCodedBytes ); - celt_encode ( CeltEncoder[iCurChanID], - &vecsSendData[0], - NULL, - &vecCeltData[0], - iCeltNumCodedBytes ); + if ( vecChannels[iCurChanID].GetNumAudioChannels() == 1 ) + { + // mono + celt_encode ( CeltEncoderMono[iCurChanID], + &vecsSendData[0], + NULL, + &vecCeltData[0], + iCeltNumCodedBytes ); + } + else + { + // stereo + celt_encode ( CeltEncoderStereo[iCurChanID], + &vecsSendData[0], + NULL, + &vecCeltData[0], + iCeltNumCodedBytes ); + } // send separate mix to current clients Socket.SendPacket ( @@ -523,39 +585,136 @@ void CServer::OnTimer() CycleTimeVariance.Update(); } -CVector CServer::ProcessData ( CVector >& vecvecsData, - CVector& vecdGains ) +CVector CServer::ProcessData ( const int iCurIndex, + CVector >& vecvecsData, + CVector& vecdGains, + CVector& vecNumAudioChannels ) { - int i; + int i, j, k; + + // get number of audio channels of current channel + const int iCurNumAudChan = vecNumAudioChannels[iCurIndex]; + + // number of samples for output vector + const int iNumOutSamples = iCurNumAudChan * SYSTEM_FRAME_SIZE_SAMPLES; // init return vector with zeros since we mix all channels on that vector - -// TODO speed optimization: avoid using the zero vector, use the first valid -// data vector for initialization instead (take care of gain of this data, too!) - - CVector vecsOutData ( SYSTEM_FRAME_SIZE_SAMPLES, 0 ); + CVector vecsOutData ( iNumOutSamples, 0 ); const int iNumClients = vecvecsData.Size(); // mix all audio data from all clients together - for ( int j = 0; j < iNumClients; j++ ) + if ( iCurNumAudChan == 1 ) { - // if channel gain is 1, avoid multiplication for speed optimization - if ( vecdGains[j] == static_cast ( 1.0 ) ) + // Mono target channel ------------------------------------------------- + for ( j = 0; j < iNumClients; j++ ) { - for ( i = 0; i < SYSTEM_FRAME_SIZE_SAMPLES; i++ ) + // if channel gain is 1, avoid multiplication for speed optimization + if ( vecdGains[j] == static_cast ( 1.0 ) ) { - vecsOutData[i] = - Double2Short ( vecsOutData[i] + vecvecsData[j][i] ); + if ( vecNumAudioChannels[j] == 1 ) + { + // mono + for ( i = 0; i < SYSTEM_FRAME_SIZE_SAMPLES; i++ ) + { + vecsOutData[i] = + Double2Short ( vecsOutData[i] + vecvecsData[j][i] ); + } + } + else + { + // stereo: apply stereo-to-mono attenuation + for ( i = 0, k = 0; i < SYSTEM_FRAME_SIZE_SAMPLES; i++, k += 2 ) + { + vecsOutData[i] = + Double2Short ( vecsOutData[i] + + ( vecvecsData[j][k] + vecvecsData[j][k + 1] ) / 2 ); + } + } + } + else + { + if ( vecNumAudioChannels[j] == 1 ) + { + // mono + for ( i = 0; i < SYSTEM_FRAME_SIZE_SAMPLES; i++ ) + { + vecsOutData[i] = + Double2Short ( vecsOutData[i] + + vecvecsData[j][i] * vecdGains[j] ); + } + } + else + { + // stereo: apply stereo-to-mono attenuation + for ( i = 0, k = 0; i < SYSTEM_FRAME_SIZE_SAMPLES; i++, k += 2 ) + { + vecsOutData[i] = + Double2Short ( vecsOutData[i] + vecdGains[j] * + ( vecvecsData[j][k] + vecvecsData[j][k + 1] ) / 2 ); + } + } } } - else + } + else + { + // Stereo target channel ----------------------------------------------- + for ( j = 0; j < iNumClients; j++ ) { - for ( i = 0; i < SYSTEM_FRAME_SIZE_SAMPLES; i++ ) + // if channel gain is 1, avoid multiplication for speed optimization + if ( vecdGains[j] == static_cast ( 1.0 ) ) { - vecsOutData[i] = - Double2Short ( vecsOutData[i] + - vecvecsData[j][i] * vecdGains[j] ); + if ( vecNumAudioChannels[j] == 1 ) + { + // mono: copy same mono data in both out stereo audio channels + for ( i = 0, k = 0; i < SYSTEM_FRAME_SIZE_SAMPLES; i++, k += 2 ) + { + // left channel + vecsOutData[k] = + Double2Short ( vecsOutData[k] + vecvecsData[j][i] ); + + // right channel + vecsOutData[k + 1] = + Double2Short ( vecsOutData[k + 1] + vecvecsData[j][i] ); + } + } + else + { + // stereo + for ( i = 0; i < iNumOutSamples; i++ ) + { + vecsOutData[i] = + Double2Short ( vecsOutData[i] + vecvecsData[j][i] ); + } + } + } + else + { + if ( vecNumAudioChannels[j] == 1 ) + { + // mono: copy same mono data in both out stereo audio channels + for ( i = 0, k = 0; i < SYSTEM_FRAME_SIZE_SAMPLES; i++, k += 2 ) + { + // left channel + vecsOutData[k] = Double2Short ( + vecsOutData[k] + vecvecsData[j][i] * vecdGains[j] ); + + // right channel + vecsOutData[k + 1] = Double2Short ( + vecsOutData[k + 1] + vecvecsData[j][i] * vecdGains[j] ); + } + } + else + { + // stereo + for ( i = 0; i < iNumOutSamples; i++ ) + { + vecsOutData[i] = + Double2Short ( vecsOutData[i] + + vecvecsData[j][i] * vecdGains[j] ); + } + } } } } diff --git a/src/server.h b/src/server.h index 3e1a4ade..2e3b2974 100755 --- a/src/server.h +++ b/src/server.h @@ -140,8 +140,10 @@ protected: const QString& strChatText ); void WriteHTMLChannelList(); - CVector ProcessData ( CVector >& vecvecsData, - CVector& vecdGains ); + CVector ProcessData ( const int iCurIndex, + CVector >& vecvecsData, + CVector& vecdGains, + CVector& vecNumAudioChannels ); virtual void customEvent ( QEvent* Event ); @@ -151,9 +153,12 @@ protected: QMutex Mutex; // audio encoder/decoder - CELTMode* CeltMode[MAX_NUM_CHANNELS]; - CELTEncoder* CeltEncoder[MAX_NUM_CHANNELS]; - CELTDecoder* CeltDecoder[MAX_NUM_CHANNELS]; + CELTMode* CeltModeMono[MAX_NUM_CHANNELS]; + CELTEncoder* CeltEncoderMono[MAX_NUM_CHANNELS]; + CELTDecoder* CeltDecoderMono[MAX_NUM_CHANNELS]; + CELTMode* CeltModeStereo[MAX_NUM_CHANNELS]; + CELTEncoder* CeltEncoderStereo[MAX_NUM_CHANNELS]; + CELTDecoder* CeltDecoderStereo[MAX_NUM_CHANNELS]; CVector vstrChatColors;