diff --git a/ChangeLog b/ChangeLog index 05d05cb7..c25a1ee5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,10 +3,19 @@ 3.4.7git +TODO add command line argument for 64/124 frame size at the server + TODO support OPUS 64 in the server as a first step towards official 64 samples frame size support +TODO for different frame sizes (64/128) the start value of 6 for the jitter buffer might be too low + TODO #12 A server in the same NAT region as the client is not visible (https://sourceforge.net/p/llcon/bugs/12/) +TODO https://sourceforge.net/p/llcon/discussion/533517/thread/62bd07fbcb/?limit=25#ea21 + 1) The column widths are slightly too narrow (UK English) for "Ping Time" and "Musicians" in the "Connection Setup panel. + 2) Spellings: It would be nice if the Location Country field had spaces in the data e.g. UnitedKingdom > United Kingdom + Instrument "Acoutic Guitar" > "Acoustic Guitar", Instrument "Accordeon" > "Accordion" + TODO in CreateLevelsForAllConChannels we should call CStereoSignalLevelMeter::CalcLogResult instead -> make it public and static TODO Multichannel CoreAudio Support #44 -> integrate important code lines in Mac sound interface diff --git a/src/buffer.h b/src/buffer.h index b92413fe..4890130e 100755 --- a/src/buffer.h +++ b/src/buffer.h @@ -475,16 +475,33 @@ public: { // allocate internal memory and reset read/write positions vecMemory.Init ( iNewMemSize ); - iMemSize = iNewMemSize; - iPutPos = 0; - iGetPos = 0; + iMemSize = iNewMemSize; + iBufferSize = iNewMemSize; + Reset(); + } + + void Reset() + { + iPutPos = 0; + iGetPos = 0; + } + + void SetBufferSize ( const int iNBSize ) + { + // if buffer size has changed, apply new value and reset the buffer pointers + if ( ( iNBSize != iBufferSize ) && ( iNBSize <= iMemSize ) ) + { + iBufferSize = iNBSize; + Reset(); + } } void PutAll ( const CVector& vecsData ) { iGetPos = 0; + std::copy ( vecsData.begin(), - vecsData.begin() + iMemSize, // note that input vector might be larger then memory size + vecsData.begin() + iBufferSize, // note that input vector might be larger then memory size vecMemory.begin() ); } @@ -495,7 +512,7 @@ public: const int iEnd = iPutPos + iVecSize; // first check for buffer overrun - if ( iEnd <= iMemSize ) + if ( iEnd <= iBufferSize ) { // copy new data in internal buffer std::copy ( vecsData.begin(), @@ -506,7 +523,7 @@ public: iPutPos = iEnd; // return "buffer is ready for readout" flag - return ( iEnd == iMemSize ); + return ( iEnd == iBufferSize ); } // buffer overrun or not initialized, return "not ready" @@ -519,6 +536,17 @@ public: return vecMemory; } + void GetAll ( CVector& vecsData, + const int iVecSize ) + { + iPutPos = 0; + + // copy data from internal buffer in given buffer + std::copy ( vecMemory.begin(), + vecMemory.begin() + iVecSize, + vecsData.begin() ); + } + bool Get ( CVector& vecsData, const int iVecSize ) { @@ -526,7 +554,7 @@ public: const int iEnd = iGetPos + iVecSize; // first check for buffer underrun - if ( iEnd <= iMemSize ) + if ( iEnd <= iBufferSize ) { // copy new data from internal buffer std::copy ( vecMemory.begin() + iGetPos, @@ -536,16 +564,17 @@ public: // set buffer pointer one block further iGetPos = iEnd; - // return "buffer is completely read" flag - return ( iEnd == iMemSize ); + // return the memory could be read + return true; } - // buffer underrun or not initialized, return "completely read" - return true; + // return that no memory could be read + return false; } protected: CVector vecMemory; int iMemSize; + int iBufferSize; int iPutPos, iGetPos; }; diff --git a/src/server.cpp b/src/server.cpp index 5873889a..466c980a 100755 --- a/src/server.cpp +++ b/src/server.cpp @@ -295,6 +295,13 @@ CServer::CServer ( const int iNewMaxNumChan, // set encoder low complexity for legacy 128 samples frame size opus_custom_encoder_ctl ( OpusEncoderMono[i], OPUS_SET_COMPLEXITY ( 1 ) ); opus_custom_encoder_ctl ( OpusEncoderStereo[i], OPUS_SET_COMPLEXITY ( 1 ) ); + + + // init double-to-normal frame size conversion buffers ----------------- + // use worst case memory initialization to avoid allocating memory in + // the time-critical thread + DoubleFrameSizeConvBufIn[i].Init ( 2 /* stereo */ * DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES /* worst case buffer size */ ); + DoubleFrameSizeConvBufOut[i].Init ( 2 /* stereo */ * DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES /* worst case buffer size */ ); } // define colors for chat window identifiers @@ -326,12 +333,13 @@ CServer::CServer ( const int iNewMaxNumChan, vecsSendData.Init ( 2 /* stereo */ * DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES /* worst case buffer size */ ); // allocate worst case memory for the temporary vectors - vecChanIDsCurConChan.Init ( iMaxNumChannels ); - vecvecdGains.Init ( iMaxNumChannels ); - vecvecsData.Init ( iMaxNumChannels ); - vecNumAudioChannels.Init ( iMaxNumChannels ); - vecNumFrameSizeConvBlocks.Init ( iMaxNumChannels ); - vecAudioComprType.Init ( iMaxNumChannels ); + vecChanIDsCurConChan.Init ( iMaxNumChannels ); + vecvecdGains.Init ( iMaxNumChannels ); + vecvecsData.Init ( iMaxNumChannels ); + vecNumAudioChannels.Init ( iMaxNumChannels ); + vecNumFrameSizeConvBlocks.Init ( iMaxNumChannels ); + vecUseDoubleSysFraSizeConvBuf.Init ( iMaxNumChannels ); + vecAudioComprType.Init ( iMaxNumChannels ); for ( i = 0; i < iMaxNumChannels; i++ ) { @@ -788,6 +796,10 @@ CreateAndSendChanListForAllConChannels(); { vecChannels[iChID].CreateLicReqMes ( eLicenceType ); } + + // reset the conversion buffers + DoubleFrameSizeConvBufIn[iChID].Reset(); + DoubleFrameSizeConvBufOut[iChID].Reset(); } void CServer::OnServerFull ( CHostAddress RecHostAddr ) @@ -907,6 +919,8 @@ JitterMeas.Measure(); vecAudioComprType[i] = vecChannels[iCurChanID].GetAudioCompressionType(); // get info about required frame size conversion properties + vecUseDoubleSysFraSizeConvBuf[i] = ( !bUseDoubleSystemFrameSize && ( vecAudioComprType[i] == CT_OPUS ) ); + if ( bUseDoubleSystemFrameSize && ( vecAudioComprType[i] == CT_OPUS64 ) ) { vecNumFrameSizeConvBlocks[i] = 2; @@ -916,6 +930,13 @@ JitterMeas.Measure(); vecNumFrameSizeConvBlocks[i] = 1; } + // update conversion buffer size (nothing will happen if the size stays the same) + if ( vecUseDoubleSysFraSizeConvBuf[i] ) + { + DoubleFrameSizeConvBufIn[iCurChanID].SetBufferSize ( DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES * vecNumAudioChannels[i] ); + DoubleFrameSizeConvBufOut[iCurChanID].SetBufferSize ( DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES * vecNumAudioChannels[i] ); + } + // select the opus decoder and raw audio frame length if ( vecAudioComprType[i] == CT_OPUS ) { @@ -961,46 +982,63 @@ JitterMeas.Measure(); vecvecdGains[i][j] *= vecChannels[vecChanIDsCurConChan[j]].GetFadeInGain(); } - // get current number of CELT coded bytes - const int iCeltNumCodedBytes = vecChannels[iCurChanID].GetNetwFrameSize(); - - for ( int iB = 0; iB < vecNumFrameSizeConvBlocks[i]; iB++ ) + // If the server frame size is smaller than the received OPUS frame size, we need a conversion + // buffer which stores the large buffer. + // Note that we have a shortcut here. If the conversion buffer is not needed, the boolean flag + // is false and the Get() function is not called at all. Therefore if the buffer is not needed + // we do not spend any time in the function but go directly inside the if condition. + if ( ( vecUseDoubleSysFraSizeConvBuf[i] == 0 ) || + !DoubleFrameSizeConvBufIn[iCurChanID].Get ( vecvecsData[i], SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i] ) ) { - // get data - const EGetDataStat eGetStat = vecChannels[iCurChanID].GetData ( vecbyCodedData, iCeltNumCodedBytes ); + // get current number of OPUS coded bytes + const int iCeltNumCodedBytes = vecChannels[iCurChanID].GetNetwFrameSize(); - // if channel was just disconnected, set flag that connected - // client list is sent to all other clients - // and emit the client disconnected signal - if ( eGetStat == GS_CHAN_NOW_DISCONNECTED ) + for ( int iB = 0; iB < vecNumFrameSizeConvBlocks[i]; iB++ ) { - if ( bEnableRecording ) + // get data + const EGetDataStat eGetStat = vecChannels[iCurChanID].GetData ( vecbyCodedData, iCeltNumCodedBytes ); + + // if channel was just disconnected, set flag that connected + // client list is sent to all other clients + // and emit the client disconnected signal + if ( eGetStat == GS_CHAN_NOW_DISCONNECTED ) { - emit ClientDisconnected ( iCurChanID ); // TODO do this outside the mutex lock? + if ( bEnableRecording ) + { + emit ClientDisconnected ( iCurChanID ); // TODO do this outside the mutex lock? + } + + bChannelIsNowDisconnected = true; } - bChannelIsNowDisconnected = true; + // get pointer to coded data + if ( eGetStat == GS_BUFFER_OK ) + { + pCurCodedData = &vecbyCodedData[0]; + } + else + { + // for lost packets use null pointer as coded input data + pCurCodedData = nullptr; + } + + // OPUS decode received data stream + if ( CurOpusDecoder != nullptr ) + { + opus_custom_decode ( CurOpusDecoder, + pCurCodedData, + iCeltNumCodedBytes, + &vecvecsData[i][iB * SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i]], + iClientFrameSizeSamples ); + } } - // get pointer to coded data - if ( eGetStat == GS_BUFFER_OK ) + // a new large frame is ready, if the conversion buffer is required, put it in the buffer + // and read out the small frame size immediately for further processing + if ( vecUseDoubleSysFraSizeConvBuf[i] != 0 ) { - pCurCodedData = &vecbyCodedData[0]; - } - else - { - // for lost packets use null pointer as coded input data - pCurCodedData = nullptr; - } - - // OPUS decode received data stream - if ( CurOpusDecoder != nullptr ) - { - opus_custom_decode ( CurOpusDecoder, - pCurCodedData, - iCeltNumCodedBytes, - &vecvecsData[i][iB * SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i]], - iClientFrameSizeSamples ); + DoubleFrameSizeConvBufIn[iCurChanID].PutAll ( vecvecsData[i] ); + DoubleFrameSizeConvBufIn[iCurChanID].Get ( vecvecsData[i], SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i] ); } } } @@ -1109,37 +1147,52 @@ JitterMeas.Measure(); CurOpusEncoder = nullptr; } - for ( int iB = 0; iB < vecNumFrameSizeConvBlocks[i]; iB++ ) + // If the server frame size is smaller than the received OPUS frame size, we need a conversion + // buffer which stores the large buffer. + // Note that we have a shortcut here. If the conversion buffer is not needed, the boolean flag + // is false and the Get() function is not called at all. Therefore if the buffer is not needed + // we do not spend any time in the function but go directly inside the if condition. + if ( ( vecUseDoubleSysFraSizeConvBuf[i] == 0 ) || + DoubleFrameSizeConvBufOut[iCurChanID].Put ( vecsSendData, SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i] ) ) { - // OPUS encoding - if ( CurOpusEncoder != nullptr ) + if ( vecUseDoubleSysFraSizeConvBuf[i] != 0 ) { + // get the large frame from the conversion buffer + DoubleFrameSizeConvBufOut[iCurChanID].GetAll ( vecsSendData, DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES * vecNumAudioChannels[i] ); + } + + for ( int iB = 0; iB < vecNumFrameSizeConvBlocks[i]; iB++ ) + { + // OPUS encoding + if ( CurOpusEncoder != nullptr ) + { // TODO find a better place than this: the setting does not change all the time // so for speed optimization it would be better to set it only if the network // frame size is changed opus_custom_encoder_ctl ( CurOpusEncoder, OPUS_SET_BITRATE ( CalcBitRateBitsPerSecFromCodedBytes ( iCeltNumCodedBytes, iClientFrameSizeSamples ) ) ); - opus_custom_encode ( CurOpusEncoder, - &vecsSendData[iB * SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i]], - iClientFrameSizeSamples, - &vecbyCodedData[0], - iCeltNumCodedBytes ); + opus_custom_encode ( CurOpusEncoder, + &vecsSendData[iB * SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i]], + iClientFrameSizeSamples, + &vecbyCodedData[0], + iCeltNumCodedBytes ); + } + + // send separate mix to current clients + vecChannels[iCurChanID].PrepAndSendPacket ( &Socket, + vecbyCodedData, + iCeltNumCodedBytes ); } - // send separate mix to current clients - vecChannels[iCurChanID].PrepAndSendPacket ( &Socket, - vecbyCodedData, - iCeltNumCodedBytes ); - } + // update socket buffer size + vecChannels[iCurChanID].UpdateSocketBufferSize(); - // update socket buffer size - vecChannels[iCurChanID].UpdateSocketBufferSize(); - - // send channel levels - if ( bSendChannelLevels && vecChannels[iCurChanID].ChannelLevelsRequired() ) - { - ConnLessProtocol.CreateCLChannelLevelListMes ( vecChannels[iCurChanID].GetAddress(), vecChannelLevels, iNumClients ); + // send channel levels + if ( bSendChannelLevels && vecChannels[iCurChanID].ChannelLevelsRequired() ) + { + ConnLessProtocol.CreateCLChannelLevelListMes ( vecChannels[iCurChanID].GetAddress(), vecChannelLevels, iNumClients ); + } } } } diff --git a/src/server.h b/src/server.h index 5496905b..988a51e4 100755 --- a/src/server.h +++ b/src/server.h @@ -35,6 +35,7 @@ # include "opus_custom.h" #endif #include "global.h" +#include "buffer.h" #include "socket.h" #include "channel.h" #include "util.h" @@ -251,6 +252,8 @@ protected: OpusCustomDecoder* OpusDecoderMono[MAX_NUM_CHANNELS]; OpusCustomEncoder* OpusEncoderStereo[MAX_NUM_CHANNELS]; OpusCustomDecoder* OpusDecoderStereo[MAX_NUM_CHANNELS]; + CConvBuf DoubleFrameSizeConvBufIn[MAX_NUM_CHANNELS]; + CConvBuf DoubleFrameSizeConvBufOut[MAX_NUM_CHANNELS]; CVector vstrChatColors; CVector vecChanIDsCurConChan; @@ -259,6 +262,7 @@ protected: CVector > vecvecsData; CVector vecNumAudioChannels; CVector vecNumFrameSizeConvBlocks; + CVector vecUseDoubleSysFraSizeConvBuf; CVector vecAudioComprType; CVector vecsSendData; CVector vecbyCodedData;