From 57f203502cb50f3d8afdbe1b1a5508612406d1fc Mon Sep 17 00:00:00 2001 From: Volker Fischer Date: Sat, 4 Apr 2020 19:03:19 +0200 Subject: [PATCH] WIP on support of 64/128 samples frame size in the server --- src/channel.cpp | 12 +- src/global.h | 4 +- src/main.cpp | 5 + src/server.cpp | 310 +++++++++++++++++++++++------------------------- src/server.h | 14 ++- 5 files changed, 170 insertions(+), 175 deletions(-) diff --git a/src/channel.cpp b/src/channel.cpp index bebdf501..9f3171bc 100755 --- a/src/channel.cpp +++ b/src/channel.cpp @@ -346,18 +346,12 @@ void CChannel::OnNetTranspPropsReceived ( CNetworkTransportProps NetworkTranspor // only the server shall act on network transport properties message if ( bIsServer ) { - // OPUS codec is the only supported codec right now -#if ( SYSTEM_FRAME_SIZE_SAMPLES == 64 ) - if ( NetworkTransportProps.eAudioCodingType != CT_OPUS64 ) + // OPUS and OPUS64 codecs are the only supported codecs right now + if ( ( NetworkTransportProps.eAudioCodingType != CT_OPUS ) && + ( NetworkTransportProps.eAudioCodingType != CT_OPUS64 ) ) { return; } -#else - if ( NetworkTransportProps.eAudioCodingType != CT_OPUS ) - { - return; - } -#endif Mutex.lock(); { diff --git a/src/global.h b/src/global.h index 6e2040ab..5b6db1d2 100755 --- a/src/global.h +++ b/src/global.h @@ -98,7 +98,9 @@ LED bar: lbr // System block size, this is the block size on which the audio coder works. // All other block sizes must be a multiple of this size. // Note that the UpdateAutoSetting() function assumes a value of 128. -#define SYSTEM_FRAME_SIZE_SAMPLES 128 +#define SYSTEM_FRAME_SIZE_SAMPLES_SMALL 64 // TODO this is temporary and shall be replaced by SYSTEM_FRAME_SIZE_SAMPLES later on +#define DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES ( 2 * SYSTEM_FRAME_SIZE_SAMPLES_SMALL ) +#define SYSTEM_FRAME_SIZE_SAMPLES 128 // default server address #define DEFAULT_SERVER_ADDRESS "jamulus.fischvolk.de" diff --git a/src/main.cpp b/src/main.cpp index adc591eb..d008d70e 100755 --- a/src/main.cpp +++ b/src/main.cpp @@ -513,6 +513,10 @@ int main ( int argc, char** argv ) } else { + +// TODO +const bool bUseDoubleSystemFrameSize = true; // TODO for now this value is fixed but has to be changeable by command line argument + // Server: // actual server object CServer Server ( iNumServerChannels, @@ -528,6 +532,7 @@ int main ( int argc, char** argv ) strRecordingDirName, bCentServPingServerInList, bDisconnectAllClients, + bUseDoubleSystemFrameSize, eLicenceType ); if ( bUseGUI ) { diff --git a/src/server.cpp b/src/server.cpp index 3cdcb096..7cba1951 100755 --- a/src/server.cpp +++ b/src/server.cpp @@ -27,11 +27,12 @@ // CHighPrecisionTimer implementation ****************************************** #ifdef _WIN32 -CHighPrecisionTimer::CHighPrecisionTimer() +CHighPrecisionTimer::CHighPrecisionTimer ( const bool bNewUseDoubleSystemFrameSize ) : + bUseDoubleSystemFrameSize ( bNewUseDoubleSystemFrameSize ) { // add some error checking, the high precision timer implementation only // supports 64 and 128 samples frame size at 48 kHz sampling rate -#if ( SYSTEM_FRAME_SIZE_SAMPLES != 64 ) && ( SYSTEM_FRAME_SIZE_SAMPLES != 128 ) +#if ( SYSTEM_FRAME_SIZE_SAMPLES_SMALL != 64 ) && ( DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES != 128 ) # error "Only system frame size of 64 and 128 samples is supported by this module" #endif #if ( SYSTEM_SAMPLE_RATE_HZ != 48000 ) @@ -68,13 +69,16 @@ void CHighPrecisionTimer::Start() iCurPosInVector = 0; iIntervalCounter = 0; -#if ( SYSTEM_FRAME_SIZE_SAMPLES == 64 ) - // start internal timer with 1 ms resolution for 64 samples frame size - Timer.start ( 1 ); -#else - // start internal timer with 2 ms resolution for 128 samples frame size - Timer.start ( 2 ); -#endif + if ( bUseDoubleSystemFrameSize ) + { + // start internal timer with 2 ms resolution for 128 samples frame size + Timer.start ( 2 ); + } + else + { + // start internal timer with 1 ms resolution for 64 samples frame size + Timer.start ( 1 ); + } } void CHighPrecisionTimer::Stop() @@ -110,13 +114,22 @@ void CHighPrecisionTimer::OnTimer() } } #else // Mac and Linux -CHighPrecisionTimer::CHighPrecisionTimer() : +CHighPrecisionTimer::CHighPrecisionTimer ( const bool bUseDoubleSystemFrameSize ) : bRun ( false ) { // calculate delay in ns - const uint64_t iNsDelay = - ( (uint64_t) SYSTEM_FRAME_SIZE_SAMPLES * 1000000000 ) / - (uint64_t) SYSTEM_SAMPLE_RATE_HZ; // in ns + const uint64_t iNsDelay; + + if ( bUseDoubleSystemFrameSize ) + { + iNsDelay = ( (uint64_t) DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES * 1000000000 ) / + (uint64_t) SYSTEM_SAMPLE_RATE_HZ; // in ns + } + else + { + iNsDelay = ( (uint64_t) SYSTEM_FRAME_SIZE_SAMPLES_SMALL * 1000000000 ) / + (uint64_t) SYSTEM_SAMPLE_RATE_HZ; // in ns + } #if defined ( __APPLE__ ) || defined ( __MACOSX ) // calculate delay in mach absolute time @@ -219,81 +232,69 @@ CServer::CServer ( const int iNewMaxNumChan, const QString& strRecordingDirName, const bool bNCentServPingServerInList, const bool bNDisconnectAllClients, + const bool bNUseDoubleSystemFrameSize, const ELicenceType eNLicenceType ) : - iMaxNumChannels ( iNewMaxNumChan ), - Socket ( this, iPortNumber ), - Logging ( iMaxDaysHistory ), - JamRecorder ( strRecordingDirName ), - bEnableRecording ( !strRecordingDirName.isEmpty() ), - bWriteStatusHTMLFile ( false ), - ServerListManager ( iPortNumber, - strCentralServer, - strServerInfo, - iNewMaxNumChan, - bNCentServPingServerInList, - &ConnLessProtocol ), - bAutoRunMinimized ( false ), - strWelcomeMessage ( strNewWelcomeMessage ), - eLicenceType ( eNLicenceType ), - bDisconnectAllClients ( bNDisconnectAllClients ) + bUseDoubleSystemFrameSize ( bNUseDoubleSystemFrameSize ), + iMaxNumChannels ( iNewMaxNumChan ), + Socket ( this, iPortNumber ), + Logging ( iMaxDaysHistory ), + JamRecorder ( strRecordingDirName ), + bEnableRecording ( !strRecordingDirName.isEmpty() ), + bWriteStatusHTMLFile ( false ), + HighPrecisionTimer ( bNUseDoubleSystemFrameSize ), + ServerListManager ( iPortNumber, + strCentralServer, + strServerInfo, + iNewMaxNumChan, + bNCentServPingServerInList, + &ConnLessProtocol ), + bAutoRunMinimized ( false ), + strWelcomeMessage ( strNewWelcomeMessage ), + eLicenceType ( eNLicenceType ), + bDisconnectAllClients ( bNDisconnectAllClients ) { int iOpusError; int i; - // create CELT encoder/decoder for each channel (must be done before + // create OPUS encoder/decoder for each channel (must be done before // enabling the channels), create a mono and stereo encoder/decoder // for each channel for ( i = 0; i < iMaxNumChannels; i++ ) { - // init audio endocder/decoder (mono) + // init OPUS ----------------------------------------------------------- OpusMode[i] = opus_custom_mode_create ( SYSTEM_SAMPLE_RATE_HZ, - SYSTEM_FRAME_SIZE_SAMPLES, + DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES, &iOpusError ); - OpusEncoderMono[i] = opus_custom_encoder_create ( OpusMode[i], - 1, - &iOpusError ); + Opus64Mode[i] = opus_custom_mode_create ( SYSTEM_SAMPLE_RATE_HZ, + SYSTEM_FRAME_SIZE_SAMPLES_SMALL, + &iOpusError ); - OpusDecoderMono[i] = opus_custom_decoder_create ( OpusMode[i], - 1, - &iOpusError ); + // init audio encoders and decoders + OpusEncoderMono[i] = opus_custom_encoder_create ( OpusMode[i], 1, &iOpusError ); // mono encoder legacy + OpusDecoderMono[i] = opus_custom_decoder_create ( OpusMode[i], 1, &iOpusError ); // mono decoder legacy + OpusEncoderStereo[i] = opus_custom_encoder_create ( OpusMode[i], 2, &iOpusError ); // stereo encoder legacy + OpusDecoderStereo[i] = opus_custom_decoder_create ( OpusMode[i], 2, &iOpusError ); // stereo decoder legacy + Opus64EncoderMono[i] = opus_custom_encoder_create ( Opus64Mode[i], 1, &iOpusError ); // mono encoder OPUS64 + Opus64DecoderMono[i] = opus_custom_decoder_create ( Opus64Mode[i], 1, &iOpusError ); // mono decoder OPUS64 + Opus64EncoderStereo[i] = opus_custom_encoder_create ( Opus64Mode[i], 2, &iOpusError ); // stereo encoder OPUS64 + Opus64DecoderStereo[i] = opus_custom_decoder_create ( Opus64Mode[i], 2, &iOpusError ); // stereo decoder OPUS64 // we require a constant bit rate - opus_custom_encoder_ctl ( OpusEncoderMono[i], - OPUS_SET_VBR ( 0 ) ); + opus_custom_encoder_ctl ( OpusEncoderMono[i], OPUS_SET_VBR ( 0 ) ); + opus_custom_encoder_ctl ( OpusEncoderStereo[i], OPUS_SET_VBR ( 0 ) ); + opus_custom_encoder_ctl ( Opus64EncoderMono[i], OPUS_SET_VBR ( 0 ) ); + opus_custom_encoder_ctl ( Opus64EncoderStereo[i], OPUS_SET_VBR ( 0 ) ); // we want as low delay as possible - opus_custom_encoder_ctl ( OpusEncoderMono[i], - OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) ); + opus_custom_encoder_ctl ( OpusEncoderMono[i], OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) ); + opus_custom_encoder_ctl ( OpusEncoderStereo[i], OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) ); + opus_custom_encoder_ctl ( Opus64EncoderMono[i], OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) ); + opus_custom_encoder_ctl ( Opus64EncoderStereo[i], OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) ); -#if ( SYSTEM_FRAME_SIZE_SAMPLES == 128 ) // set encoder low complexity for legacy 128 samples frame size - opus_custom_encoder_ctl ( OpusEncoderMono[i], - OPUS_SET_COMPLEXITY ( 1 ) ); -#endif - - // init audio endocder/decoder (stereo) - OpusEncoderStereo[i] = opus_custom_encoder_create ( OpusMode[i], - 2, - &iOpusError ); - - OpusDecoderStereo[i] = opus_custom_decoder_create ( OpusMode[i], - 2, - &iOpusError ); - - // we require a constant bit rate - opus_custom_encoder_ctl ( OpusEncoderStereo[i], - OPUS_SET_VBR ( 0 ) ); - - // we want as low delay as possible - opus_custom_encoder_ctl ( OpusEncoderStereo[i], - OPUS_SET_APPLICATION ( OPUS_APPLICATION_RESTRICTED_LOWDELAY ) ); - -#if ( SYSTEM_FRAME_SIZE_SAMPLES == 128 ) - // set encoder low complexity for legacy 128 samples frame size - opus_custom_encoder_ctl ( OpusEncoderStereo[i], - OPUS_SET_COMPLEXITY ( 1 ) ); -#endif + opus_custom_encoder_ctl ( OpusEncoderMono[i], OPUS_SET_COMPLEXITY ( 1 ) ); + opus_custom_encoder_ctl ( OpusEncoderStereo[i], OPUS_SET_COMPLEXITY ( 1 ) ); } // define colors for chat window identifiers @@ -312,7 +313,7 @@ CServer::CServer ( const int iNewMaxNumChan, // the worst case here: // we always use stereo audio buffers (which is the worst case) - vecsSendData.Init ( 2 * SYSTEM_FRAME_SIZE_SAMPLES ); + vecsSendData.Init ( 2 /* stereo */ * DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES /* worst case buffer size */ ); // allocate worst case memory for the temporary vectors vecChanIDsCurConChan.Init ( iMaxNumChannels ); @@ -326,7 +327,7 @@ CServer::CServer ( const int iNewMaxNumChan, vecvecdGains[i].Init ( iMaxNumChannels ); // we always use stereo audio buffers (see "vecsSendData") - vecvecsData[i].Init ( 2 * SYSTEM_FRAME_SIZE_SAMPLES ); + vecvecsData[i].Init ( 2 /* stereo */ * DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES /* worst case buffer size */ ); } // allocate worst case memory for the coded data @@ -845,7 +846,10 @@ void CServer::Stop() void CServer::OnTimer() { - int i, j; + int i, j; + OpusCustomDecoder* CurOpusDecoder; + OpusCustomEncoder* CurOpusEncoder; + unsigned char* pCurCodedData; /* // TEST do a timer jitter measurement @@ -882,9 +886,17 @@ JitterMeas.Measure(); // get actual ID of current channel const int iCurChanID = vecChanIDsCurConChan[i]; - // get and store number of audio channels - const int iCurNumAudChan = vecChannels[iCurChanID].GetNumAudioChannels(); - vecNumAudioChannels[i] = iCurNumAudChan; + // get and store number of audio channels and select the opus decoder + vecNumAudioChannels[i] = vecChannels[iCurChanID].GetNumAudioChannels(); + + if ( vecNumAudioChannels[i] == 1 ) + { + CurOpusDecoder = OpusDecoderMono[iCurChanID]; + } + else + { + CurOpusDecoder = OpusDecoderStereo[iCurChanID]; + } // get gains of all connected channels for ( j = 0; j < iNumClients; j++ ) @@ -902,74 +914,58 @@ JitterMeas.Measure(); // get current number of CELT coded bytes const int iCeltNumCodedBytes = vecChannels[iCurChanID].GetNetwFrameSize(); - // get data - const EGetDataStat eGetStat = - vecChannels[iCurChanID].GetData ( vecbyCodedData, - iCeltNumCodedBytes ); +// TODO +const bool bIsServerDoubleFrameSize = bUseDoubleSystemFrameSize && ( vecChannels[iCurChanID].GetAudioCompressionType() == CT_OPUS64 ); +const bool bIsClientDoubleFrameSize = !bUseDoubleSystemFrameSize && ( vecChannels[iCurChanID].GetAudioCompressionType() == CT_OPUS ); +const bool bIsCompatibleFramesSize = !( bIsServerDoubleFrameSize || bIsClientDoubleFrameSize ); +//bUseDoubleSystemFrameSize +//ConvBuf +CConvBuf ConvBufIn; // TEST NOT WORKING!!!! - // if channel was just disconnected, set flag that connected - // client list is sent to all other clients - // and emit the client disconnected signal - if ( eGetStat == GS_CHAN_NOW_DISCONNECTED ) +//ConvBufIn.Put ( ); +int iNumInBlocks = 1; + + for ( int iB = 0; iB < iNumInBlocks; iB++ ) { - if ( bEnableRecording ) + // get data + const EGetDataStat eGetStat = vecChannels[iCurChanID].GetData ( vecbyCodedData, iCeltNumCodedBytes ); + + // if channel was just disconnected, set flag that connected + // client list is sent to all other clients + // and emit the client disconnected signal + if ( eGetStat == GS_CHAN_NOW_DISCONNECTED ) { - emit ClientDisconnected ( iCurChanID ); // TODO do this outside the mutex lock? + if ( bEnableRecording ) + { + emit ClientDisconnected ( iCurChanID ); // TODO do this outside the mutex lock? + } + + bChannelIsNowDisconnected = true; } - bChannelIsNowDisconnected = true; - } + // get pointer to coded data + if ( eGetStat == GS_BUFFER_OK ) + { + pCurCodedData = &vecbyCodedData[0]; + } + else + { + // for lost packets use null pointer as coded input data + pCurCodedData = nullptr; + } + + // OPUS decode received data stream +// SYSTEM_FRAME_SIZE_SAMPLES_SMALL +// DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES - // OPUS decode received data stream - if ( eGetStat == GS_BUFFER_OK ) - { if ( ( vecChannels[iCurChanID].GetAudioCompressionType() == CT_OPUS ) || ( vecChannels[iCurChanID].GetAudioCompressionType() == CT_OPUS64 ) ) { - if ( iCurNumAudChan == 1 ) - { - // mono - opus_custom_decode ( OpusDecoderMono[iCurChanID], - &vecbyCodedData[0], - iCeltNumCodedBytes, - &vecvecsData[i][0], - SYSTEM_FRAME_SIZE_SAMPLES ); - } - else - { - // stereo - opus_custom_decode ( OpusDecoderStereo[iCurChanID], - &vecbyCodedData[0], - iCeltNumCodedBytes, - &vecvecsData[i][0], - SYSTEM_FRAME_SIZE_SAMPLES ); - } - } - } - else - { - // lost packet - if ( ( vecChannels[iCurChanID].GetAudioCompressionType() == CT_OPUS ) || - ( vecChannels[iCurChanID].GetAudioCompressionType() == CT_OPUS64 ) ) - { - if ( iCurNumAudChan == 1 ) - { - // mono - opus_custom_decode ( OpusDecoderMono[iCurChanID], - nullptr, - iCeltNumCodedBytes, - &vecvecsData[i][0], - SYSTEM_FRAME_SIZE_SAMPLES ); - } - else - { - // stereo - opus_custom_decode ( OpusDecoderStereo[iCurChanID], - nullptr, - iCeltNumCodedBytes, - &vecvecsData[i][0], - SYSTEM_FRAME_SIZE_SAMPLES ); - } + opus_custom_decode ( CurOpusDecoder, + pCurCodedData, + iCeltNumCodedBytes, + &vecvecsData[i][0], + SYSTEM_FRAME_SIZE_SAMPLES ); } } } @@ -1017,45 +1013,33 @@ JitterMeas.Measure(); iNumClients ); // get current number of CELT coded bytes - const int iCeltNumCodedBytes = - vecChannels[iCurChanID].GetNetwFrameSize(); + const int iCeltNumCodedBytes = vecChannels[iCurChanID].GetNetwFrameSize(); + + // select the opus encoder + if ( vecChannels[iCurChanID].GetNumAudioChannels() == 1 ) + { + CurOpusEncoder = OpusEncoderMono[iCurChanID]; + } + else + { + CurOpusEncoder = OpusEncoderStereo[iCurChanID]; + } // OPUS encoding if ( ( vecChannels[iCurChanID].GetAudioCompressionType() == CT_OPUS ) || ( vecChannels[iCurChanID].GetAudioCompressionType() == CT_OPUS64 ) ) { - if ( vecChannels[iCurChanID].GetNumAudioChannels() == 1 ) - { - // mono: - // TODO find a better place than this: the setting does not change all the time // so for speed optimization it would be better to set it only if the network // frame size is changed -opus_custom_encoder_ctl ( OpusEncoderMono[iCurChanID], +opus_custom_encoder_ctl ( CurOpusEncoder, OPUS_SET_BITRATE ( CalcBitRateBitsPerSecFromCodedBytes ( iCeltNumCodedBytes ) ) ); - opus_custom_encode ( OpusEncoderMono[iCurChanID], - &vecsSendData[0], - SYSTEM_FRAME_SIZE_SAMPLES, - &vecbyCodedData[0], - iCeltNumCodedBytes ); - } - else - { - // stereo: - -// TODO find a better place than this: the setting does not change all the time -// so for speed optimization it would be better to set it only if the network -// frame size is changed -opus_custom_encoder_ctl ( OpusEncoderStereo[iCurChanID], - OPUS_SET_BITRATE ( CalcBitRateBitsPerSecFromCodedBytes ( iCeltNumCodedBytes ) ) ); - - opus_custom_encode ( OpusEncoderStereo[iCurChanID], - &vecsSendData[0], - SYSTEM_FRAME_SIZE_SAMPLES, - &vecbyCodedData[0], - iCeltNumCodedBytes ); - } + opus_custom_encode ( CurOpusEncoder, + &vecsSendData[0], + SYSTEM_FRAME_SIZE_SAMPLES, + &vecbyCodedData[0], + iCeltNumCodedBytes ); } // send separate mix to current clients diff --git a/src/server.h b/src/server.h index 10800ff5..819f0016 100755 --- a/src/server.h +++ b/src/server.h @@ -55,7 +55,7 @@ class CHighPrecisionTimer : public QObject Q_OBJECT public: - CHighPrecisionTimer(); + CHighPrecisionTimer ( const bool bNewUseDoubleSystemFrameSize ); void Start(); void Stop(); @@ -66,6 +66,7 @@ protected: CVector veciTimeOutIntervals; int iCurPosInVector; int iIntervalCounter; + bool bUseDoubleSystemFrameSize; public slots: void OnTimer(); @@ -88,7 +89,7 @@ class CHighPrecisionTimer : public QThread Q_OBJECT public: - CHighPrecisionTimer(); + CHighPrecisionTimer ( const bool bUseDoubleSystemFrameSize ); void Start(); void Stop(); @@ -131,6 +132,7 @@ public: const QString& strRecordingDirName, const bool bNCentServPingServerInList, const bool bNDisconnectAllClients, + const bool bNUseDoubleSystemFrameSize, const ELicenceType eNLicenceType ); void Start(); @@ -220,6 +222,9 @@ protected: virtual void customEvent ( QEvent* pEvent ); + // if server mode is normal or double system frame size + bool bUseDoubleSystemFrameSize; + // do not use the vector class since CChannel does not have appropriate // copy constructor/operator CChannel vecChannels[MAX_NUM_CHANNELS]; @@ -228,6 +233,11 @@ protected: QMutex Mutex; // audio encoder/decoder + OpusCustomMode* Opus64Mode[MAX_NUM_CHANNELS]; + OpusCustomEncoder* Opus64EncoderMono[MAX_NUM_CHANNELS]; + OpusCustomDecoder* Opus64DecoderMono[MAX_NUM_CHANNELS]; + OpusCustomEncoder* Opus64EncoderStereo[MAX_NUM_CHANNELS]; + OpusCustomDecoder* Opus64DecoderStereo[MAX_NUM_CHANNELS]; OpusCustomMode* OpusMode[MAX_NUM_CHANNELS]; OpusCustomEncoder* OpusEncoderMono[MAX_NUM_CHANNELS]; OpusCustomDecoder* OpusDecoderMono[MAX_NUM_CHANNELS];