support 128 frame size if server operates in 64 frame size

ChangeLog

@@ -3,10 +3,19 @@
 
 3.4.7git
 
+TODO add command line argument for 64/124 frame size at the server
+
 TODO support OPUS 64 in the server as a first step towards official 64 samples frame size support
 
+TODO for different frame sizes (64/128) the start value of 6 for the jitter buffer might be too low
+
 TODO #12 A server in the same NAT region as the client is not visible (https://sourceforge.net/p/llcon/bugs/12/)
 
+TODO https://sourceforge.net/p/llcon/discussion/533517/thread/62bd07fbcb/?limit=25#ea21
+     1) The column widths are slightly too narrow (UK English) for "Ping Time" and "Musicians" in the "Connection Setup panel.
+     2) Spellings: It would be nice if the Location  Country field had spaces in the data e.g. UnitedKingdom > United Kingdom
+        Instrument "Acoutic Guitar" > "Acoustic Guitar", Instrument "Accordeon" > "Accordion"
+
 TODO in CreateLevelsForAllConChannels we should call CStereoSignalLevelMeter::CalcLogResult instead -> make it public and static
 
 TODO Multichannel CoreAudio Support #44 -> integrate important code lines in Mac sound interface

src/buffer.h

@@ -475,16 +475,33 @@ public:
     {
         // allocate internal memory and reset read/write positions
         vecMemory.Init ( iNewMemSize );
-        iMemSize = iNewMemSize;
+        iMemSize    = iNewMemSize;
-        iPutPos  = 0;
+        iBufferSize = iNewMemSize;
-        iGetPos  = 0;
+        Reset();
+    }
+
+    void Reset()
+    {
+        iPutPos = 0;
+        iGetPos = 0;
+    }
+
+    void SetBufferSize ( const int iNBSize )
+    {
+        // if buffer size has changed, apply new value and reset the buffer pointers
+        if ( ( iNBSize != iBufferSize ) && ( iNBSize <= iMemSize ) )
+        {
+            iBufferSize = iNBSize;
+            Reset();
+        }
     }
 
     void PutAll ( const CVector<TData>& vecsData )
     {
         iGetPos = 0;
+
         std::copy ( vecsData.begin(),
-                    vecsData.begin() + iMemSize, // note that input vector might be larger then memory size
+                    vecsData.begin() + iBufferSize, // note that input vector might be larger then memory size
                     vecMemory.begin() );
     }
 
@@ -495,7 +512,7 @@ public:
         const int iEnd = iPutPos + iVecSize;
 
         // first check for buffer overrun
-        if ( iEnd <= iMemSize )
+        if ( iEnd <= iBufferSize )
         {
             // copy new data in internal buffer
             std::copy ( vecsData.begin(),
@@ -506,7 +523,7 @@ public:
             iPutPos = iEnd;
 
             // return "buffer is ready for readout" flag
-            return ( iEnd == iMemSize );
+            return ( iEnd == iBufferSize );
         }
 
         // buffer overrun or not initialized, return "not ready"
@@ -519,6 +536,17 @@ public:
         return vecMemory;
     }
 
+    void GetAll ( CVector<TData>& vecsData,
+                  const int       iVecSize )
+    {
+        iPutPos = 0;
+
+        // copy data from internal buffer in given buffer
+        std::copy ( vecMemory.begin(),
+                    vecMemory.begin() + iVecSize,
+                    vecsData.begin() );
+    }
+
     bool Get ( CVector<TData>& vecsData,
                const int       iVecSize )
     {
@@ -526,7 +554,7 @@ public:
         const int iEnd = iGetPos + iVecSize;
 
         // first check for buffer underrun
-        if ( iEnd <= iMemSize )
+        if ( iEnd <= iBufferSize )
         {
             // copy new data from internal buffer
             std::copy ( vecMemory.begin() + iGetPos,
@@ -536,16 +564,17 @@ public:
             // set buffer pointer one block further
             iGetPos = iEnd;
 
-            // return "buffer is completely read" flag
+            // return the memory could be read
-            return ( iEnd == iMemSize );
+            return true;
         }
 
-        // buffer underrun or not initialized, return "completely read"
+        // return that no memory could be read
-        return true;
+        return false;
     }
 
 protected:
     CVector<TData> vecMemory;
     int            iMemSize;
+    int            iBufferSize;
     int            iPutPos, iGetPos;
 };

src/server.cpp

@@ -295,6 +295,13 @@ CServer::CServer ( const int          iNewMaxNumChan,
         // set encoder low complexity for legacy 128 samples frame size
         opus_custom_encoder_ctl ( OpusEncoderMono[i],   OPUS_SET_COMPLEXITY ( 1 ) );
         opus_custom_encoder_ctl ( OpusEncoderStereo[i], OPUS_SET_COMPLEXITY ( 1 ) );
+
+
+        // init double-to-normal frame size conversion buffers -----------------
+        // use worst case memory initialization to avoid allocating memory in
+        // the time-critical thread
+        DoubleFrameSizeConvBufIn[i].Init  ( 2 /* stereo */ * DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES /* worst case buffer size */ );
+        DoubleFrameSizeConvBufOut[i].Init ( 2 /* stereo */ * DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES /* worst case buffer size */ );
     }
 
     // define colors for chat window identifiers
@@ -326,12 +333,13 @@ CServer::CServer ( const int          iNewMaxNumChan,
     vecsSendData.Init ( 2 /* stereo */ * DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES /* worst case buffer size */ );
 
     // allocate worst case memory for the temporary vectors
-    vecChanIDsCurConChan.Init      ( iMaxNumChannels );
+    vecChanIDsCurConChan.Init          ( iMaxNumChannels );
-    vecvecdGains.Init              ( iMaxNumChannels );
+    vecvecdGains.Init                  ( iMaxNumChannels );
-    vecvecsData.Init               ( iMaxNumChannels );
+    vecvecsData.Init                   ( iMaxNumChannels );
-    vecNumAudioChannels.Init       ( iMaxNumChannels );
+    vecNumAudioChannels.Init           ( iMaxNumChannels );
-    vecNumFrameSizeConvBlocks.Init ( iMaxNumChannels );
+    vecNumFrameSizeConvBlocks.Init     ( iMaxNumChannels );
-    vecAudioComprType.Init         ( iMaxNumChannels );
+    vecUseDoubleSysFraSizeConvBuf.Init ( iMaxNumChannels );
+    vecAudioComprType.Init             ( iMaxNumChannels );
 
     for ( i = 0; i < iMaxNumChannels; i++ )
     {
@@ -788,6 +796,10 @@ CreateAndSendChanListForAllConChannels();
     {
         vecChannels[iChID].CreateLicReqMes ( eLicenceType );
     }
+
+    // reset the conversion buffers
+    DoubleFrameSizeConvBufIn[iChID].Reset();
+    DoubleFrameSizeConvBufOut[iChID].Reset();
 }
 
 void CServer::OnServerFull ( CHostAddress RecHostAddr )
@@ -907,6 +919,8 @@ JitterMeas.Measure();
             vecAudioComprType[i]   = vecChannels[iCurChanID].GetAudioCompressionType();
 
             // get info about required frame size conversion properties
+            vecUseDoubleSysFraSizeConvBuf[i] = ( !bUseDoubleSystemFrameSize && ( vecAudioComprType[i] == CT_OPUS ) );
+
             if ( bUseDoubleSystemFrameSize && ( vecAudioComprType[i] == CT_OPUS64 ) )
             {
                 vecNumFrameSizeConvBlocks[i] = 2;
@@ -916,6 +930,13 @@ JitterMeas.Measure();
                 vecNumFrameSizeConvBlocks[i] = 1;
             }
 
+            // update conversion buffer size (nothing will happen if the size stays the same)
+            if ( vecUseDoubleSysFraSizeConvBuf[i] )
+            {
+                DoubleFrameSizeConvBufIn[iCurChanID].SetBufferSize  ( DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES  * vecNumAudioChannels[i] );
+                DoubleFrameSizeConvBufOut[iCurChanID].SetBufferSize ( DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES  * vecNumAudioChannels[i] );
+            }
+
             // select the opus decoder and raw audio frame length
             if ( vecAudioComprType[i] == CT_OPUS )
             {
@@ -961,46 +982,63 @@ JitterMeas.Measure();
                 vecvecdGains[i][j] *= vecChannels[vecChanIDsCurConChan[j]].GetFadeInGain();
             }
 
-            // get current number of CELT coded bytes
+            // If the server frame size is smaller than the received OPUS frame size, we need a conversion
-            const int iCeltNumCodedBytes = vecChannels[iCurChanID].GetNetwFrameSize();
+            // buffer which stores the large buffer.
-
+            // Note that we have a shortcut here. If the conversion buffer is not needed, the boolean flag
-            for ( int iB = 0; iB < vecNumFrameSizeConvBlocks[i]; iB++ )
+            // is false and the Get() function is not called at all. Therefore if the buffer is not needed
+            // we do not spend any time in the function but go directly inside the if condition.
+            if ( ( vecUseDoubleSysFraSizeConvBuf[i] == 0 ) ||
+                 !DoubleFrameSizeConvBufIn[iCurChanID].Get ( vecvecsData[i], SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i] ) )
             {
-                // get data
+                // get current number of OPUS coded bytes
-                const EGetDataStat eGetStat = vecChannels[iCurChanID].GetData ( vecbyCodedData, iCeltNumCodedBytes );
+                const int iCeltNumCodedBytes = vecChannels[iCurChanID].GetNetwFrameSize();
 
-                // if channel was just disconnected, set flag that connected
+                for ( int iB = 0; iB < vecNumFrameSizeConvBlocks[i]; iB++ )
-                // client list is sent to all other clients
-                // and emit the client disconnected signal
-                if ( eGetStat == GS_CHAN_NOW_DISCONNECTED )
                 {
-                    if ( bEnableRecording )
+                    // get data
+                    const EGetDataStat eGetStat = vecChannels[iCurChanID].GetData ( vecbyCodedData, iCeltNumCodedBytes );
+
+                    // if channel was just disconnected, set flag that connected
+                    // client list is sent to all other clients
+                    // and emit the client disconnected signal
+                    if ( eGetStat == GS_CHAN_NOW_DISCONNECTED )
                     {
-                        emit ClientDisconnected ( iCurChanID ); // TODO do this outside the mutex lock?
+                        if ( bEnableRecording )
+                        {
+                            emit ClientDisconnected ( iCurChanID ); // TODO do this outside the mutex lock?
+                        }
+
+                        bChannelIsNowDisconnected = true;
                     }
 
-                    bChannelIsNowDisconnected = true;
+                    // get pointer to coded data
+                    if ( eGetStat == GS_BUFFER_OK )
+                    {
+                        pCurCodedData = &vecbyCodedData[0];
+                    }
+                    else
+                    {
+                        // for lost packets use null pointer as coded input data
+                        pCurCodedData = nullptr;
+                    }
+
+                    // OPUS decode received data stream
+                    if ( CurOpusDecoder != nullptr )
+                    {
+                        opus_custom_decode ( CurOpusDecoder,
+                                             pCurCodedData,
+                                             iCeltNumCodedBytes,
+                                             &vecvecsData[i][iB * SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i]],
+                                             iClientFrameSizeSamples );
+                    }
                 }
 
-                // get pointer to coded data
+                // a new large frame is ready, if the conversion buffer is required, put it in the buffer
-                if ( eGetStat == GS_BUFFER_OK )
+                // and read out the small frame size immediately for further processing
+                if ( vecUseDoubleSysFraSizeConvBuf[i] != 0 )
                 {
-                    pCurCodedData = &vecbyCodedData[0];
+                    DoubleFrameSizeConvBufIn[iCurChanID].PutAll ( vecvecsData[i] );
-                }
+                    DoubleFrameSizeConvBufIn[iCurChanID].Get ( vecvecsData[i], SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i] );
-                else
-                {
-                    // for lost packets use null pointer as coded input data
-                    pCurCodedData = nullptr;
-                }
-
-                // OPUS decode received data stream
-                if ( CurOpusDecoder != nullptr )
-                {
-                    opus_custom_decode ( CurOpusDecoder,
-                                         pCurCodedData,
-                                         iCeltNumCodedBytes,
-                                         &vecvecsData[i][iB * SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i]],
-                                         iClientFrameSizeSamples );
                 }
             }
         }
@@ -1109,37 +1147,52 @@ JitterMeas.Measure();
                 CurOpusEncoder = nullptr;
             }
 
-            for ( int iB = 0; iB < vecNumFrameSizeConvBlocks[i]; iB++ )
+            // If the server frame size is smaller than the received OPUS frame size, we need a conversion
+            // buffer which stores the large buffer.
+            // Note that we have a shortcut here. If the conversion buffer is not needed, the boolean flag
+            // is false and the Get() function is not called at all. Therefore if the buffer is not needed
+            // we do not spend any time in the function but go directly inside the if condition.
+            if ( ( vecUseDoubleSysFraSizeConvBuf[i] == 0 ) ||
+                 DoubleFrameSizeConvBufOut[iCurChanID].Put ( vecsSendData, SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i] ) )
             {
-                // OPUS encoding
+                if ( vecUseDoubleSysFraSizeConvBuf[i] != 0 )
-                if ( CurOpusEncoder != nullptr )
                 {
+                    // get the large frame from the conversion buffer
+                    DoubleFrameSizeConvBufOut[iCurChanID].GetAll ( vecsSendData, DOUBLE_SYSTEM_FRAME_SIZE_SAMPLES * vecNumAudioChannels[i] );
+                }
+
+                for ( int iB = 0; iB < vecNumFrameSizeConvBlocks[i]; iB++ )
+                {
+                    // OPUS encoding
+                    if ( CurOpusEncoder != nullptr )
+                    {
 // TODO find a better place than this: the setting does not change all the time
 //      so for speed optimization it would be better to set it only if the network
 //      frame size is changed
 opus_custom_encoder_ctl ( CurOpusEncoder,
                           OPUS_SET_BITRATE ( CalcBitRateBitsPerSecFromCodedBytes ( iCeltNumCodedBytes, iClientFrameSizeSamples ) ) );
 
-                    opus_custom_encode ( CurOpusEncoder,
+                        opus_custom_encode ( CurOpusEncoder,
-                                         &vecsSendData[iB * SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i]],
+                                             &vecsSendData[iB * SYSTEM_FRAME_SIZE_SAMPLES_SMALL * vecNumAudioChannels[i]],
-                                         iClientFrameSizeSamples,
+                                             iClientFrameSizeSamples,
-                                         &vecbyCodedData[0],
+                                             &vecbyCodedData[0],
-                                         iCeltNumCodedBytes );
+                                             iCeltNumCodedBytes );
+                    }
+
+                    // send separate mix to current clients
+                    vecChannels[iCurChanID].PrepAndSendPacket ( &Socket,
+                                                                vecbyCodedData,
+                                                                iCeltNumCodedBytes );
                 }
 
-                // send separate mix to current clients
+                // update socket buffer size
-                vecChannels[iCurChanID].PrepAndSendPacket ( &Socket,
+                vecChannels[iCurChanID].UpdateSocketBufferSize();
-                                                            vecbyCodedData,
-                                                            iCeltNumCodedBytes );
-            }
 
-            // update socket buffer size
+                // send channel levels
-            vecChannels[iCurChanID].UpdateSocketBufferSize();
+                if ( bSendChannelLevels && vecChannels[iCurChanID].ChannelLevelsRequired() )
-
+                {
-            // send channel levels
+                    ConnLessProtocol.CreateCLChannelLevelListMes ( vecChannels[iCurChanID].GetAddress(), vecChannelLevels, iNumClients );
-            if ( bSendChannelLevels && vecChannels[iCurChanID].ChannelLevelsRequired() )
+                }
-            {
-                ConnLessProtocol.CreateCLChannelLevelListMes ( vecChannels[iCurChanID].GetAddress(), vecChannelLevels, iNumClients );
             }
         }
     }

src/server.h

@@ -35,6 +35,7 @@
 # include "opus_custom.h"
 #endif
 #include "global.h"
+#include "buffer.h"
 #include "socket.h"
 #include "channel.h"
 #include "util.h"
@@ -251,6 +252,8 @@ protected:
     OpusCustomDecoder*         OpusDecoderMono[MAX_NUM_CHANNELS];
     OpusCustomEncoder*         OpusEncoderStereo[MAX_NUM_CHANNELS];
     OpusCustomDecoder*         OpusDecoderStereo[MAX_NUM_CHANNELS];
+    CConvBuf<int16_t>          DoubleFrameSizeConvBufIn[MAX_NUM_CHANNELS];
+    CConvBuf<int16_t>          DoubleFrameSizeConvBufOut[MAX_NUM_CHANNELS];
 
     CVector<QString>           vstrChatColors;
     CVector<int>               vecChanIDsCurConChan;
@@ -259,6 +262,7 @@ protected:
     CVector<CVector<int16_t> > vecvecsData;
     CVector<int>               vecNumAudioChannels;
     CVector<int>               vecNumFrameSizeConvBlocks;
+    CVector<int>               vecUseDoubleSysFraSizeConvBuf;
     CVector<EAudComprType>     vecAudioComprType;
     CVector<int16_t>           vecsSendData;
     CVector<uint8_t>           vecbyCodedData;