opalmixer.h

Go to the documentation of this file.

/*
 * opalmixer.h
 *
 * OPAL media mixers
 *
 * Open Phone Abstraction Library (OPAL)
 * Formally known as the Open H323 project.
 *
 * Copyright (C) 2007 Post Increment
 *
 * The contents of this file are subject to the Mozilla Public License
 * Version 1.0 (the "License"); you may not use this file except in
 * compliance with the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS"
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 * the License for the specific language governing rights and limitations
 * under the License.
 *
 * The Original Code is Open Phone Abstraction Library.
 *
 * The Initial Developer of the Original Code is Post Increment
 *
 * Contributor(s): Craig Southeren (craigs@postincrement.com)
 *                 Robert Jongbloed (robertj@voxlucida.com.au)
 */


#ifndef OPAL_OPAL_OPALMIXER_H
#define OPAL_OPAL_OPALMIXER_H

#ifndef _PTLIB_H
#include <ptlib.h>
#endif

#include <opal_config.h>

#if OPAL_HAS_MIXER

#include <queue>

#include <ep/localep.h>
#include <codec/vidcodec.h>


class RTP_DataFrame;
class OpalJitterBuffer;
class OpalMixerConnection;


//#define OPAL_MIXER_AUDIO_DEBUG 1


#define OPAL_MIXER_PREFIX "mcu"

#define OPAL_OPT_LISTEN_ONLY "Listen-Only"      
#define OPAL_OPT_CONF_OWNER  "Conference-Owner" 



class OpalBaseMixer : public PSmartObject
{
    PCLASSINFO(OpalBaseMixer, PSmartObject);
  public:
    OpalBaseMixer(
      bool pushThread,    
      unsigned periodMS,  
      unsigned periodTS   
    );

    virtual ~OpalBaseMixer();

    typedef PString Key_T;

    virtual bool AddStream(
      const Key_T & key   
    );

    virtual void RemoveStream(
      const Key_T & key   
    );

    virtual void RemoveAllStreams();

    virtual bool WriteStream(
      const Key_T & key,          
      const RTP_DataFrame & input 
    );

    virtual RTP_DataFrame * ReadMixed();
    virtual bool ReadMixed(RTP_DataFrame & mixed);

    virtual bool OnMixed(
      RTP_DataFrame * & mixed   
    );

    virtual void StartPushThread();

    virtual bool OnPush();

    void StopPushThread(bool lock = true);

    unsigned GetPeriodMS() const { return m_periodMS; }

    unsigned GetPeriodTS() const { return m_periodTS; }

  protected:
    struct Stream : public PObject {
      virtual ~Stream() { }
      virtual void QueuePacket(const RTP_DataFrame & rtp) = 0;
      std::queue<RTP_DataFrame> m_queue;
    };
    typedef std::map<Key_T, Stream *> StreamMap_T;

    virtual Stream * CreateStream() = 0;
    virtual bool MixStreams(RTP_DataFrame & frame) = 0;
    virtual size_t GetOutputSize() const = 0;

    void PushThreadMain();

    bool      m_pushThread;      // true if to use a thread to push data out
    unsigned  m_periodMS;        // Mixing interval in milliseconds
    unsigned  m_periodTS;        // Mixing interval in timestamp units

    StreamMap_T     m_inputStreams;     // Map of key to stream for input RTP frame queues
    unsigned        m_outputTimestamp;  // RTP timestamp for output data
    RTP_DataFrame * m_pushFrame;        // Cached frame for pushing RTP
    PThread *       m_workerThread;     // reader thread handle
    bool            m_threadRunning;    // used to stop reader thread
   PDECLARE_MUTEX(m_mutex);             // mutex for list of streams and thread handle
};


class OpalAudioMixer : public OpalBaseMixer
{
    PCLASSINFO(OpalAudioMixer, OpalBaseMixer);
  public:
    OpalAudioMixer(
      bool stereo = false,    
      unsigned sampleRate = OpalMediaFormat::AudioClockRate, 
      bool pushThread = true, 
      unsigned period = 10    
    );

    ~OpalAudioMixer() { StopPushThread(); }

    virtual void RemoveStream(
      const Key_T & key   
    );

    virtual void RemoveAllStreams();

    bool IsStereo() const { return m_stereo; }

    unsigned GetSampleRate() const { return m_sampleRate; }

    bool SetSampleRate(
      unsigned rate   
    );

    bool SetJitterBufferSize(
      const Key_T & key,                    
      const OpalJitterBuffer::Init & init   
    );

  protected:
    struct AudioStream : public Stream
    {
      AudioStream(OpalAudioMixer & mixer);
      ~AudioStream();

      virtual void QueuePacket(const RTP_DataFrame & rtp);
      const short * GetAudioDataPtr();

      OpalAudioMixer   & m_mixer;
      OpalJitterBuffer * m_jitter;
      unsigned           m_nextTimestamp;
      PShortArray        m_cacheSamples;
      size_t             m_samplesUsed;
    };

    virtual Stream * CreateStream();
    virtual bool MixStreams(RTP_DataFrame & frame);
    virtual size_t GetOutputSize() const;

    void PreMixStreams();
    void MixStereo(RTP_DataFrame & frame);
    void MixAdditive(RTP_DataFrame & frame, const short * audioToSubtract);

  protected:
    bool     m_stereo;
    unsigned m_sampleRate;

    AudioStream    * m_left;
    AudioStream    * m_right;
    std::vector<int> m_mixedAudio;
};



#if OPAL_VIDEO

class OpalVideoMixer : public OpalBaseMixer
{
    PCLASSINFO(OpalVideoMixer, OpalBaseMixer);
  public:
    enum Styles {
      eSideBySideLetterbox, 
      eSideBySideScaled,    
      eStackedPillarbox,    
      eStackedScaled,       
      eGrid,                
      eUser                 
    };

    OpalVideoMixer(
      Styles style,           
      unsigned width,         
      unsigned height,        
      unsigned rate = 15,     
      bool pushThread = true  
    );

    ~OpalVideoMixer() { StopPushThread(); }

    unsigned GetFrameWidth() const { return m_width; }

    unsigned GetFrameHeight() const { return m_height; }

    unsigned GetFrameRate() const { return 1000/m_periodMS; }

    virtual bool SetFrameRate(
      unsigned rate   // New frames per second.
    );

    bool SetFrameSize(
      unsigned width,   
      unsigned height   
    );

  protected:
    struct VideoStream : public Stream
    {
      VideoStream(OpalVideoMixer & mixer);
      virtual void QueuePacket(const RTP_DataFrame & rtp);
      void InsertVideoFrame(unsigned x, unsigned y, unsigned w, unsigned h);

      OpalVideoMixer & m_mixer;
    };

    friend struct VideoStream;

    virtual Stream * CreateStream();
    virtual bool MixStreams(RTP_DataFrame & frame);
    virtual size_t GetOutputSize() const;

    virtual bool MixVideo();
    virtual bool StartMix(unsigned & x, unsigned & y, unsigned & w, unsigned & h, unsigned & left);
    virtual bool NextMix(unsigned & x, unsigned & y, unsigned & w, unsigned & h, unsigned & left);
    void InsertVideoFrame(const StreamMap_T::iterator & it, unsigned x, unsigned y, unsigned w, unsigned h);

  protected:
    Styles     m_style;
    unsigned   m_width, m_height;
    BYTE       m_bgFillRed,m_bgFillGreen,m_bgFillBlue;

    PBYTEArray m_frameStore;
    size_t     m_lastStreamCount;
};

#endif // OPAL_VIDEO




struct OpalMixerNodeInfo
{
  OpalMixerNodeInfo(const char * name = NULL)
    : m_name(name)
    , m_closeOnEmpty(false)
    , m_listenOnly(false)
    , m_sampleRate(OpalMediaFormat::AudioClockRate)
#if OPAL_VIDEO
    , m_audioOnly(false)
    , m_style(OpalVideoMixer::eGrid)
    , m_width(PVideoFrameInfo::CIFWidth)
    , m_height(PVideoFrameInfo::CIFHeight)
    , m_rate(15)
#endif
    , m_mediaPassThru(false)
  { }

  virtual ~OpalMixerNodeInfo() { }

  virtual OpalMixerNodeInfo * Clone() const { return new OpalMixerNodeInfo(*this); }

  PString  m_name;                
  bool     m_closeOnEmpty;        
  bool     m_listenOnly;          
  unsigned m_sampleRate;          
#if OPAL_VIDEO
  bool     m_audioOnly;           
  OpalVideoMixer::Styles m_style; 
  unsigned m_width;               
  unsigned m_height;              
  unsigned m_rate;                
#endif
  bool     m_mediaPassThru;       
  PString m_displayText;          
  PString m_subject;              
  PString m_notes;                
  PString m_keywords;             
};



class OpalMixerNode;
class OpalAudioStreamMixer;
class OpalVideoStreamMixer;


class OpalMixerNodeManager
{
  public:
    OpalMixerNodeManager(OpalManager & manager);

    virtual ~OpalMixerNodeManager();

    virtual void ShutDown();

    virtual PBoolean GarbageCollection();

    virtual OpalMixerNode * CreateNode(
      OpalMixerNodeInfo * info 
    );

    virtual PSafePtr<OpalMixerNode> AddNode(
      OpalMixerNodeInfo * info 
    );

    void AddNode(OpalMixerNode * node);

    PSafePtr<OpalMixerNode> GetFirstNode(
      PSafetyMode mode = PSafeReference 
    ) const { return PSafePtr<OpalMixerNode>(m_nodesByUID, mode); }

    virtual PSafePtr<OpalMixerNode> FindNode(
      const PString & name,             
      PSafetyMode mode = PSafeReference 
    );

    virtual void RemoveNode(
      OpalMixerNode & node
    );

    bool AddNodeName(
      PString name,        
      OpalMixerNode * node 
    );

    void RemoveNodeName(
      PString name        
    );

    void RemoveNodeNames(
      const PStringSet & names   
    );

    virtual PString CreateInternalURI(
      const PGloballyUniqueID & guid
    );

    virtual void OnNodeStatusChanged(
      const OpalMixerNode & node,            
      OpalConferenceState::ChangeType change 
    );

    virtual OpalAudioStreamMixer * CreateAudioMixer(const OpalMixerNodeInfo & info);

#if OPAL_VIDEO
    virtual OpalVideoStreamMixer * CreateVideoMixer(const OpalMixerNodeInfo & info);
#endif

    OpalManager & GetManager() const { return m_manager; }

  protected:
    OpalManager & m_manager;

    PSafeDictionary<PGloballyUniqueID, OpalMixerNode> m_nodesByUID;
    PSafeDictionary<PString, OpalMixerNode>           m_nodesByName;
};



class OpalMixerEndPoint : public OpalLocalEndPoint, public OpalMixerNodeManager
{
    PCLASSINFO(OpalMixerEndPoint, OpalLocalEndPoint);
  public:
    OpalMixerEndPoint(
      OpalManager & manager,  
      const char * prefix = OPAL_MIXER_PREFIX  
    );

    ~OpalMixerEndPoint();

    virtual void ShutDown();

    virtual OpalMediaFormatList GetMediaFormats() const;

    virtual PStringList GetAvailableStringOptions() const;

    virtual PSafePtr<OpalConnection> MakeConnection(
      OpalCall & call,           
      const PString & party,     
      void * userData = NULL,    
      unsigned options = 0,      
      OpalConnection::StringOptions * stringOptions = NULL 
    );

    virtual bool GetConferenceStates(
      OpalConferenceStates & states,          
      const PString & name = PString::Empty() 
    ) const;

    virtual PBoolean GarbageCollection();

    PSafePtr<OpalMixerConnection> GetMixerConnectionWithLock(
      const PString & token,     
      PSafetyMode mode = PSafeReadWrite 
    ) { return GetConnectionWithLockAs<OpalMixerConnection>(token, mode); }

    virtual OpalMixerConnection * CreateConnection(
      PSafePtr<OpalMixerNode> node, 
      OpalCall & call,              
      void * userData,              
      unsigned options,             
      OpalConnection::StringOptions * stringOptions 
    );

    virtual OpalMixerNodeInfo * FindNodeInfo(
      const PString & name
    );

    void SetAdHocNodeInfo(
      const OpalMixerNodeInfo & info
    );
    void SetAdHocNodeInfo(
      OpalMixerNodeInfo * info
    );

    OpalMixerNodeInfo * GetAdHocNodeInfo() { return m_adHocNodeInfo; }

    void SetFactoryNodeInfo(
      const OpalMixerNodeInfo & info
    );
    void SetFactoryNodeInfo(
      OpalMixerNodeInfo * info
    );

    virtual PString GetNewFactoryName();

    OpalManager & GetManager() const { return OpalLocalEndPoint::GetManager(); }

  protected:
    virtual PString CreateInternalURI(const PGloballyUniqueID & guid);
    virtual void OnNodeStatusChanged(const OpalMixerNode & node, OpalConferenceState::ChangeType change);

    OpalMixerNodeInfo  * m_adHocNodeInfo;
    OpalMixerNodeInfo  * m_factoryNodeInfo;
    PDECLARE_MUTEX(m_infoMutex); // For above two fields
    atomic<uint32_t>     m_factoryIndex;
};



class OpalMixerConnection : public OpalLocalConnection
{
    PCLASSINFO(OpalMixerConnection, OpalLocalConnection);
  public:
    OpalMixerConnection(
      PSafePtr<OpalMixerNode> node, 
      OpalCall & call,              
      OpalMixerEndPoint & endpoint, 
      void * userData,              
      unsigned options = 0,         
      OpalConnection::StringOptions * stringOptions = NULL 
    );

    ~OpalMixerConnection();

    virtual void OnReleased();

    virtual OpalMediaFormatList GetMediaFormats() const;

    virtual OpalMediaStream * CreateMediaStream(
      const OpalMediaFormat & mediaFormat, 
      unsigned sessionID,                  
      PBoolean isSource                    
    );

    virtual void OnStartMediaPatch(
      OpalMediaPatch & patch    
    );

    virtual void OnApplyStringOptions();

    virtual PBoolean SendUserInputString(
      const PString & value                   
    );

    virtual PBoolean SendUserInputTone(
      char tone,        
      unsigned duration = 0  
    );

    virtual bool GetConferenceState(
      OpalConferenceState * state  
    ) const;

    void SetListenOnly(
      bool listenOnly   
    );

    bool GetListenOnly() const { return m_listenOnly; }

    PSafePtr<OpalMixerNode> GetNode() const { return m_node; }

  protected:
    OpalMixerEndPoint     & m_endpoint;
    PSafePtr<OpalMixerNode> m_node;
    bool                    m_listenOnly;
};


class OpalMixerMediaStream : public OpalMediaStream
{
    PCLASSINFO(OpalMixerMediaStream, OpalMediaStream);
  public:
    OpalMixerMediaStream(
      OpalConnection & conn,               
      const OpalMediaFormat & mediaFormat, 
      unsigned sessionID,                  
      bool isSource,                       
      PSafePtr<OpalMixerNode> node,        
      bool listenOnly                      
    );

    ~OpalMixerMediaStream();

    virtual PBoolean Open();

    virtual PBoolean WritePacket(
      RTP_DataFrame & packet
    );

    virtual PBoolean IsSynchronous() const;

    virtual PBoolean RequiresPatchThread() const;

    PSafePtr<OpalMixerNode> GetNode() { return m_node; }

#if OPAL_VIDEO
    bool CheckMixedVideoSize(unsigned width, unsigned height);
#endif

  protected:
    virtual void InternalClose();
    virtual bool InternalSetJitterBuffer(const OpalJitterBuffer::Init & init);

    PSafePtr<OpalMixerNode> m_node;
    bool m_listenOnly;
#if OPAL_VIDEO
    unsigned m_mixedVideoWidth;
    unsigned m_mixedVideoHeight;
#endif
};


class OpalMediaStreamMixer
{
  public:
    OpalMediaStreamMixer();
    void Append(const PSafePtr<OpalMixerMediaStream> & stream);
    void Remove(const PSafePtr<OpalMixerMediaStream> & stream) { m_outputStreams.RemoveAt(stream->GetID()); }
    void CloseOne(const PSafePtr<OpalMixerMediaStream> & stream);

  protected:
    typedef PSafeDictionary<PString, OpalMixerMediaStream> StreamDict;
    StreamDict m_outputStreams;
};

class OpalAudioStreamMixer : public OpalAudioMixer, public OpalMediaStreamMixer
{
    PCLASSINFO(OpalAudioStreamMixer, OpalAudioMixer);
  public:
    OpalAudioStreamMixer(const OpalMixerNodeInfo & info);
    ~OpalAudioStreamMixer();

    virtual bool OnPush();

  protected:
    struct CachedAudio
    {
      CachedAudio();
      ~CachedAudio();
      enum
      {
        Collecting, Collected, Completed
      } m_state;
      RTP_DataFrame    m_raw;
      RTP_DataFrame    m_encoded;
      OpalTranscoder * m_transcoder;
    };
    std::map<PString, CachedAudio> m_cache;

    void PushOne(
      PSafePtr<OpalMixerMediaStream> & stream,
      CachedAudio & cache,
      const short * audioToSubtract
    );

#ifdef OPAL_MIXER_AUDIO_DEBUG
    class PAudioMixerDebug * m_audioDebug;
#endif
};


#if OPAL_VIDEO

class OpalVideoStreamMixer : public OpalVideoMixer, public OpalMediaStreamMixer
{
    PCLASSINFO(OpalVideoStreamMixer, OpalVideoMixer);
  public:
    OpalVideoStreamMixer(const OpalMixerNodeInfo & info);
    ~OpalVideoStreamMixer();

    virtual bool SetFrameRate(unsigned rate);
    virtual bool OnMixed(RTP_DataFrame * & output);

  protected:
    typedef PDictionary<PString, OpalTranscoder> TranscoderMap;
    TranscoderMap m_transcoders;
};
#endif // OPAL_VIDEO


class OpalMixerNode : public PSafeObject
{
    PCLASSINFO(OpalMixerNode, PSafeObject);
  public:
    OpalMixerNode(
      OpalMixerNodeManager & manager, 
      OpalMixerNodeInfo * info        
    );

    ~OpalMixerNode();

    void ShutDown();

    void PrintOn(
      ostream & strm    
    ) const;

    virtual void AttachConnection(
      OpalConnection * connection  
    );

    virtual void DetachConnection(
      OpalConnection * connection  
    );

    virtual bool AttachStream(
      OpalMixerMediaStream * stream     
    );

    virtual void DetachStream(
      OpalMixerMediaStream * stream     
    );

    virtual void UseMediaPassThrough(
      unsigned sessionID,                 
      OpalConnection * connection = NULL  
    );

    bool SetJitterBufferSize(
      const OpalBaseMixer::Key_T & key,     
      const OpalJitterBuffer::Init & init   
    );

    bool WritePacket(
      const OpalMixerMediaStream & stream,  
      const RTP_DataFrame & input           
    );

    virtual void BroadcastUserInput(
      const OpalConnection * connection,      
      const PString & value                   
    );

    virtual void GetConferenceState(
      OpalConferenceState & state  
    ) const;

    const PGloballyUniqueID & GetGUID() const { return m_guid; }

    const PStringSet & GetNames() const { return m_names; }

    bool AddName(
      const PString & name
    );

    void RemoveName(
      const PString & name
    );

    PINDEX GetConnectionCount() const { return m_connections.GetSize(); }

    template <class Subclass>
    PSafePtr<Subclass> GetFirstConnectionAs(
      PSafetyMode mode = PSafeReference
    ) const { return PSafePtr<Subclass>(m_connections, mode); }

    PSafePtr<OpalConnection> GetFirstConnection(
      PSafetyMode mode = PSafeReference
    ) const { return GetFirstConnectionAs<OpalConnection>(mode); }

    const OpalMixerNodeInfo & GetNodeInfo() { return *m_info; }

    const PTime & GetCreationTime() const { return m_creationTime; }

    void SetOwnerConnection(
      const PString & connectionIdentifier
    ) { m_ownerConnection = connectionIdentifier; }

  protected:
    OpalMixerNodeManager & m_manager;
    PGloballyUniqueID      m_guid;
    PStringSet             m_names;
    OpalMixerNodeInfo    * m_info;
    PTime                  m_creationTime;
    atomic<bool>           m_shuttingDown;

    PSafeArray<OpalConnection> m_connections;
    PString                    m_ownerConnection;

    OpalAudioStreamMixer * m_audioMixer;
#if OPAL_VIDEO
    typedef std::map<OpalVideoFormat::ContentRole, OpalVideoStreamMixer *> VideoMixerMap;
    VideoMixerMap m_videoMixers;
#endif // OPAL_VIDEO

    typedef std::map<PString, OpalBaseMixer *> MixerByIdMap;
    MixerByIdMap m_mixerById;
};


#endif // OPAL_HAS_MIXER

#endif // OPAL_OPAL_OPAL_MIXER