Android Multimedia框架总结（二十二）MediaCodec中C++中创建到start过程及状态变换

上一章介绍MediaCodec中创建到start过程(到jni部分)，从今天开始，将深入源码中看看其c++过程，看下Agenda如下：

• mediacodec.h
• CreateByType
• initMediaCodec中BufferInfo内部类：
• configure过程
• start

BufferInfo在MediaCodec.h中对应是一个结构体

//create by 逆流的鱼yuiop on 2016/12/11
//blog地址：http://blog.csdn.net/hejjunlin
struct BufferInfo {
        uint32_t mBufferID;
        sp<ABuffer> mData;
        sp<ABuffer> mEncryptedData;
        sp<IMemory> mSharedEncryptedBuffer;
        sp<AMessage> mNotify;
        sp<AMessage> mFormat;
        bool mOwnedByClient;
};

mediacodec.h的方法的声明,位于\frameworks\av\include\media\stagefright下

//create by 逆流的鱼yuiop on 2016/12/11
//blog地址：http://blog.csdn.net/hejjunlin
namespace android {

struct ABuffer;
struct AMessage;
struct AReplyToken;
struct AString;
struct CodecBase;
struct IBatteryStats;
struct ICrypto;
class IMemory;
struct MemoryDealer;
class IResourceManagerClient;
class IResourceManagerService;
struct PersistentSurface;
struct SoftwareRenderer;
struct Surface;

struct MediaCodec : public AHandler {
    enum ConfigureFlags {
        CONFIGURE_FLAG_ENCODE   = 1,
    };

    enum BufferFlags {
        BUFFER_FLAG_SYNCFRAME   = 1,
        BUFFER_FLAG_CODECCONFIG = 2,
        BUFFER_FLAG_EOS         = 4,
    };

    enum {
        CB_INPUT_AVAILABLE = 1,
        CB_OUTPUT_AVAILABLE = 2,
        CB_ERROR = 3,
        CB_OUTPUT_FORMAT_CHANGED = 4,
        CB_RESOURCE_RECLAIMED = 5,
    };

    static const pid_t kNoPid = -1;

    static sp<MediaCodec> CreateByType(
            const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err = NULL,
            pid_t pid = kNoPid);

    static sp<MediaCodec> CreateByComponentName(
            const sp<ALooper> &looper, const char *name, status_t *err = NULL,
            pid_t pid = kNoPid);

    static sp<PersistentSurface> CreatePersistentInputSurface();

    status_t configure(
            const sp<AMessage> &format,
            const sp<Surface> &nativeWindow,
            const sp<ICrypto> &crypto,
            uint32_t flags);

    status_t setCallback(const sp<AMessage> &callback);

    status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);

    status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);

    status_t setInputSurface(const sp<PersistentSurface> &surface);

    status_t start();

    // Returns to a state in which the component remains allocated but
    // unconfigured.
    status_t stop();

    // Resets the codec to the INITIALIZED state.  Can be called after an error
    // has occured to make the codec usable.
    status_t reset();

    // Client MUST call release before releasing final reference to this
    // object.
    status_t release();

    status_t flush();

    status_t queueInputBuffer(
            size_t index,
            size_t offset,
            size_t size,
            int64_t presentationTimeUs,
            uint32_t flags,
            AString *errorDetailMsg = NULL);

    status_t queueSecureInputBuffer(
            size_t index,
            size_t offset,
            const CryptoPlugin::SubSample *subSamples,
            size_t numSubSamples,
            const uint8_t key[16],
            const uint8_t iv[16],
            CryptoPlugin::Mode mode,
            int64_t presentationTimeUs,
            uint32_t flags,
            AString *errorDetailMsg = NULL);

    status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);

    status_t dequeueOutputBuffer(
            size_t *index,
            size_t *offset,
            size_t *size,
            int64_t *presentationTimeUs,
            uint32_t *flags,
            int64_t timeoutUs = 0ll);

    status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
    status_t renderOutputBufferAndRelease(size_t index);
    status_t releaseOutputBuffer(size_t index);

    status_t signalEndOfInputStream();

    status_t getOutputFormat(sp<AMessage> *format) const;
    status_t getInputFormat(sp<AMessage> *format) const;

    status_t getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const;

    status_t getInputBuffers(Vector<sp<ABuffer> > *buffers) const;
    status_t getOutputBuffers(Vector<sp<ABuffer> > *buffers) const;

    status_t getOutputBuffer(size_t index, sp<ABuffer> *buffer);
    status_t getOutputFormat(size_t index, sp<AMessage> *format);
    status_t getInputBuffer(size_t index, sp<ABuffer> *buffer);

    status_t setSurface(const sp<Surface> &nativeWindow);

    status_t requestIDRFrame();

    // Notification will be posted once there "is something to do", i.e.
    // an input/output buffer has become available, a format change is
    // pending, an error is pending.
    void requestActivityNotification(const sp<AMessage> &notify);

    status_t getName(AString *componentName) const;

    status_t setParameters(const sp<AMessage> &params);

    // Create a MediaCodec notification message from a list of rendered or dropped render infos
    // by adding rendered frame information to a base notification message. Returns the number
    // of frames that were rendered.
    static size_t CreateFramesRenderedMessage(
            std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg);

protected:
    virtual ~MediaCodec();
    virtual void onMessageReceived(const sp<AMessage> &msg);

private:
    // used by ResourceManagerClient
    status_t reclaim(bool force = false);
    friend struct ResourceManagerClient;

private:
    enum State {
        UNINITIALIZED,
        INITIALIZING,
        INITIALIZED,
        CONFIGURING,
        CONFIGURED,
        STARTING,
        STARTED,
        FLUSHING,
        FLUSHED,
        STOPPING,
        RELEASING,
    };

    enum {
        kPortIndexInput         = 0,
        kPortIndexOutput        = 1,
    };

    enum {
        kWhatInit                           = 'init',
        kWhatConfigure                      = 'conf',
        kWhatSetSurface                     = 'sSur',
        kWhatCreateInputSurface             = 'cisf',
        kWhatSetInputSurface                = 'sisf',
        kWhatStart                          = 'strt',
        kWhatStop                           = 'stop',
        kWhatRelease                        = 'rele',
        kWhatDequeueInputBuffer             = 'deqI',
        kWhatQueueInputBuffer               = 'queI',
        kWhatDequeueOutputBuffer            = 'deqO',
        kWhatReleaseOutputBuffer            = 'relO',
        kWhatSignalEndOfInputStream         = 'eois',
        kWhatGetBuffers                     = 'getB',
        kWhatFlush                          = 'flus',
        kWhatGetOutputFormat                = 'getO',
        kWhatGetInputFormat                 = 'getI',
        kWhatDequeueInputTimedOut           = 'dITO',
        kWhatDequeueOutputTimedOut          = 'dOTO',
        kWhatCodecNotify                    = 'codc',
        kWhatRequestIDRFrame                = 'ridr',
        kWhatRequestActivityNotification    = 'racN',
        kWhatGetName                        = 'getN',
        kWhatSetParameters                  = 'setP',
        kWhatSetCallback                    = 'setC',
        kWhatSetNotification                = 'setN',
    };

    enum {
        kFlagUsesSoftwareRenderer       = 1,
        kFlagOutputFormatChanged        = 2,
        kFlagOutputBuffersChanged       = 4,
        kFlagStickyError                = 8,
        kFlagDequeueInputPending        = 16,
        kFlagDequeueOutputPending       = 32,
        kFlagIsSecure                   = 64,
        kFlagSawMediaServerDie          = 128,
        kFlagIsEncoder                  = 256,
        kFlagGatherCodecSpecificData    = 512,
        kFlagIsAsync                    = 1024,
        kFlagIsComponentAllocated       = 2048,
        kFlagPushBlankBuffersOnShutdown = 4096,
    };

    struct BufferInfo {
        uint32_t mBufferID;
        sp<ABuffer> mData;
        sp<ABuffer> mEncryptedData;
        sp<IMemory> mSharedEncryptedBuffer;
        sp<AMessage> mNotify;
        sp<AMessage> mFormat;
        bool mOwnedByClient;
    };

    struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
        ResourceManagerServiceProxy(pid_t pid);
        ~ResourceManagerServiceProxy();

        void init();

        // implements DeathRecipient
        virtual void binderDied(const wp<IBinder>& /*who*/);

        void addResource(
                int64_t clientId,
                const sp<IResourceManagerClient> client,
                const Vector<MediaResource> &resources);

        void removeResource(int64_t clientId);

        bool reclaimResource(const Vector<MediaResource> &resources);

    private:
        Mutex mLock;
        sp<IResourceManagerService> mService;
        pid_t mPid;
    };

    State mState;
    bool mReleasedByResourceManager;
    sp<ALooper> mLooper;
    sp<ALooper> mCodecLooper;
    sp<CodecBase> mCodec;
    AString mComponentName;
    sp<AReplyToken> mReplyID;
    uint32_t mFlags;
    status_t mStickyError;
    sp<Surface> mSurface;
    SoftwareRenderer *mSoftRenderer;

    sp<AMessage> mOutputFormat;
    sp<AMessage> mInputFormat;
    sp<AMessage> mCallback;
    sp<AMessage> mOnFrameRenderedNotification;
    sp<MemoryDealer> mDealer;

    sp<IResourceManagerClient> mResourceManagerClient;
    sp<ResourceManagerServiceProxy> mResourceManagerService;

    bool mBatteryStatNotified;
    bool mIsVideo;
    int32_t mVideoWidth;
    int32_t mVideoHeight;
    int32_t mRotationDegrees;

    // initial create parameters
    AString mInitName;
    bool mInitNameIsType;
    bool mInitIsEncoder;

    // configure parameter
    sp<AMessage> mConfigureMsg;

    // Used only to synchronize asynchronous getBufferAndFormat
    // across all the other (synchronous) buffer state change
    // operations, such as de/queueIn/OutputBuffer, start and
    // stop/flush/reset/release.
    Mutex mBufferLock;

    List<size_t> mAvailPortBuffers[2];
    Vector<BufferInfo> mPortBuffers[2];

    int32_t mDequeueInputTimeoutGeneration;
    sp<AReplyToken> mDequeueInputReplyID;

    int32_t mDequeueOutputTimeoutGeneration;
    sp<AReplyToken> mDequeueOutputReplyID;

    sp<ICrypto> mCrypto;

    List<sp<ABuffer> > mCSD;

    sp<AMessage> mActivityNotify;

    bool mHaveInputSurface;
    bool mHavePendingInputBuffers;

    MediaCodec(const sp<ALooper> &looper, pid_t pid);

    static status_t PostAndAwaitResponse(
            const sp<AMessage> &msg, sp<AMessage> *response);

    void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);

    status_t init(const AString &name, bool nameIsType, bool encoder);

    void setState(State newState);
    void returnBuffersToCodec();
    void returnBuffersToCodecOnPort(int32_t portIndex);
    size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
    status_t onQueueInputBuffer(const sp<AMessage> &msg);
    status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
    ssize_t dequeuePortBuffer(int32_t portIndex);

    status_t getBufferAndFormat(
            size_t portIndex, size_t index,
            sp<ABuffer> *buffer, sp<AMessage> *format);

    bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
    bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
    void cancelPendingDequeueOperations();

    void extractCSD(const sp<AMessage> &format);
    status_t queueCSDInputBuffer(size_t bufferIndex);

    status_t handleSetSurface(const sp<Surface> &surface);
    status_t connectToSurface(const sp<Surface> &surface);
    status_t disconnectFromSurface();

    void postActivityNotificationIfPossible();

    void onInputBufferAvailable();
    void onOutputBufferAvailable();
    void onError(status_t err, int32_t actionCode, const char *detail = NULL);
    void onOutputFormatChanged();

    status_t onSetParameters(const sp<AMessage> &params);

    status_t amendOutputFormatWithCodecSpecificData(const sp<ABuffer> &buffer);
    void updateBatteryStat();
    bool isExecuting() const;

    uint64_t getGraphicBufferSize();
    void addResource(const String8 &type, const String8 &subtype, uint64_t value);

    bool hasPendingBuffer(int portIndex);
    bool hasPendingBuffer();

    /* called to get the last codec error when the sticky flag is set.
     * if no such codec error is found, returns UNKNOWN_ERROR.
     */
    inline status_t getStickyError() const {
        return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
    }

    inline void setStickyError(status_t err) {
        mFlags |= kFlagStickyError;
        mStickyError = err;
    }

    DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
};

}  // namespace android

CreateByType

//create by 逆流的鱼yuiop on 2016/12/11
//blog地址：http://blog.csdn.net/hejjunlin
// static
sp<MediaCodec> MediaCodec::CreateByType(
        const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err, pid_t pid) {
    sp<MediaCodec> codec = new MediaCodec(looper, pid);//这果实际上new出MediaCodec对象

    const status_t ret = codec->init(mime, true /* nameIsType */, encoder);
    if (err != NULL) {
        *err = ret;
    }
    return ret == OK ? codec : NULL; // NULL deallocates codec.
}

接着到init过程

//create by 逆流的鱼yuiop on 2016/12/11
//blog地址：http://blog.csdn.net/hejjunlin
status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) {
    mResourceManagerService->init();

    // 保存 初始参数，到时用于reset
    mInitName = name;
    mInitNameIsType = nameIsType;
    mInitIsEncoder = encoder;

   // 目前视频解码器不能马上从OMX_FillThisBuffer返回，违反OpenMAX规格,直到提醒我们需要入驻另一个第三方的looper释放在事件队列中。
    if (nameIsType || !strncasecmp(name.c_str(), "omx.", 4)) {//omx.匹配
        mCodec = new ACodec;//实例化ACodec
    } else if (!nameIsType
            && !strncasecmp(name.c_str(), "android.filter.", 15)) {
        mCodec = new MediaFilter;// 实例化MediaFilter
    } else {
        return NAME_NOT_FOUND;
    }

    bool secureCodec = false;
    if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) {
        mIsVideo = true;
    } else {
        AString tmp = name;
        if (tmp.endsWith(".secure")) {
            secureCodec = true;
            tmp.erase(tmp.size() - 7, 7);
        }
        const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
        if (mcl == NULL) {
            mCodec = NULL;  // remove the codec.
            return NO_INIT; // if called from Java should raise IOException
        }
        ssize_t codecIdx = mcl->findCodecByName(tmp.c_str());
        if (codecIdx >= 0) {
            const sp<MediaCodecInfo> info = mcl->getCodecInfo(codecIdx);
            Vector<AString> mimes;
            info->getSupportedMimes(&mimes);
            for (size_t i = 0; i < mimes.size(); i++) {
                if (mimes[i].startsWith("video/")) {
                    mIsVideo = true;
                    break;
                }
            }
        }
    }

    if (mIsVideo) {
        // video codec needs dedicated looper
        if (mCodecLooper == NULL) {
            mCodecLooper = new ALooper;
            mCodecLooper->setName("CodecLooper");//设置名字为CodecLooper
            mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
        }

        mCodecLooper->registerHandler(mCodec);
    } else {
        mLooper->registerHandler(mCodec);
    }

    mLooper->registerHandler(this);

    mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, this));

    sp<AMessage> msg = new AMessage(kWhatInit, this);
    msg->setString("name", name);
    msg->setInt32("nameIsType", nameIsType);

    if (nameIsType) {
        msg->setInt32("encoder", encoder);
    }

    status_t err;
    Vector<MediaResource> resources;
    const char *type = secureCodec ? kResourceSecureCodec : kResourceNonSecureCodec;
    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
    for (int i = 0; i <= kMaxRetry; ++i) {
        if (i > 0) {
            // Don't try to reclaim resource for the first time.
            if (!mResourceManagerService->reclaimResource(resources)) {
                break;
            }
        }

        sp<AMessage> response;
        err = PostAndAwaitResponse(msg, &response);
        if (!isResourceError(err)) {
            break;
        }
    }
    return err;
}

configure过程

//create by 逆流的鱼yuiop on 2016/12/11
//blog地址：http://blog.csdn.net/hejjunlin
status_t MediaCodec::configure(
        const sp<AMessage> &format,
        const sp<Surface> &surface,
        const sp<ICrypto> &crypto,
        uint32_t flags) {
    sp<AMessage> msg = new AMessage(kWhatConfigure, this);

    if (mIsVideo) {
        format->findInt32("width", &mVideoWidth);
        format->findInt32("height", &mVideoHeight);
        if (!format->findInt32("rotation-degrees", &mRotationDegrees)) {
            mRotationDegrees = 0;
        }
    }

    msg->setMessage("format", format);
    msg->setInt32("flags", flags);
    msg->setObject("surface", surface);

    if (crypto != NULL) {
        msg->setPointer("crypto", crypto.get());
    }

    // save msg for reset
    mConfigureMsg = msg;

    status_t err;
    Vector<MediaResource> resources;
    const char *type = (mFlags & kFlagIsSecure) ?
            kResourceSecureCodec : kResourceNonSecureCodec;
    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
    // Don't know the buffer size at this point, but it's fine to use 1 because
    // the reclaimResource call doesn't consider the requester's buffer size for now.
    resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1));
    for (int i = 0; i <= kMaxRetry; ++i) {
        if (i > 0) {
            // Don't try to reclaim resource for the first time.
            if (!mResourceManagerService->reclaimResource(resources)) {
                break;
            }
        }

        sp<AMessage> response;
        err = PostAndAwaitResponse(msg, &response);
        if (err != OK && err != INVALID_OPERATION) {
            // MediaCodec now set state to UNINITIALIZED upon any fatal error.
            // To maintain backward-compatibility, do a reset() to put codec
            // back into INITIALIZED state.
            // But don't reset if the err is INVALID_OPERATION, which means
            // the configure failure is due to wrong state.

            ALOGE("configure failed with err 0x%08x, resetting...", err);
            reset();
        }
        if (!isResourceError(err)) {
            break;
        }
    }
    return err;
}

start过程

//create by 逆流的鱼yuiop on 2016/12/11
//blog地址：http://blog.csdn.net/hejjunlin
status_t MediaCodec::start() {
    sp<AMessage> msg = new AMessage(kWhatStart, this);

    status_t err;
    Vector<MediaResource> resources;
    const char *type = (mFlags & kFlagIsSecure) ?
            kResourceSecureCodec : kResourceNonSecureCodec;
    const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec;
    resources.push_back(MediaResource(String8(type), String8(subtype), 1));
    // Don't know the buffer size at this point, but it's fine to use 1 because
    // the reclaimResource call doesn't consider the requester's buffer size for now.
    resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1));
    for (int i = 0; i <= kMaxRetry; ++i) {
        if (i > 0) {
            // Don't try to reclaim resource for the first time.
            if (!mResourceManagerService->reclaimResource(resources)) {
                break;
            }
            // Recover codec from previous error before retry start.
            err = reset();
            if (err != OK) {
                ALOGE("retrying start: failed to reset codec");
                break;
            }
            sp<AMessage> response;
            err = PostAndAwaitResponse(mConfigureMsg, &response);
            if (err != OK) {
                ALOGE("retrying start: failed to configure codec");
                break;
            }
        }

        sp<AMessage> response;
        err = PostAndAwaitResponse(msg, &response);
        if (!isResourceError(err)) {
            break;
        }
    }
    return err;
}

stop过程

//create by 逆流的鱼yuiop on 2016/12/11
//blog地址：http://blog.csdn.net/hejjunlin
status_t MediaCodec::stop() {
    sp<AMessage> msg = new AMessage(kWhatStop, this);

    sp<AMessage> response;
    return PostAndAwaitResponse(msg, &response);
}

找到对应的AMessage.cpp,对应同样有一套AHandler.cpp，及ALooper.cpp，这此组成了在c++中一套机制，接口 方法的名字和Java层保持一致。 
所有message都在onMessageReceived方法中处理，MediaCodec的各个状态的相关切换。

//create by 逆流的鱼yuiop on 2016/12/11
//blog地址：http://blog.csdn.net/hejjunlin
void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
                        switch (mState) {
                        case INITIALIZING://初始化中
                        {
                            setState(UNINITIALIZED);
                            break;
                        }

                        case CONFIGURING://配置中
                        {
                            setState(actionCode == ACTION_CODE_FATAL ?
                                    UNINITIALIZED : INITIALIZED);
                            break;
                        }

                        case STARTING://start中
                        {
                            setState(actionCode == ACTION_CODE_FATAL ?
                                    UNINITIALIZED : CONFIGURED);
                            break;
                        }

                        case STOPPING://停止中
                        case RELEASING://释放中
                        {
                            // Ignore the error, assuming we'll still get
                            // the shutdown complete notification.

                            sendErrorResponse = false;

                            if (mFlags & kFlagSawMediaServerDie) {
                                // MediaServer died, there definitely won't
                                // be a shutdown complete notification after
                                // all.

                                // note that we're directly going from
                                // STOPPING->UNINITIALIZED, instead of the
                                // usual STOPPING->INITIALIZED state.
                                setState(UNINITIALIZED);
                                if (mState == RELEASING) {
                                    mComponentName.clear();
                                }
                                (new AMessage)->postReply(mReplyID);
                            }
                            break;
                        }

                        case FLUSHING://刷新中
                        {
                            if (actionCode == ACTION_CODE_FATAL) {
                                setState(UNINITIALIZED);
                            } else {
                                setState(
                                        (mFlags & kFlagIsAsync) ? FLUSHED : STARTED);
                            }
                            break;
                        }

                        case FLUSHED:
                        case STARTED:
                        {
                            sendErrorResponse = false;

                            setStickyError(err);
                            postActivityNotificationIfPossible();

                            cancelPendingDequeueOperations();

                            if (mFlags & kFlagIsAsync) {
                                onError(err, actionCode);
                            }
                            switch (actionCode) {
                            case ACTION_CODE_TRANSIENT:
                                break;
                            case ACTION_CODE_RECOVERABLE:
                                setState(INITIALIZED);
                                break;
                            default:
                                setState(UNINITIALIZED);
                                break;
                            }
                            break;
                        }

                        default:
                        {
                            sendErrorResponse = false;

                            setStickyError(err);
                            postActivityNotificationIfPossible();

                            // actionCode in an uninitialized state is always fatal.
                            if (mState == UNINITIALIZED) {
                                actionCode = ACTION_CODE_FATAL;
                            }
                            if (mFlags & kFlagIsAsync) {
                                onError(err, actionCode);
                            }
                            switch (actionCode) {
                            case ACTION_CODE_TRANSIENT:
                                break;
                            case ACTION_CODE_RECOVERABLE:
                                setState(INITIALIZED);
                                break;
                            default:
                                setState(UNINITIALIZED);
                                break;
                            }
                            break;
                        }
}