Why android mediaCodec encode low resolution video slower than high resolution

When I use Android media-codec to encode video to h264 format, I found that encode one frame of 1280 * 720 YUV420P raw image need about 10ms but one frame of 640 * 360 YUV420P raw image need about 100ms. I have tied on serval mobile with MTK or Qualcomm video encoder, all device perform like that. Can anyone tell me how to fix my following code to speed up or tell the reason why lower resolution cost more time.

private boolean openH264Encoder(int width, int height, int bitrate) throws TranscodeNativeException {
    try {
        mediaCodec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);

        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
            Log.w(TAG, "codec name " + mediaCodec.getName());
        }

        for (int i = 0; i < MediaCodecList.getCodecCount(); i++) {
            MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
            if (codecInfo.isEncoder()) {
                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
                    if (mediaCodec.getName().equalsIgnoreCase(codecInfo.getName())) {
                        MediaCodecInfo.CodecCapabilities cc = codecInfo.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC);
                        for (int c : cc.colorFormats) {
                            Log.i(TAG, String.format("color format 0x%x", c));
                            if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar == c) {
                                videoColorFormat = c;
                                Log.i(TAG, String.format("final color format 0x%x", videoColorFormat));
                                break;
                            }
                            else if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar == c) {
                                videoColorFormat = c;
                                Log.i(TAG, String.format("find color format 0x%x", videoColorFormat));
                            }
                        }
                        break;
                    }
                }
                else {
                    return false;
                }
            }
        }

        if (videoColorFormat == 0) {
            Log.e(TAG, "can't find supported color format");
            return false;
        }
        Log.i(TAG, String.format("use color format 0x%x", videoColorFormat));

        MediaFormat mediaFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
        mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, videoColorFormat);
        mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 25);
        mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 25);
        mediaFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR);

        mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);

        mediaCodec.start();

        encoderOutputBuffers = mediaCodec.getOutputBuffers();
    } catch (Exception e) {
        Log.w(TAG, "open h264 encoder failed");
        throw  new TranscodeNativeException("open h264 encoder failed");
    }

    return true;
}

public byte[] encodeH264(byte[] data, long ms) {
    if (mediaCodec == null)
        return null;

    if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar == videoColorFormat) {
        byte[] newData = new byte[data.length];

        System.arraycopy(data, 0, newData, 0, data.length * 2 / 3);
        for (int i = 0; i < data.length / 6; i++) {
            newData[data.length * 2 / 3 + i * 2] = data[data.length * 2 / 3 + i];
            newData[data.length * 2 / 3 + i * 2 + 1] = data[data.length * 5 / 6 + i];
        }

        data = newData;
    }

    MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();

    try {
        ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers();
        int inputBufferIndex = mediaCodec.dequeueInputBuffer(20 * 1000);

        if (inputBufferIndex >= 0) {
            ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
            inputBuffer.clear();
            inputBuffer.put(data);

            mediaCodec.queueInputBuffer(inputBufferIndex, 0, data.length, ms * 1000, MediaCodec.BUFFER_FLAG_CODEC_CONFIG);

            while (true) {
                int encoderStatus = mediaCodec.dequeueOutputBuffer(info, -1);
                Log.i(TAG, "encoder status " + encoderStatus);
                switch (encoderStatus) {
                    case MediaCodec.INFO_TRY_AGAIN_LATER:
                        Log.i(TAG, "why tell me try again later?");
                        return null;

                    case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: {
                        MediaFormat encformat = mediaCodec.getOutputFormat();
                        Log.i(TAG, "output format changed");
                        return new byte[0];
                    }

                    case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
                        encoderOutputBuffers = mediaCodec.getOutputBuffers();
                        Log.i(TAG, "output buffer changed");
                        continue;

                    default: {
                        ByteBuffer encoderOutputBuffer = encoderOutputBuffers[encoderStatus];
                        if (encoderOutputBuffer == null) {
                            Log.w(TAG, "output buffer is null");
                            return null;
                        }

                        byte[] outData = new byte[info.size];
                        encoderOutputBuffer.get(outData);
                        ByteBuffer byteBuffer = ByteBuffer.wrap(outData);
                        if (spsppsBuffer == null && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
                            //save sps and pps
                            if (byteBuffer.getInt() == 0x00000001) {
                                spsppsBuffer = outData;
                            }

                            String d = "";
                            for (byte i : outData)
                                d += String.format("0x%02x ", i);


                            Log.i(TAG, "got sps pps " + d);
                            mediaCodec.releaseOutputBuffer(encoderStatus, false);
                            continue;
                        }
                        else {
                            //key frame
                            if ((outData[4] & 0x1f) == 5) {
                                byte[] buffer = new byte[outData.length + spsppsBuffer.length];
                                System.arraycopy(spsppsBuffer, 0, buffer, 0, spsppsBuffer.length);
                                System.arraycopy(outData, 0, buffer, spsppsBuffer.length, outData.length);

                                Log.i(TAG, "got key frame");
                                mediaCodec.releaseOutputBuffer(encoderStatus, false);
                                return buffer;
                            }
                            else {
                                Log.i(TAG, "got non key frame");
                                mediaCodec.releaseOutputBuffer(encoderStatus, false);
                                return outData;
                            }
                        }
                    }
                }
            }
        }
        else {
            Log.w(TAG, "dequeue input buffer failed, skip one frame");

            return new byte[0];
        }
    }
    catch (Exception e) {
        e.printStackTrace();
        Log.w(TAG, "encode h264 failed");
        return null;
    }
}