int vdec_msg_queue_qbuf(struct vdec_msg_queue_ctx *msg_ctx, struct vdec_lat_buf *buf)
 {
        struct list_head *head;
+       int status;
 
        head = vdec_get_buf_list(msg_ctx->hardware_index, buf);
        if (!head) {
        msg_ctx->ready_num++;
 
        vdec_msg_queue_inc(&buf->ctx->msg_queue, msg_ctx->hardware_index);
-       if (msg_ctx->hardware_index != MTK_VDEC_CORE)
+       if (msg_ctx->hardware_index != MTK_VDEC_CORE) {
                wake_up_all(&msg_ctx->ready_to_use);
-       else
-               queue_work(buf->ctx->dev->core_workqueue,
-                          &buf->ctx->msg_queue.core_work);
+       } else {
+               if (buf->ctx->msg_queue.core_work_cnt <
+                       atomic_read(&buf->ctx->msg_queue.core_list_cnt)) {
+                       status = queue_work(buf->ctx->dev->core_workqueue,
+                                           &buf->ctx->msg_queue.core_work);
+                       if (status)
+                               buf->ctx->msg_queue.core_work_cnt++;
+               }
+       }
 
        mtk_v4l2_debug(3, "enqueue buf type: %d addr: 0x%p num: %d",
                       msg_ctx->hardware_index, buf, msg_ctx->ready_num);
                container_of(msg_queue, struct mtk_vcodec_ctx, msg_queue);
        struct mtk_vcodec_dev *dev = ctx->dev;
        struct vdec_lat_buf *lat_buf;
+       int status;
 
        lat_buf = vdec_msg_queue_dqbuf(&dev->msg_queue_core_ctx);
        if (!lat_buf)
        vdec_msg_queue_qbuf(&ctx->msg_queue.lat_ctx, lat_buf);
 
        wake_up_all(&ctx->msg_queue.core_dec_done);
-       if (atomic_read(&lat_buf->ctx->msg_queue.core_list_cnt)) {
-               mtk_v4l2_debug(3, "re-schedule to decode for core: %d",
-                              dev->msg_queue_core_ctx.ready_num);
-               queue_work(dev->core_workqueue, &msg_queue->core_work);
+       spin_lock(&dev->msg_queue_core_ctx.ready_lock);
+       lat_buf->ctx->msg_queue.core_work_cnt--;
+
+       if (lat_buf->ctx->msg_queue.core_work_cnt <
+               atomic_read(&lat_buf->ctx->msg_queue.core_list_cnt)) {
+               status = queue_work(lat_buf->ctx->dev->core_workqueue,
+                                   &lat_buf->ctx->msg_queue.core_work);
+               if (status)
+                       lat_buf->ctx->msg_queue.core_work_cnt++;
        }
+       spin_unlock(&dev->msg_queue_core_ctx.ready_lock);
 }
 
 int vdec_msg_queue_init(struct vdec_msg_queue *msg_queue,
                return 0;
 
        msg_queue->ctx = ctx;
+       msg_queue->core_work_cnt = 0;
        vdec_msg_queue_init_ctx(&msg_queue->lat_ctx, MTK_VDEC_LAT0);
        INIT_WORK(&msg_queue->core_work, vdec_msg_queue_core_work);