mutex_lock(&isp->mutex);
 
        asd->subdev.devnode = vdev;
-       /* Deferred firmware loading case. */
-       if (isp->css_env.isp_css_fw.bytes == 0) {
-               dev_err(isp->dev, "Deferred firmware load.\n");
-               isp->firmware = atomisp_load_firmware(isp);
-               if (!isp->firmware) {
-                       dev_err(isp->dev, "Failed to load ISP firmware.\n");
-                       ret = -ENOENT;
-                       goto error;
-               }
-               ret = atomisp_css_load_firmware(isp);
-               if (ret) {
-                       dev_err(isp->dev, "Failed to init css.\n");
-                       goto error;
-               }
-               /* No need to keep FW in memory anymore. */
-               release_firmware(isp->firmware);
-               isp->firmware = NULL;
-               isp->css_env.isp_css_fw.data = NULL;
-       }
 
        if (!isp->input_cnt) {
                dev_err(isp->dev, "no camera attached\n");
 
        atomisp_destroy_pipes_stream_force(asd);
 
-       if (defer_fw_load) {
-               ia_css_unload_firmware();
-               isp->css_env.isp_css_fw.data = NULL;
-               isp->css_env.isp_css_fw.bytes = 0;
-       }
-
        ret = v4l2_subdev_call(isp->flash, core, s_power, 0);
        if (ret < 0 && ret != -ENODEV && ret != -ENOIOCTLCMD)
                dev_warn(isp->dev, "Failed to power-off flash\n");
 
 module_param(skip_fwload, uint, 0644);
 MODULE_PARM_DESC(skip_fwload, "Skip atomisp firmware load");
 
-/* memory optimization: deferred firmware loading */
-bool defer_fw_load;
-module_param(defer_fw_load, bool, 0644);
-MODULE_PARM_DESC(defer_fw_load,
-                "Defer FW loading until device is opened (default:disable)");
-
 /* cross componnet debug message flag */
 int dbg_level;
 module_param(dbg_level, int, 0644);
        isp->max_isr_latency = ATOMISP_MAX_ISR_LATENCY;
 
        /* Load isp firmware from user space */
-       if (!defer_fw_load) {
-               isp->firmware = atomisp_load_firmware(isp);
-               if (!isp->firmware) {
-                       err = -ENOENT;
-                       dev_dbg(&pdev->dev, "Firmware load failed\n");
-                       goto load_fw_fail;
-               }
+       isp->firmware = atomisp_load_firmware(isp);
+       if (!isp->firmware) {
+               err = -ENOENT;
+               dev_dbg(&pdev->dev, "Firmware load failed\n");
+               goto load_fw_fail;
+       }
 
-               err = sh_css_check_firmware_version(isp->dev, isp->firmware->data);
-               if (err) {
-                       dev_dbg(&pdev->dev, "Firmware version check failed\n");
-                       goto fw_validation_fail;
-               }
-       } else {
-               dev_info(&pdev->dev, "Firmware load will be deferred\n");
+       err = sh_css_check_firmware_version(isp->dev, isp->firmware->data);
+       if (err) {
+               dev_dbg(&pdev->dev, "Firmware version check failed\n");
+               goto fw_validation_fail;
        }
 
        pci_set_master(pdev);
        }
 
        /* Load firmware into ISP memory */
-       if (!defer_fw_load) {
-               err = atomisp_css_load_firmware(isp);
-               if (err) {
-                       dev_err(&pdev->dev, "Failed to init css.\n");
-                       goto css_init_fail;
-               }
-       } else {
-               dev_dbg(&pdev->dev, "Skip css init.\n");
+       err = atomisp_css_load_firmware(isp);
+       if (err) {
+               dev_err(&pdev->dev, "Failed to init css.\n");
+               goto css_init_fail;
        }
        /* Clear FW image from memory */
        release_firmware(isp->firmware);