int
qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
- uint32_t *dsd, uint16_t tot_dsds, struct qla_tgt_cmd *tc)
+ uint32_t *cur_dsd, uint16_t tot_dsds, struct qla_tgt_cmd *tc)
{
- void *next_dsd;
- uint8_t avail_dsds = 0;
- uint32_t dsd_list_len;
- struct dsd_dma *dsd_ptr;
+ struct dsd_dma *dsd_ptr, *dif_dsd, *nxt_dsd;
struct scatterlist *sg, *sgl;
- int i;
- struct scsi_cmnd *cmd;
- uint32_t *cur_dsd = dsd;
- uint16_t used_dsds = tot_dsds;
+ struct crc_context *difctx = NULL;
struct scsi_qla_host *vha;
+ uint dsd_list_len;
+ uint avail_dsds = 0;
+ uint used_dsds = tot_dsds;
+ bool dif_local_dma_alloc = false;
+ bool direction_to_device = false;
+ int i;
if (sp) {
- cmd = GET_CMD_SP(sp);
+ struct scsi_cmnd *cmd = GET_CMD_SP(sp);
sgl = scsi_prot_sglist(cmd);
vha = sp->vha;
+ difctx = sp->u.scmd.ctx;
+ direction_to_device = cmd->sc_data_direction == DMA_TO_DEVICE;
} else if (tc) {
vha = tc->vha;
sgl = tc->prot_sg;
+ difctx = tc->ctx;
+ direction_to_device = tc->dma_data_direction == DMA_TO_DEVICE;
} else {
BUG();
return 1;
}
- ql_dbg(ql_dbg_tgt, vha, 0xe021,
- "%s: enter\n", __func__);
-
- for_each_sg(sgl, sg, tot_dsds, i) {
- dma_addr_t sle_dma;
-
- /* Allocate additional continuation packets? */
- if (avail_dsds == 0) {
- avail_dsds = (used_dsds > QLA_DSDS_PER_IOCB) ?
- QLA_DSDS_PER_IOCB : used_dsds;
- dsd_list_len = (avail_dsds + 1) * 12;
- used_dsds -= avail_dsds;
-
- /* allocate tracking DS */
- dsd_ptr = kzalloc(sizeof(struct dsd_dma), GFP_ATOMIC);
- if (!dsd_ptr)
- return 1;
-
- /* allocate new list */
- dsd_ptr->dsd_addr = next_dsd =
- dma_pool_alloc(ha->dl_dma_pool, GFP_ATOMIC,
- &dsd_ptr->dsd_list_dma);
-
- if (!next_dsd) {
- /*
- * Need to cleanup only this dsd_ptr, rest
- * will be done by sp_free_dma()
- */
- kfree(dsd_ptr);
- return 1;
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe021,
+ "%s: enter (write=%u)\n", __func__, direction_to_device);
+
+ /* if initiator doing write or target doing read */
+ if (direction_to_device) {
+ for_each_sg(sgl, sg, tot_dsds, i) {
+ dma_addr_t sle_phys = sg_phys(sg);
+
+ /* If SGE address + length flips bits in upper 32-bits */
+ if (MSD(sle_phys + sg->length) ^ MSD(sle_phys)) {
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe022,
+ "%s: page boundary crossing (phys=%llx len=%x)\n",
+ __func__, sle_phys, sg->length);
+
+ if (difctx) {
+ ha->dif_bundle_crossed_pages++;
+ dif_local_dma_alloc = true;
+ } else {
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe022,
+ "%s: difctx pointer is NULL\n", __func__);
+ }
+ break;
}
+ }
+ ha->dif_bundle_writes++;
+ } else {
+ ha->dif_bundle_reads++;
+ }
+
+ /* if test override is on, then do local dma alloc for writes */
+ if (ql2xdifbundlinginternalbuffers) {
+ dif_local_dma_alloc = direction_to_device;
+ }
+
+ if (dif_local_dma_alloc) {
+ u8 track_difbundl_buf = 0;
+ difctx->no_dif_bundl = 0;
+ difctx->dif_bundl_len = 0;
+
+ INIT_LIST_HEAD(&difctx->ldif_dsd_list); /* Track DSD buffers */
+ INIT_LIST_HEAD(&difctx->ldif_dma_hndl_list); /* Track local DMA buffers */
+
+ for_each_sg(sgl, sg, tot_dsds, i) {
+ u32 sglen = sg_dma_len(sg);
+
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe023,
+ "%s: sg[%x] (phys=%llx sglen=%x)\n",
+ __func__, i, sg_phys(sg), sglen);
+
+ while (sglen) {
+ u32 xfrlen = 0;
+
+ /* allocate list item to store the DMA buffers */
+ dsd_ptr = kzalloc(sizeof(*dsd_ptr), GFP_ATOMIC);
+ if (!dsd_ptr) {
+ ql_dbg(ql_dbg_tgt, vha, 0xe024,
+ "%s: failed alloc dsd_ptr\n", __func__);
+ return 1;
+ }
+ ha->dif_bundle_kallocs++;
+
+ /* allocate dma buffer */
+ dsd_ptr->dsd_addr = dma_pool_alloc(ha->dif_bundl_pool,
+ GFP_ATOMIC, &dsd_ptr->dsd_list_dma);
+ if (!dsd_ptr->dsd_addr) {
+ ql_dbg(ql_dbg_tgt, vha, 0xe024,
+ "%s: failed alloc ->dsd_addr\n", __func__);
+ /* need to cleanup only this dsd_ptr */
+ /* rest will be done by sp_free_dma() */
+ kfree(dsd_ptr);
+ ha->dif_bundle_kallocs--;
+ return 1;
+ }
+ ha->dif_bundle_dma_allocs++;
+
+ /* xfrlen is min of dma pool size and sglen */
+ xfrlen = (sglen > DIF_BUNDLING_DMA_POOL_SIZE) ?
+ DIF_BUNDLING_DMA_POOL_SIZE : sglen;
+
+ /* replace with local allocated dma buffer */
+ sg_pcopy_to_buffer(sgl, sg_nents(sgl),
+ dsd_ptr->dsd_addr, xfrlen, difctx->dif_bundl_len);
+ difctx->dif_bundl_len += xfrlen;
+ sglen -= xfrlen;
+ difctx->no_dif_bundl++;
- if (sp) {
list_add_tail(&dsd_ptr->list,
- &((struct crc_context *)
- sp->u.scmd.ctx)->dsd_list);
-
- sp->flags |= SRB_CRC_CTX_DSD_VALID;
- } else {
- list_add_tail(&dsd_ptr->list,
- &(tc->ctx->dsd_list));
- tc->ctx_dsd_alloced = 1;
+ &difctx->ldif_dma_hndl_list);
}
-
- /* add new list to cmd iocb or last list */
- *cur_dsd++ = cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
- *cur_dsd++ = cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
- *cur_dsd++ = dsd_list_len;
- cur_dsd = (uint32_t *)next_dsd;
}
- sle_dma = sg_dma_address(sg);
- *cur_dsd++ = cpu_to_le32(LSD(sle_dma));
- *cur_dsd++ = cpu_to_le32(MSD(sle_dma));
- *cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe025,
+ "dif_bundl_len=%x, no_dif_bundl=%x\n",
+ difctx->dif_bundl_len, difctx->no_dif_bundl);
+
+ track_difbundl_buf = used_dsds = difctx->no_dif_bundl;
+ sp->flags |= SRB_DIF_BUNDL_DMA_VALID;
+
+ list_for_each_entry_safe(dif_dsd, nxt_dsd,
+ &difctx->ldif_dma_hndl_list, list) {
+ u32 sglen = (difctx->dif_bundl_len > DIF_BUNDLING_DMA_POOL_SIZE) ?
+ DIF_BUNDLING_DMA_POOL_SIZE : difctx->dif_bundl_len;
+ BUG_ON(track_difbundl_buf == 0);
+
+ /* Allocate additional continuation packets? */
+ if (avail_dsds == 0) {
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe024,
+ "%s: adding continuation iocb's\n", __func__);
+ avail_dsds = (used_dsds > QLA_DSDS_PER_IOCB) ?
+ QLA_DSDS_PER_IOCB : used_dsds;
+ dsd_list_len = (avail_dsds + 1) * 12;
+ used_dsds -= avail_dsds;
+
+ /* allocate tracking DS */
+ dsd_ptr = kzalloc(sizeof(*dsd_ptr), GFP_ATOMIC);
+ if (!dsd_ptr) {
+ ql_dbg(ql_dbg_tgt, vha, 0xe026,
+ "%s: failed alloc dsd_ptr\n", __func__);
+ return 1;
+ }
+ ha->dif_bundle_kallocs++;
+
+ difctx->no_ldif_dsd++;
+ /* allocate new list */
+ dsd_ptr->dsd_addr =
+ dma_pool_alloc(ha->dl_dma_pool, GFP_ATOMIC,
+ &dsd_ptr->dsd_list_dma);
+ if (!dsd_ptr->dsd_addr) {
+ ql_dbg(ql_dbg_tgt, vha, 0xe026,
+ "%s: failed alloc ->dsd_addr\n", __func__);
+ /* need to cleanup only this dsd_ptr */
+ /* rest will be done by sp_free_dma() */
+ kfree(dsd_ptr);
+ ha->dif_bundle_kallocs--;
+ return 1;
+ }
+ ha->dif_bundle_dma_allocs++;
+
+ if (sp) {
+ list_add_tail(&dsd_ptr->list,
+ &difctx->ldif_dsd_list);
+ sp->flags |= SRB_CRC_CTX_DSD_VALID;
+ } else {
+ list_add_tail(&dsd_ptr->list,
+ &difctx->ldif_dsd_list);
+ tc->ctx_dsd_alloced = 1;
+ }
+
+ /* add new list to cmd iocb or last list */
+ *cur_dsd++ = cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
+ *cur_dsd++ = cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
+ *cur_dsd++ = dsd_list_len;
+ cur_dsd = dsd_ptr->dsd_addr;
+ }
+ *cur_dsd++ = cpu_to_le32(LSD(dif_dsd->dsd_list_dma));
+ *cur_dsd++ = cpu_to_le32(MSD(dif_dsd->dsd_list_dma));
+ *cur_dsd++ = cpu_to_le32(sglen);
+ avail_dsds--;
+ difctx->dif_bundl_len -= sglen;
+ track_difbundl_buf--;
+ }
- avail_dsds--;
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe026,
+ "%s: no_ldif_dsd:%x, no_dif_bundl:%x\n", __func__,
+ difctx->no_ldif_dsd, difctx->no_dif_bundl);
+ } else {
+ for_each_sg(sgl, sg, tot_dsds, i) {
+ dma_addr_t sle_dma;
+
+ /* Allocate additional continuation packets? */
+ if (avail_dsds == 0) {
+ avail_dsds = (used_dsds > QLA_DSDS_PER_IOCB) ?
+ QLA_DSDS_PER_IOCB : used_dsds;
+ dsd_list_len = (avail_dsds + 1) * 12;
+ used_dsds -= avail_dsds;
+
+ /* allocate tracking DS */
+ dsd_ptr = kzalloc(sizeof(*dsd_ptr), GFP_ATOMIC);
+ if (!dsd_ptr) {
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe027,
+ "%s: failed alloc dsd_dma...\n", __func__);
+ return 1;
+ }
+
+ /* allocate new list */
+ dsd_ptr->dsd_addr =
+ dma_pool_alloc(ha->dl_dma_pool, GFP_ATOMIC,
+ &dsd_ptr->dsd_list_dma);
+ if (!dsd_ptr->dsd_addr) {
+ /* need to cleanup only this dsd_ptr */
+ /* rest will be done by sp_free_dma() */
+ kfree(dsd_ptr);
+ return 1;
+ }
+
+ if (sp) {
+ list_add_tail(&dsd_ptr->list,
+ &difctx->dsd_list);
+ sp->flags |= SRB_CRC_CTX_DSD_VALID;
+ } else {
+ list_add_tail(&dsd_ptr->list,
+ &difctx->dsd_list);
+ tc->ctx_dsd_alloced = 1;
+ }
+
+ /* add new list to cmd iocb or last list */
+ *cur_dsd++ = cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
+ *cur_dsd++ = cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
+ *cur_dsd++ = dsd_list_len;
+ cur_dsd = dsd_ptr->dsd_addr;
+ }
+ sle_dma = sg_dma_address(sg);
+ *cur_dsd++ = cpu_to_le32(LSD(sle_dma));
+ *cur_dsd++ = cpu_to_le32(MSD(sle_dma));
+ *cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+ avail_dsds--;
+ }
}
/* Null termination */
*cur_dsd++ = 0;
if (qla24xx_walk_and_build_prot_sglist(ha, sp, cur_dsd,
tot_prot_dsds, NULL))
goto crc_queuing_error;
+
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, vha, 0xe0aa,
+ "%s: crc_ctx_pkt=%p:\n", __func__, crc_ctx_pkt);
}
return QLA_SUCCESS;
"0 (Default). Do not move IOCBs"
"1 - Move IOCBs.");
+int ql2xdifbundlinginternalbuffers;
+module_param(ql2xdifbundlinginternalbuffers, int, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(ql2xdifbundlinginternalbuffers,
+ "Force using internal buffers for DIF information"
+ "0 (Default). Based on check."
+ "1 Force using internal buffers");
+
/*
* SCSI host template entry points
*/
ctx1 = NULL;
}
+ if (sp->flags & SRB_DIF_BUNDL_DMA_VALID) {
+ struct crc_context *difctx = sp->u.scmd.ctx;
+ struct dsd_dma *dif_dsd, *nxt_dsd;
+
+ list_for_each_entry_safe(dif_dsd, nxt_dsd,
+ &difctx->ldif_dma_hndl_list, list) {
+ list_del(&dif_dsd->list);
+ dma_pool_free(ha->dif_bundl_pool, dif_dsd->dsd_addr,
+ dif_dsd->dsd_list_dma);
+ kfree(dif_dsd);
+ difctx->no_dif_bundl--;
+ }
+
+ list_for_each_entry_safe(dif_dsd, nxt_dsd,
+ &difctx->ldif_dsd_list, list) {
+ list_del(&dif_dsd->list);
+ dma_pool_free(ha->dl_dma_pool, dif_dsd->dsd_addr,
+ dif_dsd->dsd_list_dma);
+ kfree(dif_dsd);
+ difctx->no_ldif_dsd--;
+ }
+
+ if (difctx->no_ldif_dsd) {
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, sp->vha, 0xe022,
+ "%s: difctx->no_ldif_dsd=%x\n",
+ __func__, difctx->no_ldif_dsd);
+ }
+
+ if (difctx->no_dif_bundl) {
+ ql_dbg(ql_dbg_tgt+ql_dbg_verbose, sp->vha, 0xe022,
+ "%s: difctx->no_dif_bundl=%x\n",
+ __func__, difctx->no_dif_bundl);
+ }
+ }
+
CMD_SP(cmd) = NULL;
qla2x00_rel_sp(sp);
}
"Failed to allocate memory for fcp_cmnd_dma_pool.\n");
goto fail_dl_dma_pool;
}
+
+ if (ql2xenabledif) {
+ u64 bufsize = DIF_BUNDLING_DMA_POOL_SIZE;
+ struct dsd_dma *dsd, *nxt;
+ uint i;
+ /* Creata a DMA pool of buffers for DIF bundling */
+ ha->dif_bundl_pool = dma_pool_create(name, &ha->pdev->dev,
+ DIF_BUNDLING_DMA_POOL_SIZE, 8, 0);
+ if (!ha->dif_bundl_pool) {
+ ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0024,
+ "%s: failed create dif_bundl_pool\n", __func__);
+ goto fail_dif_bundl_dma_pool;
+ }
+
+ INIT_LIST_HEAD(&ha->pool.good.head);
+ INIT_LIST_HEAD(&ha->pool.unusable.head);
+ ha->pool.good.count = 0;
+ ha->pool.unusable.count = 0;
+ for (i = 0; i < 128; i++) {
+ dsd = kzalloc(sizeof(*dsd), GFP_ATOMIC);
+ if (!dsd) {
+ ql_dbg_pci(ql_dbg_init, ha->pdev, 0xe0ee,
+ "%s: failed alloc dsd\n", __func__);
+ return 1;
+ }
+ ha->dif_bundle_kallocs++;
+
+ dsd->dsd_addr = dma_pool_alloc(
+ ha->dif_bundl_pool, GFP_ATOMIC, &dsd->dsd_list_dma);
+ if (!dsd->dsd_addr) {
+ ql_dbg_pci(ql_dbg_init, ha->pdev, 0xe0ee,
+ "%s: failed alloc ->dsd_addr\n", __func__);
+ kfree(dsd);
+ ha->dif_bundle_kallocs--;
+ continue;
+ }
+ ha->dif_bundle_dma_allocs++;
+
+ /* if dma buffer crosses 4G boundary, put it on bad list */
+ if (MSD(dsd->dsd_list_dma) ^ MSD(dsd->dsd_list_dma+bufsize)) {
+ list_add_tail(&dsd->list, &ha->pool.unusable.head);
+ ha->pool.unusable.count++;
+ } else {
+ list_add_tail(&dsd->list, &ha->pool.good.head);
+ ha->pool.good.count++;
+ }
+ }
+
+ /* return the good ones back to the pool */
+ list_for_each_entry_safe(dsd, nxt, &ha->pool.good.head, list) {
+ list_del(&dsd->list);
+ dma_pool_free(ha->dif_bundl_pool, dsd->dsd_addr,
+ dsd->dsd_list_dma);
+ ha->dif_bundle_dma_allocs--;
+ kfree(dsd);
+ ha->dif_bundle_kallocs--;
+ }
+
+ ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0024,
+ "%s: dif dma pool (good=%u unusable=%u)\n",
+ __func__, ha->pool.good.count, ha->pool.unusable.count);
+ }
+
ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0025,
- "dl_dma_pool=%p fcp_cmnd_dma_pool=%p.\n",
- ha->dl_dma_pool, ha->fcp_cmnd_dma_pool);
+ "dl_dma_pool=%p fcp_cmnd_dma_pool=%p dif_bundl_pool=%p.\n",
+ ha->dl_dma_pool, ha->fcp_cmnd_dma_pool, ha->dif_bundl_pool);
}
/* Allocate memory for SNS commands */
dma_free_coherent(&ha->pdev->dev, sizeof(struct sns_cmd_pkt),
ha->sns_cmd, ha->sns_cmd_dma);
fail_dma_pool:
+ if (ql2xenabledif) {
+ struct dsd_dma *dsd, *nxt;
+ list_for_each_entry_safe(dsd, nxt, &ha->pool.unusable.head, list) {
+ list_del(&dsd->list);
+ dma_pool_free(ha->dif_bundl_pool, dsd->dsd_addr, dsd->dsd_list_dma);
+ ha->dif_bundle_dma_allocs--;
+ kfree(dsd);
+ ha->dif_bundle_kallocs--;
+ ha->pool.unusable.count--;
+ }
+ dma_pool_destroy(ha->dif_bundl_pool);
+ ha->dif_bundl_pool = NULL;
+ }
+
+fail_dif_bundl_dma_pool:
if (IS_QLA82XX(ha) || ql2xenabledif) {
dma_pool_destroy(ha->fcp_cmnd_dma_pool);
ha->fcp_cmnd_dma_pool = NULL;
if (ha->ctx_mempool)
mempool_destroy(ha->ctx_mempool);
+ if (ql2xenabledif) {
+ struct dsd_dma *dsd, *nxt;
+
+ list_for_each_entry_safe(dsd, nxt, &ha->pool.unusable.head, list) {
+ list_del(&dsd->list);
+ dma_pool_free(ha->dif_bundl_pool, dsd->dsd_addr, dsd->dsd_list_dma);
+ ha->dif_bundle_dma_allocs--;
+ kfree(dsd);
+ ha->dif_bundle_kallocs--;
+ ha->pool.unusable.count--;
+ }
+ list_for_each_entry_safe(dsd, nxt, &ha->pool.good.head, list) {
+ list_del(&dsd->list);
+ dma_pool_free(ha->dif_bundl_pool, dsd->dsd_addr, dsd->dsd_list_dma);
+ ha->dif_bundle_dma_allocs--;
+ kfree(dsd);
+ ha->dif_bundle_kallocs--;
+ }
+ }
+
+ if (ha->dif_bundl_pool)
+ dma_pool_destroy(ha->dif_bundl_pool);
+
qlt_mem_free(ha);
if (ha->init_cb)
projects / users / jedix / linux-maple.git / commitdiff