Messages in this thread | | | Date | Fri, 22 Sep 2023 15:42:45 +0800 | Subject | Re: [PATCH net-next v3 12/18] net/smc: implement DMB-related operations of loopback | From | Wen Gu <> |
| |
On 2023/9/22 07:31, Wenjia Zhang wrote: > >
<...>
>> +static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb, >> + void *client_priv) >> +{ >> + struct smc_lo_dmb_node *dmb_node, *tmp_node; >> + struct smc_lo_dev *ldev = smcd->priv; >> + int sba_idx, rc; >> + >> + /* check space for new dmb */ >> + for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LODEV_MAX_DMBS) { >> + if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask)) >> + break; >> + } >> + if (sba_idx == SMC_LODEV_MAX_DMBS) >> + return -ENOSPC; >> + >> + dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL); >> + if (!dmb_node) { >> + rc = -ENOMEM; >> + goto err_bit; >> + } >> + >> + dmb_node->sba_idx = sba_idx; >> + dmb_node->cpu_addr = kzalloc(dmb->dmb_len, GFP_KERNEL | >> + __GFP_NOWARN | __GFP_NORETRY | >> + __GFP_NOMEMALLOC); > kzalloc()/kmalloc() allocates physically contigueous memory. Are you sure it is suitable for allocating the dmb? >
Yes, physically contigueous memory is little expensive here. I initially wanted to see the best performance.
I tried using vzalloc here, and the performance dropped a bit (2%~8%) compared to kzalloc. I think it is acceptable.
- ipc-benchmark kzalloc vzalloc Message rate (msg/s) 152076 145753(-4.16%)
- sockperf kzalloc vzalloc Bandwidth(MBps) 8491.638 8002.380(-5.76%) Latency(us) 3.222 3.508(+8.88%)
- nginx/wrk kzalloc vzalloc Requests/s 272519.36 256490.94(-5.88%)
- redis-benchmark kzalloc vzalloc GET(Requests/s) 123304.56 120084.05(-2.61%) SET(Requests/s) 122062.87 118800.12(-2.67%)
>> + if (!dmb_node->cpu_addr) { >> + rc = -ENOMEM; >> + goto err_node; >> + } >> + dmb_node->len = dmb->dmb_len; >> + dmb_node->dma_addr = (dma_addr_t)dmb_node->cpu_addr; >> + >> +again: >> + /* add new dmb into hash table */ >> + get_random_bytes(&dmb_node->token, sizeof(dmb_node->token)); >> + write_lock(&ldev->dmb_ht_lock); >> + hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) { >> + if (tmp_node->token == dmb_node->token) { >> + write_unlock(&ldev->dmb_ht_lock); >> + goto again; >> + } >> + } >> + hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token); >> + write_unlock(&ldev->dmb_ht_lock); >> + >> + dmb->sba_idx = dmb_node->sba_idx; >> + dmb->dmb_tok = dmb_node->token; >> + dmb->cpu_addr = dmb_node->cpu_addr; >> + dmb->dma_addr = dmb_node->dma_addr; >> + dmb->dmb_len = dmb_node->len; >> + >> + return 0; >> + >> +err_node: >> + kfree(dmb_node); >> +err_bit: >> + clear_bit(sba_idx, ldev->sba_idx_mask); >> + return rc; >> +} >> + >> +static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb) >> +{ >> + struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node; >> + struct smc_lo_dev *ldev = smcd->priv; >> + >> + /* remove dmb from hash table */ >> + write_lock(&ldev->dmb_ht_lock); >> + hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { >> + if (tmp_node->token == dmb->dmb_tok) { >> + dmb_node = tmp_node; >> + break; >> + } >> + } >> + if (!dmb_node) { >> + write_unlock(&ldev->dmb_ht_lock); >> + return -EINVAL; >> + } >> + hash_del(&dmb_node->list); >> + write_unlock(&ldev->dmb_ht_lock); >> + >> + clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask); >> + kfree(dmb_node->cpu_addr); >> + kfree(dmb_node); >> + >> + return 0; >> +} >> +
| |