Print this page
5042 stop using deprecated atomic functions
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/hxge/hxge_send.c
+++ new/usr/src/uts/common/io/hxge/hxge_send.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 #include <hxge_impl.h>
28 28
29 29 extern uint32_t hxge_reclaim_pending;
30 30 extern uint32_t hxge_bcopy_thresh;
31 31 extern uint32_t hxge_dvma_thresh;
32 32 extern uint32_t hxge_dma_stream_thresh;
33 33 extern uint32_t hxge_tx_minfree;
34 34 extern uint32_t hxge_tx_intr_thres;
35 35 extern uint32_t hxge_tx_max_gathers;
36 36 extern uint32_t hxge_tx_tiny_pack;
37 37 extern uint32_t hxge_tx_use_bcopy;
38 38
39 39 static int hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp);
40 40
41 41 void
42 42 hxge_tx_ring_task(void *arg)
43 43 {
44 44 p_tx_ring_t ring = (p_tx_ring_t)arg;
45 45
46 46 MUTEX_ENTER(&ring->lock);
47 47 (void) hxge_txdma_reclaim(ring->hxgep, ring, 0);
48 48 MUTEX_EXIT(&ring->lock);
49 49
50 50 mac_tx_ring_update(ring->hxgep->mach, ring->ring_handle);
51 51 }
52 52
53 53 static void
54 54 hxge_tx_ring_dispatch(p_tx_ring_t ring)
55 55 {
56 56 /*
57 57 * Kick the ring task to reclaim some buffers.
58 58 */
59 59 (void) ddi_taskq_dispatch(ring->taskq,
60 60 hxge_tx_ring_task, (void *)ring, DDI_SLEEP);
61 61 }
62 62
63 63 mblk_t *
64 64 hxge_tx_ring_send(void *arg, mblk_t *mp)
65 65 {
66 66 p_hxge_ring_handle_t rhp = (p_hxge_ring_handle_t)arg;
67 67 p_hxge_t hxgep;
68 68 p_tx_ring_t tx_ring_p;
69 69 int status;
70 70
71 71 ASSERT(rhp != NULL);
72 72 ASSERT((rhp->index >= 0) && (rhp->index < HXGE_MAX_TDCS));
73 73
74 74 hxgep = rhp->hxgep;
75 75 tx_ring_p = hxgep->tx_rings->rings[rhp->index];
76 76 ASSERT(hxgep == tx_ring_p->hxgep);
77 77
78 78 status = hxge_start(hxgep, tx_ring_p, mp);
79 79 if (status != 0) {
80 80 hxge_tx_ring_dispatch(tx_ring_p);
81 81 return (mp);
82 82 }
83 83
84 84 return ((mblk_t *)NULL);
85 85 }
86 86
87 87 static int
88 88 hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
89 89 {
90 90 int dma_status, status = 0;
91 91 p_tx_desc_t tx_desc_ring_vp;
92 92 hpi_handle_t hpi_desc_handle;
93 93 hxge_os_dma_handle_t tx_desc_dma_handle;
94 94 p_tx_desc_t tx_desc_p;
95 95 p_tx_msg_t tx_msg_ring;
96 96 p_tx_msg_t tx_msg_p;
97 97 tx_desc_t tx_desc, *tmp_desc_p;
98 98 tx_desc_t sop_tx_desc, *sop_tx_desc_p;
99 99 p_tx_pkt_header_t hdrp;
100 100 p_tx_pkt_hdr_all_t pkthdrp;
101 101 uint8_t npads = 0;
102 102 uint64_t dma_ioaddr;
103 103 uint32_t dma_flags;
104 104 int last_bidx;
105 105 uint8_t *b_rptr;
106 106 caddr_t kaddr;
107 107 uint32_t nmblks;
108 108 uint32_t ngathers;
109 109 uint32_t clen;
110 110 int len;
111 111 uint32_t pkt_len, pack_len, min_len;
112 112 uint32_t bcopy_thresh;
113 113 int i, cur_index, sop_index;
114 114 uint16_t tail_index;
115 115 boolean_t tail_wrap = B_FALSE;
116 116 hxge_dma_common_t desc_area;
117 117 hxge_os_dma_handle_t dma_handle;
118 118 ddi_dma_cookie_t dma_cookie;
119 119 hpi_handle_t hpi_handle;
120 120 p_mblk_t nmp;
121 121 p_mblk_t t_mp;
122 122 uint32_t ncookies;
123 123 boolean_t good_packet;
124 124 boolean_t mark_mode = B_FALSE;
125 125 p_hxge_stats_t statsp;
126 126 p_hxge_tx_ring_stats_t tdc_stats;
127 127 t_uscalar_t start_offset = 0;
128 128 t_uscalar_t stuff_offset = 0;
129 129 t_uscalar_t end_offset = 0;
130 130 t_uscalar_t value = 0;
131 131 t_uscalar_t cksum_flags = 0;
132 132 boolean_t cksum_on = B_FALSE;
133 133 uint32_t boff = 0;
134 134 uint64_t tot_xfer_len = 0, tmp_len = 0;
135 135 boolean_t header_set = B_FALSE;
136 136 tdc_tdr_kick_t kick;
137 137 uint32_t offset;
138 138 #ifdef HXGE_DEBUG
139 139 p_tx_desc_t tx_desc_ring_pp;
140 140 p_tx_desc_t tx_desc_pp;
141 141 tx_desc_t *save_desc_p;
142 142 int dump_len;
143 143 int sad_len;
144 144 uint64_t sad;
145 145 int xfer_len;
146 146 uint32_t msgsize;
147 147 #endif
148 148
149 149 HXGE_DEBUG_MSG((hxgep, TX_CTL,
150 150 "==> hxge_start: tx dma channel %d", tx_ring_p->tdc));
151 151 HXGE_DEBUG_MSG((hxgep, TX_CTL,
152 152 "==> hxge_start: Starting tdc %d desc pending %d",
153 153 tx_ring_p->tdc, tx_ring_p->descs_pending));
154 154
155 155 statsp = hxgep->statsp;
156 156
157 157 if (hxgep->statsp->port_stats.lb_mode == hxge_lb_normal) {
158 158 if (!statsp->mac_stats.link_up) {
159 159 freemsg(mp);
160 160 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: "
161 161 "link not up or LB mode"));
162 162 goto hxge_start_fail1;
163 163 }
164 164 }
165 165
166 166 mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset, &value,
167 167 &cksum_flags);
168 168 if (!HXGE_IS_VLAN_PACKET(mp->b_rptr)) {
169 169 start_offset += sizeof (ether_header_t);
170 170 stuff_offset += sizeof (ether_header_t);
171 171 } else {
172 172 start_offset += sizeof (struct ether_vlan_header);
173 173 stuff_offset += sizeof (struct ether_vlan_header);
174 174 }
175 175
176 176 if (cksum_flags & HCK_PARTIALCKSUM) {
177 177 HXGE_DEBUG_MSG((hxgep, TX_CTL,
178 178 "==> hxge_start: mp $%p len %d "
179 179 "cksum_flags 0x%x (partial checksum) ",
180 180 mp, MBLKL(mp), cksum_flags));
181 181 cksum_on = B_TRUE;
182 182 }
183 183
184 184 MUTEX_ENTER(&tx_ring_p->lock);
185 185 start_again:
186 186 ngathers = 0;
187 187 sop_index = tx_ring_p->wr_index;
188 188 #ifdef HXGE_DEBUG
189 189 if (tx_ring_p->descs_pending) {
190 190 HXGE_DEBUG_MSG((hxgep, TX_CTL,
191 191 "==> hxge_start: desc pending %d ",
192 192 tx_ring_p->descs_pending));
193 193 }
194 194
195 195 dump_len = (int)(MBLKL(mp));
196 196 dump_len = (dump_len > 128) ? 128: dump_len;
197 197
198 198 HXGE_DEBUG_MSG((hxgep, TX_CTL,
199 199 "==> hxge_start: tdc %d: dumping ...: b_rptr $%p "
200 200 "(Before header reserve: ORIGINAL LEN %d)",
201 201 tx_ring_p->tdc, mp->b_rptr, dump_len));
202 202
203 203 HXGE_DEBUG_MSG((hxgep, TX_CTL,
204 204 "==> hxge_start: dump packets (IP ORIGINAL b_rptr $%p): %s",
205 205 mp->b_rptr, hxge_dump_packet((char *)mp->b_rptr, dump_len)));
206 206 #endif
207 207
208 208 tdc_stats = tx_ring_p->tdc_stats;
209 209 mark_mode = (tx_ring_p->descs_pending &&
210 210 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) <
211 211 hxge_tx_minfree));
|
↓ open down ↓ |
211 lines elided |
↑ open up ↑ |
212 212
213 213 HXGE_DEBUG_MSG((hxgep, TX_CTL,
214 214 "TX Descriptor ring is channel %d mark mode %d",
215 215 tx_ring_p->tdc, mark_mode));
216 216
217 217 if (!hxge_txdma_reclaim(hxgep, tx_ring_p, hxge_tx_minfree)) {
218 218 HXGE_DEBUG_MSG((hxgep, TX_CTL,
219 219 "TX Descriptor ring is full: channel %d", tx_ring_p->tdc));
220 220 HXGE_DEBUG_MSG((hxgep, TX_CTL,
221 221 "TX Descriptor ring is full: channel %d", tx_ring_p->tdc));
222 - (void) cas32((uint32_t *)&tx_ring_p->queueing, 0, 1);
222 + (void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing, 0, 1);
223 223 tdc_stats->tx_no_desc++;
224 224 MUTEX_EXIT(&tx_ring_p->lock);
225 225 status = 1;
226 226 goto hxge_start_fail1;
227 227 }
228 228
229 229 nmp = mp;
230 230 i = sop_index = tx_ring_p->wr_index;
231 231 nmblks = 0;
232 232 ngathers = 0;
233 233 pkt_len = 0;
234 234 pack_len = 0;
235 235 clen = 0;
236 236 last_bidx = -1;
237 237 good_packet = B_TRUE;
238 238
239 239 desc_area = tx_ring_p->tdc_desc;
240 240 hpi_handle = desc_area.hpi_handle;
241 241 hpi_desc_handle.regh = (hxge_os_acc_handle_t)
242 242 DMA_COMMON_ACC_HANDLE(desc_area);
243 243 hpi_desc_handle.hxgep = hxgep;
244 244 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
245 245 #ifdef HXGE_DEBUG
246 246 #if defined(__i386)
247 247 tx_desc_ring_pp = (p_tx_desc_t)(uint32_t)DMA_COMMON_IOADDR(desc_area);
248 248 #else
249 249 tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area);
250 250 #endif
251 251 #endif
252 252 tx_desc_dma_handle = (hxge_os_dma_handle_t)DMA_COMMON_HANDLE(desc_area);
253 253 tx_msg_ring = tx_ring_p->tx_msg_ring;
254 254
255 255 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: wr_index %d i %d",
256 256 sop_index, i));
257 257
258 258 #ifdef HXGE_DEBUG
259 259 msgsize = msgdsize(nmp);
260 260 HXGE_DEBUG_MSG((hxgep, TX_CTL,
261 261 "==> hxge_start(1): wr_index %d i %d msgdsize %d",
262 262 sop_index, i, msgsize));
263 263 #endif
264 264 /*
265 265 * The first 16 bytes of the premapped buffer are reserved
266 266 * for header. No padding will be used.
267 267 */
268 268 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
269 269 if (hxge_tx_use_bcopy) {
270 270 bcopy_thresh = (hxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
271 271 } else {
272 272 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
273 273 }
274 274 while (nmp) {
275 275 good_packet = B_TRUE;
276 276 b_rptr = nmp->b_rptr;
277 277 len = MBLKL(nmp);
278 278 if (len <= 0) {
279 279 nmp = nmp->b_cont;
280 280 continue;
281 281 }
282 282 nmblks++;
283 283
284 284 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): nmblks %d "
285 285 "len %d pkt_len %d pack_len %d",
286 286 nmblks, len, pkt_len, pack_len));
287 287 /*
288 288 * Hardware limits the transfer length to 4K.
289 289 * If len is more than 4K, we need to break
290 290 * nmp into two chunks: Make first chunk smaller
291 291 * than 4K. The second chunk will be broken into
292 292 * less than 4K (if needed) during the next pass.
293 293 */
294 294 if (len > (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE)) {
295 295 if ((t_mp = dupb(nmp)) != NULL) {
296 296 nmp->b_wptr = nmp->b_rptr +
297 297 (TX_MAX_TRANSFER_LENGTH -
298 298 TX_PKT_HEADER_SIZE);
299 299 t_mp->b_rptr = nmp->b_wptr;
300 300 t_mp->b_cont = nmp->b_cont;
301 301 nmp->b_cont = t_mp;
302 302 len = MBLKL(nmp);
303 303 } else {
304 304 good_packet = B_FALSE;
305 305 goto hxge_start_fail2;
306 306 }
307 307 }
308 308 tx_desc.value = 0;
309 309 tx_desc_p = &tx_desc_ring_vp[i];
310 310 #ifdef HXGE_DEBUG
311 311 tx_desc_pp = &tx_desc_ring_pp[i];
312 312 #endif
313 313 tx_msg_p = &tx_msg_ring[i];
314 314 #if defined(__i386)
315 315 hpi_desc_handle.regp = (uint32_t)tx_desc_p;
316 316 #else
317 317 hpi_desc_handle.regp = (uint64_t)tx_desc_p;
318 318 #endif
319 319 if (!header_set &&
320 320 ((!hxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
321 321 (len >= bcopy_thresh))) {
322 322 header_set = B_TRUE;
323 323 bcopy_thresh += TX_PKT_HEADER_SIZE;
324 324 boff = 0;
325 325 pack_len = 0;
326 326 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
327 327 hdrp = (p_tx_pkt_header_t)kaddr;
328 328 clen = pkt_len;
329 329 dma_handle = tx_msg_p->buf_dma_handle;
330 330 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
331 331 offset = tx_msg_p->offset_index * hxge_bcopy_thresh;
332 332 (void) ddi_dma_sync(dma_handle,
333 333 offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV);
334 334
335 335 tx_msg_p->flags.dma_type = USE_BCOPY;
336 336 goto hxge_start_control_header_only;
337 337 }
338 338
339 339 pkt_len += len;
340 340 pack_len += len;
341 341
342 342 HXGE_DEBUG_MSG((hxgep, TX_CTL,
343 343 "==> hxge_start(3): desc entry %d DESC IOADDR $%p "
344 344 "desc_vp $%p tx_desc_p $%p desc_pp $%p tx_desc_pp $%p "
345 345 "len %d pkt_len %d pack_len %d",
346 346 i,
347 347 DMA_COMMON_IOADDR(desc_area),
348 348 tx_desc_ring_vp, tx_desc_p,
349 349 tx_desc_ring_pp, tx_desc_pp,
350 350 len, pkt_len, pack_len));
351 351
352 352 if (len < bcopy_thresh) {
353 353 HXGE_DEBUG_MSG((hxgep, TX_CTL,
354 354 "==> hxge_start(4): USE BCOPY: "));
355 355 if (hxge_tx_tiny_pack) {
356 356 uint32_t blst = TXDMA_DESC_NEXT_INDEX(i, -1,
357 357 tx_ring_p->tx_wrap_mask);
358 358 HXGE_DEBUG_MSG((hxgep, TX_CTL,
359 359 "==> hxge_start(5): pack"));
360 360 if ((pack_len <= bcopy_thresh) &&
361 361 (last_bidx == blst)) {
362 362 HXGE_DEBUG_MSG((hxgep, TX_CTL,
363 363 "==> hxge_start: pack(6) "
364 364 "(pkt_len %d pack_len %d)",
365 365 pkt_len, pack_len));
366 366 i = blst;
367 367 tx_desc_p = &tx_desc_ring_vp[i];
368 368 #ifdef HXGE_DEBUG
369 369 tx_desc_pp = &tx_desc_ring_pp[i];
370 370 #endif
371 371 tx_msg_p = &tx_msg_ring[i];
372 372 boff = pack_len - len;
373 373 ngathers--;
374 374 } else if (pack_len > bcopy_thresh &&
375 375 header_set) {
376 376 pack_len = len;
377 377 boff = 0;
378 378 bcopy_thresh = hxge_bcopy_thresh;
379 379 HXGE_DEBUG_MSG((hxgep, TX_CTL,
380 380 "==> hxge_start(7): > max NEW "
381 381 "bcopy thresh %d "
382 382 "pkt_len %d pack_len %d(next)",
383 383 bcopy_thresh, pkt_len, pack_len));
384 384 }
385 385 last_bidx = i;
386 386 }
387 387 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
388 388 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
389 389 hdrp = (p_tx_pkt_header_t)kaddr;
390 390 header_set = B_TRUE;
391 391 HXGE_DEBUG_MSG((hxgep, TX_CTL,
392 392 "==> hxge_start(7_x2): "
393 393 "pkt_len %d pack_len %d (new hdrp $%p)",
394 394 pkt_len, pack_len, hdrp));
395 395 }
396 396 tx_msg_p->flags.dma_type = USE_BCOPY;
397 397 kaddr += boff;
398 398 HXGE_DEBUG_MSG((hxgep, TX_CTL,
399 399 "==> hxge_start(8): USE BCOPY: before bcopy "
400 400 "DESC IOADDR $%p entry %d bcopy packets %d "
401 401 "bcopy kaddr $%p bcopy ioaddr (SAD) $%p "
402 402 "bcopy clen %d bcopy boff %d",
403 403 DMA_COMMON_IOADDR(desc_area), i,
404 404 tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr,
405 405 clen, boff));
406 406 HXGE_DEBUG_MSG((hxgep, TX_CTL,
407 407 "==> hxge_start: 1USE BCOPY: "));
408 408 HXGE_DEBUG_MSG((hxgep, TX_CTL,
409 409 "==> hxge_start: 2USE BCOPY: "));
410 410 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: "
411 411 "last USE BCOPY: copy from b_rptr $%p "
412 412 "to KADDR $%p (len %d offset %d",
413 413 b_rptr, kaddr, len, boff));
414 414 bcopy(b_rptr, kaddr, len);
415 415 #ifdef HXGE_DEBUG
416 416 dump_len = (len > 128) ? 128: len;
417 417 HXGE_DEBUG_MSG((hxgep, TX_CTL,
418 418 "==> hxge_start: dump packets "
419 419 "(After BCOPY len %d)"
420 420 "(b_rptr $%p): %s", len, nmp->b_rptr,
421 421 hxge_dump_packet((char *)nmp->b_rptr,
422 422 dump_len)));
423 423 #endif
424 424 dma_handle = tx_msg_p->buf_dma_handle;
425 425 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
426 426 offset = tx_msg_p->offset_index * hxge_bcopy_thresh;
427 427 (void) ddi_dma_sync(dma_handle,
428 428 offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV);
429 429 clen = len + boff;
430 430 tdc_stats->tx_hdr_pkts++;
431 431 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(9): "
432 432 "USE BCOPY: DESC IOADDR $%p entry %d "
433 433 "bcopy packets %d bcopy kaddr $%p "
434 434 "bcopy ioaddr (SAD) $%p bcopy clen %d "
435 435 "bcopy boff %d",
436 436 DMA_COMMON_IOADDR(desc_area), i,
437 437 tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr,
438 438 clen, boff));
439 439 } else {
440 440 HXGE_DEBUG_MSG((hxgep, TX_CTL,
441 441 "==> hxge_start(12): USE DVMA: len %d", len));
442 442 tx_msg_p->flags.dma_type = USE_DMA;
443 443 dma_flags = DDI_DMA_WRITE;
444 444 if (len < hxge_dma_stream_thresh) {
445 445 dma_flags |= DDI_DMA_CONSISTENT;
446 446 } else {
447 447 dma_flags |= DDI_DMA_STREAMING;
448 448 }
449 449
450 450 dma_handle = tx_msg_p->dma_handle;
451 451 dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL,
452 452 (caddr_t)b_rptr, len, dma_flags,
453 453 DDI_DMA_DONTWAIT, NULL,
454 454 &dma_cookie, &ncookies);
455 455 if (dma_status == DDI_DMA_MAPPED) {
456 456 dma_ioaddr = dma_cookie.dmac_laddress;
457 457 len = (int)dma_cookie.dmac_size;
458 458 clen = (uint32_t)dma_cookie.dmac_size;
459 459 HXGE_DEBUG_MSG((hxgep, TX_CTL,
460 460 "==> hxge_start(12_1): "
461 461 "USE DVMA: len %d clen %d ngathers %d",
462 462 len, clen, ngathers));
463 463 #if defined(__i386)
464 464 hpi_desc_handle.regp = (uint32_t)tx_desc_p;
465 465 #else
466 466 hpi_desc_handle.regp = (uint64_t)tx_desc_p;
467 467 #endif
468 468 while (ncookies > 1) {
469 469 ngathers++;
470 470 /*
471 471 * this is the fix for multiple
472 472 * cookies, which are basically
473 473 * a descriptor entry, we don't set
474 474 * SOP bit as well as related fields
475 475 */
476 476
477 477 (void) hpi_txdma_desc_gather_set(
478 478 hpi_desc_handle, &tx_desc,
479 479 (ngathers -1), mark_mode,
480 480 ngathers, dma_ioaddr, clen);
481 481 tx_msg_p->tx_msg_size = clen;
482 482 HXGE_DEBUG_MSG((hxgep, TX_CTL,
483 483 "==> hxge_start: DMA "
484 484 "ncookie %d ngathers %d "
485 485 "dma_ioaddr $%p len %d"
486 486 "desc $%p descp $%p (%d)",
487 487 ncookies, ngathers,
488 488 dma_ioaddr, clen,
489 489 *tx_desc_p, tx_desc_p, i));
490 490
491 491 ddi_dma_nextcookie(dma_handle,
492 492 &dma_cookie);
493 493 dma_ioaddr = dma_cookie.dmac_laddress;
494 494
495 495 len = (int)dma_cookie.dmac_size;
496 496 clen = (uint32_t)dma_cookie.dmac_size;
497 497 HXGE_DEBUG_MSG((hxgep, TX_CTL,
498 498 "==> hxge_start(12_2): "
499 499 "USE DVMA: len %d clen %d ",
500 500 len, clen));
501 501
502 502 i = TXDMA_DESC_NEXT_INDEX(i, 1,
503 503 tx_ring_p->tx_wrap_mask);
504 504 tx_desc_p = &tx_desc_ring_vp[i];
505 505
506 506 hpi_desc_handle.regp =
507 507 #if defined(__i386)
508 508 (uint32_t)tx_desc_p;
509 509 #else
510 510 (uint64_t)tx_desc_p;
511 511 #endif
512 512 tx_msg_p = &tx_msg_ring[i];
513 513 tx_msg_p->flags.dma_type = USE_NONE;
514 514 tx_desc.value = 0;
515 515 ncookies--;
516 516 }
517 517 tdc_stats->tx_ddi_pkts++;
518 518 HXGE_DEBUG_MSG((hxgep, TX_CTL,
519 519 "==> hxge_start: DMA: ddi packets %d",
520 520 tdc_stats->tx_ddi_pkts));
521 521 } else {
522 522 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
523 523 "dma mapping failed for %d "
524 524 "bytes addr $%p flags %x (%d)",
525 525 len, b_rptr, status, status));
526 526 good_packet = B_FALSE;
527 527 tdc_stats->tx_dma_bind_fail++;
528 528 tx_msg_p->flags.dma_type = USE_NONE;
529 529 status = 1;
530 530 goto hxge_start_fail2;
531 531 }
532 532 } /* ddi dvma */
533 533
534 534 nmp = nmp->b_cont;
535 535 hxge_start_control_header_only:
536 536 #if defined(__i386)
537 537 hpi_desc_handle.regp = (uint32_t)tx_desc_p;
538 538 #else
539 539 hpi_desc_handle.regp = (uint64_t)tx_desc_p;
540 540 #endif
541 541 ngathers++;
542 542
543 543 if (ngathers == 1) {
544 544 #ifdef HXGE_DEBUG
545 545 save_desc_p = &sop_tx_desc;
546 546 #endif
547 547 sop_tx_desc_p = &sop_tx_desc;
548 548 sop_tx_desc_p->value = 0;
549 549 sop_tx_desc_p->bits.tr_len = clen;
550 550 sop_tx_desc_p->bits.sad = dma_ioaddr >> 32;
551 551 sop_tx_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff;
552 552 } else {
553 553 #ifdef HXGE_DEBUG
554 554 save_desc_p = &tx_desc;
555 555 #endif
556 556 tmp_desc_p = &tx_desc;
557 557 tmp_desc_p->value = 0;
558 558 tmp_desc_p->bits.tr_len = clen;
559 559 tmp_desc_p->bits.sad = dma_ioaddr >> 32;
560 560 tmp_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff;
561 561
562 562 tx_desc_p->value = tmp_desc_p->value;
563 563 }
564 564
565 565 HXGE_DEBUG_MSG((hxgep, TX_CTL,
566 566 "==> hxge_start(13): Desc_entry %d ngathers %d "
567 567 "desc_vp $%p tx_desc_p $%p "
568 568 "len %d clen %d pkt_len %d pack_len %d nmblks %d "
569 569 "dma_ioaddr (SAD) $%p mark %d",
570 570 i, ngathers, tx_desc_ring_vp, tx_desc_p,
571 571 len, clen, pkt_len, pack_len, nmblks,
572 572 dma_ioaddr, mark_mode));
573 573
574 574 #ifdef HXGE_DEBUG
575 575 hpi_desc_handle.hxgep = hxgep;
576 576 hpi_desc_handle.function.function = 0;
577 577 hpi_desc_handle.function.instance = hxgep->instance;
578 578 sad = save_desc_p->bits.sad;
579 579 sad = (sad << 32) | save_desc_p->bits.sad_l;
580 580 xfer_len = save_desc_p->bits.tr_len;
581 581
582 582 HXGE_DEBUG_MSG((hxgep, TX_CTL, "\n\t: value 0x%llx\n"
583 583 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
584 584 "mark %d sop %d\n",
585 585 save_desc_p->value, sad, save_desc_p->bits.tr_len,
586 586 xfer_len, save_desc_p->bits.num_ptr,
587 587 save_desc_p->bits.mark, save_desc_p->bits.sop));
588 588
589 589 hpi_txdma_dump_desc_one(hpi_desc_handle, NULL, i);
590 590 #endif
591 591
592 592 tx_msg_p->tx_msg_size = clen;
593 593 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
594 594 if (ngathers > hxge_tx_max_gathers) {
595 595 good_packet = B_FALSE;
596 596 mac_hcksum_get(mp, &start_offset, &stuff_offset,
597 597 &end_offset, &value, &cksum_flags);
598 598
599 599 HXGE_DEBUG_MSG((NULL, TX_CTL,
600 600 "==> hxge_start(14): pull msg - "
601 601 "len %d pkt_len %d ngathers %d",
602 602 len, pkt_len, ngathers));
603 603 goto hxge_start_fail2;
604 604 }
605 605 } /* while (nmp) */
606 606
607 607 tx_msg_p->tx_message = mp;
608 608 tx_desc_p = &tx_desc_ring_vp[sop_index];
609 609 #if defined(__i386)
610 610 hpi_desc_handle.regp = (uint32_t)tx_desc_p;
611 611 #else
612 612 hpi_desc_handle.regp = (uint64_t)tx_desc_p;
613 613 #endif
614 614
615 615 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
616 616 pkthdrp->reserved = 0;
617 617 hdrp->value = 0;
618 618 (void) hxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
619 619 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp);
620 620
621 621 /*
622 622 * Hardware header should not be counted as part of the frame
623 623 * when determining the frame size
624 624 */
625 625 if ((pkt_len - TX_PKT_HEADER_SIZE) > (STD_FRAME_SIZE - ETHERFCSL)) {
626 626 tdc_stats->tx_jumbo_pkts++;
627 627 }
628 628
629 629 min_len = (hxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2));
630 630 if (pkt_len < min_len) {
631 631 /* Assume we use bcopy to premapped buffers */
632 632 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
633 633 HXGE_DEBUG_MSG((NULL, TX_CTL,
634 634 "==> hxge_start(14-1): < (msg_min + 16)"
635 635 "len %d pkt_len %d min_len %d bzero %d ngathers %d",
636 636 len, pkt_len, min_len, (min_len - pkt_len), ngathers));
637 637 bzero((kaddr + pkt_len), (min_len - pkt_len));
638 638 pkt_len = tx_msg_p->tx_msg_size = min_len;
639 639
640 640 sop_tx_desc_p->bits.tr_len = min_len;
641 641
642 642 HXGE_MEM_PIO_WRITE64(hpi_desc_handle, sop_tx_desc_p->value);
643 643 tx_desc_p->value = sop_tx_desc_p->value;
644 644
645 645 HXGE_DEBUG_MSG((NULL, TX_CTL,
646 646 "==> hxge_start(14-2): < msg_min - "
647 647 "len %d pkt_len %d min_len %d ngathers %d",
648 648 len, pkt_len, min_len, ngathers));
649 649 }
650 650
651 651 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: cksum_flags 0x%x ",
652 652 cksum_flags));
653 653 if (cksum_flags & HCK_PARTIALCKSUM) {
654 654 HXGE_DEBUG_MSG((hxgep, TX_CTL,
655 655 "==> hxge_start: cksum_flags 0x%x (partial checksum) ",
656 656 cksum_flags));
657 657 cksum_on = B_TRUE;
658 658 HXGE_DEBUG_MSG((hxgep, TX_CTL,
659 659 "==> hxge_start: from IP cksum_flags 0x%x "
660 660 "(partial checksum) "
661 661 "start_offset %d stuff_offset %d",
662 662 cksum_flags, start_offset, stuff_offset));
663 663 tmp_len = (uint64_t)(start_offset >> 1);
664 664 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT);
665 665 tmp_len = (uint64_t)(stuff_offset >> 1);
666 666 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT);
667 667
668 668 HXGE_DEBUG_MSG((hxgep, TX_CTL,
669 669 "==> hxge_start: from IP cksum_flags 0x%x "
670 670 "(partial checksum) "
671 671 "after SHIFT start_offset %d stuff_offset %d",
672 672 cksum_flags, start_offset, stuff_offset));
673 673 }
674 674
675 675 /*
676 676 * pkt_len already includes 16 + paddings!!
677 677 * Update the control header length
678 678 */
679 679
680 680 /*
681 681 * Note that Hydra is different from Neptune where
682 682 * tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
683 683 */
684 684 tot_xfer_len = pkt_len;
685 685 tmp_len = hdrp->value |
686 686 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
687 687
688 688 HXGE_DEBUG_MSG((hxgep, TX_CTL,
689 689 "==> hxge_start(15_x1): setting SOP "
690 690 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
691 691 "0x%llx hdrp->value 0x%llx",
692 692 tot_xfer_len, tot_xfer_len, pkt_len, tmp_len, hdrp->value));
693 693 #if defined(_BIG_ENDIAN)
694 694 hdrp->value = ddi_swap64(tmp_len);
695 695 #else
696 696 hdrp->value = tmp_len;
697 697 #endif
698 698 HXGE_DEBUG_MSG((hxgep,
699 699 TX_CTL, "==> hxge_start(15_x2): setting SOP "
700 700 "after SWAP: tot_xfer_len 0x%llx pkt_len %d "
701 701 "tmp_len 0x%llx hdrp->value 0x%llx",
702 702 tot_xfer_len, pkt_len, tmp_len, hdrp->value));
703 703
704 704 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(15): setting SOP "
705 705 "wr_index %d tot_xfer_len (%d) pkt_len %d npads %d",
706 706 sop_index, tot_xfer_len, pkt_len, npads));
707 707
708 708 sop_tx_desc_p->bits.sop = 1;
709 709 sop_tx_desc_p->bits.mark = mark_mode;
710 710 sop_tx_desc_p->bits.num_ptr = ngathers;
711 711
712 712 if (mark_mode)
713 713 tdc_stats->tx_marks++;
714 714
715 715 HXGE_MEM_PIO_WRITE64(hpi_desc_handle, sop_tx_desc_p->value);
716 716 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(16): set SOP done"));
717 717
718 718 #ifdef HXGE_DEBUG
719 719 hpi_desc_handle.hxgep = hxgep;
720 720 hpi_desc_handle.function.function = 0;
721 721 hpi_desc_handle.function.instance = hxgep->instance;
722 722
723 723 HXGE_DEBUG_MSG((hxgep, TX_CTL, "\n\t: value 0x%llx\n"
724 724 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
725 725 save_desc_p->value, sad, save_desc_p->bits.tr_len,
726 726 xfer_len, save_desc_p->bits.num_ptr, save_desc_p->bits.mark,
727 727 save_desc_p->bits.sop));
728 728 (void) hpi_txdma_dump_desc_one(hpi_desc_handle, NULL, sop_index);
729 729
730 730 dump_len = (pkt_len > 128) ? 128: pkt_len;
731 731 HXGE_DEBUG_MSG((hxgep, TX_CTL,
732 732 "==> hxge_start: dump packets(17) (after sop set, len "
733 733 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
734 734 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
735 735 (char *)hdrp, hxge_dump_packet((char *)hdrp, dump_len)));
736 736 HXGE_DEBUG_MSG((hxgep, TX_CTL,
737 737 "==> hxge_start(18): TX desc sync: sop_index %d", sop_index));
738 738 #endif
739 739
740 740 if ((ngathers == 1) || tx_ring_p->wr_index < i) {
741 741 (void) ddi_dma_sync(tx_desc_dma_handle,
742 742 sop_index * sizeof (tx_desc_t),
743 743 ngathers * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV);
744 744
745 745 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(19): sync 1 "
746 746 "cs_off = 0x%02X cs_s_off = 0x%02X "
747 747 "pkt_len %d ngathers %d sop_index %d\n",
748 748 stuff_offset, start_offset,
749 749 pkt_len, ngathers, sop_index));
750 750 } else { /* more than one descriptor and wrap around */
751 751 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
752 752 (void) ddi_dma_sync(tx_desc_dma_handle,
753 753 sop_index * sizeof (tx_desc_t),
754 754 nsdescs * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV);
755 755 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(20): sync 1 "
756 756 "cs_off = 0x%02X cs_s_off = 0x%02X "
757 757 "pkt_len %d ngathers %d sop_index %d\n",
758 758 stuff_offset, start_offset, pkt_len, ngathers, sop_index));
759 759
760 760 (void) ddi_dma_sync(tx_desc_dma_handle, 0,
761 761 (ngathers - nsdescs) * sizeof (tx_desc_t),
762 762 DDI_DMA_SYNC_FORDEV);
763 763 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(21): sync 2 "
764 764 "cs_off = 0x%02X cs_s_off = 0x%02X "
765 765 "pkt_len %d ngathers %d sop_index %d\n",
766 766 stuff_offset, start_offset,
767 767 pkt_len, ngathers, sop_index));
768 768 }
769 769
770 770 tail_index = tx_ring_p->wr_index;
771 771 tail_wrap = tx_ring_p->wr_index_wrap;
772 772
773 773 tx_ring_p->wr_index = i;
774 774 if (tx_ring_p->wr_index <= tail_index) {
775 775 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
776 776 B_FALSE : B_TRUE);
777 777 }
778 778
779 779 tx_ring_p->descs_pending += ngathers;
780 780 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: TX kick: "
781 781 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
782 782 tx_ring_p->tdc, tx_ring_p->wr_index, tx_ring_p->wr_index_wrap,
783 783 ngathers, tx_ring_p->descs_pending));
784 784 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: TX KICKING: "));
785 785
786 786 kick.value = 0;
787 787 kick.bits.wrap = tx_ring_p->wr_index_wrap;
788 788 kick.bits.tail = (uint16_t)tx_ring_p->wr_index;
789 789
790 790 /* Kick start the Transmit kick register */
791 791 TXDMA_REG_WRITE64(HXGE_DEV_HPI_HANDLE(hxgep),
792 792 TDC_TDR_KICK, (uint8_t)tx_ring_p->tdc, kick.value);
793 793 tdc_stats->tx_starts++;
794 794 MUTEX_EXIT(&tx_ring_p->lock);
795 795 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_start"));
796 796 return (status);
797 797
798 798 hxge_start_fail2:
799 799 if (good_packet == B_FALSE) {
800 800 cur_index = sop_index;
801 801 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: clean up"));
802 802 for (i = 0; i < ngathers; i++) {
803 803 tx_desc_p = &tx_desc_ring_vp[cur_index];
804 804 #if defined(__i386)
805 805 hpi_handle.regp = (uint32_t)tx_desc_p;
806 806 #else
807 807 hpi_handle.regp = (uint64_t)tx_desc_p;
808 808 #endif
809 809 tx_msg_p = &tx_msg_ring[cur_index];
810 810 (void) hpi_txdma_desc_set_zero(hpi_handle, 1);
811 811 if (tx_msg_p->flags.dma_type == USE_DVMA) {
812 812 HXGE_DEBUG_MSG((hxgep, TX_CTL,
813 813 "tx_desc_p = %X index = %d",
814 814 tx_desc_p, tx_ring_p->rd_index));
815 815 (void) dvma_unload(tx_msg_p->dvma_handle,
816 816 0, -1);
817 817 tx_msg_p->dvma_handle = NULL;
818 818 if (tx_ring_p->dvma_wr_index ==
819 819 tx_ring_p->dvma_wrap_mask)
820 820 tx_ring_p->dvma_wr_index = 0;
821 821 else
822 822 tx_ring_p->dvma_wr_index++;
823 823 tx_ring_p->dvma_pending--;
824 824 } else if (tx_msg_p->flags.dma_type == USE_DMA) {
825 825 if (ddi_dma_unbind_handle(
826 826 tx_msg_p->dma_handle)) {
827 827 cmn_err(CE_WARN, "hxge_start: "
828 828 "ddi_dma_unbind_handle failed");
829 829 }
830 830 }
831 831 tx_msg_p->flags.dma_type = USE_NONE;
832 832 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
833 833 tx_ring_p->tx_wrap_mask);
834 834
835 835 }
836 836 }
837 837
838 838 MUTEX_EXIT(&tx_ring_p->lock);
839 839
840 840 hxge_start_fail1:
841 841 /* Add FMA to check the access handle hxge_hregh */
842 842 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_start"));
843 843 return (status);
844 844 }
|
↓ open down ↓ |
612 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX