+TCP STM32F7 Network buffer release

I’m having an issue with network buffers getting hung up, so I’ve built monitoring in the cli, which I’ll attach to show how I’m doing this.

What I’m trying to understand is the flow of release, why do some buffers show as not free when in fact they are not accepted? In this example dump here

MON:[01](IPV4)581122D6230D->FCC23D0B6A84 192.168.0.120 -> 192.168.0.77(TCP) 9491 -> 80
MON:[02](IPV4)581122D6230D->FCC23D0B6A84 192.168.0.120 -> 192.168.0.77(TCP) 9293 -> 80
MON:[04](IPV4)FCC23D0B6A84->581122D6230D 192.168.0.77 -> 192.168.0.120(TCP) 80 -> 9291
MON:[06](IPV4)FCC23D0B6A84->581122D6230D 192.168.0.77 -> 192.168.0.120(TCP) 80 -> 9491
MON:[07](0x0069)245A4C8B7DFF->0180C2000000 4242 -> ...
MON:[12]( ARP)FCC23D0B6A84->FFFFFFFFFFFF 192.168.0.77 -> 192.168.0.77
MON:[16](IPV4)FCC23D0B6A84->581122D6230D 192.168.0.77 -> 192.168.0.120(TCP) 80 -> 9291
MON:[22](IPV4)FCC23D0B6A84->581122D6230D 192.168.0.77 -> 192.168.0.120(TCP) 80 -> 9491

MON[07] is a Spanning Tree Protocol packet, which gets rejected in prvNetworkInterfaceInput at

        if( ( pxDMARxDescriptor->Status & ( ETH_DMARXDESC_CE | ETH_DMARXDESC_IPV4HCE | ETH_DMARXDESC_FT ) ) != ETH_DMARXDESC_FT )
        {
            /* Not an Ethernet frame-type or a checksum error. */
            xAccepted = pdFALSE;
        }

So why do these show up at all as non free as they never were really allocated? Or is the current descriptor ready to capture the next rx always listed as non free? If so, why would I ever get multiples, like this dump at startup?

MON:[00](0x88CC)245A4C8B7DFF->0180C200000E 0207 -> ...
MON:[01](0x0069)245A4C8B7DFF->0180C2000000 4242 -> ...
MON:[02](0x0069)245A4C8B7DFF->0180C2000000 4242 -> ...
MON:[03](IPV4)FCC23D0B6A84->581122D6230D 192.168.0.77 -> 192.168.0.120(TCP) 80 -> 9293
MON:[04](IPV4)FCC23D0B6A84->581122D6230D 192.168.0.77 -> 192.168.0.120(TCP) 80 -> 9293
MON:[05](IPV4)FCC23D0B6A84->581122D6230D 192.168.0.77 -> 192.168.0.120(TCP) 80 -> 9491
MON:[06](IPV4)581122D6230D->FCC23D0B6A84 192.168.0.120 -> 192.168.0.77(TCP) 9713 -> 80
MON:[07](IPV4)581122D6230D->FCC23D0B6A84 192.168.0.120 -> 192.168.0.77(TCP) 9491 -> 80

Corresponding code:

#if ENABLE_NET_BUF_VIEW == 1
static BaseType_t viewNetBuffers(char *pcWriteBuffer, size_t xWriteBufferLen,
                                 const char *pcCommandString,
                                 CLI_CmdState *state) {
  int i, len;
  int count = 0;
  char *buf = x_malloc(128, XMEM_HEAP_SLOW);
  char *ipSrc = x_malloc(32, XMEM_HEAP_SLOW);
  char *ipDst = x_malloc(32, XMEM_HEAP_SLOW);
  char *frameType = x_malloc(32, XMEM_HEAP_SLOW);
  char *out = pcWriteBuffer;
  // typedef struct xIP_PACKET *IPPacket;
  IPPacket_t *IPPacket;
  TCPPacket_t *TCPPacket;
  UDPPacket_t *UDPPacket;
  ICMPPacket_t *ICMPPacket;
  ARPPacket_t * ARPPacket;
  NetworkBufferDescriptor_t *bd = debug_xNetworkBuffers();
  for (i = 0; i < ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS; i++) {
    if (!isBufferFree(&bd[i])) {
      count++;
      vTaskSuspendAll();
      IPPacket = (IPPacket_t *)bd[i].pucEthernetBuffer;
      ARPPacket = (ARPPacket_t *)bd[i].pucEthernetBuffer;
      switch (IPPacket->xEthernetHeader.usFrameType) {
      case ipARP_FRAME_TYPE:
        sprintf(frameType, " ARP");
        sprintf(ipSrc, "%i.%i.%i.%i",
                ARPPacket->xARPHeader.ucSenderProtocolAddress[0],
                ARPPacket->xARPHeader.ucSenderProtocolAddress[1],
                ARPPacket->xARPHeader.ucSenderProtocolAddress[2],
                ARPPacket->xARPHeader.ucSenderProtocolAddress[3]);
        FreeRTOS_inet_ntoa(ARPPacket->xARPHeader.ulTargetProtocolAddress,
                           ipDst);
        break;
      case ipIPv4_FRAME_TYPE:
        sprintf(frameType, "IPV4");
        FreeRTOS_inet_ntoa(IPPacket->xIPHeader.ulSourceIPAddress,
                           (char *)ipSrc);
        FreeRTOS_inet_ntoa(IPPacket->xIPHeader.ulDestinationIPAddress,
                           (char *)ipDst);
        break;
      case ipIPv6_FRAME_TYPE:
        sprintf(frameType, "IPV6");
        sprintf(ipSrc, "...");
        sprintf(ipDst, "...");
        break;
      default:
        sprintf(frameType, "0x%02X%02X",
                IPPacket->xEthernetHeader.usFrameType & 0xFF,
                IPPacket->xEthernetHeader.usFrameType >> 8);
        sprintf(ipSrc, "%02X%02X", IPPacket->xIPHeader.ucVersionHeaderLength,
                IPPacket->xIPHeader.ucDifferentiatedServicesCode);
        sprintf(ipDst, "...");

        break;
      }
      len = sprintf(
          buf,
          "MON:[%02i](%s)%02X%02X%02X%02X%02X%02X->%02X%02X%02X%02X%02X%"
          "02X %s -> %s",
          i, frameType, IPPacket->xEthernetHeader.xSourceAddress.ucBytes[0],
          IPPacket->xEthernetHeader.xSourceAddress.ucBytes[1],
          IPPacket->xEthernetHeader.xSourceAddress.ucBytes[2],
          IPPacket->xEthernetHeader.xSourceAddress.ucBytes[3],
          IPPacket->xEthernetHeader.xSourceAddress.ucBytes[4],
          IPPacket->xEthernetHeader.xSourceAddress.ucBytes[5],
          IPPacket->xEthernetHeader.xDestinationAddress.ucBytes[0],
          IPPacket->xEthernetHeader.xDestinationAddress.ucBytes[1],
          IPPacket->xEthernetHeader.xDestinationAddress.ucBytes[2],
          IPPacket->xEthernetHeader.xDestinationAddress.ucBytes[3],
          IPPacket->xEthernetHeader.xDestinationAddress.ucBytes[4],
          IPPacket->xEthernetHeader.xDestinationAddress.ucBytes[5], ipSrc,
          ipDst);

      if (IPPacket->xEthernetHeader.usFrameType == ipIPv4_FRAME_TYPE) {
        switch (IPPacket->xIPHeader.ucProtocol) {
        case FREERTOS_IPPROTO_TCP:
          TCPPacket = (TCPPacket_t *)IPPacket;
          len +=
              sprintf(&buf[len], "(TCP) %i -> %i\r\n",
                      FreeRTOS_htons(TCPPacket->xTCPHeader.usSourcePort),
                      FreeRTOS_htons(TCPPacket->xTCPHeader.usDestinationPort));
          break;
        case FREERTOS_IPPROTO_UDP:
          UDPPacket = (UDPPacket_t *)IPPacket;
          len +=
              sprintf(&buf[len], "(UDP) %i -> %i\r\n",
                      FreeRTOS_htons(UDPPacket->xUDPHeader.usSourcePort),
                      FreeRTOS_htons(UDPPacket->xUDPHeader.usDestinationPort));
          break;
        case FREERTOS_SOCK_STREAM: // ICMP
          ICMPPacket = (ICMPPacket_t *)IPPacket;
          len += sprintf(&buf[len], "(ICMP) msg=%i type=%i\r\n",
                         ICMPPacket->xICMPHeader.ucTypeOfMessage,
                         ICMPPacket->xICMPHeader.ucTypeOfService);
          break;
        default:
          len += sprintf(&buf[len], "(%i)\r\n", IPPacket->xIPHeader.ucProtocol);
          break;
        }
      } else {
        len += sprintf(&buf[len], "\r\n");
      }
      xTaskResumeAll();
      out += cliSafePrint(out, &xWriteBufferLen, buf);
    }
  }
  if (count == 0) {
    out += cliSafePrint(out, &xWriteBufferLen, "No used buffers\r\n");
  }
  *out = 0;
  x_free(buf);
  x_free(ipSrc);
  x_free(ipDst);
  x_free(frameType);

  return pdFALSE;
}
#endif

Hello Erik,

When using the zero-copy scheme, it is normal to see this:

  MON:[00](0x88CC)245A4C8B7DFF->0180C200000E 0207 -> ...
  MON:[01](0x0069)245A4C8B7DFF->0180C2000000 4242 -> ...
  MON:[02](0x0069)245A4C8B7DFF->0180C2000000 4242 -> ...

These packets were received, stored into buffers (0, 1, and 2). But they were not passed to the application. It was concluded that xAccepted is false, and so the pucEthernetBuffer will be reused/refilled later.

What I don’t know is why packets seem to arrive “twice”. I have seen that behaviour before, a long time ago.

A few questions:

  • Are you using BufferAllocation 1 or 2 ?

  • Is your isBufferFree() using the original prvIsFreeBuffer() ?

  • Do you have ipconfigTCP_IP_SANITY defined (recommended)
    Note : it shows warning through FreeRTOS_debug_printf()

  • Do you haveipconfigZERO_COPY_RX_DRIVER set to 1 ?

  • Are you using the older STM32 driver from :
    NetworkInterface/STM32/Legacy/STM32Fxx
    or the newer one?

  • Are you using BufferAllocation 1 or 2 ?

I’m using BufferAllocation_1

  • Is your isBufferFree() using the original prvIsFreeBuffer() ?

It’s a little different, I forget how I came by this.

int isBufferFree(NetworkBufferDescriptor_t *pxNetworkBuffer) {
  return listIS_CONTAINED_WITHIN(&xFreeBuffersList, &(pxNetworkBuffer->xBufferListItem));
}
  • Do you have ipconfigTCP_IP_SANITY defined (recommended)

No I don’t. But I do have similar functionality in a monitor thread that tells me when I’m out of buffers, basically dumps the contents similar to above. I’m checking uxGetNumberOfFreeNetworkBuffers() and triggering at a threshold of 5 in this case. When it triggers its always final, the network is dead and won’t recover. (This could be a result of another threading issue elsewhere, I’m unclear)

  • Do you haveipconfigZERO_COPY_RX_DRIVER set to 1 ?

Yes, ipconfigZERO_COPY_RX_DRIVER is set to 1

  • Are you using the older STM32 driver from :
    NetworkInterface/STM32/Legacy/STM32Fxx
    or the newer one?

I don’t know for sure, probably the older one, it has FreeRTOS+TCP V2.3.4 in the header.

Do you suggest updating here? I’m on +TCP 2.3.3.

I’ve been a little hesitant to update because I’ve been bitten by new “security” features that broke DHCP for some routers, etc.

I think that I answered that question:

When a correct packet is received using zero-copy, the ownership will be transferred from DMA to the IP-task. At the same time, a new buffer is allocated and assigned to DMA.

So when you pause the application, you may see packets that were rejected, because they have a “wrong” frame-type. They are still owned by DMA.

(
In earlier versions of BufferAllocation_1.c I had registered a name to each buffer, so I could see where a problem had started,

Buffer allocation needs some discipline: we must always make sure that a buffer is owned by a single instance: NetworkInterface DMA, IP-task reception, IP-task transmission, etc. and make sure that every buffer will be released when ready.
)

Back to your question:

I’m having an issue with network buffers getting hung up

What do you mean with buffer getting hung up? Does pxGetNetworkBufferWithDescriptor() return NULL?

In that case you could add a new parameter to pxGetNetworkBufferWithDescriptor() to indicated the location where it was created, e.g. FreeRTOS_TCP_Transmission.c:683.

I work very often with STM32Fxx, and I have not seen problems with buffer allocation.

I just tested with the latest FreeRTOS+TCP (V4.2.2), but the NetworkInterface uses the Legacy, because STM32\Drivers\F7 does not compile.

I think I’m understanding the workings better, so ultimately the exact number of dma allocations isn’t really set if I understand this correctly. Ideally my debug code would show whether DMA or the IP task owned it.

I’m not sure about the buffers get hung up, but I’m ultimately checking with uxGetNumberOfFreeNetworkBuffers() which ends at 0. However, at this point I think it may have been caused by an improperly terminated vApplicationMallocFailedHook, which allowed the higher priority threads to continue running and fill up all the available buffers.

I’m just trying to fully understand what has happened well enough from the in the field logs to create a build that may fix some issues. At this point it appears I haven’t had enough heap to handle all the potential HTTP connections. As a side question, is there a decent way to gracefully time out http connections using FreeRTOS_TCP_server and FreeRTOS_HTTP_server based code so browsers don’t hog every single socket?

Would FREERTOS_SO_RCVTIMEO solve your usecase - FreeRTOS_setsockopt() - FreeRTOS™?

@friesen wrote:

I think I’m understanding the workings better, so ultimately the exact number of dma allocations isn’t really set if I understand this correctly. Ideally my debug code would show whether DMA or the IP task owned it

That’s right! You can not tell if a buffer pointer has been passed from the NetworkInterface to the IP-task or vv.

If you want to know the current owner of a network buffer, you’d also have to monitor the calls of type:

    eNetworkRxEvent
    eNetworkTxEvent
    eStackTxEvent  // UDP only

because each type passes a network buffer to the IP-task, for instance here :

static void prvSendRxEvent( NetworkBufferDescriptor_t * const pxDescriptor )
{
    const IPStackEvent_t xRxEvent =
    {
        .eEventType = eNetworkRxEvent,
        .pvData     = ( void * ) pxDescriptor
    };
    if( xSendEventStructToIPTask( &xRxEvent, pdMS_TO_TICKS( MAX_TIME_MS ) ) != pdPASS )
    {
        prvReleaseNetworkBufferDescriptor( pxDescriptor );
    }
}

Mind you that both FreeRTOS_recvfrom() or FreeRTOS_sendto() may change the buffer ownership when the FREERTOS_ZERO_COPY flag set.

About browsers holding unused sockets:

As you know, an HTTP server is “passive”, it receives connections as long as resources allow.
Also the parameter xBacklog of FreeRTOS_listen determines the maximum number of connected sockets.
Once full, the newcomers get a “connection reset”.

When my HTTP server receives a GET command, it answers with:

HTTP/1.1 200 OK
Content-Type: text/html
Connection: keep-alive
Content-Length: 11926


<!doctype html>
<html>
...

The keep-alive makes sure that the socket will be re-used, thus increasing the performance and lowering the use of resources.

I have monitored the number of TCP sockets: about 6 for a complex embedded JQuery application. My server allows up to 12 sockets, which is enough for two parallel HTTP sessions.

If you still encounter problems, you could consider adding a “last access time” property to each HTTP client. But I must say it is quite unusual to throw out TCP clients. All modern browsers clean up their unused sockets.

And if you decide to throw out a connection, it is enough to call FreeRTOS_shutdown() once and let the driver do the rest.

Sorry for the long text :slight_smile:

I have a new dump from a problem in the field, I know this one hasn’t run out of memory, although it still uses the original dump code as shown. In this case, all 24 descriptors are used, and the device loses connectivity until a reboot.

The background, this particular device keeps losing connectivity when connected to a specific switch. I think the switch may be misconfigured because the port is seeing all this mDns and DNS-SD dante traffic, but nevertheless I don’t want the device crashing anyway.

<1> 06/18/25 15:07:07.165 MON:[00](IPV4)001DC19A11B5->01005E0000FB 10.27.129.91 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.165 MON:[01](IPV4)001DC19A10E9->01005E0000FB 10.27.129.209 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.165 MON:[02](IPV4)001DC19A11CD->01005E0000FB 10.27.129.27 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.165 MON:[03](IPV4)001DC19A0F03->01005E0000FB 10.27.129.11 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.165 MON:[04](IPV4)001DC19A0E57->01005E0000FB 10.27.129.207 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.166 MON:[05](IPV4)001DC19A0ECD->01005E0000FB 10.27.129.120 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.166 MON:[06](IPV4)001DC19A1110->01005E0000FB 10.27.129.127 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.166 MON:[07](IPV4)001DC19A10C2->01005E0000FB 10.27.129.142 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.166 MON:[08](IPV4)001DC19A0EE0->01005E0000FB 10.27.129.112 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.166 MON:[09](IPV4)001DC19A0EEF->01005E0000FB 10.27.129.129 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.167 MON:[10](IPV4)001DC19A0FE4->01005E0000FB 10.27.129.25 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.168 MON:[11](IPV4)001DC19A0EB6->01005E0000FB 10.27.129.139 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.168 MON:[12](IPV4)001DC19A10FD->01005E0000FB 10.27.129.135 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.168 MON:[13](IPV4)001DC19A0F21->01005E0000FB 10.27.129.214 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.168 MON:[14](IPV4)001DC199293B->01005E0000FB 10.27.129.143 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.169 MON:[15](IPV4)001DC19A102C->01005E0000FB 10.27.129.118 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.170 MON:[16](IPV4)001DC19A10BE->01005E0000FB 10.27.129.43 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.171 MON:[17](IPV4)001DC19A0EC3->01005E0000FB 10.27.129.114 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.172 MON:[18](IPV4)001DC19A1050->01005E0000FB 10.27.129.136 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.173 MON:[19](IPV4)001DC19A0EBC->01005E0000FB 10.27.129.12 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.174 MON:[20](IPV4)001DC19A1109->01005E0000FB 10.27.129.14 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.175 MON:[21](IPV4)001DC19A1107->01005E0000FB 10.27.129.30 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.176 MON:[22](IPV4)001DC19A1116->01005E0000FB 10.27.129.92 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.193 MON:[23](IPV4)001DC19A0F0F->01005E0000FB 10.27.129.140 -> 224.0.0.251(UDP) 5353 -> 5353
<1> 06/18/25 15:07:07.194 MON:[24](IPV4)001DC19A10EE->01005E0000FB 10.27.129.132 -> 224.0.0.251(UDP) 5353 -> 5353

Is there anything I can look at, or add to debug this further? Is there any way that if all descriptors are used by rejected packets that something like this could happen?

Edit: these probably aren’t directly rejected but have to be rejected in the IP stack.

I always have ipconfigUSE_MDNS enabled, so when a lookup is received, it will be processed and answered.

There is a possible risk in a huge network, where many devices are doing lookups: your DUT may run out of resources.

Now I’m not sure what happens in the version of your stack: do you have ipconfigUSE_MDNS enabled? And it not, the packet should be discarded.

Also, are you using the Legacy version of the driver, or the new multi-platform STM driver?

I am also using STM32F7, and in the function xMayAcceptPacket()I see that mDNS packets are dropped:

#if ipconfigUSE_MDNS == 1
    && ( usDestinationPort != ipMDNS_PORT ) &&
       ( usSourcePort != ipMDNS_PORT )
#endif

Can you try to follow one mDNS lookup? From reception to sending a possible answer?

This is a customers network that isn’t easily accessed so I’m basically left to work with logs.

I don’t have ipconfigUSE_MDNS enabled.

I’m using the Legacy version of the driver.

Some indications could point to a misconfigured multicast network. In this case they tell us it is sharing a VLAN with Dante and Video multicast. I can recreate this in my network if I use scapy to flood udp packets, however the network does recover after the flood stops.

>>> pkt = Ether()/IP(dst="192.168.0.77")/UDP(dport=12345,sport=54321)/Raw(load="abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzsfadfasdfasdfasdfasdfas")

>>> sendpfast(pkt * 10000)

We have another customer with similar issues that resolved after they reconfigured multicast.

I don’t have ipconfigETHERNET_DRIVER_FILTERS_PACKETS enabled, is there a significant performance advantage for this?

I just remember this line:

    /* Allow all multicast addresses while testing. */
    /* macinit.MulticastFramesFilter = ETH_MULTICASTFRAMESFILTER_PERFECT; */
    macinit.MulticastFramesFilter = ETH_MULTICASTFRAMESFILTER_NONE;

I think it means that the multicast filter is not active, so mDNS packets will go to the IP-task.

These are the choices:

#define ETH_MULTICASTFRAMESFILTER_PERFECTHASHTABLE    0x00000404
#define ETH_MULTICASTFRAMESFILTER_HASHTABLE           0x00000004
#define ETH_MULTICASTFRAMESFILTER_PERFECT             0x00000000
#define ETH_MULTICASTFRAMESFILTER_NONE                0x00000010

There are only 8 address locations for perfect matches.

I recommend to use the ETH_MULTICASTFRAMESFILTER_PERFECT, so that mDNS packets are never seen.

Unfortunately I can not reproduce the problem you describe.

I also recommend trying a more rigorous testing setup, by adding and maintaining new fields:

typedef struct xNETWORK_BUFFER
{
    ...
    #if ( ipconfigNETWORK_BUFFER_DEBUGGING != 0 )
        const char * pcWhereCreated;
        const char * pcWhereReleased;
    #endif
}

I was able to find any leak in the buffer handling.