I am running FreeRTOS+TCP on a Cortex-M4F with an ENC28J60 ethernet controller via SPI. After a while network buffers are exhausted.
I am using stock BufferAllocation1.c
(I only added printf macros to track allocations and deallocations).
Increasing ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS
only delays the time to exhaust. This bug happens reliably every time. DHCP works okay and sending UDP frames also works fine so most of the network driver is definitely correct. I have the usual network noise, nothing out of the ordinary. The bug does not go away after unplugging the network cable so I think it is not just a surge of traffic.
buffer allocation function
void vNetworkInterfaceAllocateRAMToBuffers(NetworkBufferDescriptor_t pxNetworkBuffers[ ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS ]){
// Taken from:
// https://github.com/FreeRTOS/FreeRTOS-Plus-TCP/blob/51590f627e9fe830fabd51958df250181f2712d2/portable/NetworkInterface/xilinx_ultrascale/NetworkInterface.c#L326
#define niBUFFER_1_PACKET_SIZE (ipTOTAL_ETHERNET_FRAME_SIZE + ipBUFFER_PADDING)
static uint8_t ucNetworkPackets[ ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS * niBUFFER_1_PACKET_SIZE ] __attribute__((aligned(4)));
uint8_t *ucRAMBuffer = ucNetworkPackets;
uint32_t ul;
for( ul = 0; ul < ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS; ul++ ){
pxNetworkBuffers[ ul ].pucEthernetBuffer = ucRAMBuffer + ipBUFFER_PADDING;
*( ( uintptr_t * ) ucRAMBuffer ) = ( uintptr_t ) &( pxNetworkBuffers[ ul ] );
ucRAMBuffer += niBUFFER_1_PACKET_SIZE;
}
debugf("buffers allocated");
}
send function
ipconfigZERO_COPY_TX_DRIVER is set to 1 (otherwise the breakpoint would be hit)
BaseType_t xNetworkInterfaceOutput(NetworkBufferDescriptor_t * const pxNetworkBuffer, BaseType_t xReleaseAfterSend){
debugf("output len %d", (unsigned int)pxNetworkBuffer->xDataLength);
enc28j60_send_packet(pxNetworkBuffer->pucEthernetBuffer, pxNetworkBuffer->xDataLength );
/* Call the standard trace macro to log the send event. */
iptraceNETWORK_INTERFACE_TRANSMIT();
if (xReleaseAfterSend){
vReleaseNetworkBufferAndDescriptor( pxNetworkBuffer );
} else {
__BKPT(101); //network stack is misconfigured?!
}
return pdTRUE;
}
send function (lower level)
The only hack is that I am using data[-1]
to store an SPI opcode to transmit the whole thing in a single DMA transfer (and it works fine). Config has #define ipconfigBUFFER_PADDING 4
.
void enc28j60_send_packet(uint8_t *data, uint16_t len){
uint32_t tickstart = xTaskGetTickCount();
while(enc28j60_rcr(ECON1) & ECON1_TXRTS){
// TXRTS may not clear - ENC28J60 bug. We must reset
// transmit logic in cause of Tx error
if(enc28j60_rcr(EIR) & EIR_TXERIF){
enc28j60_bfs(ECON1, ECON1_TXRST);
enc28j60_bfc(ECON1, ECON1_TXRST);
}
//If previous packet can't be sent within 100 ms, abort previous packet, ENC28J60 errata #12
if((xTaskGetTickCount() - tickstart) > pdMS_TO_TICKS(100)){
enc28j60_bfc(ECON1, ECON1_TXRTS);
debugf("Errata #12 workaround");
}
}
enc28j60_wcr16(EWRPT, ENC28J60_TXSTART);
pin_eth_cs_low();
uint8_t buffer[] = { ENC28J60_SPI_WBM, 0 };
spi_transfer_blocking(sizeof(buffer), buffer);
pin_eth_cs_high();
pin_eth_cs_low();
//The buffer does have space in front of it, see vNetworkInterfaceAllocateRAMToBuffers()
data[-1] = ENC28J60_SPI_WBM;
spi_transfer_blocking(len + 1, &data[-1]);
pin_eth_cs_high();
enc28j60_wcr16(ETXST, ENC28J60_TXSTART);
enc28j60_wcr16(ETXND, ENC28J60_TXSTART + len);
enc28j60_bfs(ECON1, ECON1_TXRTS); // Request packet send
}
receive function
It is scheduled from GPIO interrupt handler via xTimerPendFunctionCallFromISR
.
void prvReceivePacket(void *unused1 __attribute__((unused)), uint32_t unused2 __attribute__((unused))){
enc28j60_bfc(EIE, EIE_INTIE); // mask enc28j60 interrupts
volatile uint8_t eir_flags = enc28j60_rcr(EIR);
if (eir_flags & EIR_PKTIF){ /* if there is pending packet */
// retrieve packet from enc28j60
uint16_t xBytesReceived = 0;
uint16_t rxlen = 0;
uint16_t status = 0;
uint16_t temp;
xNetworkBufferDescriptor_t *pxBufferDescriptor;
xIPStackEvent_t xRxEvent;
enc28j60_read_buffer((void *)(&enc28j60_rxrdpt), sizeof(enc28j60_rxrdpt));
enc28j60_read_buffer((void *)(&rxlen), sizeof(rxlen));
enc28j60_read_buffer((void *)(&status), sizeof(status));
//debugf("Packet pending, rxrdpt %d, len %d, status %d", enc28j60_rxrdpt, rxlen, status);
if(status & 0x80){ //success
// Throw out crc
xBytesReceived = rxlen - 4;
// Allocate buffer for packet
pxBufferDescriptor = pxGetNetworkBufferWithDescriptor( xBytesReceived, 0 );
if( pxBufferDescriptor != NULL ){
// Read packet content
enc28j60_read_buffer( pxBufferDescriptor->pucEthernetBuffer, xBytesReceived );
pxBufferDescriptor->xDataLength = xBytesReceived;
/* The event about to be sent to the TCP/IP is an Rx event. */
xRxEvent.eEventType = eNetworkRxEvent;
/* pvData is used to point to the network buffer descriptor that
now references the received data. */
xRxEvent.pvData = ( void * ) pxBufferDescriptor;
/* Send the data to the TCP/IP stack. */
if( xSendEventStructToIPTask( &xRxEvent, 0 ) == pdFALSE ){
/* The buffer could not be sent to the IP task so the buffer
must be released. */
vReleaseNetworkBufferAndDescriptor( pxBufferDescriptor );
/* Make a call to the standard trace macro to log the
occurrence. */
iptraceETHERNET_RX_EVENT_LOST();
debugf("Stack did not accept incoming packet");
} else {
/* The message was successfully sent to the TCP/IP stack.
Call the standard trace macro to log the occurrence. */
iptraceNETWORK_INTERFACE_RECEIVE();
}
} else {
/* The event was lost because a network buffer was not available.
Call the standard trace macro to log the occurrence. */
iptraceETHERNET_RX_EVENT_LOST();
debugf("Couldn't allocate buffer for incoming packet ******************************");
}
} else {
debugf("Wrong status?");
}
// Set Rx buffer guard to next packet
if (enc28j60_rxrdpt == ENC28J60_RXSTART){
temp = ENC28J60_RXEND;
} else {
temp = enc28j60_rxrdpt - 1;
}
enc28j60_wcr16(ERXRDPT, temp);
// Set Rx pointer to next packet
enc28j60_wcr16(ERDPT, enc28j60_rxrdpt);
// Decrement packet counter
enc28j60_bfs(ECON2, ECON2_PKTDEC);
} else if (eir_flags & EIR_TXIF) {
//debugf("enc28j60: transmit done\n");
enc28j60_bfc(EIR, EIR_TXIF);
} else if (eir_flags & EIR_TXERIF) {
debugf("enc28j60: transmit error !!\n");
enc28j60_bfc(EIR, EIR_TXERIF);
} else if (eir_flags & EIR_RXERIF) {
debugf("enc28j60: receive error !!\n");
enc28j60_bfc(EIR, EIR_RXERIF);
} else {
debugf("enc28j60: unknown interrupt flag, we shouldn't be here\n");
}
enc28j60_bfs(EIE, EIE_INTIE); // unmask enc28j60 interrupts
}
typical log
The numbers 2001xxxx next to allocation/deallocation are simply the pointers to pxNetworkBuffer
. IP addresses make sense for my network.
0 576:vNetworkInterfaceAllocateRAMToBuffers buffers allocated
0 90:main RTOS start
0 397:prvIPTask prvIPTask started
0 543:xNetworkInterfaceInitialise ----------- network interface init ----------------
0 534:enc28j60_init Ethernet MAC readback aa:bb:cc:dd:ee:aa
0 779:prvInitialiseDHCP prvInitialiseDHCP: start after 100 ticks
0 53:xApplicationDHCPHook DHCP phase 0 address 192.168.9.2
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015790 allocated from prvCreatePartDHCPMessage
0 1248:prvSendDHCPDiscover vDHCPProcess: discover
0 548:xNetworkInterfaceOutput output len 300
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015790 released by xNetworkInterfaceOutput
0 3062:vPrintResourceStats Network buffers: 15 lowest 14
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200157B4 allocated from prvReceivePacket
0 1038:prvProcessDHCPReplies vDHCPProcess: offer ac10c9c0ip
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 00000000 released by FreeRTOS_ReleaseUDPPayloadBuffer
0 53:xApplicationDHCPHook DHCP phase 1 address 172.16.201.192
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200157D8 allocated from prvCreatePartDHCPMessage
0 1203:prvSendDHCPRequest vDHCPProcess: reply ac10c9c0ip
0 548:xNetworkInterfaceOutput output len 307
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157D8 released by xNetworkInterfaceOutput
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200157FC allocated from prvReceivePacket
0 1038:prvProcessDHCPReplies vDHCPProcess: offer ac10c9c0ip
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 00000000 released by FreeRTOS_ReleaseUDPPayloadBuffer
0 519:vDHCPProcess vDHCPProcess: acked ac10c9c0ip
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015820 allocated from FreeRTOS_OutputARPRequest
0 548:xNetworkInterfaceOutput output len 42
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015820 released by xNetworkInterfaceOutput
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015844 allocated from prvReceivePacket
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015844 released by prvProcessEthernetPacket
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015868 allocated from prvReceivePacket
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015868 released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 2001588C allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 2001588C released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200158B0 allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158B0 released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200158D4 allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158D4 released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200158F8 allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158F8 released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 2001591C allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 2001591C released by prvProcessEthernetPacket
2 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015940 allocated from prvReceivePacket
2 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015940 released by prvProcessEthernetPacket
2 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015964 allocated from prvReceivePacket
2 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015964 released by prvProcessEthernetPacket
2 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015988 allocated from prvReceivePacket
2 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015988 released by prvProcessEthernetPacket
3 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015790 allocated from prvReceivePacket
3 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015790 released by prvProcessEthernetPacket
3 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
3 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
3 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
3 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
3 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
3 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
4 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
4 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
4 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
4 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
4 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
4 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
5 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
5 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
5 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
5 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
5 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
5 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
Basically It looks like I can use every buffer just once. What am I doing wrong?
Many thanks