+TCP buffers exhausted after a while

I am running FreeRTOS+TCP on a Cortex-M4F with an ENC28J60 ethernet controller via SPI. After a while network buffers are exhausted.

I am using stock BufferAllocation1.c (I only added printf macros to track allocations and deallocations).

Increasing ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS only delays the time to exhaust. This bug happens reliably every time. DHCP works okay and sending UDP frames also works fine so most of the network driver is definitely correct. I have the usual network noise, nothing out of the ordinary. The bug does not go away after unplugging the network cable so I think it is not just a surge of traffic.

buffer allocation function
void vNetworkInterfaceAllocateRAMToBuffers(NetworkBufferDescriptor_t pxNetworkBuffers[ ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS ]){
    // Taken from:
    // https://github.com/FreeRTOS/FreeRTOS-Plus-TCP/blob/51590f627e9fe830fabd51958df250181f2712d2/portable/NetworkInterface/xilinx_ultrascale/NetworkInterface.c#L326
    #define niBUFFER_1_PACKET_SIZE (ipTOTAL_ETHERNET_FRAME_SIZE + ipBUFFER_PADDING)
    static uint8_t ucNetworkPackets[ ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS * niBUFFER_1_PACKET_SIZE ] __attribute__((aligned(4)));
    uint8_t *ucRAMBuffer = ucNetworkPackets;
    uint32_t ul;

    for( ul = 0; ul < ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS; ul++ ){
        pxNetworkBuffers[ ul ].pucEthernetBuffer = ucRAMBuffer + ipBUFFER_PADDING;
        *( ( uintptr_t * ) ucRAMBuffer ) = ( uintptr_t ) &( pxNetworkBuffers[ ul ] );
        ucRAMBuffer += niBUFFER_1_PACKET_SIZE;
    }
    debugf("buffers allocated");
}
send function

ipconfigZERO_COPY_TX_DRIVER is set to 1 (otherwise the breakpoint would be hit)

BaseType_t xNetworkInterfaceOutput(NetworkBufferDescriptor_t * const pxNetworkBuffer, BaseType_t xReleaseAfterSend){
    debugf("output len %d", (unsigned int)pxNetworkBuffer->xDataLength);
    enc28j60_send_packet(pxNetworkBuffer->pucEthernetBuffer, pxNetworkBuffer->xDataLength );

    /* Call the standard trace macro to log the send event. */
    iptraceNETWORK_INTERFACE_TRANSMIT();

    if (xReleaseAfterSend){
        vReleaseNetworkBufferAndDescriptor( pxNetworkBuffer );
    } else {
        __BKPT(101); //network stack is misconfigured?!
    }
    return pdTRUE;
}
send function (lower level)

The only hack is that I am using data[-1] to store an SPI opcode to transmit the whole thing in a single DMA transfer (and it works fine). Config has #define ipconfigBUFFER_PADDING 4.

void enc28j60_send_packet(uint8_t *data, uint16_t len){
    uint32_t tickstart = xTaskGetTickCount();
    while(enc28j60_rcr(ECON1) & ECON1_TXRTS){
        // TXRTS may not clear - ENC28J60 bug. We must reset
        // transmit logic in cause of Tx error
        if(enc28j60_rcr(EIR) & EIR_TXERIF){
            enc28j60_bfs(ECON1, ECON1_TXRST);
            enc28j60_bfc(ECON1, ECON1_TXRST);
        }

        //If previous packet can't be sent within 100 ms, abort previous packet, ENC28J60 errata #12
        if((xTaskGetTickCount() - tickstart) > pdMS_TO_TICKS(100)){
            enc28j60_bfc(ECON1, ECON1_TXRTS);
            debugf("Errata #12 workaround");
        }
    }

    enc28j60_wcr16(EWRPT, ENC28J60_TXSTART);

    pin_eth_cs_low();
    uint8_t buffer[] = { ENC28J60_SPI_WBM, 0 };
    spi_transfer_blocking(sizeof(buffer), buffer);
    pin_eth_cs_high();

    pin_eth_cs_low();
    //The buffer does have space in front of it, see vNetworkInterfaceAllocateRAMToBuffers()
    data[-1] = ENC28J60_SPI_WBM;
    spi_transfer_blocking(len + 1, &data[-1]);
    pin_eth_cs_high();

    enc28j60_wcr16(ETXST, ENC28J60_TXSTART);
    enc28j60_wcr16(ETXND, ENC28J60_TXSTART + len);

    enc28j60_bfs(ECON1, ECON1_TXRTS); // Request packet send
}
receive function

It is scheduled from GPIO interrupt handler via xTimerPendFunctionCallFromISR.

void prvReceivePacket(void *unused1 __attribute__((unused)), uint32_t unused2 __attribute__((unused))){
    enc28j60_bfc(EIE, EIE_INTIE); // mask enc28j60 interrupts

    volatile uint8_t eir_flags = enc28j60_rcr(EIR);

    if (eir_flags & EIR_PKTIF){ /* if there is pending packet */
        // retrieve packet from enc28j60
        uint16_t xBytesReceived = 0;
        uint16_t rxlen = 0;
        uint16_t status = 0;
        uint16_t temp;
        xNetworkBufferDescriptor_t *pxBufferDescriptor;
        xIPStackEvent_t xRxEvent;

        enc28j60_read_buffer((void *)(&enc28j60_rxrdpt), sizeof(enc28j60_rxrdpt));
        enc28j60_read_buffer((void *)(&rxlen), sizeof(rxlen));
        enc28j60_read_buffer((void *)(&status), sizeof(status));

        //debugf("Packet pending, rxrdpt %d, len %d, status %d", enc28j60_rxrdpt, rxlen, status);

        if(status & 0x80){ //success
            // Throw out crc
            xBytesReceived = rxlen - 4;

            // Allocate buffer for packet
            pxBufferDescriptor = pxGetNetworkBufferWithDescriptor( xBytesReceived, 0 );
            if( pxBufferDescriptor != NULL ){

                // Read packet content
                enc28j60_read_buffer( pxBufferDescriptor->pucEthernetBuffer, xBytesReceived );
                pxBufferDescriptor->xDataLength = xBytesReceived;

                /* The event about to be sent to the TCP/IP is an Rx event. */
                xRxEvent.eEventType = eNetworkRxEvent;

                /* pvData is used to point to the network buffer descriptor that
                       now references the received data. */
                xRxEvent.pvData = ( void * ) pxBufferDescriptor;

                /* Send the data to the TCP/IP stack. */
                if( xSendEventStructToIPTask( &xRxEvent, 0 ) == pdFALSE ){
                    /* The buffer could not be sent to the IP task so the buffer
                           must be released. */
                    vReleaseNetworkBufferAndDescriptor( pxBufferDescriptor );

                    /* Make a call to the standard trace macro to log the
                           occurrence. */
                    iptraceETHERNET_RX_EVENT_LOST();

                    debugf("Stack did not accept incoming packet");

                } else {
                    /* The message was successfully sent to the TCP/IP stack.
                           Call the standard trace macro to log the occurrence. */
                    iptraceNETWORK_INTERFACE_RECEIVE();
                }
            } else {
                /* The event was lost because a network buffer was not available.
                       Call the standard trace macro to log the occurrence. */
                iptraceETHERNET_RX_EVENT_LOST();
                debugf("Couldn't allocate buffer for incoming packet ******************************");
            }
        } else {
            debugf("Wrong status?");
        }

        // Set Rx buffer guard to next packet
        if (enc28j60_rxrdpt == ENC28J60_RXSTART){
            temp = ENC28J60_RXEND;
        } else {
            temp = enc28j60_rxrdpt - 1;
        }

        enc28j60_wcr16(ERXRDPT, temp);

        // Set Rx pointer to next packet
        enc28j60_wcr16(ERDPT, enc28j60_rxrdpt);

        // Decrement packet counter
        enc28j60_bfs(ECON2, ECON2_PKTDEC);

    } else if (eir_flags & EIR_TXIF) {
        //debugf("enc28j60: transmit done\n");
        enc28j60_bfc(EIR, EIR_TXIF);
    } else if (eir_flags & EIR_TXERIF) {
        debugf("enc28j60: transmit error !!\n");
        enc28j60_bfc(EIR, EIR_TXERIF);
    } else if (eir_flags & EIR_RXERIF) {
        debugf("enc28j60: receive error !!\n");
        enc28j60_bfc(EIR, EIR_RXERIF);
    } else {
        debugf("enc28j60: unknown interrupt flag, we shouldn't be here\n");
    }

    enc28j60_bfs(EIE, EIE_INTIE); // unmask enc28j60 interrupts
}
typical log

The numbers 2001xxxx next to allocation/deallocation are simply the pointers to pxNetworkBuffer. IP addresses make sense for my network.

0 576:vNetworkInterfaceAllocateRAMToBuffers buffers allocated
0 90:main RTOS start
0 397:prvIPTask prvIPTask started
0 543:xNetworkInterfaceInitialise ----------- network interface init ----------------
0 534:enc28j60_init Ethernet MAC readback aa:bb:cc:dd:ee:aa
0 779:prvInitialiseDHCP prvInitialiseDHCP: start after 100 ticks
0 53:xApplicationDHCPHook DHCP phase 0 address 192.168.9.2
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015790 allocated from prvCreatePartDHCPMessage
0 1248:prvSendDHCPDiscover vDHCPProcess: discover
0 548:xNetworkInterfaceOutput output len 300
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015790 released by xNetworkInterfaceOutput
0 3062:vPrintResourceStats Network buffers: 15 lowest 14
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200157B4 allocated from prvReceivePacket
0 1038:prvProcessDHCPReplies vDHCPProcess: offer ac10c9c0ip
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 00000000 released by FreeRTOS_ReleaseUDPPayloadBuffer
0 53:xApplicationDHCPHook DHCP phase 1 address 172.16.201.192
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200157D8 allocated from prvCreatePartDHCPMessage
0 1203:prvSendDHCPRequest vDHCPProcess: reply ac10c9c0ip
0 548:xNetworkInterfaceOutput output len 307
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157D8 released by xNetworkInterfaceOutput
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200157FC allocated from prvReceivePacket
0 1038:prvProcessDHCPReplies vDHCPProcess: offer ac10c9c0ip
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 00000000 released by FreeRTOS_ReleaseUDPPayloadBuffer
0 519:vDHCPProcess vDHCPProcess: acked ac10c9c0ip
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015820 allocated from FreeRTOS_OutputARPRequest
0 548:xNetworkInterfaceOutput output len 42
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015820 released by xNetworkInterfaceOutput
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015844 allocated from prvReceivePacket
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015844 released by prvProcessEthernetPacket
0 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015868 allocated from prvReceivePacket
0 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015868 released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 2001588C allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 2001588C released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200158B0 allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158B0 released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200158D4 allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158D4 released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 200158F8 allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158F8 released by prvProcessEthernetPacket
1 301:pxGetNetworkBufferWithDescriptor_INTERNAL 2001591C allocated from prvReceivePacket
1 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 2001591C released by prvProcessEthernetPacket
2 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015940 allocated from prvReceivePacket
2 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015940 released by prvProcessEthernetPacket
2 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015964 allocated from prvReceivePacket
2 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015964 released by prvProcessEthernetPacket
2 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015988 allocated from prvReceivePacket
2 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015988 released by prvProcessEthernetPacket
3 301:pxGetNetworkBufferWithDescriptor_INTERNAL 20015790 allocated from prvReceivePacket
3 369:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015790 released by prvProcessEthernetPacket
3 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
3 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
3 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
3 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
3 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
3 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
4 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
4 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
4 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
4 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
4 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
4 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
5 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
5 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
5 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
5 262:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BFA9 (valid 1)
5 301:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated from prvReceivePacket
5 360:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************

Basically It looks like I can use every buffer just once. What am I doing wrong?
Many thanks

There is nothing obvious so I will ask some questions that may be helpful only if they make you think of different ways to debug:

  • I see 26 allocations and 16 releases. At this point how many free buffers do you expect to see? Have any of those released buffers ever got reused?
  • If you step into pxGetNetworkBufferWithDescriptor() can you see why it says the buffer is invalid? For example, is it just because the semaphore count is unexpectedly zero or because something is corrected?
  • Is 4 byte alignment adequate for the buffers?

I found something “interesting”. Buffers work perfectly fine when DHCP is disabled. The numbers in ( ) are absolute allocation and deallocation counters, so I see that the buffers are deallocated as many times as they are allocated.

without DHCP

#define ipconfigUSE_DHCP 0

0 564:vNetworkInterfaceAllocateRAMToBuffers buffers allocated
0 397:prvIPTask prvIPTask started
0 531:xNetworkInterfaceInitialise ----------- network interface init ----------------
6 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015760 allocated (0) from prvReceivePacket
6 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015760 released (0) by prvProcessEthernetPacket
6 3062:vPrintResourceStats Network buffers: 15 lowest 14
6 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015784 allocated (1) from prvReceivePacket
6 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015784 released (1) by prvProcessEthernetPacket
6 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200157A8 allocated (2) from prvReceivePacket
6 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157A8 released (2) by prvProcessEthernetPacket
6 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200157CC allocated (3) from prvReceivePacket
6 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157CC released (3) by prvProcessEthernetPacket
7 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200157F0 allocated (4) from prvReceivePacket
7 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157F0 released (4) by prvProcessEthernetPacket
8 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015814 allocated (5) from prvReceivePacket
8 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015814 released (5) by prvProcessEthernetPacket
9 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015838 allocated (6) from prvReceivePacket
9 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015838 released (6) by prvProcessEthernetPacket
9 302:pxGetNetworkBufferWithDescriptor_INTERNAL 2001585C allocated (7) from prvReceivePacket
9 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 2001585C released (7) by prvProcessEthernetPacket
9 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015880 allocated (8) from prvReceivePacket
9 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015880 released (8) by prvProcessEthernetPacket
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200158A4 allocated (9) from prvReceivePacket
10 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158A4 released (9) by prvProcessEthernetPacket
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200158C8 allocated (10) from prvReceivePacket
10 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158C8 released (10) by prvProcessEthernetPacket
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200158EC allocated (11) from prvReceivePacket
10 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158EC released (11) by prvProcessEthernetPacket
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015910 allocated (12) from FreeRTOS_OutputARPRequest
10 536:xNetworkInterfaceOutput output len 42
10 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015910 released (12) by xNetworkInterfaceOutput
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015934 allocated (13) from prvReceivePacket
10 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015934 released (13) by prvProcessEthernetPacket
11 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015958 allocated (14) from prvReceivePacket
11 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015760 allocated (15) from prvReceivePacket
11 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015958 released (14) by prvProcessEthernetPacket
11 3062:vPrintResourceStats Network buffers: 14 lowest 13
11 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015760 released (15) by prvProcessEthernetPacket
12 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015784 allocated (16) from prvReceivePacket
12 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015784 released (16) by prvProcessEthernetPacket
12 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200157A8 allocated (17) from prvReceivePacket
12 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157A8 released (17) by prvProcessEthernetPacket
12 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200157CC allocated (18) from prvReceivePacket
12 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157CC released (18) by prvProcessEthernetPacket
13 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200157F0 allocated (19) from prvReceivePacket
13 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157F0 released (19) by prvProcessEthernetPacket
13 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015814 allocated (20) from prvReceivePacket
13 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015814 released (20) by prvProcessEthernetPacket
13 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015838 allocated (21) from prvReceivePacket
13 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015838 released (21) by prvProcessEthernetPacket
13 302:pxGetNetworkBufferWithDescriptor_INTERNAL 2001585C allocated (22) from prvReceivePacket
13 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 2001585C released (22) by prvProcessEthernetPacket
13 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015880 allocated (23) from prvReceivePacket
13 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015880 released (23) by prvProcessEthernetPacket
14 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200158A4 allocated (24) from prvReceivePacket
14 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158A4 released (24) by prvProcessEthernetPacket
with DHCP

with #define ipconfigUSE_DHCP 1 (DHCP hook is 0)

0 564:vNetworkInterfaceAllocateRAMToBuffers buffers allocated
0 397:prvIPTask prvIPTask started
0 531:xNetworkInterfaceInitialise ----------- network interface init ----------------
0 779:prvInitialiseDHCP prvInitialiseDHCP: start after 100 ticks
0 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015798 allocated (0) from prvCreatePartDHCPMessage
0 1248:prvSendDHCPDiscover vDHCPProcess: discover
0 536:xNetworkInterfaceOutput output len 300
0 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015798 released (0) by xNetworkInterfaceOutput
0 3062:vPrintResourceStats Network buffers: 15 lowest 14
5 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200157BC allocated (1) from prvCreatePartDHCPMessage
5 1248:prvSendDHCPDiscover vDHCPProcess: discover
5 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 00000000 released (1) by FreeRTOS_ReleaseUDPPayloadBuffer
5 469:vDHCPProcess vDHCPProcess: timeout 4000 ticks
5 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200157E0 allocated (2) from prvReceivePacket
5 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200157E0 released (2) by prvProcessEthernetPacket
6 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015804 allocated (3) from prvReceivePacket
6 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015804 released (3) by prvProcessEthernetPacket
6 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015828 allocated (4) from prvReceivePacket
6 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015828 released (4) by prvProcessEthernetPacket
6 302:pxGetNetworkBufferWithDescriptor_INTERNAL 2001584C allocated (5) from prvReceivePacket
6 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 2001584C released (5) by prvProcessEthernetPacket
7 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015870 allocated (6) from prvReceivePacket
7 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015870 released (6) by prvProcessEthernetPacket
7 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015894 allocated (7) from prvReceivePacket
7 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200158B8 allocated (8) from prvReceivePacket
7 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015894 released (7) by prvProcessEthernetPacket
7 3062:vPrintResourceStats Network buffers: 14 lowest 13
7 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158B8 released (8) by prvProcessEthernetPacket
7 302:pxGetNetworkBufferWithDescriptor_INTERNAL 200158DC allocated (9) from prvReceivePacket
7 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 200158DC released (9) by prvProcessEthernetPacket
7 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015900 allocated (10) from prvReceivePacket
7 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015900 released (10) by prvProcessEthernetPacket
7 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015924 allocated (11) from prvReceivePacket
7 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015924 released (11) by prvProcessEthernetPacket
7 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015948 allocated (12) from prvReceivePacket
7 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015948 released (12) by prvProcessEthernetPacket
8 302:pxGetNetworkBufferWithDescriptor_INTERNAL 2001596C allocated (13) from prvReceivePacket
8 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 2001596C released (13) by prvProcessEthernetPacket
8 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015990 allocated (14) from prvReceivePacket
8 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015990 released (14) by prvProcessEthernetPacket
8 302:pxGetNetworkBufferWithDescriptor_INTERNAL 20015798 allocated (15) from prvReceivePacket
8 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
8 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (16) from prvReceivePacket
8 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
8 372:vReleaseNetworkBufferAndDescriptor_INTERNAL 20015798 released (15) by prvProcessEthernetPacket
8 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
8 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (17) from prvReceivePacket
8 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
8 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
8 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (18) from prvReceivePacket
8 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
9 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
9 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (19) from prvReceivePacket
9 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
9 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
9 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (20) from prvReceivePacket
9 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
10 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (21) from prvReceivePacket
10 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
10 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (22) from prvReceivePacket
10 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
10 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (23) from prvReceivePacket
10 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
10 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (24) from prvReceivePacket
10 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
10 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (25) from prvReceivePacket
10 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
10 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (26) from prvReceivePacket
10 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************
10 263:pxGetNetworkBufferWithDescriptor_INTERNAL pxGetNetworkBufferWithDescriptor: INVALID BUFFER: 0000BF0D (valid 1)
10 302:pxGetNetworkBufferWithDescriptor_INTERNAL 00000000 allocated (27) from prvReceivePacket
10 354:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************

The problem appears after the last buffer has been used once and the first one should be reused. The check in pxGetNetworkBufferWithDescriptor that fails is listIS_CONTAINED_WITHIN( &xFreeBuffersList, &( pxReturn->xBufferListItem ) )

I also found out that the last “working” uxListRemove( &( pxReturn->xBufferListItem ) ); in https://github.com/FreeRTOS/FreeRTOS-Plus-TCP/blob/3372cd57cf9f18fdd0f02313c47ce6fd08b92e3c/portable/BufferManagement/BufferAllocation_1.c#L244 sets xFreeBuffersList.xListEnd.pxNext to zero, so obviously next time the list is accessed it has no next element.

I also tried to blindly increase the stack sizes of the IP task, idle task, timer tasks - no difference.

4 byte alignment is fine in my case.

I #defined configUSE_LIST_DATA_INTEGRITY_CHECK_BYTES 1 and I am hitting configASSERT in prvDetermineSocketSize https://github.com/FreeRTOS/FreeRTOS-Plus-TCP/blob/3372cd57cf9f18fdd0f02313c47ce6fd08b92e3c/FreeRTOS_Sockets.c#L331 (call stack: prvIPTask->vDHCPProcess->prvInitialiseDHCP->prvCreateDHCPSocket->FreeRTOS_Socket)

Hello!

This is a very interesting problem. It seems that the buffers are not being released properly and thus not being added to the list of free buffers.

Can you try to use version 2.3.1 of FreeRTOS+TCP and see whether this problem persists? (The version can be found here).
If the problem is with the current version of the TCP stack, it will help us zero in on the problem.

Till that time, I shall try to find out what is wrong here.

Where did you get a network interface for ENC28J60? Is that public, could I have a look at it?
Thanks.

I updated FreeRTOS to v10.4.3 and TCP to v2.3.1. I am mpw hitting this assert listTEST_LIST_ITEM_INTEGRITY( pxNewListItem ); in https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/578d040659fb7d0da039f662baca2cc2fe535563/list.c#L94

This is called from vReleaseNetworkBufferAndDescriptor(), called from prvSendDHCPCDiscover() https://github.com/FreeRTOS/FreeRTOS-Plus-TCP/blob/95a5df5473727cce2fd3a34ac5a437fcb285e3a8/FreeRTOS_DHCP.c#L1141. It happens on the second deallocation (no matter how many buffers I have).

Everything else apart from the network stack works fine (tasks, queues, semaphores) so I guess I would run into much bigger trouble earlier if I had something grossly misconfigured (it is a bog standard M4F with gcc).

What is even more strange - I have switched to BufferAllocation_2.c (with an 8KB heap). The device starts up fine, gets address from DHCP. I can ping it. After a longer while it also runs out of buffers.

My driver is based on https://github.com/v-zaburdaev/FreeRTOS-TCP-enc28j60

header
#pragma once
#include <stdint.h>

#define ENC28J60_BUFSIZE    0x2000
#define ENC28J60_RXSIZE     0x1A00
#define ENC28J60_BUFEND     (ENC28J60_BUFSIZE-1)

#define ENC28J60_MAXFRAME   1518

#define ENC28J60_RXSTART    0
#define ENC28J60_RXEND      (ENC28J60_RXSIZE-1)
#define ENC28J60_TXSTART    ENC28J60_RXSIZE

#define ENC28J60_SPI_RCR    0x00
#define ENC28J60_SPI_RBM    0x3A
#define ENC28J60_SPI_WCR    0x40
#define ENC28J60_SPI_WBM    0x7A
#define ENC28J60_SPI_BFS    0x80
#define ENC28J60_SPI_BFC    0xA0
#define ENC28J60_SPI_SC     0xFF

#define ENC28J60_ADDR_MASK  0x1F
#define ENC28J60_COMMON_CR  0x1B

/*
 * Main registers
 */

#define EIE                 0x1B
#define EIR                 0x1C
#define ESTAT               0x1D
#define ECON2               0x1E
#define ECON1               0x1F


// Buffer read pointer
#define ERDPTL              0x00
#define ERDPTH              0x01
#define ERDPT               ERDPTL

// Buffer write pointer
#define EWRPTL              0x02
#define EWRPTH              0x03
#define EWRPT               EWRPTL

// Tx packet start pointer
#define ETXSTL              0x04
#define ETXSTH              0x05
#define ETXST               ETXSTL

// Tx packet end pointer
#define ETXNDL              0x06
#define ETXNDH              0x07
#define ETXND               ETXNDL

// Rx FIFO start pointer
#define ERXSTL              0x08
#define ERXSTH              0x09
#define ERXST               ERXSTL

// Rx FIFO end pointer
#define ERXNDL              0x0A
#define ERXNDH              0x0B
#define ERXND               ERXNDL

// Rx FIFO read pointer
#define ERXRDPTL            0x0C
#define ERXRDPTH            0x0D
#define ERXRDPT             ERXRDPTL

// Rx FIFO write pointer
#define ERXWRPTL            0x0E
#define ERXWRPTH            0x0F
#define ERXWRPT             ERXWRPTL

// DMA source block start pointer
#define EDMASTL             0x10
#define EDMASTH             0x11
#define EDMAST              EDMASTL

// DMA source block end pointer
#define EDMANDL             0x12
#define EDMANDH             0x13
#define EDMAND              EDMANDL

// DMA destination pointer
#define EDMADSTL            0x14
#define EDMADSTH            0x15
#define EDMADST             EDMADSTL

// DMA checksum
#define EDMACSL             0x16
#define EDMACSH             0x17
#define EDMACS              EDMACSL



// Hash table registers
#define EHT0                (0x00 | 0x20)
#define EHT1                (0x01 | 0x20)
#define EHT2                (0x02 | 0x20)
#define EHT3                (0x03 | 0x20)
#define EHT4                (0x04 | 0x20)
#define EHT5                (0x05 | 0x20)
#define EHT6                (0x06 | 0x20)
#define EHT7                (0x07 | 0x20)

// Pattern match registers
#define EPMM0               (0x08 | 0x20)
#define EPMM1               (0x09 | 0x20)
#define EPMM2               (0x0A | 0x20)
#define EPMM3               (0x0B | 0x20)
#define EPMM4               (0x0C | 0x20)
#define EPMM5               (0x0D | 0x20)
#define EPMM6               (0x0E | 0x20)
#define EPMM7               (0x0F | 0x20)
#define EPMCSL              (0x10 | 0x20)
#define EPMCSH              (0x11 | 0x20)
#define EPMOL               (0x14 | 0x20)
#define EPMOH               (0x15 | 0x20)

// Wake-on-LAN interrupt registers
#define EWOLIE              (0x16 | 0x20)
#define EWOLIR              (0x17 | 0x20)

// Receive filters mask
#define ERXFCON             (0x18 | 0x20)

// Packet counter
#define EPKTCNT             (0x19 | 0x20)


// MAC control registers
#define MACON1              (0x00 | 0x40 | 0x80)
#define MACON2              (0x01 | 0x40 | 0x80)
#define MACON3              (0x02 | 0x40 | 0x80)
#define MACON4              (0x03 | 0x40 | 0x80)

// MAC Back-to-back gap
#define MABBIPG             (0x04 | 0x40 | 0x80)

// MAC Non back-to-back gap
#define MAIPGL              (0x06 | 0x40 | 0x80)
#define MAIPGH              (0x07 | 0x40 | 0x80)

// Collision window & rexmit timer
#define MACLCON1            (0x08 | 0x40 | 0x80)
#define MACLCON2            (0x09 | 0x40 | 0x80)

// Max frame length
#define MAMXFLL             (0x0A | 0x40 | 0x80)
#define MAMXFLH             (0x0B | 0x40 | 0x80)
#define MAMXFL              MAMXFLL

// MAC-PHY support register
#define MAPHSUP             (0x0D | 0x40 | 0x80)
#define MICON               (0x11 | 0x40 | 0x80)

// MII registers
#define MICMD               (0x12 | 0x40 | 0x80)
#define MIREGADR            (0x14 | 0x40 | 0x80)

#define MIWRL               (0x16 | 0x40 | 0x80)
#define MIWRH               (0x17 | 0x40 | 0x80)
#define MIWR                MIWRL

#define MIRDL               (0x18 | 0x40 | 0x80)
#define MIRDH               (0x19 | 0x40 | 0x80)
#define MIRD                MIRDL

// MAC Address
#define MAADR1              (0x00 | 0x60 | 0x80)
#define MAADR0              (0x01 | 0x60 | 0x80)
#define MAADR3              (0x02 | 0x60 | 0x80)
#define MAADR2              (0x03 | 0x60 | 0x80)
#define MAADR5              (0x04 | 0x60 | 0x80)
#define MAADR4              (0x05 | 0x60 | 0x80)

// Built-in self-test
#define EBSTSD              (0x06 | 0x60)
#define EBSTCON             (0x07 | 0x60)
#define EBSTCSL             (0x08 | 0x60)
#define EBSTCSH             (0x09 | 0x60)
#define MISTAT              (0x0A | 0x60 | 0x80)

// Revision ID
#define EREVID              (0x12 | 0x60)

// Clock output control register
#define ECOCON              (0x15 | 0x60)

// Flow control registers
#define EFLOCON             (0x17 | 0x60)
#define EPAUSL              (0x18 | 0x60)
#define EPAUSH              (0x19 | 0x60)

// PHY registers
#define PHCON1              0x00
#define PHSTAT1             0x01
#define PHID1               0x02
#define PHID2               0x03
#define PHCON2              0x10
#define PHSTAT2             0x11
#define PHIE                0x12
#define PHIR                0x13
#define PHLCON              0x14

// EIE
#define EIE_INTIE           0x80
#define EIE_PKTIE           0x40
#define EIE_DMAIE           0x20
#define EIE_LINKIE          0x10
#define EIE_TXIE            0x08
#define EIE_WOLIE           0x04
#define EIE_TXERIE          0x02
#define EIE_RXERIE          0x01

// EIR
#define EIR_PKTIF           0x40
#define EIR_DMAIF           0x20
#define EIR_LINKIF          0x10
#define EIR_TXIF            0x08
#define EIR_WOLIF           0x04
#define EIR_TXERIF          0x02
#define EIR_RXERIF          0x01

// ESTAT
#define ESTAT_INT           0x80
#define ESTAT_LATECOL       0x10
#define ESTAT_RXBUSY        0x04
#define ESTAT_TXABRT        0x02
#define ESTAT_CLKRDY        0x01

// ECON2
#define ECON2_AUTOINC       0x80
#define ECON2_PKTDEC        0x40
#define ECON2_PWRSV         0x20
#define ECON2_VRPS          0x08

// ECON1
#define ECON1_TXRST         0x80
#define ECON1_RXRST         0x40
#define ECON1_DMAST         0x20
#define ECON1_CSUMEN        0x10
#define ECON1_TXRTS         0x08
#define ECON1_RXEN          0x04
#define ECON1_BSEL1         0x02
#define ECON1_BSEL0         0x01

// EWOLIE
#define EWOLIE_UCWOLIE      0x80
#define EWOLIE_AWOLIE       0x40
#define EWOLIE_PMWOLIE      0x10
#define EWOLIE_MPWOLIE      0x08
#define EWOLIE_HTWOLIE      0x04
#define EWOLIE_MCWOLIE      0x02
#define EWOLIE_BCWOLIE      0x01

// EWOLIR
#define EWOLIR_UCWOLIF      0x80
#define EWOLIR_AWOLIF       0x40
#define EWOLIR_PMWOLIF      0x10
#define EWOLIR_MPWOLIF      0x08
#define EWOLIR_HTWOLIF      0x04
#define EWOLIR_MCWOLIF      0x02
#define EWOLIR_BCWOLIF      0x01

// ERXFCON
#define ERXFCON_UCEN        0x80
#define ERXFCON_ANDOR       0x40
#define ERXFCON_CRCEN       0x20
#define ERXFCON_PMEN        0x10
#define ERXFCON_MPEN        0x08
#define ERXFCON_HTEN        0x04
#define ERXFCON_MCEN        0x02
#define ERXFCON_BCEN        0x01

// MACON1
#define MACON1_LOOPBK       0x10
#define MACON1_TXPAUS       0x08
#define MACON1_RXPAUS       0x04
#define MACON1_PASSALL      0x02
#define MACON1_MARXEN       0x01

// MACON2
#define MACON2_MARST        0x80
#define MACON2_RNDRST       0x40
#define MACON2_MARXRST      0x08
#define MACON2_RFUNRST      0x04
#define MACON2_MATXRST      0x02
#define MACON2_TFUNRST      0x01

// MACON3
#define MACON3_PADCFG2      0x80
#define MACON3_PADCFG1      0x40
#define MACON3_PADCFG0      0x20
#define MACON3_TXCRCEN      0x10
#define MACON3_PHDRLEN      0x08
#define MACON3_HFRMEN       0x04
#define MACON3_FRMLNEN      0x02
#define MACON3_FULDPX       0x01

// MACON4
#define MACON4_DEFER        0x40
#define MACON4_BPEN         0x20
#define MACON4_NOBKOFF      0x10
#define MACON4_LONGPRE      0x02
#define MACON4_PUREPRE      0x01

// MAPHSUP
#define MAPHSUP_RSTINTFC    0x80
#define MAPHSUP_RSTRMII     0x08

// MICON
#define MICON_RSTMII        0x80

// MICMD
#define MICMD_MIISCAN       0x02
#define MICMD_MIIRD         0x01

// EBSTCON
#define EBSTCON_PSV2        0x80
#define EBSTCON_PSV1        0x40
#define EBSTCON_PSV0        0x20
#define EBSTCON_PSEL        0x10
#define EBSTCON_TMSEL1      0x08
#define EBSTCON_TMSEL0      0x04
#define EBSTCON_TME         0x02
#define EBSTCON_BISTST      0x01

// MISTAT
#define MISTAT_NVALID       0x04
#define MISTAT_SCAN         0x02
#define MISTAT_BUSY         0x01

// ECOCON
#define ECOCON_COCON2       0x04
#define ECOCON_COCON1       0x02
#define ECOCON_COCON0       0x01

// EFLOCON
#define EFLOCON_FULDPXS     0x04
#define EFLOCON_FCEN1       0x02
#define EFLOCON_FCEN0       0x01

// PHCON1
#define PHCON1_PRST         0x8000
#define PHCON1_PLOOPBK      0x4000
#define PHCON1_PPWRSV       0x0800
#define PHCON1_PDPXMD       0x0100

// PHSTAT1
#define PHSTAT1_PFDPX       0x1000
#define PHSTAT1_PHDPX       0x0800
#define PHSTAT1_LLSTAT      0x0004
#define PHSTAT1_JBSTAT      0x0002

// PHCON2
#define PHCON2_FRCLNK       0x4000
#define PHCON2_TXDIS        0x2000
#define PHCON2_JABBER       0x0400
#define PHCON2_HDLDIS       0x0100

// PHSTAT2
#define PHSTAT2_TXSTAT      0x2000
#define PHSTAT2_RXSTAT      0x1000
#define PHSTAT2_COLSTAT     0x0800
#define PHSTAT2_LSTAT       0x0400
#define PHSTAT2_DPXSTAT     0x0200
#define PHSTAT2_PLRITY      0x0010

// PHIE
#define PHIE_PLNKIE         0x0010
#define PHIE_PGEIE          0x0002

// PHIR
#define PHIR_PLNKIF         0x0010
#define PHIR_PGIF           0x0004

// PHLCON
#define PHLCON_LACFG3       0x0800
#define PHLCON_LACFG2       0x0400
#define PHLCON_LACFG1       0x0200
#define PHLCON_LACFG0       0x0100
#define PHLCON_LBCFG3       0x0080
#define PHLCON_LBCFG2       0x0040
#define PHLCON_LBCFG1       0x0020
#define PHLCON_LBCFG0       0x0010
#define PHLCON_LFRQ1        0x0008
#define PHLCON_LFRQ0        0x0004
#define PHLCON_STRCH        0x0002
source
/*
 * FreeRTOS+TCP V2.3.2
 * Copyright (C) 2020 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 * the Software, and to permit persons to whom the Software is furnished to do so,
 * subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * http://aws.amazon.com/freertos
 * http://www.FreeRTOS.org
 */

#include "FreeRTOS.h"
#include "list.h"
#include "FreeRTOS_IP.h"
#include "FreeRTOS_IP_Private.h"
#include "enc28j60_registers.h"
#include "network.h"
#include "NetworkBufferManagement.h"
#include "pins.h"
#include "rtos_common.h"
#include <spidrv.h>

//Ethernet SPI uses USART2 at location #4
#define USART_ETH USART2
#define USART_ETH_LOCATION 4

static SemaphoreHandle_t _spi_semaphore;
static SPIDRV_HandleData_t spiHandleData;
static SPIDRV_Handle_t _spi_handle = &spiHandleData;

/* If ipconfigETHERNET_DRIVER_FILTERS_FRAME_TYPES is set to 1, then the Ethernet
 * driver will filter incoming packets and only pass the stack those packets it
 * considers need processing. */
#if ( ipconfigETHERNET_DRIVER_FILTERS_FRAME_TYPES == 0 )
#define ipCONSIDER_FRAME_FOR_PROCESSING( pucEthernetBuffer )    eProcessBuffer
#else
#define ipCONSIDER_FRAME_FOR_PROCESSING( pucEthernetBuffer )    eConsiderFrameForProcessing( ( pucEthernetBuffer ) )
#endif

// -------------------------- ENC28J60 driver functions -----------------------
static volatile uint8_t enc28j60_current_bank = 0;
static volatile uint16_t enc28j60_rxrdpt = 0;

static void SPI_transfer_complete_cb_fromISR(SPIDRV_Handle_t spi_handle __attribute__((unused)), Ecode_t transferStatus, int itemsTransferred __attribute__((unused))){
    if ( transferStatus != ECODE_EMDRV_SPIDRV_OK){
        __BKPT(1); //what happened?!
    }
    BaseType_t xHigherPriorityTaskWoken = pdFALSE;
    xSemaphoreGiveFromISR(_spi_semaphore, &xHigherPriorityTaskWoken);
    portYIELD_FROM_ISR(xHigherPriorityTaskWoken);
}

static void spi_transfer_blocking(uint32_t length, uint8_t *data){
    SPIDRV_MTransfer(_spi_handle, data/*tx buffer*/, data/*rx buffer*/, length, SPI_transfer_complete_cb_fromISR);
    xSemaphoreTake(_spi_semaphore, portMAX_DELAY); //the DMA callback will unlock the semaphore
}

void enc28j60_init_once(void){

    static StaticSemaphore_t eth_spi_semaphore_buffer;
    _spi_semaphore = xSemaphoreCreateBinaryStatic(&eth_spi_semaphore_buffer);

    SPIDRV_Init_t init = {
            .port = USART_ETH,
            .portLocationClk = USART_ETH_LOCATION,
            .portLocationTx = USART_ETH_LOCATION,
            .portLocationRx = USART_ETH_LOCATION,
            .bitRate = 8000000, //8MHz is good for ENC28J60 errata #1 (if we bought an old revision of the chip)
            .frameLength = 8,
            .dummyTxValue = 0x00,
            .type = spidrvMaster,
            .bitOrder = spidrvBitOrderMsbFirst,
            .clockMode = spidrvClockMode0,
            .csControl = spidrvCsControlApplication,
    };
    SPIDRV_Init(_spi_handle, &init);

    GPIO_PinModeSet(PORT_ETH_CS, PIN_ETH_CS, gpioModePushPull, 1/*output level*/);

    //Strange - after the driver initializes the peripheral the MISO pin has to be reconfigured again
    GPIO_PinModeSet(PORT_ETH_MISO, PIN_ETH_MISO, gpioModeInputPull, 1/*pullup*/);

    debugf("Ethernet controller SPI initialized");

    //configure PF2 for IRQ
    GPIO_IntConfig(PORT_ETH_INT, PIN_ETH_INT, false/*rising*/, true/*falling*/, true/*enable*/);
    NVIC_SetPriority(GPIO_EVEN_IRQn, 7); //low priority
    NVIC_EnableIRQ(GPIO_EVEN_IRQn);
}


// Generic SPI read command
static uint8_t enc28j60_read_op(uint8_t cmd, uint8_t adr){
    pin_eth_cs_low();
    //always transmit 3 bytes
    uint8_t buffer[] = {
            cmd | (adr & ENC28J60_ADDR_MASK),
            0xFF,
            0xFF
    };
    //debugf("TX %02X %02X %02X", buffer[0], buffer[1], buffer[2]);
    spi_transfer_blocking(sizeof(buffer), buffer);
    pin_eth_cs_high();
    //debugf("RX %02X %02X %02X", buffer[0], buffer[1], buffer[2]);

    if(adr & 0x80){ // throw out dummy byte when reading MII/MAC register
        return buffer[2];
    }
    return buffer[1];
}

// Generic SPI write command
static void enc28j60_write_op(uint8_t cmd, uint8_t adr, uint8_t data){
    pin_eth_cs_low();

    uint8_t buffer[] = {
            cmd | (adr & ENC28J60_ADDR_MASK),
            data
    };
    spi_transfer_blocking(sizeof(buffer), buffer);
    pin_eth_cs_high();
}

// Set register bank
static void enc28j60_set_bank(uint8_t adr){
    uint8_t bank;

    if( (adr & ENC28J60_ADDR_MASK) < ENC28J60_COMMON_CR )
    {
        bank = (adr >> 5) & 0x03; //BSEL1|BSEL0=0x03
        if(bank != enc28j60_current_bank)
        {
            enc28j60_write_op(ENC28J60_SPI_BFC, ECON1, 0x03);
            enc28j60_write_op(ENC28J60_SPI_BFS, ECON1, bank);
            enc28j60_current_bank = bank;
        }
    }
}

// Read register
static uint8_t enc28j60_rcr(uint8_t adr){
    enc28j60_set_bank(adr);
    return enc28j60_read_op(ENC28J60_SPI_RCR, adr);
}

// Read register pair
static uint16_t enc28j60_rcr16(uint8_t adr){
    enc28j60_set_bank(adr);
    return enc28j60_read_op(ENC28J60_SPI_RCR, adr) |
            (enc28j60_read_op(ENC28J60_SPI_RCR, adr+1) << 8);
}

// Write register
static void enc28j60_wcr(uint8_t adr, uint8_t arg){
    enc28j60_set_bank(adr);
    enc28j60_write_op(ENC28J60_SPI_WCR, adr, arg);
}

// Write register pair
static void enc28j60_wcr16(uint8_t adr, uint16_t arg){
    enc28j60_set_bank(adr);
    enc28j60_write_op(ENC28J60_SPI_WCR, adr, arg);
    enc28j60_write_op(ENC28J60_SPI_WCR, adr+1, arg>>8);
}

// Clear bits in register (reg &= ~mask)
static void enc28j60_bfc(uint8_t adr, uint8_t mask){
    enc28j60_set_bank(adr);
    enc28j60_write_op(ENC28J60_SPI_BFC, adr, mask);
}

// Set bits in register (reg |= mask)
static void enc28j60_bfs(uint8_t adr, uint8_t mask){
    enc28j60_set_bank(adr);
    enc28j60_write_op(ENC28J60_SPI_BFS, adr, mask);
}

//TODO: find out where to hook up this function
void prvSoftTimerInt(void *unused __attribute__((unused)), uint32_t unused2 __attribute__((unused)))
{
    if (enc28j60_rcr(EIR) == 0xFF) {
        if (xTimerPendFunctionCall( prvSoftTimerInt, NULL, 0, 10) == pdFALSE) {
            debugf("PANIC: enc28j60: daemon's queue full\n");
        }
    } else {
        taskENTER_CRITICAL();
        GPIO_EVEN_IRQHandler();
        taskEXIT_CRITICAL();
    }

}

// Read Rx/Tx buffer (at ERDPT)
static void enc28j60_read_buffer(uint8_t *buf, uint16_t len){
    //debugf("ptr %08X, len %d", (unsigned int)buf, len);
    pin_eth_cs_low();
    //The buffer does have space in front of it, see vNetworkInterfaceAllocateRAMToBuffers()
    uint8_t opcode[] = { ENC28J60_SPI_RBM };
    spi_transfer_blocking(sizeof(opcode), opcode);
    spi_transfer_blocking(len, buf);
    pin_eth_cs_high();
}

// Read PHY register
static uint16_t enc28j60_read_phy(uint8_t adr){
    enc28j60_wcr(MIREGADR, adr);
    enc28j60_bfs(MICMD, MICMD_MIIRD);
    while(enc28j60_rcr(MISTAT) & MISTAT_BUSY){}
    enc28j60_bfc(MICMD, MICMD_MIIRD);
    return enc28j60_rcr16(MIRD);
}

// Write PHY register
static void enc28j60_write_phy(uint8_t adr, uint16_t data){
    enc28j60_wcr(MIREGADR, adr);
    enc28j60_wcr16(MIWR, data);
    while(enc28j60_rcr(MISTAT) & MISTAT_BUSY) {} //function call will block anyway, so no sleeps are required in this loop
}

static void enc28j60_soft_reset(){
    pin_eth_cs_low();
    uint8_t buffer[] = {
            ENC28J60_SPI_SC
    };
    spi_transfer_blocking(sizeof(buffer), buffer);
    pin_eth_cs_high();

    enc28j60_current_bank = 0;
    rtos_sleep_ms(5);
}

void prvReceivePacket(void *unused1 __attribute__((unused)), uint32_t unused2 __attribute__((unused))){
    //debugf("rx from service task");
    enc28j60_bfc(EIE, EIE_INTIE); // mask enc28j60 interrupts

    volatile uint8_t eir_flags = enc28j60_rcr(EIR);

    if (eir_flags & EIR_PKTIF){ /* if there is pending packet */
        // retrieve packet from enc28j60
        uint16_t xBytesReceived = 0;
        uint16_t rxlen = 0;
        uint16_t status = 0;
        uint16_t temp;
        xNetworkBufferDescriptor_t *pxBufferDescriptor;
        xIPStackEvent_t xRxEvent;

        enc28j60_read_buffer((void *)(&enc28j60_rxrdpt), sizeof(enc28j60_rxrdpt));
        enc28j60_read_buffer((void *)(&rxlen), sizeof(rxlen));
        enc28j60_read_buffer((void *)(&status), sizeof(status));

        //debugf("Packet pending, rxrdpt %d, len %d, status %d", enc28j60_rxrdpt, rxlen, status);

        if(status & 0x80){ //success
            // Throw out crc
            xBytesReceived = rxlen - 4;

            // Allocate buffer for packet
            pxBufferDescriptor = pxGetNetworkBufferWithDescriptor( xBytesReceived, 0 );
            if( pxBufferDescriptor != NULL ){

                // Read packet content
                enc28j60_read_buffer( pxBufferDescriptor->pucEthernetBuffer, xBytesReceived );
                pxBufferDescriptor->xDataLength = xBytesReceived;

                /* The event about to be sent to the TCP/IP is an Rx event. */
                xRxEvent.eEventType = eNetworkRxEvent;

                /* pvData is used to point to the network buffer descriptor that
                       now references the received data. */
                xRxEvent.pvData = ( void * ) pxBufferDescriptor;

                /* Send the data to the TCP/IP stack. */
                if( xSendEventStructToIPTask( &xRxEvent, 0 ) == pdFALSE ){
                    /* The buffer could not be sent to the IP task so the buffer
                           must be released. */
                    vReleaseNetworkBufferAndDescriptor( pxBufferDescriptor );

                    /* Make a call to the standard trace macro to log the
                           occurrence. */
                    iptraceETHERNET_RX_EVENT_LOST();

                    debugf("Stack did not accept incoming packet");

                } else {
                    /* The message was successfully sent to the TCP/IP stack.
                           Call the standard trace macro to log the occurrence. */
                    iptraceNETWORK_INTERFACE_RECEIVE();
                }
            } else {
                /* The event was lost because a network buffer was not available.
                       Call the standard trace macro to log the occurrence. */
                iptraceETHERNET_RX_EVENT_LOST();
                debugf("Couldn't allocate buffer for incoming packet ******************************");
            }
        } else {
            debugf("Wrong status?");
        }

        // Set Rx buffer guard to next packet
        if (enc28j60_rxrdpt == ENC28J60_RXSTART){
            temp = ENC28J60_RXEND;
        } else {
            temp = enc28j60_rxrdpt - 1;
        }

        enc28j60_wcr16(ERXRDPT, temp);

        // Set Rx pointer to next packet
        enc28j60_wcr16(ERDPT, enc28j60_rxrdpt);

        // Decrement packet counter
        enc28j60_bfs(ECON2, ECON2_PKTDEC);

    } else if (eir_flags & EIR_TXIF) {
        //debugf("enc28j60: transmit done\n");
        enc28j60_bfc(EIR, EIR_TXIF);
    } else if (eir_flags & EIR_TXERIF) {
        debugf("enc28j60: transmit error !!\n");
        enc28j60_bfc(EIR, EIR_TXERIF);
    } else if (eir_flags & EIR_RXERIF) {
        debugf("enc28j60: receive error !!\n");
        enc28j60_bfc(EIR, EIR_RXERIF);
    } else {
        debugf("enc28j60: unknown interrupt flag, we shouldn't be here\n");
    }

    enc28j60_bfs(EIE, EIE_INTIE); // unmask enc28j60 interrupts
}

void GPIO_EVEN_IRQHandler(void){
    GPIO_IntClear(0x04); //pin2 generates 0x4 interrupt mask, strange, see figure 34.6 in reference manual
    //SEGGER_RTT_printf(0, "IF %08X", GPIO->IF); //don't use debugf in ISR

    BaseType_t xHigherPriorityTaskWoken = pdFALSE;
    if (xTimerPendFunctionCallFromISR( prvReceivePacket, NULL, 0, &xHigherPriorityTaskWoken ) == pdFALSE) {
        __BKPT(170);
        debugf("PANIC: enc28j60: daemon's queue full\n");
        while(1);
    }
    portYIELD_FROM_ISR( xHigherPriorityTaskWoken );
}

void enc28j60_send_packet(uint8_t *data, uint16_t len){
    uint32_t tickstart = xTaskGetTickCount();
    while(enc28j60_rcr(ECON1) & ECON1_TXRTS){
        // TXRTS may not clear - ENC28J60 bug. We must reset
        // transmit logic in cause of Tx error

        if(enc28j60_rcr(EIR) & EIR_TXERIF){
            enc28j60_bfs(ECON1, ECON1_TXRST);
            enc28j60_bfc(ECON1, ECON1_TXRST);
        }

        //If previous packet can't be sent within 100 ms, abort previous packet, ENC28J60 errata #12
        if((xTaskGetTickCount() - tickstart) > pdMS_TO_TICKS(100)){
            enc28j60_bfc(ECON1, ECON1_TXRTS);
            debugf("Errata #12 workaround");
        }
    }

    enc28j60_wcr16(EWRPT, ENC28J60_TXSTART);

    pin_eth_cs_low();
    uint8_t buffer[] = { ENC28J60_SPI_WBM, 0 };
    spi_transfer_blocking(sizeof(buffer), buffer);
    pin_eth_cs_high();

    pin_eth_cs_low();
    //The buffer does have space in front of it, see vNetworkInterfaceAllocateRAMToBuffers()
    data[-1] = ENC28J60_SPI_WBM;
    spi_transfer_blocking(len + 1, &data[-1]);
    pin_eth_cs_high();

    enc28j60_wcr16(ETXST, ENC28J60_TXSTART);
    enc28j60_wcr16(ETXND, ENC28J60_TXSTART + len);

    enc28j60_bfs(ECON1, ECON1_TXRTS); // Request packet send
}

uint8_t enc28j60_init(uint8_t *macadr){

    //this function is called by the TCP stack

    //pins have been initialized by pins_init()
    //SPI has been initialized by enc28j60_init_once() called from network.c

    //reset is held low by pins_init()
    rtos_sleep_ms(50);
    pin_eth_rst_high();
    rtos_sleep_ms(50);
#if 0
    debugf("ECON1 =  %02X", enc28j60_read_op(ENC28J60_SPI_RCR, ECON1));
#endif
    // Reset ENC28J60
    enc28j60_soft_reset();
#if 0
    //trivial test if we can switch register banks, this proves that SPI is okay
    debugf("ECON1 =  %02X", enc28j60_read_op(ENC28J60_SPI_RCR, ECON1));
    enc28j60_write_op(ENC28J60_SPI_WCR, ECON1, 0x01);
    debugf("ECON1 =  %02X", enc28j60_read_op(ENC28J60_SPI_RCR, ECON1));
    enc28j60_write_op(ENC28J60_SPI_WCR, ECON1, 0x02);
    debugf("ECON1 =  %02X", enc28j60_read_op(ENC28J60_SPI_RCR, ECON1));
    enc28j60_write_op(ENC28J60_SPI_WCR, ECON1, 0x03);
    debugf("ECON1 =  %02X", enc28j60_read_op(ENC28J60_SPI_RCR, ECON1));
#endif
    // Setup Rx/Tx buffer
    enc28j60_wcr16(ERXST, ENC28J60_RXSTART);
    enc28j60_rcr16(ERXST);
    enc28j60_wcr16(ERXRDPT, ENC28J60_RXSTART);
    enc28j60_wcr16(ERXND, ENC28J60_RXEND);
    enc28j60_rxrdpt = ENC28J60_RXSTART;

    // set read packet pointer
    enc28j60_wcr16(ERDPT, enc28j60_rxrdpt);

    // set autoincrement of pointers mode
    enc28j60_bfs(ECON2, ECON2_AUTOINC);

    // Setup MAC
    enc28j60_wcr(MACON1, MACON1_TXPAUS| // Enable flow control
            MACON1_RXPAUS|MACON1_MARXEN); // Enable MAC Rx
    enc28j60_wcr(MACON2, 0); // Clear reset
    enc28j60_wcr(MACON3,
            MACON3_PADCFG0 | MACON3_PADCFG2 | // Enable padding and automatic vlan frames recognition
            MACON3_TXCRCEN | MACON3_FRMLNEN | MACON3_FULDPX); // Enable crc & frame len chk
    enc28j60_wcr16(MAMXFL, ENC28J60_MAXFRAME);
    enc28j60_wcr(MABBIPG, 0x15); // Set inter-frame gap
    enc28j60_wcr(MAIPGL, 0x12);
    enc28j60_wcr(MAIPGH, 0x0c); // ICE
    enc28j60_wcr(MAADR5, macadr[0]); // Set MAC address
    enc28j60_wcr(MAADR4, macadr[1]);
    enc28j60_wcr(MAADR3, macadr[2]);
    enc28j60_wcr(MAADR2, macadr[3]);
    enc28j60_wcr(MAADR1, macadr[4]);
    enc28j60_wcr(MAADR0, macadr[5]);

    // Setup PHY
    enc28j60_write_phy(PHCON1, PHCON1_PDPXMD); // Force full-duplex mode
    enc28j60_write_phy(PHCON2, PHCON2_HDLDIS); // Disable loopback
    enc28j60_write_phy(PHLCON, PHLCON_LACFG2| // Configure LED ctrl
            PHLCON_LBCFG2|PHLCON_LBCFG1|PHLCON_LBCFG0|
            PHLCON_LFRQ0|PHLCON_STRCH);

    // Enable interrupt line
    //HAL_NVIC_EnableIRQ(irqn_line);

    // Enable enc28j60 receive packet pending interrupt
    // and transmit and receive error interrupt
    enc28j60_bfs(EIE, EIE_INTIE | EIE_TXIE | EIE_PKTIE | EIE_TXERIE | EIE_RXERIE);
    // Enable Rx packets
    // enc28j60_wcr(ERXFCON, 0x9F); // packet filtering
    enc28j60_bfs(ECON1, ECON1_RXEN);

    debugf("Ethernet MAC readback %02X:%02X:%02X:%02X:%02X:%02X",
            enc28j60_rcr(MAADR5),
            enc28j60_rcr(MAADR4),
            enc28j60_rcr(MAADR3),
            enc28j60_rcr(MAADR2),
            enc28j60_rcr(MAADR1),
            enc28j60_rcr(MAADR0)
    );

    return pdPASS;
}

// -------------------------- FreeRTOS+TCP functions --------------------------

BaseType_t xNetworkInterfaceInitialise( void )
{
    debugf("----------- network interface init ----------------");
    return enc28j60_init(GLOBAL_ethernet_mac);
}

BaseType_t xNetworkInterfaceOutput(NetworkBufferDescriptor_t * const pxNetworkBuffer, BaseType_t xReleaseAfterSend){
    debugf("output len %d", (unsigned int)pxNetworkBuffer->xDataLength);
    enc28j60_send_packet(pxNetworkBuffer->pucEthernetBuffer, pxNetworkBuffer->xDataLength );

    /* Call the standard trace macro to log the send event. */
    iptraceNETWORK_INTERFACE_TRANSMIT();

    if (xReleaseAfterSend){
        vReleaseNetworkBufferAndDescriptor( pxNetworkBuffer );
    } else {
        __BKPT(101); //network stack is misconfigured?!
    }
    return pdTRUE;
}

// Doc: https://www.freertos.org/FreeRTOS-Plus/FreeRTOS_Plus_TCP/Embedded_Ethernet_Porting.html#vNetworkInterfaceAllocateRAMToBuffers
void vNetworkInterfaceAllocateRAMToBuffers(NetworkBufferDescriptor_t pxNetworkBuffers[ ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS ]){
    // Taken from:
    // https://github.com/FreeRTOS/FreeRTOS-Plus-TCP/blob/51590f627e9fe830fabd51958df250181f2712d2/portable/NetworkInterface/xilinx_ultrascale/NetworkInterface.c#L326
#define niBUFFER_1_PACKET_SIZE (ipTOTAL_ETHERNET_FRAME_SIZE + ipBUFFER_PADDING)
    static uint8_t ucNetworkPackets[ ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS * niBUFFER_1_PACKET_SIZE ] __attribute__((aligned(4)));
    uint8_t *ucRAMBuffer = ucNetworkPackets;
    uint32_t ul;

    for( ul = 0; ul < ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS; ul++ ){
        pxNetworkBuffers[ ul ].pucEthernetBuffer = ucRAMBuffer + ipBUFFER_PADDING;
        *( ( uintptr_t * ) ucRAMBuffer ) = ( uintptr_t ) &( pxNetworkBuffers[ ul ] );
        ucRAMBuffer += niBUFFER_1_PACKET_SIZE;
    }
    debugf("buffers allocated");
}

thanks for sharing the driver. It looks good, no obvious problems. Yet a few remarks:

At the moment enc28j60_send_packet() returns, is the transmission already ready?
In other words, is it OK to release the descriptor immediately?

if (xReleaseAfterSend){
    vReleaseNetworkBufferAndDescriptor( pxNetworkBuffer );
} else {
    __BKPT(101); //network stack is misconfigured?!
}

xReleaseAfterSend is always true in case you have a zero-copy network driver. The idea is that the driver takes over ownership of the packet descriptor.
If you have a non-zero-copy driver, xReleaseAfterSend may sometimes be false.

void GPIO_EVEN_IRQHandler(void){
    GPIO_IntClear(0x04); //pin2 generates 0x4 interrupt mask, strange, see figure 34.6 in reference manual
    //SEGGER_RTT_printf(0, "IF %08X", GPIO->IF); //don't use debugf in ISR

    BaseType_t xHigherPriorityTaskWoken = pdFALSE;
    if (xTimerPendFunctionCallFromISR( prvReceivePacket, NULL, 0, &xHigherPriorityTaskWoken ) == pdFALSE) {

I would consider creating a normal task that handles the incoming packets, other than a timer task. I think that it will be faster, and also it can do more administration, like e.g. checking the Link Status. You can wake up the task with a tasknotify function. See for instance this driver.

static uint16_t enc28j60_read_phy(uint8_t adr){
static void enc28j60_write_phy(uint8_t adr, uint16_t data){

You are declaring two static functions that are not used?

Tomorrow more about your DHCP observations…

send_packet() is fully blocking because spi_transfer_blocking() is fully blocking, so it is okay to release the buffer immediately. Tomorrow I will try disabling sending and reception in the driver altogether to see if the stack alone (without interacting with the actual Ethernet chip & SPI & DMA) hits any assertions (though I’m pretty confident of the SPI routine because I am using it for other peripherals in the same fashion, but you never know…).

I will certainly end up with a dedicated network task of some sort but for now I am not concerned at all about performance.

Dangling declarations are probably leftovers from the original driver from github.

The FreeRTOS+TCP library is not known to leak network buffers. The library keeps well track of the ownership of all network buffer.

Now about DHCP : it uses UDP packets. When a device receives a UDP packet for which it has no open UDP socket, the packet will be dropped. If there is a UDP socket bound the right port number, the packet will be transferred to that socket, the socket will own it.
If the application doesn’t read the packets by calling recvrfrom(), the buffer pool will get exhausted. This may happen very quickly.

When using a debugger, you might want to watch UDP sockets and look at xSocket.u.xUDP.xWaitingPacketsList.uxNumberOfItems, especially for the socket xDHCPSocket ( defined in FreeRTOS_DHCP.c ).

I have disabled all other RTOS tasks plus sending and reception in the driver itself (to rule out plain memory corruption or DMA screwing things up). I still have the problem.

I narrowed it down to prvPacketBuffer_to_NetworkBuffer() returning a null descriptor for an ethernet buffer when called by FreeRTOS_ReleaseUDPPayloadBuffer() like:

1100:FreeRTOS_ReleaseUDPPayloadBuffer udp buffer at 200072C4, network descriptor at 00000000

I am not sure if I understand the logic in prvPacketBuffer_to_NetworkBuffer(). Must the first 4 bytes of the ethernet buffer (after ipBUFFER_PADDING) always hold the “back” pointer to the network descriptor? Is this the rationale behind this line https://github.com/FreeRTOS/FreeRTOS-Plus-TCP/blob/51590f627e9fe830fabd51958df250181f2712d2/portable/NetworkInterface/xilinx_ultrascale/NetworkInterface.c#L335 ?

I planned to use ipBUFFER_PADDING to store SPI opcode so the DMA transfer looks like this: { opcode, data1, data2, data3 etc.}.

Very good that you found this, great. This subject is a bit complicated. Because of the zero-copy method, packets are translated to payload buffers, and back to network packets. This was a simple conversion, but since IPv6 the IP-headers have a variable length.

You are not the only developer who wants to use the space before pucEthernetBuffer for your own driver, and that is surely possible.

However, please increase the size of ipconfigBUFFER_PADDING from 10 to 14 or more, because the IPv6 library already uses the other bytes.

Here is a layout of the bytes before pucEthernetBuffer[0] :

pucEthernetBuffer -14  : for your your driver, or more if you want to
pucEthernetBuffer -10  : a pointer to the containing `NetworkBufferDescriptor_t`
                         On a 64-bit platform, the pointer will be 8-bytes long.
pucEthernetBuffer  -6  : For IPv6 : store the frame type
pucEthernetBuffer  -2  : ipconfigPACKET_FILLER_SIZE ( 2 )
                         I think it is OK to use the bytes at -1 and -2 for your driver.

From here the normal packet data:

pucEthernetBuffer  0   : MACAddress_t xDestinationAddress;
pucEthernetBuffer  6   : MACAddress_t xSourceAddress;
pucEthernetBuffer  10  : uint16_t usFrameType;

From here on, the fields are 4-byte aligned:

pucEthernetBuffer  12  : uint8_t ucVersionHeaderLength;

See also this comment in FreeRTOS_IP.c :

/* For an IPv4 packet, pucIPType points to 6 bytes before the pucEthernetBuffer,
 * for a IPv6 packet, pucIPType will point to the first byte of the IP-header:
 * 'ucVersionTrafficClass'. */

I hope this all works for you! When I’m not clear, please ask more.

Thank you for the explanation. I only really need the [-1] byte in the buffer.

Could the problem have anything to do with my #define ipconfigBUFFER_PADDING 4 in the config? :unamused: I thought I have to explicitly reserve bytes in front of the payload for my driver. This would explain why the whole thing was failing with buffer allocation 1 and 2.

FreeRTOSIPConfig.h
/*
 * FreeRTOS+TCP V2.3.2
 * Copyright (C) 2020 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 * the Software, and to permit persons to whom the Software is furnished to do so,
 * subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * http://www.FreeRTOS.org
 * http://aws.amazon.com/freertos
 *
 * 1 tab == 4 spaces!
 */

#ifndef FREERTOS_IP_CONFIG_H
#define FREERTOS_IP_CONFIG_H

#include <common/rtos_common.h>

/* The error numbers defined in this file will be moved to the core FreeRTOS
 * code in future versions of FreeRTOS - at which time the following header file
 * will be removed. */
#include "FreeRTOS_errno_TCP.h"

/* This file provides default values for configuration options that are missing
 * from the FreeRTOSIPConfig.h configuration header file. */

/* These macros are used to define away static keyword for CBMC proofs */
#ifndef _static
#define _static    static
#endif

/*
 * FreeRTOS debug logging routine (proposal)
 * The macro will be called in the printf() style. Users can define
 * their own logging routine as:
 *
 *     #define FreeRTOS_debug_printf( MSG )         my_printf MSG
 *
 * The FreeRTOS_debug_printf() must be thread-safe but does not have to be
 * interrupt-safe.
 */
#define ipconfigHAS_DEBUG_PRINTF 1
#define DEBUG_ID DEBUG_ID_NETWORK
#include <common/debug.h>
#define FreeRTOS_debug_printf(MSG) debugf MSG
#define FreeRTOS_printf(MSG)       debugf MSG
#define ipconfigHAS_PRINTF 1

/*
 * At several places within the library, random numbers are needed:
 * - DHCP:    For creating a DHCP transaction number
 * - TCP:     Set the Initial Sequence Number: this is the value of the first outgoing
 *            sequence number being used when connecting to a peer.
 *            Having a well randomized ISN is important to avoid spoofing
 * - UDP/TCP: for setting the first port number to be used, in case a socket
 *            uses a 'random' or anonymous port number
 */
#include <common/rng.h>
#define ipconfigRAND32()    rng_get()

#define ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS   ( 25 )
#define ipconfigEVENT_QUEUE_LENGTH    ( ipconfigNUM_NETWORK_BUFFER_DESCRIPTORS + 5 )

#define ipconfigUSE_DHCP_HOOK   ( 1 )

#define ipconfigNETWORK_MTU   ( 1500 )

/*
 * Only applicable when DHCP is in use:
 * If no DHCP server responds, use "Auto-IP" : the
 * device will allocate a random LinkLayer IP address.
 */
#define ipconfigDHCP_FALL_BACK_AUTO_IP    ( 0 )

#define ipconfigUSE_DHCP   ( 0 )

#define ipconfigUSE_DNS_CACHE   ( 1 )

#define ipconfigREPLY_TO_INCOMING_PINGS   ( 1 )

/* When non-zero, the buffers passed to the SEND routine may be passed
 * to DMA. As soon as sending is ready, the buffers must be released by
 * calling vReleaseNetworkBufferAndDescriptor(), */
#define ipconfigZERO_COPY_TX_DRIVER    ( 1 )

/* This define doesn't mean much to the driver, except that it makes
 * sure that pxPacketBuffer_to_NetworkBuffer() will be included. */
#define ipconfigZERO_COPY_RX_DRIVER    ( 1 )

#endif /* FREERTOS_DEFAULT_IP_CONFIG_H */

Even if I define padding to 128 I get list assertions when freeing from FreeRTOS_ReleaseUDPPayloadBuffer().

If I now remove the define then everything works for much longer until I see a stream of allocations without deallocations:

Log
36 378:vReleaseNetworkBufferAndDescriptor 20006AB0 released (268)

36 307:pxGetNetworkBufferWithDescriptor 20006ADC allocated (269)

36 378:vReleaseNetworkBufferAndDescriptor 20006ADC released (269)

37 307:pxGetNetworkBufferWithDescriptor 20006B08 allocated (270)

37 378:vReleaseNetworkBufferAndDescriptor 20006B08 released (270)

37 307:pxGetNetworkBufferWithDescriptor 20006B34 allocated (271)

37 378:vReleaseNetworkBufferAndDescriptor 20006B34 released (271)

37 307:pxGetNetworkBufferWithDescriptor 20006B60 allocated (272)

37 378:vReleaseNetworkBufferAndDescriptor 20006B60 released (272)

37 307:pxGetNetworkBufferWithDescriptor 20006B8C allocated (273)

37 307:pxGetNetworkBufferWithDescriptor 20006BB8 allocated (274)

37 307:pxGetNetworkBufferWithDescriptor 20006798 allocated (275)

37 307:pxGetNetworkBufferWithDescriptor 200067C4 allocated (276)

37 307:pxGetNetworkBufferWithDescriptor 200067F0 allocated (277)

37 307:pxGetNetworkBufferWithDescriptor 2000681C allocated (278)

38 307:pxGetNetworkBufferWithDescriptor 20006848 allocated (279)

38 307:pxGetNetworkBufferWithDescriptor 20006874 allocated (280)

38 307:pxGetNetworkBufferWithDescriptor 200068A0 allocated (281)

38 307:pxGetNetworkBufferWithDescriptor 200068CC allocated (282)

38 307:pxGetNetworkBufferWithDescriptor 200068F8 allocated (283)

38 307:pxGetNetworkBufferWithDescriptor 20006924 allocated (284)

38 307:pxGetNetworkBufferWithDescriptor 20006950 allocated (285)

38 307:pxGetNetworkBufferWithDescriptor 2000697C allocated (286)

38 307:pxGetNetworkBufferWithDescriptor 200069A8 allocated (287)

38 307:pxGetNetworkBufferWithDescriptor 200069D4 allocated (288)

39 307:pxGetNetworkBufferWithDescriptor 20006A00 allocated (289)

39 307:pxGetNetworkBufferWithDescriptor 20006A2C allocated (290)

39 307:pxGetNetworkBufferWithDescriptor 20006A58 allocated (291)

39 307:pxGetNetworkBufferWithDescriptor 20006A84 allocated (292)

39 307:pxGetNetworkBufferWithDescriptor 20006AB0 allocated (293)

39 307:pxGetNetworkBufferWithDescriptor 20006ADC allocated (294)

39 307:pxGetNetworkBufferWithDescriptor 20006B08 allocated (295)

39 307:pxGetNetworkBufferWithDescriptor 20006B34 allocated (296)

39 307:pxGetNetworkBufferWithDescriptor 20006B60 allocated (297)

39 307:pxGetNetworkBufferWithDescriptor 00000000 allocated (298)

39 308:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************

40 307:pxGetNetworkBufferWithDescriptor 00000000 allocated (299)

40 308:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************

40 307:pxGetNetworkBufferWithDescriptor 00000000 allocated (300)

40 308:prvReceivePacket Couldn't allocate buffer for incoming packet ******************************

This is not just a temporary traffic burst. I’m okay to temporarily drop packets, but the problem does not go away. This also happens with DHCP disabled. I am not opening any sockets in the application whatsoever so the stack should quickly process and discard any traffic.

Should I try to dump that traffic burst?