Cellular - how to correctly implement binary read?

How should I implement handling of a command AT+HTTPREAD=0,1024 that replies with:

OK\r\n
+HTTPREAD: 1024\r\n
here comes the binary payload...
+HTTPREAD: 0\r\n

I tried following Cellular_SocketRecv and this thread.

I try to receive the prefix and binary payload in the prefix handler (I could do that in the data handler callback as well). I get all the data (after returning CELLULAR_PKT_STATUS_SIZE_MISMATCH several times)
but then the library starts parsing the same line again on its own. I get a second prefix callback, then I see the URC handler. I have tried CELLULAR_AT_NO_RESULT, CELLULAR_AT_MULTI_WO_PREFIX, CELLULAR_AT_MULTI_DATA_WO_PREFIX. How can I tell the library that I have consumed the data?

2781:HTTPreadDataPrefix ctx 0x562691428b38 pLine 0x562691385c41 <OK
+HTTPREAD: 1024
payload...............
>
2782:HTTPreadDataPrefix ppDataStart 0x5626914ab088 pDataLength 0
2817:HTTPreadDataPrefix prefix matched
2844:HTTPreadDataPrefix copied number 4 <1024> read length 1024
2864:HTTPreadDataPrefix okay, read 1047 bytes
2871:HTTPreadDataPrefix line at 0x562691385c41, payload at 0x562691385c56, len 1024
2873:HTTPreadDataPrefix return 0
658:_handleMsgType Allocate at response 0x562691386c90
661:_handleMsgType AT solicited Resp[OK]
361:_Cellular_ProcessLine Final AT response is SUCCESS [OK]
946:_getNextLine Garbage collection
2781:HTTPreadDataPrefix ctx 0x562691428b38 pLine 0x562691385c42 <+HTTPREAD: 1024
payload...............
>
2782:HTTPreadDataPrefix ppDataStart 0x5626914ab088 pDataLength 1024
2817:HTTPreadDataPrefix prefix matched
2844:HTTPreadDataPrefix copied number 4 <1024> read length 1024
2864:HTTPreadDataPrefix okay, read 1043 bytes
2871:HTTPreadDataPrefix line at 0x562691385c42, payload at 0x562691385c53, len 1024
2873:HTTPreadDataPrefix return 0
144:_processUrcPacket Next URC token to parse [+HTTPREAD: 1024]
179:_processUrcPacket No URC Callback func avail HTTPREAD, now trying generic URC Callback
2781:HTTPreadDataPrefix ctx 0x562691428b38 pLine 0x562691385c53 <payload...............
>
2782:HTTPreadDataPrefix ppDataStart 0x5626914ab088 pDataLength 1024
2813:HTTPreadDataPrefix prefix not matched
2873:HTTPreadDataPrefix return 5

The code:

    uint8_t *dst;
    uint32_t bytes_to_read;
    bool prefix_found;
} httpread_ctx_t;

httpread_ctx_t *_httpread_ctx;

static CellularPktStatus_t _Cellular_HTTPREADFuncData(
    CellularContext_t * pContext __attribute__((unused)),
    const CellularATCommandResponse_t * pAtResp,
    void * pData,
    uint16_t dataLen){

    LogInfo(("nothing to do"));
    return CELLULAR_PKT_STATUS_OK;
}

#define MAX_HTTPREAD_STRING_PREFIX_STRING 24 /* +HTTPREAD: 1024   */

static CellularPktStatus_t HTTPreadDataPrefix(
                                        void * pCallbackContext,
                                        char * pLine,
                                        uint32_t lineLength,
                                        char ** ppDataStart,
                                        uint32_t * pDataLength){
    CellularATError_t atResult = CELLULAR_AT_SUCCESS;
    CellularPktStatus_t pktStatus = CELLULAR_PKT_STATUS_OK;

    static const char HTTPREAD_PREFIX[] = "+HTTPREAD:";
    const size_t HTTPREAD_PREFIX_LEN = strlen(HTTPREAD_PREFIX);

    static const char HTTPREAD_PREFIX_STRING_CHANGELINE[] = "\r\n";
    const size_t HTTPREAD_PREFIX_STRING_CHANGELINE_LENGTH = strlen(HTTPREAD_PREFIX_STRING_CHANGELINE);

    char *prefix_start;
    char *eol_start;
    int32_t httpread_length = 0;
    httpread_ctx_t *ctx;
    ptrdiff_t prefix_len;
    size_t number_length;

    if ((pLine == NULL) || (ppDataStart == NULL) || (pDataLength == NULL) || (pCallbackContext == NULL)){
        pktStatus = CELLULAR_PKT_STATUS_BAD_PARAM;
        LogInfo(("bad param %p %p %p %p", pLine, ppDataStart, pDataLength, pCallbackContext));
        goto end;
    }

    ctx = pCallbackContext;
    LogInfo(("ctx %p pLine %p <%.*s>", pCallbackContext, pLine, lineLength, pLine));
    LogInfo(("ppDataStart %p pDataLength %d", ppDataStart, *pDataLength));

    if (ctx->bytes_to_read == 0){
        pktStatus = CELLULAR_PKT_STATUS_BAD_PARAM;
        LogInfo(("bad requested read length? %d", ctx->bytes_to_read));
        goto end;
    }
    // We are looking for "+HTTPREAD: 1024\r\n" (or another number)

    if (lineLength < HTTPREAD_PREFIX_LEN + 1/*space*/ + 1/*at least one digit*/ + 2/*\r\n*/){
        pktStatus = CELLULAR_PKT_STATUS_SIZE_MISMATCH;
        LogInfo(("not enough data, len is %d", (int)lineLength));
        goto end;
    }

    /* Check if the message is a data response. */
    prefix_start = memmem( // binary-safe strstr()
                        pLine,
                        lineLength,
                        HTTPREAD_PREFIX,
                        HTTPREAD_PREFIX_LEN);
    if (prefix_start == NULL){
        pktStatus = CELLULAR_PKT_STATUS_SIZE_MISMATCH;
        LogInfo(("prefix not matched"));
        goto end;
    }

    LogInfo(("prefix matched"));

    // By now we have at least "+HTTPREAD:", let's see if the second newline is there
    eol_start = memmem( // binary-safe strstr()
        prefix_start + HTTPREAD_PREFIX_LEN,
        lineLength - HTTPREAD_PREFIX_LEN,
        HTTPREAD_PREFIX_STRING_CHANGELINE,
        HTTPREAD_PREFIX_STRING_CHANGELINE_LENGTH);

    if (eol_start == NULL){
        LogInfo(("2nd newline missing, need more data"));
        pktStatus = CELLULAR_PKT_STATUS_SIZE_MISMATCH;
        goto end;
    }

    // we have a newline so we should have received "+HTTPREAD: xxxx\r\n"
    // and there should be the length number as well
    char number_buffer[6];
    number_length = eol_start - prefix_start - HTTPREAD_PREFIX_LEN - 1;
    if (number_length > sizeof(number_buffer)-1){
        LogInfo(("HTTPREAD length number too large?!"));
        pktStatus = CELLULAR_PKT_STATUS_BAD_RESPONSE;
        goto end;
    }
    memcpy(number_buffer, &prefix_start[HTTPREAD_PREFIX_LEN+1], number_length);
    number_buffer[number_length] = '\0';
    atResult = Cellular_ATStrtoi(number_buffer, 10, &httpread_length);
    LogInfo(("copied number %d <%s> read length %d", (int)number_length, number_buffer, httpread_length));

    if (atResult != CELLULAR_AT_SUCCESS){
        LogInfo(("could not parse response length, err %d", atResult));
        pktStatus = CELLULAR_PKT_STATUS_BAD_RESPONSE;
        goto end;
    }
    if (httpread_length != (int)ctx->bytes_to_read){
        pktStatus = CELLULAR_PKT_STATUS_BAD_RESPONSE;
        LogInfo(("read length mistmatch %d vs %d", httpread_length, ctx->bytes_to_read));
        goto end;
    }
    // now we may have the payload after \r\n
    prefix_len = eol_start - pLine + HTTPREAD_PREFIX_STRING_CHANGELINE_LENGTH;

    if (lineLength < prefix_len + httpread_length){
        pktStatus = CELLULAR_PKT_STATUS_SIZE_MISMATCH;
        LogInfo(("not enough data, is %d, need %d", lineLength, prefix_len + httpread_length));
        goto end;
    }
    LogInfo(("okay, read %d bytes",lineLength));

    *ppDataStart = pLine + prefix_len; // payload starts here
    *pDataLength = httpread_length;    // and will be this long

    ctx->prefix_found = true;

    LogInfo(("line at %p, payload at %p, len %d", pLine, *ppDataStart, *pDataLength));
end:
    LogInfo(("return %d", pktStatus));
    return pktStatus;
}

CellularError_t Cellular_HTTPread(CellularHandle_t cellularHandle,
                                      uint32_t readOffset,
                                      uint8_t* pBuffer,
                                      uint32_t bytesToRead){

    CellularContext_t* pContext = (CellularContext_t*)cellularHandle;
    CellularError_t cellularStatus = CELLULAR_SUCCESS;
    CellularPktStatus_t pktStatus = CELLULAR_PKT_STATUS_OK;
    httpread_ctx_t ctx = {
        .dst = pBuffer,
        .bytes_to_read = bytesToRead,
        .prefix_found = false,
    };
    _httpread_ctx = &ctx;

    char cmdBuf[CELLULAR_AT_CMD_TYPICAL_MAX_SIZE] = {'\0'};
    uint32_t recvTimeout = DATA_READ_TIMEOUT_MS;

    /* Form the AT command. */
    (void)snprintf_(cmdBuf, CELLULAR_AT_CMD_TYPICAL_MAX_SIZE,
                    "AT+HTTPREAD=%ld,%ld", readOffset, bytesToRead);

    CellularAtReq_t atReqSocketRecv = {
        cmdBuf,
        CELLULAR_AT_MULTI_DATA_WO_PREFIX,
        "+HTTPREAD",
        _Cellular_HTTPREADFuncData,
        pBuffer,
        bytesToRead,
    };
    LogInfo(("ctx %p, req %p, %s", &ctx, &atReqSocketRecv, cmdBuf));

    cellularStatus = _Cellular_CheckLibraryStatus(pContext);

    if (cellularStatus != CELLULAR_SUCCESS) {
        LogDebug(("_Cellular_CheckLibraryStatus failed"));
        goto end;
    }
    if ((pBuffer == NULL) || (bytesToRead == 0U)) {
        LogError(("_Cellular_RecvData: Bad input Param"));
        cellularStatus = CELLULAR_BAD_PARAMETER;
        goto end;
    }

    LogDebug(("Waiting for +HTTPREAD"));
    pktStatus = _Cellular_TimeoutAtcmdDataRecvRequestWithCallback(
                            pContext,
                            atReqSocketRecv,
                            recvTimeout,
                            HTTPreadDataPrefix,
                            &ctx);

    if (pktStatus != CELLULAR_PKT_STATUS_OK) {
        /* Reset data handling parameters. */
        LogError(("HTTPREAD fail, pktStatus: %d", pktStatus));
        cellularStatus = _Cellular_TranslatePktStatus(pktStatus);
    }
end:
    return cellularStatus;
}

This loop reads response one line at a time and this is where a line is read. Try to see if your comm interface is retruning the same data again.

I finally implemented it by registering a URC handler for +HTTPREAD that sets the expected length and calls _Cellular_RegisterInputBufferCallback.

The binary read handler then returns CELLULAR_PKT_STATUS_SIZE_MISMATCH until there is enough data and then calls _Cellular_RegisterInputBufferCallback to de-register itself.

Thank you for sharing your solution!