Converting a Makefile build to CMake using the QEMU Demo - HardFault_Handler called with my CMake build

Hello folks,

I’m trying to create a CMake build for the CORTEX_MPU_M3_MPS2_QEMU_GCC demo. In case it matters, I’m doing this because I intend to use FreeRTOS and QEMU in another CMake based project of mine and am using this as a learning step to get the build working.

I’ve uploaded my mini project to github KartikAiyer/freertos_qemu_demo for your reference (new user so can’t add links)

I have a toolchain file that will download the ARM toolchain v13.2.Rel1 and use it for the CMake build. I have also provided a env.sh which you can source to setup your path to run the FreeRTOS supplied makefile based build of the demo.

I’m currently trying this on an M1 based Macbook air but I’ve setup the build to work on Linux
as well (though I haven’t tried it on Linux).

My Problem

I generate the build using both the FreeRTOS supplied Makefile for the Demo and with my CMake build. When I launch my CMake based and connect GDB to it, I can see that my code calls the HardFault_Handler the moment it tries to create the stack variable

TaskParameters_t xROAccessTaskParameters =
    {
        .pvTaskCode     = prvROAccessTask,
        .pcName         = "ROAccess",
        .usStackDepth   = configMINIMAL_STACK_SIZE,
        .pvParameters   = NULL,
        .uxPriority     = tskIDLE_PRIORITY,
        .puxStackBuffer = xROAccessTaskStack,
        .xRegions       =
        {
            { ucSharedMemory,                  SHARED_MEMORY_SIZE,
              portMPU_REGION_PRIVILEGED_READ_WRITE_UNPRIV_READ_ONLY |
              portMPU_REGION_EXECUTE_NEVER },
            { ( void * ) ucROTaskFaultTracker, SHARED_MEMORY_SIZE,
              portMPU_REGION_READ_WRITE | portMPU_REGION_EXECUTE_NEVER },
            { 0,                               0,                 0},
        }
    };

When I launch the build generated using the FreeRTOS based Makefile, the code does not crash.

I’ve tried to make my CMakeLists.txt file do exactly what the Makefile does and I’m not able to
trace why there is a difference. I tried comparing the output.map files from both builds it looks
like symbols are created at different addresses.

I could really use some help with debugging this.

Build Files

In case you can’t look up the GitHub repo, I’m providing my CMakeLists.txt file here

cmake_minimum_required(VERSION 3.22)
project(qemu_demo C)


set(FREERTOS_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/FreeRTOS/FreeRTOS/Source)
set(FREERTOS_PORTABLE_DIR "${FREERTOS_SOURCES_DIR}/portable")
set(FREERTOS_PORTABLE_M3_DIR "${FREERTOS_PORTABLE_DIR}/GCC/ARM_CM3_MPU")
set(FREERTOS_QEMU_MPS2_DIR ${CMAKE_CURRENT_SOURCE_DIR}/FreeRTOS/FreeRTOS/Demo/CORTEX_MPU_M3_MPS2_QEMU_GCC)

list(APPEND 
  sources 
  "${FREERTOS_QEMU_MPS2_DIR}/init/startup.c" 
  "${FREERTOS_QEMU_MPS2_DIR}/syscall.c" 
  "${FREERTOS_QEMU_MPS2_DIR}/main.c" 
  "${FREERTOS_QEMU_MPS2_DIR}/app_main.c" 
  "${FREERTOS_QEMU_MPS2_DIR}/mpu_demo.c" 
  "${FREERTOS_PORTABLE_M3_DIR}/port.c"
  "${FREERTOS_PORTABLE_M3_DIR}/mpu_wrappers_v2_asm.c"
  "${FREERTOS_PORTABLE_DIR}/Common/mpu_wrappers.c"
  "${FREERTOS_PORTABLE_DIR}/Common/mpu_wrappers_v2.c"
  "${FREERTOS_PORTABLE_DIR}/MemMang/heap_4.c"
  "${FREERTOS_SOURCES_DIR}/list.c"
  "${FREERTOS_SOURCES_DIR}/tasks.c"
  "${FREERTOS_SOURCES_DIR}/queue.c"
  "${FREERTOS_SOURCES_DIR}/timers.c"
  "${FREERTOS_SOURCES_DIR}/event_groups.c"
  "${FREERTOS_SOURCES_DIR}/stream_buffer.c"
)

list(APPEND 
  includeDirs
  "${FREERTOS_QEMU_MPS2_DIR}"
  "${FREERTOS_QEMU_MPS2_DIR}/CMSIS"
  "${FREERTOS_SOURCES_DIR}/include"
  "${FREERTOS_PORTABLE_M3_DIR}"
)

list(APPEND
  DEFINES
  -DHEAP_4
)

list(APPEND
  CFLAGS
  -mthumb
  -mcpu=cortex-m3
  -Wall
  -Wextra
  -Wshadow
  -Wno-unused-parameter
  -g3
  -Og
  -ffunction-sections
  -fdata-sections
  -save-temps=obj
)

list(APPEND
  LDFLAGS
  "${CFLAGS}"
  -T ${FREERTOS_QEMU_MPS2_DIR}/scripts/mps2_m3.ld
  -Wl,-Map=${CMAKE_BINARY_DIR}/output.map,--gc-sections
  -nostartfiles -nostdlib -nolibc -nodefaultlibs
)

add_executable(qemu_demo "${sources}")
target_include_directories(qemu_demo PRIVATE "${includeDirs}")
target_compile_definitions(qemu_demo PRIVATE "${DEFINES}")
target_compile_options(qemu_demo PRIVATE "${CFLAGS}")
target_link_options(qemu_demo PRIVATE "${LDFLAGS}")

And this the Corresponding Makefile

CC                     =    arm-none-eabi-gcc
SIZE                   =    arm-none-eabi-size
BIN                   :=    RTOSDemo.axf

BUILD_DIR             :=    build

FREERTOS_DIR_REL      :=    ../../../FreeRTOS
FREERTOS_DIR          :=    $(abspath $(FREERTOS_DIR_REL))
KERNEL_DIR            :=    $(FREERTOS_DIR)/Source

# Startup sources
SOURCE_FILES          +=    init/startup.c  syscall.c main.c

# platform portable source file
SOURCE_FILES          +=    $(KERNEL_DIR)/portable/GCC/ARM_CM3_MPU/port.c
SOURCE_FILES          +=    $(KERNEL_DIR)/portable/GCC/ARM_CM3_MPU/mpu_wrappers_v2_asm.c

# Kernel source files
SOURCE_FILES          +=    $(KERNEL_DIR)/portable/Common/mpu_wrappers.c
SOURCE_FILES          +=    $(KERNEL_DIR)/portable/Common/mpu_wrappers_v2.c
SOURCE_FILES          +=    $(KERNEL_DIR)/tasks.c
SOURCE_FILES          +=    $(KERNEL_DIR)/list.c
SOURCE_FILES          +=    $(KERNEL_DIR)/queue.c
SOURCE_FILES          +=    $(KERNEL_DIR)/timers.c
SOURCE_FILES          +=    $(KERNEL_DIR)/event_groups.c
SOURCE_FILES          +=    ${KERNEL_DIR}/portable/MemMang/heap_4.c
SOURCE_FILES          +=    $(KERNEL_DIR)/stream_buffer.c

# application source files
SOURCE_FILES          +=    app_main.c
SOURCE_FILES          +=    mpu_demo.c

INCLUDE_DIRS          +=    -I$(FREERTOS_DIR)/Demo/CORTEX_MPU_M3_MPS2_QEMU_GCC
INCLUDE_DIRS          +=    -I$(FREERTOS_DIR)/Demo/CORTEX_MPU_M3_MPS2_QEMU_GCC/CMSIS
INCLUDE_DIRS          +=    -I$(KERNEL_DIR)/include
INCLUDE_DIRS          +=    -I$(KERNEL_DIR)/portable/GCC/ARM_CM3_MPU

DEFINES               :=    -DHEAP4
CPPFLAGS              +=    $(DEFINES)

CFLAGS                +=    -mthumb -mcpu=cortex-m3
CFLAGS                +=    -Wall -Wextra -Wshadow -Wno-unused-parameter
#CFLAGS                +=    -Wpedantic -fanalyzer
CFLAGS                +=    $(INCLUDE_DIRS)

LDFLAGS                =    -T ./scripts/mps2_m3.ld
LDFLAGS               +=    -Xlinker -Map=${BUILD_DIR}/output.map
LDFLAGS               +=    -Xlinker --gc-sections
LDFLAGS               +=    -nostartfiles -nostdlib -nolibc -nodefaultlibs

ifeq ($(DEBUG), 1)
    CFLAGS                +=     -g3 -Og -ffunction-sections -fdata-sections -save-temps=obj
else
    CFLAGS                +=     -Os -ffunction-sections -fdata-sections
endif

ifeq ($(PICOLIBC), 1)
    CFLAGS                +=     --specs=picolibc.specs -DPICOLIBC_INTEGER_PRINTF_SCANF
   LDFLAGS                +=     --specs=picolibc.specs -DPICOLIBC_INTEGER_PRINTF_SCANF
endif

OBJ_FILES             :=    $(SOURCE_FILES:%.c=$(BUILD_DIR)/%.o)

.PHONY: clean

$(BUILD_DIR)/$(BIN) : $(OBJ_FILES)
	$(CC) $(CFLAGS) $(LDFLAGS) $+ -o $(@)
	$(SIZE) $(@)

%.d: %.c
	@set -e; rm -f $@; \
	$(CC) -M $(CPPFLAGS) $< > $@.$$$$; \
	sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \
	rm -f $@.$$$$

INCLUDES              :=     $(SOURCE_FILES:%.c=$(BUILD_DIR)/%.d)
-include $(INCLUDES)

${BUILD_DIR}/%.o : %.c Makefile
	-mkdir -p $(@D)
	$(CC) $(CFLAGS) $(CPPFLAGS) -MMD -c $< -o $@

clean:
	-rm -rf build

This is the toolchain.cmake file

# Set the desired ARM toolchain version
set(ARM_TOOLCHAIN_VERSION "13.2.Rel1")

# Set the download directory
set(ARM_TOOLCHAIN_PARENT_DIR "${CMAKE_BINARY_DIR}/arm-toolchain")

message(STATUS "Looking and downloading for toolchain")

set(HOST_CHIP_VERSION "")
if(APPLE)
  message(STATUS "Selecting Apple Silicon host compiler toolchain for ${CMAKE_HOST_SYSTEM_PROCESSOR}")
  set(HOST_CHIP_VERSION "darwin-${CMAKE_HOST_SYSTEM_PROCESSOR}")
elseif(CMAKE_HOST_WIN32)
  if(CMAKE_GENERATOR MATCHES "MinGW")
    set(HOST_CHIP_VERSION "mingw-w64-i686")
  else()
    message(FATAL_ERROR "Only MinGW Windows generator supported")
  endif()
else()
  message(
    STATUS "Selecting Linux Host Chip version ${CMAKE_HOST_SYSTEM_PROCESSOR}")
  set(HOST_CHIP_VERSION "${CMAKE_HOST_SYSTEM_PROCESSOR}")
endif()

# Set the URL to download the ARM toolchain
set(ARM_TOOLCHAIN_URL
    "https://developer.arm.com/-/media/Files/downloads/gnu/${ARM_TOOLCHAIN_VERSION}/binrel/arm-gnu-toolchain-${ARM_TOOLCHAIN_VERSION}-${HOST_CHIP_VERSION}-arm-none-eabi.tar.xz"
)
set(ARM_TOOLCHAIN_FULL_NAME_DIR "${ARM_TOOLCHAIN_PARENT_DIR}/arm-gnu-toolchain-${ARM_TOOLCHAIN_VERSION}-${HOST_CHIP_VERSION}-arm-none-eabi")
set(ARM_TOOLCHAIN_DIR "${ARM_TOOLCHAIN_PARENT_DIR}/arm-gnu-toolchain")
# Set the toolchain paths
set(ARM_TOOLCHAIN_BIN_DIR
    "${ARM_TOOLCHAIN_DIR}/bin"
)
# Create a custom command to download the ARM toolchain
if(NOT EXISTS "${ARM_TOOLCHAIN_BIN_DIR}/arm-none-eabi-gcc")
  message(STATUS "Downloading ARM toolchain from ${ARM_TOOLCHAIN_URL}")
  file(DOWNLOAD "${ARM_TOOLCHAIN_URL}"
       "${ARM_TOOLCHAIN_PARENT_DIR}/arm-toolchain.tar.xz" SHOW_PROGRESS)
  execute_process(
    COMMAND ${CMAKE_COMMAND} -E tar xf
            "${ARM_TOOLCHAIN_PARENT_DIR}/arm-toolchain.tar.xz"
    WORKING_DIRECTORY "${ARM_TOOLCHAIN_PARENT_DIR}")
  execute_process(COMMAND mv "${ARM_TOOLCHAIN_FULL_NAME_DIR}" "${ARM_TOOLCHAIN_DIR}" WORKING_DIRECTORY "${ARM_TOOLCHAIN_PARENT_DIR}")
endif()

# Set the target system root
set(CMAKE_FIND_ROOT_PATH "${ARM_TOOLCHAIN_DIR}")

# Search for programs only in the build host directories
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
# Disable compiler checks
set(CMAKE_C_COMPILER_WORKS TRUE)
set(CMAKE_CXX_COMPILER_WORKS TRUE)
set(CMAKE_C_ABI_COMPILED TRUE)
set(CMAKE_CXX_ABI_COMPILED TRUE)

set(CMAKE_C_COMPILER "${ARM_TOOLCHAIN_BIN_DIR}/arm-none-eabi-gcc")
set(CMAKE_CXX_COMPILER "${ARM_TOOLCHAIN_BIN_DIR}/arm-none-eabi-g++")
set(CMAKE_ASM_COMPILER "${ARM_TOOLCHAIN_BIN_DIR}/arm-none-eabi-gcc")

# Set the target system and architecture
set(CMAKE_SYSTEM_NAME Generic)
set(CMAKE_SYSTEM_PROCESSOR arm)

# Add the toolchain include directories
include_directories(
  "${ARM_TOOLCHAIN_DIR}/arm-none-eabi/include"
)

# Add the toolchain library directories
link_directories(
  "${ARM_TOOLCHAIN_DIR}/arm-none-eabi/lib"
)

Running My CMAKE Build

Launching QEMU

qemu-system-arm -machine mps2-an385 -monitor null -semihosting --semihosting-config enable=on,target=native -kernel ./build/qemu_demo -serial stdio -nographic -s -S

I then launch GDB

arm-none-eabi-gdb -q ./build/qemu_demo
Reading symbols from ./build/qemu_demo...
(gdb) target remote:1234
Remote debugging using :1234
Reset_Handler () at /Users/kartikaiyer/fun/qemu_demo/FreeRTOS/FreeRTOS/Demo/CORTEX_MPU_M3_MPS2_QEMU_GCC/init/startup.c:50
50          __asm volatile ( "ldr r0, =_estack" );
(gdb) b HardFault_Handler
Breakpoint 1 at 0x10584: file /Users/kartikaiyer/fun/qemu_demo/FreeRTOS/FreeRTOS/Demo/CORTEX_MPU_M3_MPS2_QEMU_GCC/init/startup.c, line 130.
(gdb) c
Continuing.

Breakpoint 1, HardFault_Handler ()
    at /Users/kartikaiyer/fun/qemu_demo/FreeRTOS/FreeRTOS/Demo/CORTEX_MPU_M3_MPS2_QEMU_GCC/init/startup.c:130
130         __asm volatile
(gdb)

Running the Makefile Build

The following is executed in the Demo folder

Launch QEMU

qemu-system-arm -machine mps2-an385 -monitor null -semihosting --semihosting-config enable=on,target=native -kernel ./build/RTOSDemo.axf -serial stdio -nographic -s -S

Launching GDB

arm-none-eabi-gdb -q ./build/RTOSDemo.axf
Reading symbols from ./build/RTOSDemo.axf...
(gdb) target remote:1234
Remote debugging using :1234
Reset_Handler () at init/startup.c:50
50          __asm volatile ( "ldr r0, =_estack" );
(gdb) b HardFault_Handler
Breakpoint 1 at 0x10614: file init/startup.c, line 130.
(gdb) c
Continuing.

As you can see above, there is no crash.

Can you share both the map files?

Can you step through the assembly and try to find out the exact instruction that is crashing?

Hi Gaurav,
Thanks a lot for responding. I think I found the problem. To answer your question about displaying the assembly instructions that the code is crashing on, I actually found that there is a call to memset and a call to _free_r and it is here that the code crashes on an undefined instruction. Please refer to the snapshots below


│B+  0x10788 <vStartMPUDemo>         stmdb   sp!, {r4, r5, r6, r7, r8, lr}                                                                       │
│    0x1078c <vStartMPUDemo+4>       sub     sp, #128        @ 0x80                                                                              │
│    0x1078e <vStartMPUDemo+6>       mov.w   r8, #64 @ 0x40                                                                                      │
│    0x10792 <vStartMPUDemo+10>      mov     r2, r8                                                                                              │
│    0x10794 <vStartMPUDemo+12>      movs    r1, #0                                                                                              │
│    0x10796 <vStartMPUDemo+14>      add.w   r0, sp, r8                                                                                          │
│  > 0x1079a <vStartMPUDemo+18>      bl      0x11474 <memset>                                                                                    │
│    0x1079e <vStartMPUDemo+22>      ldr     r3, [pc, #96]   @ (0x10800 <vStartMPUDemo+120>)                                                     │
│    0x107a0 <vStartMPUDemo+24>      str     r3, [sp, #64]   @ 0x40                                                                              │
│    0x107a2 <vStartMPUDemo+26>      ldr     r3, [pc, #96]   @ (0x10804 <vStartMPUDemo+124>)                                                     │
│    0x107a4 <vStartMPUDemo+28>      str     r3, [sp, #68]   @ 0x44                                                                              │
│    0x107a6 <vStartMPUDemo+30>      mov.w   r7, #256        @ 0x100                                                                             │
│    0x107aa <vStartMPUDemo+34>      str     r7, [sp, #72]   @ 0x48                                                                              │
│    0x107ac <vStartMPUDemo+36>      ldr     r3, [pc, #88]   @ (0x10808 <vStartMPUDemo+128>)                                                     │
│    0x107ae <vStartMPUDemo+38>      str     r3, [sp, #84]   @ 0x54                                                                              │
│    0x107b0 <vStartMPUDemo+40>      ldr     r6, [pc, #88]   @ (0x1080c <vStartMPUDemo+132>)                                                     │
│    0x107b2 <vStartMPUDemo+42>      str     r6, [sp, #88]   @ 0x58                                                                              │
│    0x107b4 <vStartMPUDemo+44>      movs    r4, #32                                                                                             │
│    0x107b6 <vStartMPUDemo+46>      str     r4, [sp, #92]   @ 0x5c                                                                              │
│    0x107b8 <vStartMPUDemo+48>      mov.w   r3, #301989888  @ 0x12000000                                                                        │
│    0x107bc <vStartMPUDemo+52>      str     r3, [sp, #96]   @ 0x60                                                                              │
└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
remote Thread 1.1 In: vStartMPUDemo                                                                                             L212  PC: 0x1079a
(gdb) si
│  > 0x11474 <memset>        movs    r3, r0                                                                                                      │
│    0x11476 <memset+2>      b.n     0x11a9a <_free_r+418>                                                                                       │
│    0x11478 <memset+4>      ands    r0, r6                                                                                                      │
│    0x1147a <memset+6>      stmdb   sp!, {r1, r2, r4, r5}                                                                                       │
│    0x1147e <memset+10>     lsrs    r0, r0, #8                                                                                                  │
│    0x11480 <memset+12>     movs    r0, r0                                                                                                      │
│    0x11482 <memset+14>     b.n     0x11b2a <_free_r+562>                                                                                       │
│    0x11484 <memset+16>     adds    r0, #1                                                                                                      │
│    0x11486 <memset+18>     b.n     0x1190e <_free_r+22>                                                                                        │
│    0x11488 <memset+20>     movs    r7, r5                                                                                                      │
│    0x1148a <memset+22>     lsrs    r0, r0, #8                                                                                                  │
│    0x1148c <memset+24>     b.n     0x11490 <memset+28>                                                                                         │
│    0x1148e <memset+26>     b.n     0x117d2 <strlen+78>                                                                                         │
│    0x11490 <memset+28>     movs    r0, #255        @ 0xff                                                                                      │

The crashing line


│    0x11a9a <_free_r+418>   asrs    r4, r1, #22                                                                                                 │
│    0x11a9c <_free_r+420>   asrs    r0, r0, #32                                                                                                 │
│    0x11a9e <_free_r+422>   asrs    r6, r1, #22                                                                                                 │
│  > 0x11aa0 <_free_r+424>                   @ <UNDEFINED> instruction: 0xffc81aff                                                               │
│    0x11aa4 <_free_r+428>   adds    r0, #1                                                                                                      │

Given that Im compiling without stdlib and libc I figured there was a problem here. I removed the these lines from my toolchain cmake file that was linking in the libraries from the toolchain and the code no longer crashed.

## Add the toolchain include directories
#include_directories(
#  "${ARM_TOOLCHAIN_DIR}/arm-none-eabi/include"
#)

## Add the toolchain library directories
#link_directories(
#  "${ARM_TOOLCHAIN_DIR}/arm-none-eabi/lib"
#)

I have a few questions that maybe you can help me with:

  1. Are there FreeRTOS implementations to memcpy and free that are being called in the demo ?
  2. I want to eventually use FreeRTOS in another project where I only use static allocation but still want to use libc for things like memcpy etc. Can you comment on any steps that i should take in order to correctly link in libC and avoid calling code that will likely fault when I don’t intend to use a heap ?

Thanks

Hi, for dynamic memory allocations, FreeRTOS has a few heap choices. It seems you are linking in heap4, which allocates out of a static buffer, and thus FreeRTOS APIs will not result in a call to free or malloc, unless done so by the port, which your port does not seem to do.

The FreeRTOS kernel does however contain calls to memcpy.

For using FreeRTOS without dynamic allocations, you can enable configSUPPORT_STATIC_ALLOCATION and disable configSUPPORT_DYNAMIC_ALLOCATION. This will entail using the static versions of the API calls and defining some functions for providing memory to the idle and timer tasks. Documentation is at FreeRTOS Static Memory Allocation.

1 Like

Thank you for sharing your solution.

As @archigup mentioned, enabling static allocation to not use a heap, should not cause any problem. Let us know if you face any issue.