How do you prevent CMake from adding C flags to linker flags when linking with GCC?

My toolchain:

set(CMAKE_C_FLAGS_INIT -D__ATmega64M1__)
list(APPEND CMAKE_C_FLAGS_INIT -mmcu=atmega64m1)
list(APPEND CMAKE_C_FLAGS_INIT -x c)
list(APPEND CMAKE_C_FLAGS_INIT -funsigned-char)
list(APPEND CMAKE_C_FLAGS_INIT -funsigned-bitfields)
list(APPEND CMAKE_C_FLAGS_INIT -ffunction-sections)
list(APPEND CMAKE_C_FLAGS_INIT -fdata-sections)
list(APPEND CMAKE_C_FLAGS_INIT -fpack-struct)
list(APPEND CMAKE_C_FLAGS_INIT -fshort-enums)
list(APPEND CMAKE_C_FLAGS_INIT -Wall)
list(APPEND CMAKE_C_FLAGS_INIT -Wextra)
list(APPEND CMAKE_C_FLAGS_INIT -std=c11)
list(JOIN CMAKE_C_FLAGS_INIT " " CMAKE_C_FLAGS_INIT)

set(CMAKE_EXE_LINKER_FLAGS "-Wl,--gc-sections -static -Wl,--start-group -lc -lm -Wl,--end-group -Wl,--print-memory-usage")

When the linker is run on my executable target, the compiler binary avr-gcc is used and no matter what I try, it is using both the C flags and linker flags I set in the toolchain. I want the linker flags to be completely separate. How do you do this?