dma.h

Go to the documentation of this file.

/*
 * Copyright (c) 2020 Raspberry Pi (Trading) Ltd.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */
 
#ifndef _HARDWARE_DMA_H
#define _HARDWARE_DMA_H
 
#include "pico.h"
#include "hardware/structs/dma.h"
#include "hardware/regs/dreq.h"
#include "pico/assert.h"
#include "hardware/regs/intctrl.h"
 
#ifdef __cplusplus
extern "C" {
#endif

 
// PICO_CONFIG: PARAM_ASSERTIONS_ENABLED_HARDWARE_DMA, Enable/disable hardware_dma assertions, type=bool, default=0, group=hardware_dma
#ifndef PARAM_ASSERTIONS_ENABLED_HARDWARE_DMA
#ifdef PARAM_ASSERTIONS_ENABLED_DMA // backwards compatibility with SDK < 2.0.0
#define PARAM_ASSERTIONS_ENABLED_HARDWARE_DMA PARAM_ASSERTIONS_ENABLED_DMA
#else
#define PARAM_ASSERTIONS_ENABLED_HARDWARE_DMA 0
#endif
#endif

#ifndef DMA_IRQ_NUM
#define DMA_IRQ_NUM(irq_index) (DMA_IRQ_0 + (irq_index))
#endif
 
static inline void check_dma_channel_param(__unused uint channel) {
#if PARAM_ASSERTIONS_ENABLED(HARDWARE_DMA)
    // this method is used a lot by inline functions so avoid code bloat by deferring to function
    extern void check_dma_channel_param_impl(uint channel);
    check_dma_channel_param_impl(channel);
#endif
}
 
static inline void check_dma_timer_param(__unused uint timer_num) {
    valid_params_if(HARDWARE_DMA, timer_num < NUM_DMA_TIMERS);
}
 
inline static dma_channel_hw_t *dma_channel_hw_addr(uint channel) {
    check_dma_channel_param(channel);
    return &dma_hw->ch[channel];
}

void dma_channel_claim(uint channel);

void dma_claim_mask(uint32_t channel_mask);

void dma_channel_unclaim(uint channel);

void dma_unclaim_mask(uint32_t channel_mask);

int dma_claim_unused_channel(bool required);

bool dma_channel_is_claimed(uint channel);


typedef enum dma_channel_transfer_size {
    DMA_SIZE_8 = 0,    
    DMA_SIZE_16 = 1,   
    DMA_SIZE_32 = 2    
} dma_channel_transfer_size_t;

typedef enum dma_address_update_type {
    DMA_ADDRESS_UPDATE_NONE = 0,                
    DMA_ADDRESS_UPDATE_INCREMENT = 1,           
#if !PICO_RP2040
    DMA_ADDRESS_UPDATE_INCREMENT_BY_TWO = 2,    
    DMA_ADDRESS_UPDATE_DECREMENT = 3,           
#endif
} dma_address_update_type_t;

typedef struct {
    uint32_t ctrl;
} dma_channel_config_t;
 
// backwards compatibility
typedef dma_channel_config_t dma_channel_config;
 
#ifndef DMA_ADDRESS_UPDATE_TYPE_TO_DMA_CH_CTRL_READ_BITS
#if PICO_RP2040
#define DMA_ADDRESS_UPDATE_TYPE_TO_DMA_CH_CTRL_READ_BITS(u) (((u)&1) << DMA_CH0_CTRL_TRIG_INCR_READ_LSB)
#else
#define DMA_ADDRESS_UPDATE_TYPE_TO_DMA_CH_CTRL_READ_BITS(u) \
    ((((u)&1) << DMA_CH0_CTRL_TRIG_INCR_READ_LSB) | \
    (((u)&2) << (DMA_CH0_CTRL_TRIG_INCR_READ_REV_LSB - 1)))
#endif
#endif
 
#ifndef DMA_ADDRESS_UPDATE_TYPE_TO_DMA_CH_CTRL_WRITE_BITS
#if PICO_RP2040
#define DMA_ADDRESS_UPDATE_TYPE_TO_DMA_CH_CTRL_WRITE_BITS(u) (((u)&1) << DMA_CH0_CTRL_TRIG_INCR_WRITE_LSB)
#else
#define DMA_ADDRESS_UPDATE_TYPE_TO_DMA_CH_CTRL_WRITE_BITS(u) \
((((u)&1) << DMA_CH0_CTRL_TRIG_INCR_WRITE_LSB) | \
(((u)&2) << (DMA_CH0_CTRL_TRIG_INCR_WRITE_REV_LSB - 1)))
#endif
#endif
 
#ifndef DMA_CH_CTRL_ALL_ADDRESS_UPDATE_READ_BITS
#if PICO_RP2040
#define DMA_CH_CTRL_ALL_ADDRESS_UPDATE_READ_BITS DMA_CH0_CTRL_TRIG_INCR_READ_BITS
#else
#define DMA_CH_CTRL_ALL_ADDRESS_UPDATE_READ_BITS (DMA_CH0_CTRL_TRIG_INCR_READ_BITS | DMA_CH0_CTRL_TRIG_INCR_READ_REV_BITS)
#endif
#endif
 
#ifndef DMA_CH_CTRL_ALL_ADDRESS_UPDATE_WRITE_BITS
#if PICO_RP2040
#define DMA_CH_CTRL_ALL_ADDRESS_UPDATE_WRITE_BITS DMA_CH0_CTRL_TRIG_INCR_WRITE_BITS
#else
#define DMA_CH_CTRL_ALL_ADDRESS_UPDATE_WRITE_BITS (DMA_CH0_CTRL_TRIG_INCR_WRITE_BITS | DMA_CH0_CTRL_TRIG_INCR_WRITE_REV_BITS)
#endif
#endif

static inline void channel_config_set_read_address_update_type(dma_channel_config_t *c, dma_address_update_type_t update_type) {
    c->ctrl = (c->ctrl & ~DMA_CH_CTRL_ALL_ADDRESS_UPDATE_READ_BITS) |
        DMA_ADDRESS_UPDATE_TYPE_TO_DMA_CH_CTRL_READ_BITS(update_type);
}

static inline void channel_config_set_write_address_update_type(dma_channel_config_t *c, dma_address_update_type_t update_type) {
    c->ctrl = (c->ctrl & ~DMA_CH_CTRL_ALL_ADDRESS_UPDATE_WRITE_BITS) |
        DMA_ADDRESS_UPDATE_TYPE_TO_DMA_CH_CTRL_WRITE_BITS(update_type);
}

static inline void channel_config_set_read_increment(dma_channel_config_t *c, bool incr) {
    channel_config_set_read_address_update_type(c, incr ? DMA_ADDRESS_UPDATE_INCREMENT : DMA_ADDRESS_UPDATE_NONE);
}

static inline void channel_config_set_write_increment(dma_channel_config_t *c, bool incr) {
    channel_config_set_write_address_update_type(c, incr ? DMA_ADDRESS_UPDATE_INCREMENT : DMA_ADDRESS_UPDATE_NONE);
}

static inline void channel_config_set_dreq(dma_channel_config_t *c, uint dreq) {
    assert(dreq <= DREQ_FORCE);
    c->ctrl = (c->ctrl & ~DMA_CH0_CTRL_TRIG_TREQ_SEL_BITS) | (dreq << DMA_CH0_CTRL_TRIG_TREQ_SEL_LSB);
}

static inline void channel_config_set_chain_to(dma_channel_config_t *c, uint chain_to) {
    assert(chain_to <= NUM_DMA_CHANNELS);
    c->ctrl = (c->ctrl & ~DMA_CH0_CTRL_TRIG_CHAIN_TO_BITS) | (chain_to << DMA_CH0_CTRL_TRIG_CHAIN_TO_LSB);
}

static inline void channel_config_set_transfer_data_size(dma_channel_config_t *c, dma_channel_transfer_size_t size) {
    assert(size == DMA_SIZE_8 || size == DMA_SIZE_16 || size == DMA_SIZE_32);
    c->ctrl = (c->ctrl & ~DMA_CH0_CTRL_TRIG_DATA_SIZE_BITS) | (((uint)size) << DMA_CH0_CTRL_TRIG_DATA_SIZE_LSB);
}

static inline void channel_config_set_ring(dma_channel_config_t *c, bool write, uint size_bits) {
    assert(size_bits < 32);
    c->ctrl = (c->ctrl & ~(DMA_CH0_CTRL_TRIG_RING_SIZE_BITS | DMA_CH0_CTRL_TRIG_RING_SEL_BITS)) |
              (size_bits << DMA_CH0_CTRL_TRIG_RING_SIZE_LSB) |
              (write ? DMA_CH0_CTRL_TRIG_RING_SEL_BITS : 0);
}

static inline void channel_config_set_bswap(dma_channel_config_t *c, bool bswap) {
    c->ctrl = bswap ? (c->ctrl | DMA_CH0_CTRL_TRIG_BSWAP_BITS) : (c->ctrl & ~DMA_CH0_CTRL_TRIG_BSWAP_BITS);
}

static inline void channel_config_set_irq_quiet(dma_channel_config_t *c, bool irq_quiet) {
    c->ctrl = irq_quiet ? (c->ctrl | DMA_CH0_CTRL_TRIG_IRQ_QUIET_BITS) : (c->ctrl & ~DMA_CH0_CTRL_TRIG_IRQ_QUIET_BITS);
}

static inline void channel_config_set_high_priority(dma_channel_config_t *c, bool high_priority) {
    c->ctrl = high_priority ? (c->ctrl | DMA_CH0_CTRL_TRIG_HIGH_PRIORITY_BITS) : (c->ctrl & ~DMA_CH0_CTRL_TRIG_HIGH_PRIORITY_BITS);
}

static inline void channel_config_set_enable(dma_channel_config_t *c, bool enable) {
    c->ctrl = enable ? (c->ctrl | DMA_CH0_CTRL_TRIG_EN_BITS) : (c->ctrl & ~DMA_CH0_CTRL_TRIG_EN_BITS);
}

static inline void channel_config_set_sniff_enable(dma_channel_config_t *c, bool sniff_enable) {
    c->ctrl = sniff_enable ? (c->ctrl | DMA_CH0_CTRL_TRIG_SNIFF_EN_BITS) : (c->ctrl &
                                                                             ~DMA_CH0_CTRL_TRIG_SNIFF_EN_BITS);
}

static inline dma_channel_config_t dma_channel_get_default_config(uint channel) {
    dma_channel_config_t c = {0};
    channel_config_set_read_increment(&c, true);
    channel_config_set_write_increment(&c, false);
    channel_config_set_dreq(&c, DREQ_FORCE);
    channel_config_set_chain_to(&c, channel);
    channel_config_set_transfer_data_size(&c, DMA_SIZE_32);
    channel_config_set_ring(&c, false, 0);
    channel_config_set_bswap(&c, false);
    channel_config_set_irq_quiet(&c, false);
    channel_config_set_enable(&c, true);
    channel_config_set_sniff_enable(&c, false);
    channel_config_set_high_priority( &c, false);
    return c;
}

static inline dma_channel_config_t dma_get_channel_config(uint channel) {
    dma_channel_config_t c;
    c.ctrl = dma_channel_hw_addr(channel)->ctrl_trig;
    return c;
}

static inline uint32_t channel_config_get_ctrl_value(const dma_channel_config_t *config) {
    return config->ctrl;
}

static inline void dma_channel_set_config(uint channel, const dma_channel_config_t *config, bool trigger) {
    // Don't use CTRL_TRIG since we don't want to start a transfer
    if (!trigger) {
        dma_channel_hw_addr(channel)->al1_ctrl = channel_config_get_ctrl_value(config);
    } else {
        dma_channel_hw_addr(channel)->ctrl_trig = channel_config_get_ctrl_value(config);
    }
}

static inline void dma_channel_set_read_addr(uint channel, const volatile void *read_addr, bool trigger) {
    if (!trigger) {
        dma_channel_hw_addr(channel)->read_addr = (uintptr_t) read_addr;
    } else {
        dma_channel_hw_addr(channel)->al3_read_addr_trig = (uintptr_t) read_addr;
    }
}

static inline void dma_channel_set_write_addr(uint channel, volatile void *write_addr, bool trigger) {
    if (!trigger) {
        dma_channel_hw_addr(channel)->write_addr = (uintptr_t) write_addr;
    } else {
        dma_channel_hw_addr(channel)->al2_write_addr_trig = (uintptr_t) write_addr;
    }
}

static inline uint32_t dma_encode_transfer_count(uint transfer_count) {
#if !PICO_RP2040
    invalid_params_if(HARDWARE_DMA, transfer_count & DMA_CH0_TRANS_COUNT_MODE_BITS);
    return transfer_count & DMA_CH0_TRANS_COUNT_COUNT_BITS;
#else
    return transfer_count;
#endif
}

static inline uint32_t dma_encode_transfer_count_with_self_trigger(uint transfer_count) {
#if PICO_RP2040
    panic_unsupported();
#else
    return dma_encode_transfer_count(transfer_count) | (DMA_CH0_TRANS_COUNT_MODE_VALUE_TRIGGER_SELF << DMA_CH0_TRANS_COUNT_MODE_LSB);
#endif
}

static inline uint32_t dma_encode_endless_transfer_count(void) {
#if PICO_RP2040
    panic_unsupported();
#else
    static_assert(DMA_CH0_TRANS_COUNT_MODE_VALUE_ENDLESS == 0xf);
    static_assert(DMA_CH0_TRANS_COUNT_MODE_LSB == 28);
    return 0xffffffffu;
#endif
}

static inline void dma_channel_set_transfer_count(uint channel, uint32_t encoded_transfer_count, bool trigger) {
    if (!trigger) {
        dma_channel_hw_addr(channel)->transfer_count = encoded_transfer_count;
    } else {
        dma_channel_hw_addr(channel)->al1_transfer_count_trig = encoded_transfer_count;
    }
}
 
// backwards compatibility with SDK < 2.2.0
static inline void dma_channel_set_trans_count(uint channel, uint32_t trans_count, bool trigger) {
    dma_channel_set_transfer_count(channel, trans_count, trigger);
}

static inline void dma_channel_configure(uint channel, const dma_channel_config_t *config, volatile void *write_addr,
                                         const volatile void *read_addr,
                                         uint32_t encoded_transfer_count, bool trigger) {
    dma_channel_set_read_addr(channel, read_addr, false);
    dma_channel_set_write_addr(channel, write_addr, false);
    dma_channel_set_transfer_count(channel, encoded_transfer_count, false);
    dma_channel_set_config(channel, config, trigger);
}

inline static void __attribute__((always_inline)) dma_channel_transfer_from_buffer_now(uint channel, 
                                                                                       const volatile void *read_addr,
                                                                                       uint32_t encoded_transfer_count) {
//    check_dma_channel_param(channel);
    dma_channel_hw_t *hw = dma_channel_hw_addr(channel);
    hw->read_addr = (uintptr_t) read_addr;
    hw->al1_transfer_count_trig = encoded_transfer_count;
}

inline static void dma_channel_transfer_to_buffer_now(uint channel, volatile void *write_addr, uint32_t encoded_transfer_count) {
    dma_channel_hw_t *hw = dma_channel_hw_addr(channel);
    hw->write_addr = (uintptr_t) write_addr;
    hw->al1_transfer_count_trig = encoded_transfer_count;
}

static inline void dma_start_channel_mask(uint32_t chan_mask) {
    valid_params_if(HARDWARE_DMA, chan_mask && chan_mask < (1u << NUM_DMA_CHANNELS));
    dma_hw->multi_channel_trigger = chan_mask;
}

static inline void dma_channel_start(uint channel) {
    dma_start_channel_mask(1u << channel);
}

static inline void dma_channel_abort(uint channel) {
    check_dma_channel_param(channel);
    dma_hw->abort = 1u << channel;
    // Bit will go 0 once channel has reached safe state
    // (i.e. any in-flight transfers have retired)
    while (dma_hw->ch[channel].ctrl_trig & DMA_CH0_CTRL_TRIG_BUSY_BITS) tight_loop_contents();
}

static inline void dma_channel_set_irq0_enabled(uint channel, bool enabled) {
    check_dma_channel_param(channel);
    check_hw_layout(dma_hw_t, inte0, DMA_INTE0_OFFSET);
    if (enabled)
        hw_set_bits(&dma_hw->inte0, 1u << channel);
    else
        hw_clear_bits(&dma_hw->inte0, 1u << channel);
}

static inline void dma_set_irq0_channel_mask_enabled(uint32_t channel_mask, bool enabled) {
    if (enabled) {
        hw_set_bits(&dma_hw->inte0, channel_mask);
    } else {
        hw_clear_bits(&dma_hw->inte0, channel_mask);
    }
}

static inline void dma_channel_set_irq1_enabled(uint channel, bool enabled) {
    check_dma_channel_param(channel);
    check_hw_layout(dma_hw_t, inte1, DMA_INTE1_OFFSET);
    if (enabled)
        hw_set_bits(&dma_hw->inte1, 1u << channel);
    else
        hw_clear_bits(&dma_hw->inte1, 1u << channel);
}

static inline void dma_set_irq1_channel_mask_enabled(uint32_t channel_mask, bool enabled) {
    if (enabled) {
        hw_set_bits(&dma_hw->inte1, channel_mask);
    } else {
        hw_clear_bits(&dma_hw->inte1, channel_mask);
    }
}

static inline void dma_irqn_set_channel_enabled(uint irq_index, uint channel, bool enabled) {
    invalid_params_if(HARDWARE_DMA, irq_index >= NUM_DMA_IRQS);
 
    if (enabled)
        hw_set_bits(&dma_hw->irq_ctrl[irq_index].inte, 1u << channel);
    else
        hw_clear_bits(&dma_hw->irq_ctrl[irq_index].inte, 1u << channel);
}

static inline void dma_irqn_set_channel_mask_enabled(uint irq_index, uint32_t channel_mask,  bool enabled) {
    invalid_params_if(HARDWARE_DMA, irq_index >= NUM_DMA_IRQS);
    if (enabled) {
        hw_set_bits(&dma_hw->irq_ctrl[irq_index].inte, channel_mask);
    } else {
        hw_clear_bits(&dma_hw->irq_ctrl[irq_index].inte, channel_mask);
    }
}

static inline bool dma_channel_get_irq0_status(uint channel) {
    check_dma_channel_param(channel);
    return dma_hw->ints0 & (1u << channel);
}

static inline bool dma_channel_get_irq1_status(uint channel) {
    check_dma_channel_param(channel);
    return dma_hw->ints1 & (1u << channel);
}

static inline bool dma_irqn_get_channel_status(uint irq_index, uint channel) {
    invalid_params_if(HARDWARE_DMA, irq_index >= NUM_DMA_IRQS);
    check_dma_channel_param(channel);
    return dma_hw->irq_ctrl[irq_index].ints & (1u << channel);
}

static inline void dma_channel_acknowledge_irq0(uint channel) {
    check_dma_channel_param(channel);
    dma_hw->ints0 = 1u << channel;
}

static inline void dma_channel_acknowledge_irq1(uint channel) {
    check_dma_channel_param(channel);
    dma_hw->ints1 = 1u << channel;
}

static inline void dma_irqn_acknowledge_channel(uint irq_index, uint channel) {
    invalid_params_if(HARDWARE_DMA, irq_index >= NUM_DMA_IRQS);
    check_dma_channel_param(channel);
    dma_hw->irq_ctrl[irq_index].ints = 1u << channel;
}

inline static bool dma_channel_is_busy(uint channel) {
    check_dma_channel_param(channel);
    return dma_hw->ch[channel].al1_ctrl & DMA_CH0_CTRL_TRIG_BUSY_BITS;
}

inline static void dma_channel_wait_for_finish_blocking(uint channel) {
    while (dma_channel_is_busy(channel)) tight_loop_contents();
    // stop the compiler hoisting a non-volatile buffer access above the DMA completion.
    __compiler_memory_barrier();
}

inline static void dma_sniffer_enable(uint channel, uint mode, bool force_channel_enable) {
    check_dma_channel_param(channel);
    check_hw_layout(dma_hw_t, sniff_ctrl, DMA_SNIFF_CTRL_OFFSET);
    if (force_channel_enable) {
        hw_set_bits(&dma_hw->ch[channel].al1_ctrl, DMA_CH0_CTRL_TRIG_SNIFF_EN_BITS);
    }
    hw_write_masked(&dma_hw->sniff_ctrl,
        (((channel << DMA_SNIFF_CTRL_DMACH_LSB) & DMA_SNIFF_CTRL_DMACH_BITS) |
         ((mode << DMA_SNIFF_CTRL_CALC_LSB) & DMA_SNIFF_CTRL_CALC_BITS) |
         DMA_SNIFF_CTRL_EN_BITS),
        (DMA_SNIFF_CTRL_DMACH_BITS |
         DMA_SNIFF_CTRL_CALC_BITS |
         DMA_SNIFF_CTRL_EN_BITS));
}

inline static void dma_sniffer_set_byte_swap_enabled(bool swap) {
    if (swap)
        hw_set_bits(&dma_hw->sniff_ctrl, DMA_SNIFF_CTRL_BSWAP_BITS);
    else
        hw_clear_bits(&dma_hw->sniff_ctrl, DMA_SNIFF_CTRL_BSWAP_BITS);
}

inline static void dma_sniffer_set_output_invert_enabled(bool invert) {
    if (invert)
        hw_set_bits(&dma_hw->sniff_ctrl, DMA_SNIFF_CTRL_OUT_INV_BITS);
    else
        hw_clear_bits(&dma_hw->sniff_ctrl, DMA_SNIFF_CTRL_OUT_INV_BITS);
}

inline static void dma_sniffer_set_output_reverse_enabled(bool reverse) {
    if (reverse)
        hw_set_bits(&dma_hw->sniff_ctrl, DMA_SNIFF_CTRL_OUT_REV_BITS);
    else
        hw_clear_bits(&dma_hw->sniff_ctrl, DMA_SNIFF_CTRL_OUT_REV_BITS);
}

inline static void dma_sniffer_disable(void) {
    dma_hw->sniff_ctrl = 0;
}

inline static void dma_sniffer_set_data_accumulator(uint32_t seed_value) {
    dma_hw->sniff_data = seed_value;
}

inline static uint32_t dma_sniffer_get_data_accumulator(void) {
    return dma_hw->sniff_data;
}

void dma_timer_claim(uint timer);

void dma_timer_unclaim(uint timer);

int dma_claim_unused_timer(bool required);

bool dma_timer_is_claimed(uint timer);

static inline void dma_timer_set_fraction(uint timer, uint16_t numerator, uint16_t denominator) {
    check_dma_timer_param(timer);
    invalid_params_if(HARDWARE_DMA, numerator > denominator);
    dma_hw->timer[timer] = (((uint32_t)numerator) << DMA_TIMER0_X_LSB) | (((uint32_t)denominator) << DMA_TIMER0_Y_LSB);
}

static inline uint dma_get_timer_dreq(uint timer_num) {
    static_assert(DREQ_DMA_TIMER1 == DREQ_DMA_TIMER0 + 1, "");
    static_assert(DREQ_DMA_TIMER2 == DREQ_DMA_TIMER0 + 2, "");
    static_assert(DREQ_DMA_TIMER3 == DREQ_DMA_TIMER0 + 3, "");
    check_dma_timer_param(timer_num);
    return DREQ_DMA_TIMER0 + timer_num;
}

static inline int dma_get_irq_num(uint irq_index) {
    valid_params_if(HARDWARE_DMA, irq_index < NUM_DMA_IRQS);
    return DMA_IRQ_NUM(irq_index);
}

void dma_channel_cleanup(uint channel);
 
#ifndef NDEBUG
void print_dma_ctrl(dma_channel_hw_t *channel);
#endif
 
#ifdef __cplusplus
}
#endif
 
#endif