1 /* SPDX-License-Identifier: GPL-2.0
3 * Copyright 2019-2020 HabanaLabs, Ltd.
11 #include <uapi/misc/habanalabs.h>
12 #include "../common/habanalabs.h"
13 #include "../include/common/hl_boot_if.h"
14 #include "../include/gaudi/gaudi_packets.h"
15 #include "../include/gaudi/gaudi.h"
16 #include "../include/gaudi/gaudi_async_events.h"
17 #include "../include/gaudi/gaudi_fw_if.h"
19 #define NUMBER_OF_EXT_HW_QUEUES 8
20 #define NUMBER_OF_CMPLT_QUEUES NUMBER_OF_EXT_HW_QUEUES
21 #define NUMBER_OF_CPU_HW_QUEUES 1
22 #define NUMBER_OF_INT_HW_QUEUES 100
23 #define NUMBER_OF_HW_QUEUES (NUMBER_OF_EXT_HW_QUEUES + \
24 NUMBER_OF_CPU_HW_QUEUES + \
25 NUMBER_OF_INT_HW_QUEUES)
27 /* 10 NIC QMANs, DMA5 QMAN, TPC7 QMAN */
28 #define NUMBER_OF_COLLECTIVE_QUEUES 12
29 #define NUMBER_OF_SOBS_IN_GRP 11
32 * Number of MSI interrupts IDS:
33 * Each completion queue has 1 ID
34 * The event queue has 1 ID
36 #define NUMBER_OF_INTERRUPTS (NUMBER_OF_CMPLT_QUEUES + \
37 NUMBER_OF_CPU_HW_QUEUES)
39 #if (NUMBER_OF_INTERRUPTS > GAUDI_MSI_ENTRIES)
40 #error "Number of MSI interrupts must be smaller or equal to GAUDI_MSI_ENTRIES"
43 #define CORESIGHT_TIMEOUT_USEC 100000 /* 100 ms */
45 #define GAUDI_MAX_CLK_FREQ 2200000000ull /* 2200 MHz */
47 #define MAX_POWER_DEFAULT_PCI 200000 /* 200W */
48 #define MAX_POWER_DEFAULT_PMC 350000 /* 350W */
50 #define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */
52 #define TPC_ENABLED_MASK 0xFF
54 #define GAUDI_HBM_SIZE_32GB 0x800000000ull
55 #define GAUDI_HBM_DEVICES 4
56 #define GAUDI_HBM_CHANNELS 8
57 #define GAUDI_HBM_CFG_BASE (mmHBM0_BASE - CFG_BASE)
58 #define GAUDI_HBM_CFG_OFFSET (mmHBM1_BASE - mmHBM0_BASE)
60 #define DMA_MAX_TRANSFER_SIZE U32_MAX
62 #define GAUDI_DEFAULT_CARD_NAME "HL2000"
64 #define GAUDI_MAX_PENDING_CS SZ_16K
66 #if !IS_MAX_PENDING_CS_VALID(GAUDI_MAX_PENDING_CS)
67 #error "GAUDI_MAX_PENDING_CS must be power of 2 and greater than 1"
70 #define PCI_DMA_NUMBER_OF_CHNLS 2
71 #define HBM_DMA_NUMBER_OF_CHNLS 6
72 #define DMA_NUMBER_OF_CHNLS (PCI_DMA_NUMBER_OF_CHNLS + \
73 HBM_DMA_NUMBER_OF_CHNLS)
75 #define MME_NUMBER_OF_SLAVE_ENGINES 2
76 #define MME_NUMBER_OF_ENGINES (MME_NUMBER_OF_MASTER_ENGINES + \
77 MME_NUMBER_OF_SLAVE_ENGINES)
78 #define MME_NUMBER_OF_QMANS (MME_NUMBER_OF_MASTER_ENGINES * \
81 #define QMAN_STREAMS 4
83 #define DMA_QMAN_OFFSET (mmDMA1_QM_BASE - mmDMA0_QM_BASE)
84 #define TPC_QMAN_OFFSET (mmTPC1_QM_BASE - mmTPC0_QM_BASE)
85 #define MME_QMAN_OFFSET (mmMME1_QM_BASE - mmMME0_QM_BASE)
86 #define NIC_MACRO_QMAN_OFFSET (mmNIC1_QM0_BASE - mmNIC0_QM0_BASE)
87 #define NIC_ENGINE_QMAN_OFFSET (mmNIC0_QM1_BASE - mmNIC0_QM0_BASE)
89 #define TPC_CFG_OFFSET (mmTPC1_CFG_BASE - mmTPC0_CFG_BASE)
91 #define DMA_CORE_OFFSET (mmDMA1_CORE_BASE - mmDMA0_CORE_BASE)
93 #define QMAN_LDMA_SRC_OFFSET (mmDMA0_CORE_SRC_BASE_LO - mmDMA0_CORE_CFG_0)
94 #define QMAN_LDMA_DST_OFFSET (mmDMA0_CORE_DST_BASE_LO - mmDMA0_CORE_CFG_0)
95 #define QMAN_LDMA_SIZE_OFFSET (mmDMA0_CORE_DST_TSIZE_0 - mmDMA0_CORE_CFG_0)
97 #define QMAN_CPDMA_SRC_OFFSET (mmDMA0_QM_CQ_PTR_LO_4 - mmDMA0_CORE_CFG_0)
98 #define QMAN_CPDMA_DST_OFFSET (mmDMA0_CORE_DST_BASE_LO - mmDMA0_CORE_CFG_0)
99 #define QMAN_CPDMA_SIZE_OFFSET (mmDMA0_QM_CQ_TSIZE_4 - mmDMA0_CORE_CFG_0)
101 #define SIF_RTR_CTRL_OFFSET (mmSIF_RTR_CTRL_1_BASE - mmSIF_RTR_CTRL_0_BASE)
103 #define NIF_RTR_CTRL_OFFSET (mmNIF_RTR_CTRL_1_BASE - mmNIF_RTR_CTRL_0_BASE)
105 #define MME_ACC_OFFSET (mmMME1_ACC_BASE - mmMME0_ACC_BASE)
106 #define SRAM_BANK_OFFSET (mmSRAM_Y0_X1_RTR_BASE - mmSRAM_Y0_X0_RTR_BASE)
108 #define PLL_NR_OFFSET 0
109 #define PLL_NF_OFFSET (mmPSOC_CPU_PLL_NF - mmPSOC_CPU_PLL_NR)
110 #define PLL_OD_OFFSET (mmPSOC_CPU_PLL_OD - mmPSOC_CPU_PLL_NR)
111 #define PLL_DIV_FACTOR_0_OFFSET (mmPSOC_CPU_PLL_DIV_FACTOR_0 - \
113 #define PLL_DIV_SEL_0_OFFSET (mmPSOC_CPU_PLL_DIV_SEL_0 - mmPSOC_CPU_PLL_NR)
115 #define NUM_OF_SOB_IN_BLOCK \
116 (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 - \
117 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0) + 4) >> 2)
119 #define NUM_OF_MONITORS_IN_BLOCK \
120 (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 - \
121 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0) + 4) >> 2)
124 /* DRAM Memory Map */
126 #define CPU_FW_IMAGE_SIZE 0x10000000 /* 256MB */
127 #define MMU_PAGE_TABLES_SIZE 0x0BF00000 /* 191MB */
128 #define MMU_CACHE_MNG_SIZE 0x00100000 /* 1MB */
129 #define RESERVED 0x04000000 /* 64MB */
131 #define CPU_FW_IMAGE_ADDR DRAM_PHYS_BASE
132 #define MMU_PAGE_TABLES_ADDR (CPU_FW_IMAGE_ADDR + CPU_FW_IMAGE_SIZE)
133 #define MMU_CACHE_MNG_ADDR (MMU_PAGE_TABLES_ADDR + MMU_PAGE_TABLES_SIZE)
135 #define DRAM_DRIVER_END_ADDR (MMU_CACHE_MNG_ADDR + MMU_CACHE_MNG_SIZE +\
138 #define DRAM_BASE_ADDR_USER 0x20000000
140 #if (DRAM_DRIVER_END_ADDR > DRAM_BASE_ADDR_USER)
141 #error "Driver must reserve no more than 512MB"
144 /* Internal QMANs PQ sizes */
146 #define MME_QMAN_LENGTH 1024
147 #define MME_QMAN_SIZE_IN_BYTES (MME_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
149 #define HBM_DMA_QMAN_LENGTH 1024
150 #define HBM_DMA_QMAN_SIZE_IN_BYTES \
151 (HBM_DMA_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
153 #define TPC_QMAN_LENGTH 1024
154 #define TPC_QMAN_SIZE_IN_BYTES (TPC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
156 #define NIC_QMAN_LENGTH 1024
157 #define NIC_QMAN_SIZE_IN_BYTES (NIC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
160 #define SRAM_USER_BASE_OFFSET GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START
162 /* Virtual address space */
163 #define VA_HOST_SPACE_START 0x1000000000000ull /* 256TB */
164 #define VA_HOST_SPACE_END 0x3FF8000000000ull /* 1PB - 512GB */
165 #define VA_HOST_SPACE_SIZE (VA_HOST_SPACE_END - \
166 VA_HOST_SPACE_START) /* 767TB */
167 #define HOST_SPACE_INTERNAL_CB_SZ SZ_2M
169 #define HW_CAP_PLL BIT(0)
170 #define HW_CAP_HBM BIT(1)
171 #define HW_CAP_MMU BIT(2)
172 #define HW_CAP_MME BIT(3)
173 #define HW_CAP_CPU BIT(4)
174 #define HW_CAP_PCI_DMA BIT(5)
175 #define HW_CAP_MSI BIT(6)
176 #define HW_CAP_CPU_Q BIT(7)
177 #define HW_CAP_HBM_DMA BIT(8)
178 #define HW_CAP_CLK_GATE BIT(9)
179 #define HW_CAP_SRAM_SCRAMBLER BIT(10)
180 #define HW_CAP_HBM_SCRAMBLER BIT(11)
182 #define HW_CAP_NIC0 BIT(14)
183 #define HW_CAP_NIC1 BIT(15)
184 #define HW_CAP_NIC2 BIT(16)
185 #define HW_CAP_NIC3 BIT(17)
186 #define HW_CAP_NIC4 BIT(18)
187 #define HW_CAP_NIC5 BIT(19)
188 #define HW_CAP_NIC6 BIT(20)
189 #define HW_CAP_NIC7 BIT(21)
190 #define HW_CAP_NIC8 BIT(22)
191 #define HW_CAP_NIC9 BIT(23)
192 #define HW_CAP_NIC_MASK GENMASK(23, 14)
193 #define HW_CAP_NIC_SHIFT 14
195 #define HW_CAP_TPC0 BIT(24)
196 #define HW_CAP_TPC1 BIT(25)
197 #define HW_CAP_TPC2 BIT(26)
198 #define HW_CAP_TPC3 BIT(27)
199 #define HW_CAP_TPC4 BIT(28)
200 #define HW_CAP_TPC5 BIT(29)
201 #define HW_CAP_TPC6 BIT(30)
202 #define HW_CAP_TPC7 BIT(31)
203 #define HW_CAP_TPC_MASK GENMASK(31, 24)
204 #define HW_CAP_TPC_SHIFT 24
206 #define GAUDI_CPU_PCI_MSB_ADDR(addr) (((addr) & GENMASK_ULL(49, 39)) >> 39)
207 #define GAUDI_PCI_TO_CPU_ADDR(addr) \
209 (addr) &= ~GENMASK_ULL(49, 39); \
210 (addr) |= BIT_ULL(39); \
212 #define GAUDI_CPU_TO_PCI_ADDR(addr, extension) \
214 (addr) &= ~GENMASK_ULL(49, 39); \
215 (addr) |= (u64) (extension) << 39; \
218 enum gaudi_dma_channels {
230 enum gaudi_tpc_mask {
231 GAUDI_TPC_MASK_TPC0 = 0x01,
232 GAUDI_TPC_MASK_TPC1 = 0x02,
233 GAUDI_TPC_MASK_TPC2 = 0x04,
234 GAUDI_TPC_MASK_TPC3 = 0x08,
235 GAUDI_TPC_MASK_TPC4 = 0x10,
236 GAUDI_TPC_MASK_TPC5 = 0x20,
237 GAUDI_TPC_MASK_TPC6 = 0x40,
238 GAUDI_TPC_MASK_TPC7 = 0x80,
239 GAUDI_TPC_MASK_ALL = 0xFF
242 enum gaudi_nic_mask {
243 GAUDI_NIC_MASK_NIC0 = 0x01,
244 GAUDI_NIC_MASK_NIC1 = 0x02,
245 GAUDI_NIC_MASK_NIC2 = 0x04,
246 GAUDI_NIC_MASK_NIC3 = 0x08,
247 GAUDI_NIC_MASK_NIC4 = 0x10,
248 GAUDI_NIC_MASK_NIC5 = 0x20,
249 GAUDI_NIC_MASK_NIC6 = 0x40,
250 GAUDI_NIC_MASK_NIC7 = 0x80,
251 GAUDI_NIC_MASK_NIC8 = 0x100,
252 GAUDI_NIC_MASK_NIC9 = 0x200,
253 GAUDI_NIC_MASK_ALL = 0x3FF
257 * struct gaudi_hw_sob_group - H/W SOB group info.
258 * @hdev: habanalabs device structure.
259 * @kref: refcount of this SOB group. group will reset once refcount is zero.
260 * @base_sob_id: base sob id of this SOB group.
262 struct gaudi_hw_sob_group {
263 struct hl_device *hdev;
268 #define NUM_SOB_GROUPS (HL_RSVD_SOBS * QMAN_STREAMS)
270 * struct gaudi_collective_properties -
271 * holds all SOB groups and queues info reserved for the collective
272 * @hw_sob_group: H/W SOB groups.
273 * @next_sob_group_val: the next value to use for the currently used SOB group.
274 * @curr_sob_group_idx: the index of the currently used SOB group.
275 * @mstr_sob_mask: pre-defined masks for collective master monitors
277 struct gaudi_collective_properties {
278 struct gaudi_hw_sob_group hw_sob_group[NUM_SOB_GROUPS];
279 u16 next_sob_group_val[QMAN_STREAMS];
280 u8 curr_sob_group_idx[QMAN_STREAMS];
281 u8 mstr_sob_mask[HL_COLLECTIVE_RSVD_MSTR_MONS];
285 * struct gaudi_internal_qman_info - Internal QMAN information.
286 * @pq_kernel_addr: Kernel address of the PQ memory area in the host.
287 * @pq_dma_addr: DMA address of the PQ memory area in the host.
288 * @pq_size: Size of allocated host memory for PQ.
290 struct gaudi_internal_qman_info {
291 void *pq_kernel_addr;
292 dma_addr_t pq_dma_addr;
297 * struct gaudi_device - ASIC specific manage structure.
298 * @cpucp_info_get: get information on device from CPU-CP
299 * @hw_queues_lock: protects the H/W queues from concurrent access.
300 * @clk_gate_mutex: protects code areas that require clock gating to be disabled
302 * @internal_qmans: Internal QMANs information. The array size is larger than
303 * the actual number of internal queues because they are not in
305 * @hbm_bar_cur_addr: current address of HBM PCI bar.
306 * @max_freq_value: current max clk frequency.
307 * @events: array that holds all event id's
308 * @events_stat: array that holds histogram of all received events.
309 * @events_stat_aggregate: same as events_stat but doesn't get cleared on reset
310 * @hw_cap_initialized: This field contains a bit per H/W engine. When that
311 * engine is initialized, that bit is set by the driver to
312 * signal we can use this engine in later code paths.
313 * Each bit is cleared upon reset of its corresponding H/W
315 * @multi_msi_mode: whether we are working in multi MSI single MSI mode.
316 * Multi MSI is possible only with IOMMU enabled.
317 * @mmu_cache_inv_pi: PI for MMU cache invalidation flow. The H/W expects an
318 * 8-bit value so use u8.
320 struct gaudi_device {
321 int (*cpucp_info_get)(struct hl_device *hdev);
323 /* TODO: remove hw_queues_lock after moving to scheduler code */
324 spinlock_t hw_queues_lock;
325 struct mutex clk_gate_mutex;
327 struct gaudi_internal_qman_info internal_qmans[GAUDI_QUEUE_ID_SIZE];
329 struct gaudi_collective_properties collective_props;
331 u64 hbm_bar_cur_addr;
334 u32 events[GAUDI_EVENT_SIZE];
335 u32 events_stat[GAUDI_EVENT_SIZE];
336 u32 events_stat_aggregate[GAUDI_EVENT_SIZE];
337 u32 hw_cap_initialized;
342 void gaudi_init_security(struct hl_device *hdev);
343 void gaudi_add_device_attr(struct hl_device *hdev,
344 struct attribute_group *dev_attr_grp);
345 void gaudi_set_pll_profile(struct hl_device *hdev, enum hl_pll_frequency freq);
346 int gaudi_debug_coresight(struct hl_device *hdev, void *data);
347 void gaudi_halt_coresight(struct hl_device *hdev);
348 int gaudi_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk);
349 void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid);
351 #endif /* GAUDIP_H_ */