1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved.
7 #include <linux/of_device.h>
8 #include <linux/slab.h>
10 #include <dt-bindings/memory/tegra30-mc.h>
14 static const unsigned long tegra30_mc_emem_regs[] = {
16 MC_EMEM_ARB_OUTSTANDING_REQ,
17 MC_EMEM_ARB_TIMING_RCD,
18 MC_EMEM_ARB_TIMING_RP,
19 MC_EMEM_ARB_TIMING_RC,
20 MC_EMEM_ARB_TIMING_RAS,
21 MC_EMEM_ARB_TIMING_FAW,
22 MC_EMEM_ARB_TIMING_RRD,
23 MC_EMEM_ARB_TIMING_RAP2PRE,
24 MC_EMEM_ARB_TIMING_WAP2PRE,
25 MC_EMEM_ARB_TIMING_R2R,
26 MC_EMEM_ARB_TIMING_W2W,
27 MC_EMEM_ARB_TIMING_R2W,
28 MC_EMEM_ARB_TIMING_W2R,
30 MC_EMEM_ARB_DA_COVERS,
32 MC_EMEM_ARB_RING1_THROTTLE,
35 static const struct tegra_mc_client tegra30_mc_clients[] = {
39 .swgroup = TEGRA_SWGROUP_PTC,
52 .swgroup = TEGRA_SWGROUP_DC,
65 .fifo_size = 16 * 128,
69 .swgroup = TEGRA_SWGROUP_DCB,
82 .fifo_size = 16 * 128,
86 .swgroup = TEGRA_SWGROUP_DC,
102 .name = "display0bb",
103 .swgroup = TEGRA_SWGROUP_DCB,
116 .fifo_size = 16 * 64,
120 .swgroup = TEGRA_SWGROUP_DC,
133 .fifo_size = 16 * 128,
136 .name = "display0cb",
137 .swgroup = TEGRA_SWGROUP_DCB,
150 .fifo_size = 16 * 128,
154 .swgroup = TEGRA_SWGROUP_DC,
167 .fifo_size = 16 * 64,
170 .name = "display1bb",
171 .swgroup = TEGRA_SWGROUP_DCB,
184 .fifo_size = 16 * 64,
188 .swgroup = TEGRA_SWGROUP_EPP,
205 .swgroup = TEGRA_SWGROUP_G2,
218 .fifo_size = 16 * 64,
222 .swgroup = TEGRA_SWGROUP_G2,
235 .fifo_size = 16 * 64,
239 .swgroup = TEGRA_SWGROUP_MPE,
256 .swgroup = TEGRA_SWGROUP_VI,
273 .swgroup = TEGRA_SWGROUP_AFI,
286 .fifo_size = 16 * 32,
290 .swgroup = TEGRA_SWGROUP_AVPC,
307 .swgroup = TEGRA_SWGROUP_DC,
323 .name = "displayhcb",
324 .swgroup = TEGRA_SWGROUP_DCB,
341 .swgroup = TEGRA_SWGROUP_NV,
354 .fifo_size = 16 * 48,
358 .swgroup = TEGRA_SWGROUP_NV2,
371 .fifo_size = 16 * 48,
375 .swgroup = TEGRA_SWGROUP_G2,
388 .fifo_size = 16 * 48,
392 .swgroup = TEGRA_SWGROUP_HDA,
405 .fifo_size = 16 * 16,
408 .name = "host1xdmar",
409 .swgroup = TEGRA_SWGROUP_HC,
422 .fifo_size = 16 * 16,
426 .swgroup = TEGRA_SWGROUP_HC,
443 .swgroup = TEGRA_SWGROUP_NV,
456 .fifo_size = 16 * 64,
460 .swgroup = TEGRA_SWGROUP_NV2,
473 .fifo_size = 16 * 64,
477 .swgroup = TEGRA_SWGROUP_MPE,
494 .swgroup = TEGRA_SWGROUP_MPE,
507 .fifo_size = 16 * 64,
511 .swgroup = TEGRA_SWGROUP_MPE,
527 .name = "ppcsahbdmar",
528 .swgroup = TEGRA_SWGROUP_PPCS,
544 .name = "ppcsahbslvr",
545 .swgroup = TEGRA_SWGROUP_PPCS,
562 .swgroup = TEGRA_SWGROUP_SATA,
575 .fifo_size = 16 * 32,
579 .swgroup = TEGRA_SWGROUP_NV,
592 .fifo_size = 16 * 64,
596 .swgroup = TEGRA_SWGROUP_NV2,
609 .fifo_size = 16 * 64,
613 .swgroup = TEGRA_SWGROUP_VDE,
630 .swgroup = TEGRA_SWGROUP_VDE,
647 .swgroup = TEGRA_SWGROUP_VDE,
660 .fifo_size = 16 * 16,
664 .swgroup = TEGRA_SWGROUP_VDE,
677 .fifo_size = 16 * 16,
681 .swgroup = TEGRA_SWGROUP_MPCORELP,
690 .fifo_size = 16 * 14,
694 .swgroup = TEGRA_SWGROUP_MPCORE,
703 .fifo_size = 16 * 14,
707 .swgroup = TEGRA_SWGROUP_EPP,
720 .fifo_size = 16 * 64,
724 .swgroup = TEGRA_SWGROUP_EPP,
737 .fifo_size = 16 * 64,
741 .swgroup = TEGRA_SWGROUP_EPP,
754 .fifo_size = 16 * 64,
758 .swgroup = TEGRA_SWGROUP_MPE,
775 .swgroup = TEGRA_SWGROUP_VI,
788 .fifo_size = 16 * 64,
792 .swgroup = TEGRA_SWGROUP_VI,
805 .fifo_size = 16 * 64,
809 .swgroup = TEGRA_SWGROUP_VI,
822 .fifo_size = 16 * 64,
826 .swgroup = TEGRA_SWGROUP_VI,
839 .fifo_size = 16 * 64,
843 .swgroup = TEGRA_SWGROUP_G2,
856 .fifo_size = 16 * 128,
860 .swgroup = TEGRA_SWGROUP_AFI,
873 .fifo_size = 16 * 32,
877 .swgroup = TEGRA_SWGROUP_AVPC,
894 .swgroup = TEGRA_SWGROUP_NV,
907 .fifo_size = 16 * 48,
911 .swgroup = TEGRA_SWGROUP_NV2,
924 .fifo_size = 16 * 48,
928 .swgroup = TEGRA_SWGROUP_HDA,
941 .fifo_size = 16 * 16,
945 .swgroup = TEGRA_SWGROUP_HC,
958 .fifo_size = 16 * 32,
962 .swgroup = TEGRA_SWGROUP_ISP,
975 .fifo_size = 16 * 64,
979 .swgroup = TEGRA_SWGROUP_MPCORELP,
988 .fifo_size = 16 * 24,
992 .swgroup = TEGRA_SWGROUP_MPCORE,
1001 .fifo_size = 16 * 24,
1005 .swgroup = TEGRA_SWGROUP_MPE,
1018 .fifo_size = 16 * 8,
1021 .name = "ppcsahbdmaw",
1022 .swgroup = TEGRA_SWGROUP_PPCS,
1035 .fifo_size = 16 * 2,
1038 .name = "ppcsahbslvw",
1039 .swgroup = TEGRA_SWGROUP_PPCS,
1052 .fifo_size = 16 * 4,
1056 .swgroup = TEGRA_SWGROUP_SATA,
1069 .fifo_size = 16 * 32,
1073 .swgroup = TEGRA_SWGROUP_VDE,
1086 .fifo_size = 16 * 4,
1090 .swgroup = TEGRA_SWGROUP_VDE,
1103 .fifo_size = 16 * 16,
1107 .swgroup = TEGRA_SWGROUP_VDE,
1120 .fifo_size = 16 * 2,
1124 .swgroup = TEGRA_SWGROUP_VDE,
1137 .fifo_size = 16 * 16,
1141 static const struct tegra_smmu_swgroup tegra30_swgroups[] = {
1142 { .name = "dc", .swgroup = TEGRA_SWGROUP_DC, .reg = 0x240 },
1143 { .name = "dcb", .swgroup = TEGRA_SWGROUP_DCB, .reg = 0x244 },
1144 { .name = "epp", .swgroup = TEGRA_SWGROUP_EPP, .reg = 0x248 },
1145 { .name = "g2", .swgroup = TEGRA_SWGROUP_G2, .reg = 0x24c },
1146 { .name = "mpe", .swgroup = TEGRA_SWGROUP_MPE, .reg = 0x264 },
1147 { .name = "vi", .swgroup = TEGRA_SWGROUP_VI, .reg = 0x280 },
1148 { .name = "afi", .swgroup = TEGRA_SWGROUP_AFI, .reg = 0x238 },
1149 { .name = "avpc", .swgroup = TEGRA_SWGROUP_AVPC, .reg = 0x23c },
1150 { .name = "nv", .swgroup = TEGRA_SWGROUP_NV, .reg = 0x268 },
1151 { .name = "nv2", .swgroup = TEGRA_SWGROUP_NV2, .reg = 0x26c },
1152 { .name = "hda", .swgroup = TEGRA_SWGROUP_HDA, .reg = 0x254 },
1153 { .name = "hc", .swgroup = TEGRA_SWGROUP_HC, .reg = 0x250 },
1154 { .name = "ppcs", .swgroup = TEGRA_SWGROUP_PPCS, .reg = 0x270 },
1155 { .name = "sata", .swgroup = TEGRA_SWGROUP_SATA, .reg = 0x278 },
1156 { .name = "vde", .swgroup = TEGRA_SWGROUP_VDE, .reg = 0x27c },
1157 { .name = "isp", .swgroup = TEGRA_SWGROUP_ISP, .reg = 0x258 },
1160 static const unsigned int tegra30_group_drm[] = {
1168 static const struct tegra_smmu_group_soc tegra30_groups[] = {
1171 .swgroups = tegra30_group_drm,
1172 .num_swgroups = ARRAY_SIZE(tegra30_group_drm),
1176 static const struct tegra_smmu_soc tegra30_smmu_soc = {
1177 .clients = tegra30_mc_clients,
1178 .num_clients = ARRAY_SIZE(tegra30_mc_clients),
1179 .swgroups = tegra30_swgroups,
1180 .num_swgroups = ARRAY_SIZE(tegra30_swgroups),
1181 .groups = tegra30_groups,
1182 .num_groups = ARRAY_SIZE(tegra30_groups),
1183 .supports_round_robin_arbitration = false,
1184 .supports_request_limit = false,
1185 .num_tlb_lines = 16,
1189 #define TEGRA30_MC_RESET(_name, _control, _status, _bit) \
1192 .id = TEGRA30_MC_RESET_##_name, \
1193 .control = _control, \
1194 .status = _status, \
1198 static const struct tegra_mc_reset tegra30_mc_resets[] = {
1199 TEGRA30_MC_RESET(AFI, 0x200, 0x204, 0),
1200 TEGRA30_MC_RESET(AVPC, 0x200, 0x204, 1),
1201 TEGRA30_MC_RESET(DC, 0x200, 0x204, 2),
1202 TEGRA30_MC_RESET(DCB, 0x200, 0x204, 3),
1203 TEGRA30_MC_RESET(EPP, 0x200, 0x204, 4),
1204 TEGRA30_MC_RESET(2D, 0x200, 0x204, 5),
1205 TEGRA30_MC_RESET(HC, 0x200, 0x204, 6),
1206 TEGRA30_MC_RESET(HDA, 0x200, 0x204, 7),
1207 TEGRA30_MC_RESET(ISP, 0x200, 0x204, 8),
1208 TEGRA30_MC_RESET(MPCORE, 0x200, 0x204, 9),
1209 TEGRA30_MC_RESET(MPCORELP, 0x200, 0x204, 10),
1210 TEGRA30_MC_RESET(MPE, 0x200, 0x204, 11),
1211 TEGRA30_MC_RESET(3D, 0x200, 0x204, 12),
1212 TEGRA30_MC_RESET(3D2, 0x200, 0x204, 13),
1213 TEGRA30_MC_RESET(PPCS, 0x200, 0x204, 14),
1214 TEGRA30_MC_RESET(SATA, 0x200, 0x204, 15),
1215 TEGRA30_MC_RESET(VDE, 0x200, 0x204, 16),
1216 TEGRA30_MC_RESET(VI, 0x200, 0x204, 17),
1219 static void tegra30_mc_tune_client_latency(struct tegra_mc *mc,
1220 const struct tegra_mc_client *client,
1221 unsigned int bandwidth_mbytes_sec)
1223 u32 arb_tolerance_compensation_nsec, arb_tolerance_compensation_div;
1224 unsigned int fifo_size = client->fifo_size;
1225 u32 arb_nsec, la_ticks, value;
1227 /* see 18.4.1 Client Configuration in Tegra3 TRM v03p */
1228 if (bandwidth_mbytes_sec)
1229 arb_nsec = fifo_size * NSEC_PER_USEC / bandwidth_mbytes_sec;
1234 * Latency allowness should be set with consideration for the module's
1235 * latency tolerance and internal buffering capabilities.
1237 * Display memory clients use isochronous transfers and have very low
1238 * tolerance to a belated transfers. Hence we need to compensate the
1239 * memory arbitration imperfection for them in order to prevent FIFO
1240 * underflow condition when memory bus is busy.
1242 * VI clients also need a stronger compensation.
1244 switch (client->swgroup) {
1245 case TEGRA_SWGROUP_MPCORE:
1246 case TEGRA_SWGROUP_PTC:
1248 * We always want lower latency for these clients, hence
1253 case TEGRA_SWGROUP_DC:
1254 case TEGRA_SWGROUP_DCB:
1255 arb_tolerance_compensation_nsec = 1050;
1256 arb_tolerance_compensation_div = 2;
1259 case TEGRA_SWGROUP_VI:
1260 arb_tolerance_compensation_nsec = 1050;
1261 arb_tolerance_compensation_div = 1;
1265 arb_tolerance_compensation_nsec = 150;
1266 arb_tolerance_compensation_div = 1;
1270 if (arb_nsec > arb_tolerance_compensation_nsec)
1271 arb_nsec -= arb_tolerance_compensation_nsec;
1275 arb_nsec /= arb_tolerance_compensation_div;
1278 * Latency allowance is a number of ticks a request from a particular
1279 * client may wait in the EMEM arbiter before it becomes a high-priority
1282 la_ticks = arb_nsec / mc->tick;
1283 la_ticks = min(la_ticks, client->regs.la.mask);
1285 value = mc_readl(mc, client->regs.la.reg);
1286 value &= ~(client->regs.la.mask << client->regs.la.shift);
1287 value |= la_ticks << client->regs.la.shift;
1288 mc_writel(mc, value, client->regs.la.reg);
1291 static int tegra30_mc_icc_set(struct icc_node *src, struct icc_node *dst)
1293 struct tegra_mc *mc = icc_provider_to_tegra_mc(src->provider);
1294 const struct tegra_mc_client *client = &mc->soc->clients[src->id];
1295 u64 peak_bandwidth = icc_units_to_bps(src->peak_bw);
1298 * Skip pre-initialization that is done by icc_node_add(), which sets
1299 * bandwidth to maximum for all clients before drivers are loaded.
1301 * This doesn't make sense for us because we don't have drivers for all
1302 * clients and it's okay to keep configuration left from bootloader
1303 * during boot, at least for today.
1308 /* convert bytes/sec to megabytes/sec */
1309 do_div(peak_bandwidth, 1000000);
1311 tegra30_mc_tune_client_latency(mc, client, peak_bandwidth);
1316 static int tegra30_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
1317 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
1320 * ISO clients need to reserve extra bandwidth up-front because
1321 * there could be high bandwidth pressure during initial filling
1322 * of the client's FIFO buffers. Secondly, we need to take into
1323 * account impurities of the memory subsystem.
1325 if (tag & TEGRA_MC_ICC_TAG_ISO)
1326 peak_bw = tegra_mc_scale_percents(peak_bw, 400);
1329 *agg_peak = max(*agg_peak, peak_bw);
1334 static struct icc_node_data *
1335 tegra30_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data)
1337 struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
1338 const struct tegra_mc_client *client;
1339 unsigned int i, idx = spec->args[0];
1340 struct icc_node_data *ndata;
1341 struct icc_node *node;
1343 list_for_each_entry(node, &mc->provider.nodes, node_list) {
1344 if (node->id != idx)
1347 ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);
1349 return ERR_PTR(-ENOMEM);
1351 client = &mc->soc->clients[idx];
1354 switch (client->swgroup) {
1355 case TEGRA_SWGROUP_DC:
1356 case TEGRA_SWGROUP_DCB:
1357 case TEGRA_SWGROUP_PTC:
1358 case TEGRA_SWGROUP_VI:
1359 /* these clients are isochronous by default */
1360 ndata->tag = TEGRA_MC_ICC_TAG_ISO;
1364 ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
1371 for (i = 0; i < mc->soc->num_clients; i++) {
1372 if (mc->soc->clients[i].id == idx)
1373 return ERR_PTR(-EPROBE_DEFER);
1376 dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
1378 return ERR_PTR(-EINVAL);
1381 static const struct tegra_mc_icc_ops tegra30_mc_icc_ops = {
1382 .xlate_extended = tegra30_mc_of_icc_xlate_extended,
1383 .aggregate = tegra30_mc_icc_aggreate,
1384 .set = tegra30_mc_icc_set,
1387 const struct tegra_mc_soc tegra30_mc_soc = {
1388 .clients = tegra30_mc_clients,
1389 .num_clients = ARRAY_SIZE(tegra30_mc_clients),
1390 .num_address_bits = 32,
1392 .client_id_mask = 0x7f,
1393 .smmu = &tegra30_smmu_soc,
1394 .emem_regs = tegra30_mc_emem_regs,
1395 .num_emem_regs = ARRAY_SIZE(tegra30_mc_emem_regs),
1396 .intmask = MC_INT_INVALID_SMMU_PAGE | MC_INT_SECURITY_VIOLATION |
1398 .reset_ops = &tegra_mc_reset_ops_common,
1399 .resets = tegra30_mc_resets,
1400 .num_resets = ARRAY_SIZE(tegra30_mc_resets),
1401 .icc_ops = &tegra30_mc_icc_ops,
1402 .ops = &tegra30_mc_ops,