1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved.
7 #include <linux/of_device.h>
8 #include <linux/slab.h>
10 #include <dt-bindings/memory/tegra30-mc.h>
14 static const unsigned long tegra30_mc_emem_regs[] = {
16 MC_EMEM_ARB_OUTSTANDING_REQ,
17 MC_EMEM_ARB_TIMING_RCD,
18 MC_EMEM_ARB_TIMING_RP,
19 MC_EMEM_ARB_TIMING_RC,
20 MC_EMEM_ARB_TIMING_RAS,
21 MC_EMEM_ARB_TIMING_FAW,
22 MC_EMEM_ARB_TIMING_RRD,
23 MC_EMEM_ARB_TIMING_RAP2PRE,
24 MC_EMEM_ARB_TIMING_WAP2PRE,
25 MC_EMEM_ARB_TIMING_R2R,
26 MC_EMEM_ARB_TIMING_W2W,
27 MC_EMEM_ARB_TIMING_R2W,
28 MC_EMEM_ARB_TIMING_W2R,
30 MC_EMEM_ARB_DA_COVERS,
32 MC_EMEM_ARB_RING1_THROTTLE,
35 static const struct tegra_mc_client tegra30_mc_clients[] = {
39 .swgroup = TEGRA_SWGROUP_PTC,
50 .swgroup = TEGRA_SWGROUP_DC,
61 .fifo_size = 16 * 128,
65 .swgroup = TEGRA_SWGROUP_DCB,
76 .fifo_size = 16 * 128,
80 .swgroup = TEGRA_SWGROUP_DC,
95 .swgroup = TEGRA_SWGROUP_DCB,
106 .fifo_size = 16 * 64,
110 .swgroup = TEGRA_SWGROUP_DC,
121 .fifo_size = 16 * 128,
124 .name = "display0cb",
125 .swgroup = TEGRA_SWGROUP_DCB,
136 .fifo_size = 16 * 128,
140 .swgroup = TEGRA_SWGROUP_DC,
151 .fifo_size = 16 * 64,
154 .name = "display1bb",
155 .swgroup = TEGRA_SWGROUP_DCB,
166 .fifo_size = 16 * 64,
170 .swgroup = TEGRA_SWGROUP_EPP,
185 .swgroup = TEGRA_SWGROUP_G2,
196 .fifo_size = 16 * 64,
200 .swgroup = TEGRA_SWGROUP_G2,
211 .fifo_size = 16 * 64,
215 .swgroup = TEGRA_SWGROUP_MPE,
230 .swgroup = TEGRA_SWGROUP_VI,
245 .swgroup = TEGRA_SWGROUP_AFI,
256 .fifo_size = 16 * 32,
260 .swgroup = TEGRA_SWGROUP_AVPC,
275 .swgroup = TEGRA_SWGROUP_DC,
289 .name = "displayhcb",
290 .swgroup = TEGRA_SWGROUP_DCB,
305 .swgroup = TEGRA_SWGROUP_NV,
316 .fifo_size = 16 * 48,
320 .swgroup = TEGRA_SWGROUP_NV2,
331 .fifo_size = 16 * 48,
335 .swgroup = TEGRA_SWGROUP_G2,
346 .fifo_size = 16 * 48,
350 .swgroup = TEGRA_SWGROUP_HDA,
361 .fifo_size = 16 * 16,
364 .name = "host1xdmar",
365 .swgroup = TEGRA_SWGROUP_HC,
376 .fifo_size = 16 * 16,
380 .swgroup = TEGRA_SWGROUP_HC,
395 .swgroup = TEGRA_SWGROUP_NV,
406 .fifo_size = 16 * 64,
410 .swgroup = TEGRA_SWGROUP_NV2,
421 .fifo_size = 16 * 64,
425 .swgroup = TEGRA_SWGROUP_MPE,
440 .swgroup = TEGRA_SWGROUP_MPE,
451 .fifo_size = 16 * 64,
455 .swgroup = TEGRA_SWGROUP_MPE,
469 .name = "ppcsahbdmar",
470 .swgroup = TEGRA_SWGROUP_PPCS,
484 .name = "ppcsahbslvr",
485 .swgroup = TEGRA_SWGROUP_PPCS,
500 .swgroup = TEGRA_SWGROUP_SATA,
511 .fifo_size = 16 * 32,
515 .swgroup = TEGRA_SWGROUP_NV,
526 .fifo_size = 16 * 64,
530 .swgroup = TEGRA_SWGROUP_NV2,
541 .fifo_size = 16 * 64,
545 .swgroup = TEGRA_SWGROUP_VDE,
560 .swgroup = TEGRA_SWGROUP_VDE,
575 .swgroup = TEGRA_SWGROUP_VDE,
586 .fifo_size = 16 * 16,
590 .swgroup = TEGRA_SWGROUP_VDE,
601 .fifo_size = 16 * 16,
605 .swgroup = TEGRA_SWGROUP_MPCORELP,
612 .fifo_size = 16 * 14,
616 .swgroup = TEGRA_SWGROUP_MPCORE,
623 .fifo_size = 16 * 14,
627 .swgroup = TEGRA_SWGROUP_EPP,
638 .fifo_size = 16 * 64,
642 .swgroup = TEGRA_SWGROUP_EPP,
653 .fifo_size = 16 * 64,
657 .swgroup = TEGRA_SWGROUP_EPP,
668 .fifo_size = 16 * 64,
672 .swgroup = TEGRA_SWGROUP_MPE,
687 .swgroup = TEGRA_SWGROUP_VI,
698 .fifo_size = 16 * 64,
702 .swgroup = TEGRA_SWGROUP_VI,
713 .fifo_size = 16 * 64,
717 .swgroup = TEGRA_SWGROUP_VI,
728 .fifo_size = 16 * 64,
732 .swgroup = TEGRA_SWGROUP_VI,
743 .fifo_size = 16 * 64,
747 .swgroup = TEGRA_SWGROUP_G2,
758 .fifo_size = 16 * 128,
762 .swgroup = TEGRA_SWGROUP_AFI,
773 .fifo_size = 16 * 32,
777 .swgroup = TEGRA_SWGROUP_AVPC,
792 .swgroup = TEGRA_SWGROUP_NV,
803 .fifo_size = 16 * 48,
807 .swgroup = TEGRA_SWGROUP_NV2,
818 .fifo_size = 16 * 48,
822 .swgroup = TEGRA_SWGROUP_HDA,
833 .fifo_size = 16 * 16,
837 .swgroup = TEGRA_SWGROUP_HC,
848 .fifo_size = 16 * 32,
852 .swgroup = TEGRA_SWGROUP_ISP,
863 .fifo_size = 16 * 64,
867 .swgroup = TEGRA_SWGROUP_MPCORELP,
874 .fifo_size = 16 * 24,
878 .swgroup = TEGRA_SWGROUP_MPCORE,
885 .fifo_size = 16 * 24,
889 .swgroup = TEGRA_SWGROUP_MPE,
903 .name = "ppcsahbdmaw",
904 .swgroup = TEGRA_SWGROUP_PPCS,
918 .name = "ppcsahbslvw",
919 .swgroup = TEGRA_SWGROUP_PPCS,
934 .swgroup = TEGRA_SWGROUP_SATA,
945 .fifo_size = 16 * 32,
949 .swgroup = TEGRA_SWGROUP_VDE,
964 .swgroup = TEGRA_SWGROUP_VDE,
975 .fifo_size = 16 * 16,
979 .swgroup = TEGRA_SWGROUP_VDE,
994 .swgroup = TEGRA_SWGROUP_VDE,
1005 .fifo_size = 16 * 16,
1009 static const struct tegra_smmu_swgroup tegra30_swgroups[] = {
1010 { .name = "dc", .swgroup = TEGRA_SWGROUP_DC, .reg = 0x240 },
1011 { .name = "dcb", .swgroup = TEGRA_SWGROUP_DCB, .reg = 0x244 },
1012 { .name = "epp", .swgroup = TEGRA_SWGROUP_EPP, .reg = 0x248 },
1013 { .name = "g2", .swgroup = TEGRA_SWGROUP_G2, .reg = 0x24c },
1014 { .name = "mpe", .swgroup = TEGRA_SWGROUP_MPE, .reg = 0x264 },
1015 { .name = "vi", .swgroup = TEGRA_SWGROUP_VI, .reg = 0x280 },
1016 { .name = "afi", .swgroup = TEGRA_SWGROUP_AFI, .reg = 0x238 },
1017 { .name = "avpc", .swgroup = TEGRA_SWGROUP_AVPC, .reg = 0x23c },
1018 { .name = "nv", .swgroup = TEGRA_SWGROUP_NV, .reg = 0x268 },
1019 { .name = "nv2", .swgroup = TEGRA_SWGROUP_NV2, .reg = 0x26c },
1020 { .name = "hda", .swgroup = TEGRA_SWGROUP_HDA, .reg = 0x254 },
1021 { .name = "hc", .swgroup = TEGRA_SWGROUP_HC, .reg = 0x250 },
1022 { .name = "ppcs", .swgroup = TEGRA_SWGROUP_PPCS, .reg = 0x270 },
1023 { .name = "sata", .swgroup = TEGRA_SWGROUP_SATA, .reg = 0x278 },
1024 { .name = "vde", .swgroup = TEGRA_SWGROUP_VDE, .reg = 0x27c },
1025 { .name = "isp", .swgroup = TEGRA_SWGROUP_ISP, .reg = 0x258 },
1028 static const unsigned int tegra30_group_drm[] = {
1036 static const struct tegra_smmu_group_soc tegra30_groups[] = {
1039 .swgroups = tegra30_group_drm,
1040 .num_swgroups = ARRAY_SIZE(tegra30_group_drm),
1044 static const struct tegra_smmu_soc tegra30_smmu_soc = {
1045 .clients = tegra30_mc_clients,
1046 .num_clients = ARRAY_SIZE(tegra30_mc_clients),
1047 .swgroups = tegra30_swgroups,
1048 .num_swgroups = ARRAY_SIZE(tegra30_swgroups),
1049 .groups = tegra30_groups,
1050 .num_groups = ARRAY_SIZE(tegra30_groups),
1051 .supports_round_robin_arbitration = false,
1052 .supports_request_limit = false,
1053 .num_tlb_lines = 16,
1057 #define TEGRA30_MC_RESET(_name, _control, _status, _bit) \
1060 .id = TEGRA30_MC_RESET_##_name, \
1061 .control = _control, \
1062 .status = _status, \
1066 static const struct tegra_mc_reset tegra30_mc_resets[] = {
1067 TEGRA30_MC_RESET(AFI, 0x200, 0x204, 0),
1068 TEGRA30_MC_RESET(AVPC, 0x200, 0x204, 1),
1069 TEGRA30_MC_RESET(DC, 0x200, 0x204, 2),
1070 TEGRA30_MC_RESET(DCB, 0x200, 0x204, 3),
1071 TEGRA30_MC_RESET(EPP, 0x200, 0x204, 4),
1072 TEGRA30_MC_RESET(2D, 0x200, 0x204, 5),
1073 TEGRA30_MC_RESET(HC, 0x200, 0x204, 6),
1074 TEGRA30_MC_RESET(HDA, 0x200, 0x204, 7),
1075 TEGRA30_MC_RESET(ISP, 0x200, 0x204, 8),
1076 TEGRA30_MC_RESET(MPCORE, 0x200, 0x204, 9),
1077 TEGRA30_MC_RESET(MPCORELP, 0x200, 0x204, 10),
1078 TEGRA30_MC_RESET(MPE, 0x200, 0x204, 11),
1079 TEGRA30_MC_RESET(3D, 0x200, 0x204, 12),
1080 TEGRA30_MC_RESET(3D2, 0x200, 0x204, 13),
1081 TEGRA30_MC_RESET(PPCS, 0x200, 0x204, 14),
1082 TEGRA30_MC_RESET(SATA, 0x200, 0x204, 15),
1083 TEGRA30_MC_RESET(VDE, 0x200, 0x204, 16),
1084 TEGRA30_MC_RESET(VI, 0x200, 0x204, 17),
1087 static void tegra30_mc_tune_client_latency(struct tegra_mc *mc,
1088 const struct tegra_mc_client *client,
1089 unsigned int bandwidth_mbytes_sec)
1091 u32 arb_tolerance_compensation_nsec, arb_tolerance_compensation_div;
1092 const struct tegra_mc_la *la = &client->la;
1093 unsigned int fifo_size = client->fifo_size;
1094 u32 arb_nsec, la_ticks, value;
1096 /* see 18.4.1 Client Configuration in Tegra3 TRM v03p */
1097 if (bandwidth_mbytes_sec)
1098 arb_nsec = fifo_size * NSEC_PER_USEC / bandwidth_mbytes_sec;
1103 * Latency allowness should be set with consideration for the module's
1104 * latency tolerance and internal buffering capabilities.
1106 * Display memory clients use isochronous transfers and have very low
1107 * tolerance to a belated transfers. Hence we need to compensate the
1108 * memory arbitration imperfection for them in order to prevent FIFO
1109 * underflow condition when memory bus is busy.
1111 * VI clients also need a stronger compensation.
1113 switch (client->swgroup) {
1114 case TEGRA_SWGROUP_MPCORE:
1115 case TEGRA_SWGROUP_PTC:
1117 * We always want lower latency for these clients, hence
1122 case TEGRA_SWGROUP_DC:
1123 case TEGRA_SWGROUP_DCB:
1124 arb_tolerance_compensation_nsec = 1050;
1125 arb_tolerance_compensation_div = 2;
1128 case TEGRA_SWGROUP_VI:
1129 arb_tolerance_compensation_nsec = 1050;
1130 arb_tolerance_compensation_div = 1;
1134 arb_tolerance_compensation_nsec = 150;
1135 arb_tolerance_compensation_div = 1;
1139 if (arb_nsec > arb_tolerance_compensation_nsec)
1140 arb_nsec -= arb_tolerance_compensation_nsec;
1144 arb_nsec /= arb_tolerance_compensation_div;
1147 * Latency allowance is a number of ticks a request from a particular
1148 * client may wait in the EMEM arbiter before it becomes a high-priority
1151 la_ticks = arb_nsec / mc->tick;
1152 la_ticks = min(la_ticks, la->mask);
1154 value = mc_readl(mc, la->reg);
1155 value &= ~(la->mask << la->shift);
1156 value |= la_ticks << la->shift;
1157 mc_writel(mc, value, la->reg);
1160 static int tegra30_mc_icc_set(struct icc_node *src, struct icc_node *dst)
1162 struct tegra_mc *mc = icc_provider_to_tegra_mc(src->provider);
1163 const struct tegra_mc_client *client = &mc->soc->clients[src->id];
1164 u64 peak_bandwidth = icc_units_to_bps(src->peak_bw);
1167 * Skip pre-initialization that is done by icc_node_add(), which sets
1168 * bandwidth to maximum for all clients before drivers are loaded.
1170 * This doesn't make sense for us because we don't have drivers for all
1171 * clients and it's okay to keep configuration left from bootloader
1172 * during boot, at least for today.
1177 /* convert bytes/sec to megabytes/sec */
1178 do_div(peak_bandwidth, 1000000);
1180 tegra30_mc_tune_client_latency(mc, client, peak_bandwidth);
1185 static int tegra30_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
1186 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
1189 * ISO clients need to reserve extra bandwidth up-front because
1190 * there could be high bandwidth pressure during initial filling
1191 * of the client's FIFO buffers. Secondly, we need to take into
1192 * account impurities of the memory subsystem.
1194 if (tag & TEGRA_MC_ICC_TAG_ISO)
1195 peak_bw = tegra_mc_scale_percents(peak_bw, 400);
1198 *agg_peak = max(*agg_peak, peak_bw);
1203 static struct icc_node_data *
1204 tegra30_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data)
1206 struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
1207 const struct tegra_mc_client *client;
1208 unsigned int i, idx = spec->args[0];
1209 struct icc_node_data *ndata;
1210 struct icc_node *node;
1212 list_for_each_entry(node, &mc->provider.nodes, node_list) {
1213 if (node->id != idx)
1216 ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);
1218 return ERR_PTR(-ENOMEM);
1220 client = &mc->soc->clients[idx];
1223 switch (client->swgroup) {
1224 case TEGRA_SWGROUP_DC:
1225 case TEGRA_SWGROUP_DCB:
1226 case TEGRA_SWGROUP_PTC:
1227 case TEGRA_SWGROUP_VI:
1228 /* these clients are isochronous by default */
1229 ndata->tag = TEGRA_MC_ICC_TAG_ISO;
1233 ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
1240 for (i = 0; i < mc->soc->num_clients; i++) {
1241 if (mc->soc->clients[i].id == idx)
1242 return ERR_PTR(-EPROBE_DEFER);
1245 dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
1247 return ERR_PTR(-EINVAL);
1250 static const struct tegra_mc_icc_ops tegra30_mc_icc_ops = {
1251 .xlate_extended = tegra30_mc_of_icc_xlate_extended,
1252 .aggregate = tegra30_mc_icc_aggreate,
1253 .set = tegra30_mc_icc_set,
1256 const struct tegra_mc_soc tegra30_mc_soc = {
1257 .clients = tegra30_mc_clients,
1258 .num_clients = ARRAY_SIZE(tegra30_mc_clients),
1259 .num_address_bits = 32,
1261 .client_id_mask = 0x7f,
1262 .smmu = &tegra30_smmu_soc,
1263 .emem_regs = tegra30_mc_emem_regs,
1264 .num_emem_regs = ARRAY_SIZE(tegra30_mc_emem_regs),
1265 .intmask = MC_INT_INVALID_SMMU_PAGE | MC_INT_SECURITY_VIOLATION |
1267 .reset_ops = &tegra_mc_reset_ops_common,
1268 .resets = tegra30_mc_resets,
1269 .num_resets = ARRAY_SIZE(tegra30_mc_resets),
1270 .icc_ops = &tegra30_mc_icc_ops,