git.itanic.dy.fi Git - linux-stable/blob - drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "mmhub_v1_0.h"
  25
  26 #include "mmhub/mmhub_1_0_offset.h"
  27 #include "mmhub/mmhub_1_0_sh_mask.h"
  28 #include "mmhub/mmhub_1_0_default.h"
  29 #include "athub/athub_1_0_offset.h"
  30 #include "athub/athub_1_0_sh_mask.h"
  31 #include "vega10_enum.h"
  32
  33 #include "soc15_common.h"
  34
  35 #define mmDAGB0_CNTL_MISC2_RV 0x008f
  36 #define mmDAGB0_CNTL_MISC2_RV_BASE_IDX 0
  37
  38 u64 mmhub_v1_0_get_fb_location(struct amdgpu_device *adev)
  39 {
  40         u64 base = RREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE);
  41
  42         base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
  43         base <<= 24;
  44
  45         return base;
  46 }
  47
  48 static void mmhub_v1_0_init_gart_pt_regs(struct amdgpu_device *adev)
  49 {
  50         uint64_t value;
  51
  52         BUG_ON(adev->gart.table_addr & (~0x0000FFFFFFFFF000ULL));
  53         value = adev->gart.table_addr - adev->gmc.vram_start +
  54                 adev->vm_manager.vram_base_offset;
  55         value &= 0x0000FFFFFFFFF000ULL;
  56         value |= 0x1; /* valid bit */
  57
  58         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
  59                      lower_32_bits(value));
  60
  61         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
  62                      upper_32_bits(value));
  63 }
  64
  65 static void mmhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
  66 {
  67         mmhub_v1_0_init_gart_pt_regs(adev);
  68
  69         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
  70                      (u32)(adev->gmc.gart_start >> 12));
  71         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
  72                      (u32)(adev->gmc.gart_start >> 44));
  73
  74         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
  75                      (u32)(adev->gmc.gart_end >> 12));
  76         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
  77                      (u32)(adev->gmc.gart_end >> 44));
  78 }
  79
  80 static void mmhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
  81 {
  82         uint64_t value;
  83         uint32_t tmp;
  84
  85         /* Disable AGP. */
  86         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_BASE, 0);
  87         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_TOP, 0);
  88         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_BOT, 0x00FFFFFF);
  89
  90         /* Program the system aperture low logical page number. */
  91         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
  92                      adev->gmc.vram_start >> 18);
  93         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
  94                      adev->gmc.vram_end >> 18);
  95
  96         /* Set default page address. */
  97         value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start +
  98                 adev->vm_manager.vram_base_offset;
  99         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
 100                      (u32)(value >> 12));
 101         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
 102                      (u32)(value >> 44));
 103
 104         /* Program "protection fault". */
 105         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
 106                      (u32)(adev->dummy_page_addr >> 12));
 107         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 108                      (u32)((u64)adev->dummy_page_addr >> 44));
 109
 110         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL2);
 111         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
 112                             ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
 113         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 114 }
 115
 116 static void mmhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
 117 {
 118         uint32_t tmp;
 119
 120         /* Setup TLB control */
 121         tmp = RREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL);
 122
 123         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
 124         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
 125         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 126                             ENABLE_ADVANCED_DRIVER_MODEL, 1);
 127         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 128                             SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
 129         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
 130         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 131                             MTYPE, MTYPE_UC);/* XXX for emulation. */
 132         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
 133
 134         WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
 135 }
 136
 137 static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 138 {
 139         uint32_t tmp;
 140
 141         /* Setup L2 cache */
 142         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
 143         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
 144         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
 145         /* XXX for emulation, Refer to closed source code.*/
 146         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
 147                             0);
 148         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
 149         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 150         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
 151         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
 152
 153         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2);
 154         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
 155         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
 156         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2, tmp);
 157
 158         tmp = mmVM_L2_CNTL3_DEFAULT;
 159         if (adev->gmc.translate_further) {
 160                 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
 161                 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
 162                                     L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
 163         } else {
 164                 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
 165                 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
 166                                     L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
 167         }
 168         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, tmp);
 169
 170         tmp = mmVM_L2_CNTL4_DEFAULT;
 171         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
 172         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
 173         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL4, tmp);
 174 }
 175
 176 static void mmhub_v1_0_enable_system_domain(struct amdgpu_device *adev)
 177 {
 178         uint32_t tmp;
 179
 180         tmp = RREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_CNTL);
 181         tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
 182         tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
 183         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_CNTL, tmp);
 184 }
 185
 186 static void mmhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
 187 {
 188         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
 189                      0XFFFFFFFF);
 190         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
 191                      0x0000000F);
 192
 193         WREG32_SOC15(MMHUB, 0,
 194                      mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0);
 195         WREG32_SOC15(MMHUB, 0,
 196                      mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0);
 197
 198         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
 199                      0);
 200         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
 201                      0);
 202 }
 203
 204 static void mmhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 205 {
 206         unsigned num_level, block_size;
 207         uint32_t tmp;
 208         int i;
 209
 210         num_level = adev->vm_manager.num_level;
 211         block_size = adev->vm_manager.block_size;
 212         if (adev->gmc.translate_further)
 213                 num_level -= 1;
 214         else
 215                 block_size -= 9;
 216
 217         for (i = 0; i <= 14; i++) {
 218                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i);
 219                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
 220                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
 221                                     num_level);
 222                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 223                                     RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 224                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 225                                     DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 226                                     1);
 227                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 228                                     PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 229                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 230                                     VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 231                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 232                                     READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 233                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 234                                     WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 235                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 236                                     EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 237                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 238                                     PAGE_TABLE_BLOCK_SIZE,
 239                                     block_size);
 240                 /* Send no-retry XNACK on fault to suppress VM fault storm. */
 241                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 242                                     RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
 243                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i, tmp);
 244                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
 245                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
 246                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, i*2,
 247                         lower_32_bits(adev->vm_manager.max_pfn - 1));
 248                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
 249                         upper_32_bits(adev->vm_manager.max_pfn - 1));
 250         }
 251 }
 252
 253 static void mmhub_v1_0_program_invalidation(struct amdgpu_device *adev)
 254 {
 255         unsigned i;
 256
 257         for (i = 0; i < 18; ++i) {
 258                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
 259                                     2 * i, 0xffffffff);
 260                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
 261                                     2 * i, 0x1f);
 262         }
 263 }
 264
 265 struct pctl_data {
 266         uint32_t index;
 267         uint32_t data;
 268 };
 269
 270 static const struct pctl_data pctl0_data[] = {
 271         {0x0, 0x7a640},
 272         {0x9, 0x2a64a},
 273         {0xd, 0x2a680},
 274         {0x11, 0x6a684},
 275         {0x19, 0xea68e},
 276         {0x29, 0xa69e},
 277         {0x2b, 0x0010a6c0},
 278         {0x3d, 0x83a707},
 279         {0xc2, 0x8a7a4},
 280         {0xcc, 0x1a7b8},
 281         {0xcf, 0xfa7cc},
 282         {0xe0, 0x17a7dd},
 283         {0xf9, 0xa7dc},
 284         {0xfb, 0x12a7f5},
 285         {0x10f, 0xa808},
 286         {0x111, 0x12a810},
 287         {0x125, 0x7a82c}
 288 };
 289 #define PCTL0_DATA_LEN (ARRAY_SIZE(pctl0_data))
 290
 291 #define PCTL0_RENG_EXEC_END_PTR 0x12d
 292 #define PCTL0_STCTRL_REG_SAVE_RANGE0_BASE  0xa640
 293 #define PCTL0_STCTRL_REG_SAVE_RANGE0_LIMIT 0xa833
 294
 295 static const struct pctl_data pctl1_data[] = {
 296         {0x0, 0x39a000},
 297         {0x3b, 0x44a040},
 298         {0x81, 0x2a08d},
 299         {0x85, 0x6ba094},
 300         {0xf2, 0x18a100},
 301         {0x10c, 0x4a132},
 302         {0x112, 0xca141},
 303         {0x120, 0x2fa158},
 304         {0x151, 0x17a1d0},
 305         {0x16a, 0x1a1e9},
 306         {0x16d, 0x13a1ec},
 307         {0x182, 0x7a201},
 308         {0x18b, 0x3a20a},
 309         {0x190, 0x7a580},
 310         {0x199, 0xa590},
 311         {0x19b, 0x4a594},
 312         {0x1a1, 0x1a59c},
 313         {0x1a4, 0x7a82c},
 314         {0x1ad, 0xfa7cc},
 315         {0x1be, 0x17a7dd},
 316         {0x1d7, 0x12a810},
 317         {0x1eb, 0x4000a7e1},
 318         {0x1ec, 0x5000a7f5},
 319         {0x1ed, 0x4000a7e2},
 320         {0x1ee, 0x5000a7dc},
 321         {0x1ef, 0x4000a7e3},
 322         {0x1f0, 0x5000a7f6},
 323         {0x1f1, 0x5000a7e4}
 324 };
 325 #define PCTL1_DATA_LEN (ARRAY_SIZE(pctl1_data))
 326
 327 #define PCTL1_RENG_EXEC_END_PTR 0x1f1
 328 #define PCTL1_STCTRL_REG_SAVE_RANGE0_BASE  0xa000
 329 #define PCTL1_STCTRL_REG_SAVE_RANGE0_LIMIT 0xa20d
 330 #define PCTL1_STCTRL_REG_SAVE_RANGE1_BASE  0xa580
 331 #define PCTL1_STCTRL_REG_SAVE_RANGE1_LIMIT 0xa59d
 332 #define PCTL1_STCTRL_REG_SAVE_RANGE2_BASE  0xa82c
 333 #define PCTL1_STCTRL_REG_SAVE_RANGE2_LIMIT 0xa833
 334
 335 static void mmhub_v1_0_power_gating_write_save_ranges(struct amdgpu_device *adev)
 336 {
 337         uint32_t tmp = 0;
 338
 339         /* PCTL0_STCTRL_REGISTER_SAVE_RANGE0 */
 340         tmp = REG_SET_FIELD(tmp, PCTL0_STCTRL_REGISTER_SAVE_RANGE0,
 341                         STCTRL_REGISTER_SAVE_BASE,
 342                         PCTL0_STCTRL_REG_SAVE_RANGE0_BASE);
 343         tmp = REG_SET_FIELD(tmp, PCTL0_STCTRL_REGISTER_SAVE_RANGE0,
 344                         STCTRL_REGISTER_SAVE_LIMIT,
 345                         PCTL0_STCTRL_REG_SAVE_RANGE0_LIMIT);
 346         WREG32_SOC15(MMHUB, 0, mmPCTL0_STCTRL_REGISTER_SAVE_RANGE0, tmp);
 347
 348         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE0 */
 349         tmp = 0;
 350         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE0,
 351                         STCTRL_REGISTER_SAVE_BASE,
 352                         PCTL1_STCTRL_REG_SAVE_RANGE0_BASE);
 353         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE0,
 354                         STCTRL_REGISTER_SAVE_LIMIT,
 355                         PCTL1_STCTRL_REG_SAVE_RANGE0_LIMIT);
 356         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE0, tmp);
 357
 358         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE1 */
 359         tmp = 0;
 360         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE1,
 361                         STCTRL_REGISTER_SAVE_BASE,
 362                         PCTL1_STCTRL_REG_SAVE_RANGE1_BASE);
 363         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE1,
 364                         STCTRL_REGISTER_SAVE_LIMIT,
 365                         PCTL1_STCTRL_REG_SAVE_RANGE1_LIMIT);
 366         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE1, tmp);
 367
 368         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE2 */
 369         tmp = 0;
 370         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE2,
 371                         STCTRL_REGISTER_SAVE_BASE,
 372                         PCTL1_STCTRL_REG_SAVE_RANGE2_BASE);
 373         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE2,
 374                         STCTRL_REGISTER_SAVE_LIMIT,
 375                         PCTL1_STCTRL_REG_SAVE_RANGE2_LIMIT);
 376         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE2, tmp);
 377 }
 378
 379 void mmhub_v1_0_initialize_power_gating(struct amdgpu_device *adev)
 380 {
 381         uint32_t pctl0_misc = 0;
 382         uint32_t pctl0_reng_execute = 0;
 383         uint32_t pctl1_misc = 0;
 384         uint32_t pctl1_reng_execute = 0;
 385         int i = 0;
 386
 387         if (amdgpu_sriov_vf(adev))
 388                 return;
 389
 390         /****************** pctl0 **********************/
 391         pctl0_misc = RREG32_SOC15(MMHUB, 0, mmPCTL0_MISC);
 392         pctl0_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE);
 393
 394         /* Light sleep must be disabled before writing to pctl0 registers */
 395         pctl0_misc &= ~PCTL0_MISC__RENG_MEM_LS_ENABLE_MASK;
 396         WREG32_SOC15(MMHUB, 0, mmPCTL0_MISC, pctl0_misc);
 397
 398         /* Write data used to access ram of register engine */
 399         for (i = 0; i < PCTL0_DATA_LEN; i++) {
 400                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_RAM_INDEX,
 401                         pctl0_data[i].index);
 402                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_RAM_DATA,
 403                         pctl0_data[i].data);
 404         }
 405
 406         /* Re-enable light sleep */
 407         pctl0_misc |= PCTL0_MISC__RENG_MEM_LS_ENABLE_MASK;
 408         WREG32_SOC15(MMHUB, 0, mmPCTL0_MISC, pctl0_misc);
 409
 410         /****************** pctl1 **********************/
 411         pctl1_misc = RREG32_SOC15(MMHUB, 0, mmPCTL1_MISC);
 412         pctl1_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE);
 413
 414         /* Light sleep must be disabled before writing to pctl1 registers */
 415         pctl1_misc &= ~PCTL1_MISC__RENG_MEM_LS_ENABLE_MASK;
 416         WREG32_SOC15(MMHUB, 0, mmPCTL1_MISC, pctl1_misc);
 417
 418         /* Write data used to access ram of register engine */
 419         for (i = 0; i < PCTL1_DATA_LEN; i++) {
 420                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_RAM_INDEX,
 421                         pctl1_data[i].index);
 422                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_RAM_DATA,
 423                         pctl1_data[i].data);
 424         }
 425
 426         /* Re-enable light sleep */
 427         pctl1_misc |= PCTL1_MISC__RENG_MEM_LS_ENABLE_MASK;
 428         WREG32_SOC15(MMHUB, 0, mmPCTL1_MISC, pctl1_misc);
 429
 430         mmhub_v1_0_power_gating_write_save_ranges(adev);
 431
 432         /* Set the reng execute end ptr for pctl0 */
 433         pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 434                                         PCTL0_RENG_EXECUTE,
 435                                         RENG_EXECUTE_END_PTR,
 436                                         PCTL0_RENG_EXEC_END_PTR);
 437         WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 438
 439         /* Set the reng execute end ptr for pctl1 */
 440         pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 441                                         PCTL1_RENG_EXECUTE,
 442                                         RENG_EXECUTE_END_PTR,
 443                                         PCTL1_RENG_EXEC_END_PTR);
 444         WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 445 }
 446
 447 void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev,
 448                                 bool enable)
 449 {
 450         uint32_t pctl0_reng_execute = 0;
 451         uint32_t pctl1_reng_execute = 0;
 452
 453         if (amdgpu_sriov_vf(adev))
 454                 return;
 455
 456         pctl0_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE);
 457         pctl1_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE);
 458
 459         if (enable && adev->pg_flags & AMD_PG_SUPPORT_MMHUB) {
 460                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 461                                                 PCTL0_RENG_EXECUTE,
 462                                                 RENG_EXECUTE_ON_PWR_UP, 1);
 463                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 464                                                 PCTL0_RENG_EXECUTE,
 465                                                 RENG_EXECUTE_ON_REG_UPDATE, 1);
 466                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 467
 468                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 469                                                 PCTL1_RENG_EXECUTE,
 470                                                 RENG_EXECUTE_ON_PWR_UP, 1);
 471                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 472                                                 PCTL1_RENG_EXECUTE,
 473                                                 RENG_EXECUTE_ON_REG_UPDATE, 1);
 474                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 475
 476                 if (adev->powerplay.pp_funcs->set_powergating_by_smu)
 477                         amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GMC, true);
 478
 479         } else {
 480                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 481                                                 PCTL0_RENG_EXECUTE,
 482                                                 RENG_EXECUTE_ON_PWR_UP, 0);
 483                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 484                                                 PCTL0_RENG_EXECUTE,
 485                                                 RENG_EXECUTE_ON_REG_UPDATE, 0);
 486                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 487
 488                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 489                                                 PCTL1_RENG_EXECUTE,
 490                                                 RENG_EXECUTE_ON_PWR_UP, 0);
 491                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 492                                                 PCTL1_RENG_EXECUTE,
 493                                                 RENG_EXECUTE_ON_REG_UPDATE, 0);
 494                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 495         }
 496 }
 497
 498 int mmhub_v1_0_gart_enable(struct amdgpu_device *adev)
 499 {
 500         if (amdgpu_sriov_vf(adev)) {
 501                 /*
 502                  * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
 503                  * VF copy registers so vbios post doesn't program them, for
 504                  * SRIOV driver need to program them
 505                  */
 506                 WREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE,
 507                              adev->gmc.vram_start >> 24);
 508                 WREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_TOP,
 509                              adev->gmc.vram_end >> 24);
 510         }
 511
 512         /* GART Enable. */
 513         mmhub_v1_0_init_gart_aperture_regs(adev);
 514         mmhub_v1_0_init_system_aperture_regs(adev);
 515         mmhub_v1_0_init_tlb_regs(adev);
 516         mmhub_v1_0_init_cache_regs(adev);
 517
 518         mmhub_v1_0_enable_system_domain(adev);
 519         mmhub_v1_0_disable_identity_aperture(adev);
 520         mmhub_v1_0_setup_vmid_config(adev);
 521         mmhub_v1_0_program_invalidation(adev);
 522
 523         return 0;
 524 }
 525
 526 void mmhub_v1_0_gart_disable(struct amdgpu_device *adev)
 527 {
 528         u32 tmp;
 529         u32 i;
 530
 531         /* Disable all tables */
 532         for (i = 0; i < 16; i++)
 533                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT0_CNTL, i, 0);
 534
 535         /* Setup TLB control */
 536         tmp = RREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL);
 537         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
 538         tmp = REG_SET_FIELD(tmp,
 539                                 MC_VM_MX_L1_TLB_CNTL,
 540                                 ENABLE_ADVANCED_DRIVER_MODEL,
 541                                 0);
 542         WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
 543
 544         /* Setup L2 cache */
 545         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
 546         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
 547         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
 548         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, 0);
 549 }
 550
 551 /**
 552  * mmhub_v1_0_set_fault_enable_default - update GART/VM fault handling
 553  *
 554  * @adev: amdgpu_device pointer
 555  * @value: true redirects VM faults to the default page
 556  */
 557 void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value)
 558 {
 559         u32 tmp;
 560         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
 561         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 562                         RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 563         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 564                         PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 565         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 566                         PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 567         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 568                         PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 569         tmp = REG_SET_FIELD(tmp,
 570                         VM_L2_PROTECTION_FAULT_CNTL,
 571                         TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
 572                         value);
 573         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 574                         NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 575         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 576                         DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 577         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 578                         VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 579         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 580                         READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 581         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 582                         WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 583         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 584                         EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 585         if (!value) {
 586                 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 587                                 CRASH_ON_NO_RETRY_FAULT, 1);
 588                 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 589                                 CRASH_ON_RETRY_FAULT, 1);
 590     }
 591
 592         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
 593 }
 594
 595 void mmhub_v1_0_init(struct amdgpu_device *adev)
 596 {
 597         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB];
 598
 599         hub->ctx0_ptb_addr_lo32 =
 600                 SOC15_REG_OFFSET(MMHUB, 0,
 601                                  mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
 602         hub->ctx0_ptb_addr_hi32 =
 603                 SOC15_REG_OFFSET(MMHUB, 0,
 604                                  mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
 605         hub->vm_inv_eng0_req =
 606                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_REQ);
 607         hub->vm_inv_eng0_ack =
 608                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ACK);
 609         hub->vm_context0_cntl =
 610                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT0_CNTL);
 611         hub->vm_l2_pro_fault_status =
 612                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_STATUS);
 613         hub->vm_l2_pro_fault_cntl =
 614                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
 615
 616 }
 617
 618 static void mmhub_v1_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 619                                                         bool enable)
 620 {
 621         uint32_t def, data, def1, data1, def2 = 0, data2 = 0;
 622
 623         def  = data  = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 624
 625         if (adev->asic_type != CHIP_RAVEN) {
 626                 def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2);
 627                 def2 = data2 = RREG32_SOC15(MMHUB, 0, mmDAGB1_CNTL_MISC2);
 628         } else
 629                 def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_RV);
 630
 631         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
 632                 data |= ATC_L2_MISC_CG__ENABLE_MASK;
 633
 634                 data1 &= ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 635                            DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 636                            DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 637                            DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 638                            DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 639                            DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 640
 641                 if (adev->asic_type != CHIP_RAVEN)
 642                         data2 &= ~(DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 643                                    DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 644                                    DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 645                                    DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 646                                    DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 647                                    DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 648         } else {
 649                 data &= ~ATC_L2_MISC_CG__ENABLE_MASK;
 650
 651                 data1 |= (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 652                           DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 653                           DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 654                           DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 655                           DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 656                           DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 657
 658                 if (adev->asic_type != CHIP_RAVEN)
 659                         data2 |= (DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 660                                   DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 661                                   DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 662                                   DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 663                                   DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 664                                   DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 665         }
 666
 667         if (def != data)
 668                 WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data);
 669
 670         if (def1 != data1) {
 671                 if (adev->asic_type != CHIP_RAVEN)
 672                         WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2, data1);
 673                 else
 674                         WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_RV, data1);
 675         }
 676
 677         if (adev->asic_type != CHIP_RAVEN && def2 != data2)
 678                 WREG32_SOC15(MMHUB, 0, mmDAGB1_CNTL_MISC2, data2);
 679 }
 680
 681 static void athub_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 682                                                    bool enable)
 683 {
 684         uint32_t def, data;
 685
 686         def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 687
 688         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
 689                 data |= ATHUB_MISC_CNTL__CG_ENABLE_MASK;
 690         else
 691                 data &= ~ATHUB_MISC_CNTL__CG_ENABLE_MASK;
 692
 693         if (def != data)
 694                 WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data);
 695 }
 696
 697 static void mmhub_v1_0_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 698                                                        bool enable)
 699 {
 700         uint32_t def, data;
 701
 702         def = data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 703
 704         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
 705                 data |= ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 706         else
 707                 data &= ~ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 708
 709         if (def != data)
 710                 WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data);
 711 }
 712
 713 static void athub_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 714                                                   bool enable)
 715 {
 716         uint32_t def, data;
 717
 718         def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 719
 720         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) &&
 721             (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
 722                 data |= ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK;
 723         else
 724                 data &= ~ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK;
 725
 726         if(def != data)
 727                 WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data);
 728 }
 729
 730 int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev,
 731                                enum amd_clockgating_state state)
 732 {
 733         if (amdgpu_sriov_vf(adev))
 734                 return 0;
 735
 736         switch (adev->asic_type) {
 737         case CHIP_VEGA10:
 738         case CHIP_VEGA12:
 739         case CHIP_VEGA20:
 740         case CHIP_RAVEN:
 741                 mmhub_v1_0_update_medium_grain_clock_gating(adev,
 742                                 state == AMD_CG_STATE_GATE ? true : false);
 743                 athub_update_medium_grain_clock_gating(adev,
 744                                 state == AMD_CG_STATE_GATE ? true : false);
 745                 mmhub_v1_0_update_medium_grain_light_sleep(adev,
 746                                 state == AMD_CG_STATE_GATE ? true : false);
 747                 athub_update_medium_grain_light_sleep(adev,
 748                                 state == AMD_CG_STATE_GATE ? true : false);
 749                 break;
 750         default:
 751                 break;
 752         }
 753
 754         return 0;
 755 }
 756
 757 void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
 758 {
 759         int data;
 760
 761         if (amdgpu_sriov_vf(adev))
 762                 *flags = 0;
 763
 764         /* AMD_CG_SUPPORT_MC_MGCG */
 765         data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 766         if (data & ATHUB_MISC_CNTL__CG_ENABLE_MASK)
 767                 *flags |= AMD_CG_SUPPORT_MC_MGCG;
 768
 769         /* AMD_CG_SUPPORT_MC_LS */
 770         data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 771         if (data & ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
 772                 *flags |= AMD_CG_SUPPORT_MC_LS;
 773 }