git.itanic.dy.fi Git - linux-stable/blob - drivers/cxl/core/hdm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
   3 #include <linux/seq_file.h>
   4 #include <linux/device.h>
   5 #include <linux/delay.h>
   6
   7 #include "cxlmem.h"
   8 #include "core.h"
   9
  10 /**
  11  * DOC: cxl core hdm
  12  *
  13  * Compute Express Link Host Managed Device Memory, starting with the
  14  * CXL 2.0 specification, is managed by an array of HDM Decoder register
  15  * instances per CXL port and per CXL endpoint. Define common helpers
  16  * for enumerating these registers and capabilities.
  17  */
  18
  19 DECLARE_RWSEM(cxl_dpa_rwsem);
  20
  21 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
  22                            int *target_map)
  23 {
  24         int rc;
  25
  26         rc = cxl_decoder_add_locked(cxld, target_map);
  27         if (rc) {
  28                 put_device(&cxld->dev);
  29                 dev_err(&port->dev, "Failed to add decoder\n");
  30                 return rc;
  31         }
  32
  33         rc = cxl_decoder_autoremove(&port->dev, cxld);
  34         if (rc)
  35                 return rc;
  36
  37         dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
  38
  39         return 0;
  40 }
  41
  42 /*
  43  * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
  44  * single ported host-bridges need not publish a decoder capability when a
  45  * passthrough decode can be assumed, i.e. all transactions that the uport sees
  46  * are claimed and passed to the single dport. Disable the range until the first
  47  * CXL region is enumerated / activated.
  48  */
  49 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
  50 {
  51         struct cxl_switch_decoder *cxlsd;
  52         struct cxl_dport *dport = NULL;
  53         int single_port_map[1];
  54         unsigned long index;
  55
  56         cxlsd = cxl_switch_decoder_alloc(port, 1);
  57         if (IS_ERR(cxlsd))
  58                 return PTR_ERR(cxlsd);
  59
  60         device_lock_assert(&port->dev);
  61
  62         xa_for_each(&port->dports, index, dport)
  63                 break;
  64         single_port_map[0] = dport->port_id;
  65
  66         return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
  67 }
  68 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
  69
  70 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
  71 {
  72         u32 hdm_cap;
  73
  74         hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
  75         cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
  76         cxlhdm->target_count =
  77                 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
  78         if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
  79                 cxlhdm->interleave_mask |= GENMASK(11, 8);
  80         if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
  81                 cxlhdm->interleave_mask |= GENMASK(14, 12);
  82 }
  83
  84 static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb,
  85                                 struct cxl_component_regs *regs)
  86 {
  87         struct cxl_register_map map = {
  88                 .dev = &port->dev,
  89                 .resource = port->component_reg_phys,
  90                 .base = crb,
  91                 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
  92         };
  93
  94         cxl_probe_component_regs(&port->dev, crb, &map.component_map);
  95         if (!map.component_map.hdm_decoder.valid) {
  96                 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
  97                 /* unique error code to indicate no HDM decoder capability */
  98                 return -ENODEV;
  99         }
 100
 101         return cxl_map_component_regs(&map, regs, BIT(CXL_CM_CAP_CAP_ID_HDM));
 102 }
 103
 104 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
 105 {
 106         struct cxl_hdm *cxlhdm;
 107         void __iomem *hdm;
 108         u32 ctrl;
 109         int i;
 110
 111         if (!info)
 112                 return false;
 113
 114         cxlhdm = dev_get_drvdata(&info->port->dev);
 115         hdm = cxlhdm->regs.hdm_decoder;
 116
 117         if (!hdm)
 118                 return true;
 119
 120         /*
 121          * If HDM decoders are present and the driver is in control of
 122          * Mem_Enable skip DVSEC based emulation
 123          */
 124         if (!info->mem_enabled)
 125                 return false;
 126
 127         /*
 128          * If any decoders are committed already, there should not be any
 129          * emulated DVSEC decoders.
 130          */
 131         for (i = 0; i < cxlhdm->decoder_count; i++) {
 132                 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
 133                 dev_dbg(&info->port->dev,
 134                         "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
 135                         info->port->id, i,
 136                         FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
 137                         readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
 138                         readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
 139                         readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
 140                         readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
 141                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
 142                         return false;
 143         }
 144
 145         return true;
 146 }
 147
 148 /**
 149  * devm_cxl_setup_hdm - map HDM decoder component registers
 150  * @port: cxl_port to map
 151  * @info: cached DVSEC range register info
 152  */
 153 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
 154                                    struct cxl_endpoint_dvsec_info *info)
 155 {
 156         struct device *dev = &port->dev;
 157         struct cxl_hdm *cxlhdm;
 158         void __iomem *crb;
 159         int rc;
 160
 161         cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
 162         if (!cxlhdm)
 163                 return ERR_PTR(-ENOMEM);
 164         cxlhdm->port = port;
 165         dev_set_drvdata(dev, cxlhdm);
 166
 167         crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE);
 168         if (!crb && info && info->mem_enabled) {
 169                 cxlhdm->decoder_count = info->ranges;
 170                 return cxlhdm;
 171         } else if (!crb) {
 172                 dev_err(dev, "No component registers mapped\n");
 173                 return ERR_PTR(-ENXIO);
 174         }
 175
 176         rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs);
 177         iounmap(crb);
 178         if (rc)
 179                 return ERR_PTR(rc);
 180
 181         parse_hdm_decoder_caps(cxlhdm);
 182         if (cxlhdm->decoder_count == 0) {
 183                 dev_err(dev, "Spec violation. Caps invalid\n");
 184                 return ERR_PTR(-ENXIO);
 185         }
 186
 187         /*
 188          * Now that the hdm capability is parsed, decide if range
 189          * register emulation is needed and fixup cxlhdm accordingly.
 190          */
 191         if (should_emulate_decoders(info)) {
 192                 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
 193                         info->ranges > 1 ? "s" : "");
 194                 cxlhdm->decoder_count = info->ranges;
 195         }
 196
 197         return cxlhdm;
 198 }
 199 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
 200
 201 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
 202 {
 203         unsigned long long start = r->start, end = r->end;
 204
 205         seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
 206                    r->name);
 207 }
 208
 209 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
 210 {
 211         struct resource *p1, *p2;
 212
 213         down_read(&cxl_dpa_rwsem);
 214         for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
 215                 __cxl_dpa_debug(file, p1, 0);
 216                 for (p2 = p1->child; p2; p2 = p2->sibling)
 217                         __cxl_dpa_debug(file, p2, 1);
 218         }
 219         up_read(&cxl_dpa_rwsem);
 220 }
 221 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
 222
 223 /*
 224  * Must be called in a context that synchronizes against this decoder's
 225  * port ->remove() callback (like an endpoint decoder sysfs attribute)
 226  */
 227 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
 228 {
 229         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 230         struct cxl_port *port = cxled_to_port(cxled);
 231         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 232         struct resource *res = cxled->dpa_res;
 233         resource_size_t skip_start;
 234
 235         lockdep_assert_held_write(&cxl_dpa_rwsem);
 236
 237         /* save @skip_start, before @res is released */
 238         skip_start = res->start - cxled->skip;
 239         __release_region(&cxlds->dpa_res, res->start, resource_size(res));
 240         if (cxled->skip)
 241                 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
 242         cxled->skip = 0;
 243         cxled->dpa_res = NULL;
 244         put_device(&cxled->cxld.dev);
 245         port->hdm_end--;
 246 }
 247
 248 static void cxl_dpa_release(void *cxled)
 249 {
 250         down_write(&cxl_dpa_rwsem);
 251         __cxl_dpa_release(cxled);
 252         up_write(&cxl_dpa_rwsem);
 253 }
 254
 255 /*
 256  * Must be called from context that will not race port device
 257  * unregistration, like decoder sysfs attribute methods
 258  */
 259 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
 260 {
 261         struct cxl_port *port = cxled_to_port(cxled);
 262
 263         lockdep_assert_held_write(&cxl_dpa_rwsem);
 264         devm_remove_action(&port->dev, cxl_dpa_release, cxled);
 265         __cxl_dpa_release(cxled);
 266 }
 267
 268 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 269                              resource_size_t base, resource_size_t len,
 270                              resource_size_t skipped)
 271 {
 272         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 273         struct cxl_port *port = cxled_to_port(cxled);
 274         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 275         struct device *dev = &port->dev;
 276         struct resource *res;
 277
 278         lockdep_assert_held_write(&cxl_dpa_rwsem);
 279
 280         if (!len) {
 281                 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
 282                          port->id, cxled->cxld.id);
 283                 return -EINVAL;
 284         }
 285
 286         if (cxled->dpa_res) {
 287                 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
 288                         port->id, cxled->cxld.id, cxled->dpa_res);
 289                 return -EBUSY;
 290         }
 291
 292         if (port->hdm_end + 1 != cxled->cxld.id) {
 293                 /*
 294                  * Assumes alloc and commit order is always in hardware instance
 295                  * order per expectations from 8.2.5.12.20 Committing Decoder
 296                  * Programming that enforce decoder[m] committed before
 297                  * decoder[m+1] commit start.
 298                  */
 299                 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
 300                         cxled->cxld.id, port->id, port->hdm_end + 1);
 301                 return -EBUSY;
 302         }
 303
 304         if (skipped) {
 305                 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
 306                                        dev_name(&cxled->cxld.dev), 0);
 307                 if (!res) {
 308                         dev_dbg(dev,
 309                                 "decoder%d.%d: failed to reserve skipped space\n",
 310                                 port->id, cxled->cxld.id);
 311                         return -EBUSY;
 312                 }
 313         }
 314         res = __request_region(&cxlds->dpa_res, base, len,
 315                                dev_name(&cxled->cxld.dev), 0);
 316         if (!res) {
 317                 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
 318                         port->id, cxled->cxld.id);
 319                 if (skipped)
 320                         __release_region(&cxlds->dpa_res, base - skipped,
 321                                          skipped);
 322                 return -EBUSY;
 323         }
 324         cxled->dpa_res = res;
 325         cxled->skip = skipped;
 326
 327         if (resource_contains(&cxlds->pmem_res, res))
 328                 cxled->mode = CXL_DECODER_PMEM;
 329         else if (resource_contains(&cxlds->ram_res, res))
 330                 cxled->mode = CXL_DECODER_RAM;
 331         else {
 332                 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id,
 333                         cxled->cxld.id, cxled->dpa_res);
 334                 cxled->mode = CXL_DECODER_MIXED;
 335         }
 336
 337         port->hdm_end++;
 338         get_device(&cxled->cxld.dev);
 339         return 0;
 340 }
 341
 342 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 343                                 resource_size_t base, resource_size_t len,
 344                                 resource_size_t skipped)
 345 {
 346         struct cxl_port *port = cxled_to_port(cxled);
 347         int rc;
 348
 349         down_write(&cxl_dpa_rwsem);
 350         rc = __cxl_dpa_reserve(cxled, base, len, skipped);
 351         up_write(&cxl_dpa_rwsem);
 352
 353         if (rc)
 354                 return rc;
 355
 356         return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
 357 }
 358 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL);
 359
 360 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
 361 {
 362         resource_size_t size = 0;
 363
 364         down_read(&cxl_dpa_rwsem);
 365         if (cxled->dpa_res)
 366                 size = resource_size(cxled->dpa_res);
 367         up_read(&cxl_dpa_rwsem);
 368
 369         return size;
 370 }
 371
 372 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
 373 {
 374         resource_size_t base = -1;
 375
 376         down_read(&cxl_dpa_rwsem);
 377         if (cxled->dpa_res)
 378                 base = cxled->dpa_res->start;
 379         up_read(&cxl_dpa_rwsem);
 380
 381         return base;
 382 }
 383
 384 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
 385 {
 386         struct cxl_port *port = cxled_to_port(cxled);
 387         struct device *dev = &cxled->cxld.dev;
 388         int rc;
 389
 390         down_write(&cxl_dpa_rwsem);
 391         if (!cxled->dpa_res) {
 392                 rc = 0;
 393                 goto out;
 394         }
 395         if (cxled->cxld.region) {
 396                 dev_dbg(dev, "decoder assigned to: %s\n",
 397                         dev_name(&cxled->cxld.region->dev));
 398                 rc = -EBUSY;
 399                 goto out;
 400         }
 401         if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
 402                 dev_dbg(dev, "decoder enabled\n");
 403                 rc = -EBUSY;
 404                 goto out;
 405         }
 406         if (cxled->cxld.id != port->hdm_end) {
 407                 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
 408                         port->hdm_end);
 409                 rc = -EBUSY;
 410                 goto out;
 411         }
 412         devm_cxl_dpa_release(cxled);
 413         rc = 0;
 414 out:
 415         up_write(&cxl_dpa_rwsem);
 416         return rc;
 417 }
 418
 419 int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
 420                      enum cxl_decoder_mode mode)
 421 {
 422         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 423         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 424         struct device *dev = &cxled->cxld.dev;
 425         int rc;
 426
 427         switch (mode) {
 428         case CXL_DECODER_RAM:
 429         case CXL_DECODER_PMEM:
 430                 break;
 431         default:
 432                 dev_dbg(dev, "unsupported mode: %d\n", mode);
 433                 return -EINVAL;
 434         }
 435
 436         down_write(&cxl_dpa_rwsem);
 437         if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
 438                 rc = -EBUSY;
 439                 goto out;
 440         }
 441
 442         /*
 443          * Only allow modes that are supported by the current partition
 444          * configuration
 445          */
 446         if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
 447                 dev_dbg(dev, "no available pmem capacity\n");
 448                 rc = -ENXIO;
 449                 goto out;
 450         }
 451         if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
 452                 dev_dbg(dev, "no available ram capacity\n");
 453                 rc = -ENXIO;
 454                 goto out;
 455         }
 456
 457         cxled->mode = mode;
 458         rc = 0;
 459 out:
 460         up_write(&cxl_dpa_rwsem);
 461
 462         return rc;
 463 }
 464
 465 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
 466 {
 467         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 468         resource_size_t free_ram_start, free_pmem_start;
 469         struct cxl_port *port = cxled_to_port(cxled);
 470         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 471         struct device *dev = &cxled->cxld.dev;
 472         resource_size_t start, avail, skip;
 473         struct resource *p, *last;
 474         int rc;
 475
 476         down_write(&cxl_dpa_rwsem);
 477         if (cxled->cxld.region) {
 478                 dev_dbg(dev, "decoder attached to %s\n",
 479                         dev_name(&cxled->cxld.region->dev));
 480                 rc = -EBUSY;
 481                 goto out;
 482         }
 483
 484         if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
 485                 dev_dbg(dev, "decoder enabled\n");
 486                 rc = -EBUSY;
 487                 goto out;
 488         }
 489
 490         for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
 491                 last = p;
 492         if (last)
 493                 free_ram_start = last->end + 1;
 494         else
 495                 free_ram_start = cxlds->ram_res.start;
 496
 497         for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
 498                 last = p;
 499         if (last)
 500                 free_pmem_start = last->end + 1;
 501         else
 502                 free_pmem_start = cxlds->pmem_res.start;
 503
 504         if (cxled->mode == CXL_DECODER_RAM) {
 505                 start = free_ram_start;
 506                 avail = cxlds->ram_res.end - start + 1;
 507                 skip = 0;
 508         } else if (cxled->mode == CXL_DECODER_PMEM) {
 509                 resource_size_t skip_start, skip_end;
 510
 511                 start = free_pmem_start;
 512                 avail = cxlds->pmem_res.end - start + 1;
 513                 skip_start = free_ram_start;
 514
 515                 /*
 516                  * If some pmem is already allocated, then that allocation
 517                  * already handled the skip.
 518                  */
 519                 if (cxlds->pmem_res.child &&
 520                     skip_start == cxlds->pmem_res.child->start)
 521                         skip_end = skip_start - 1;
 522                 else
 523                         skip_end = start - 1;
 524                 skip = skip_end - skip_start + 1;
 525         } else {
 526                 dev_dbg(dev, "mode not set\n");
 527                 rc = -EINVAL;
 528                 goto out;
 529         }
 530
 531         if (size > avail) {
 532                 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
 533                         cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
 534                         &avail);
 535                 rc = -ENOSPC;
 536                 goto out;
 537         }
 538
 539         rc = __cxl_dpa_reserve(cxled, start, size, skip);
 540 out:
 541         up_write(&cxl_dpa_rwsem);
 542
 543         if (rc)
 544                 return rc;
 545
 546         return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
 547 }
 548
 549 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
 550 {
 551         u16 eig;
 552         u8 eiw;
 553
 554         /*
 555          * Input validation ensures these warns never fire, but otherwise
 556          * suppress unititalized variable usage warnings.
 557          */
 558         if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
 559                       "invalid interleave_ways: %d\n", cxld->interleave_ways))
 560                 return;
 561         if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
 562                       "invalid interleave_granularity: %d\n",
 563                       cxld->interleave_granularity))
 564                 return;
 565
 566         u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
 567         u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
 568         *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
 569 }
 570
 571 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
 572 {
 573         u32p_replace_bits(ctrl,
 574                           !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
 575                           CXL_HDM_DECODER0_CTRL_HOSTONLY);
 576 }
 577
 578 static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
 579 {
 580         struct cxl_dport **t = &cxlsd->target[0];
 581         int ways = cxlsd->cxld.interleave_ways;
 582
 583         if (dev_WARN_ONCE(&cxlsd->cxld.dev,
 584                           ways > 8 || ways > cxlsd->nr_targets,
 585                           "ways: %d overflows targets: %d\n", ways,
 586                           cxlsd->nr_targets))
 587                 return -ENXIO;
 588
 589         *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
 590         if (ways > 1)
 591                 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
 592         if (ways > 2)
 593                 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
 594         if (ways > 3)
 595                 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
 596         if (ways > 4)
 597                 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
 598         if (ways > 5)
 599                 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
 600         if (ways > 6)
 601                 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
 602         if (ways > 7)
 603                 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
 604
 605         return 0;
 606 }
 607
 608 /*
 609  * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
 610  * committed or error within 10ms, but just be generous with 20ms to account for
 611  * clock skew and other marginal behavior
 612  */
 613 #define COMMIT_TIMEOUT_MS 20
 614 static int cxld_await_commit(void __iomem *hdm, int id)
 615 {
 616         u32 ctrl;
 617         int i;
 618
 619         for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
 620                 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 621                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
 622                         ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
 623                         writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 624                         return -EIO;
 625                 }
 626                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
 627                         return 0;
 628                 fsleep(1000);
 629         }
 630
 631         return -ETIMEDOUT;
 632 }
 633
 634 static int cxl_decoder_commit(struct cxl_decoder *cxld)
 635 {
 636         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 637         struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
 638         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 639         int id = cxld->id, rc;
 640         u64 base, size;
 641         u32 ctrl;
 642
 643         if (cxld->flags & CXL_DECODER_F_ENABLE)
 644                 return 0;
 645
 646         if (port->commit_end + 1 != id) {
 647                 dev_dbg(&port->dev,
 648                         "%s: out of order commit, expected decoder%d.%d\n",
 649                         dev_name(&cxld->dev), port->id, port->commit_end + 1);
 650                 return -EBUSY;
 651         }
 652
 653         down_read(&cxl_dpa_rwsem);
 654         /* common decoder settings */
 655         ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
 656         cxld_set_interleave(cxld, &ctrl);
 657         cxld_set_type(cxld, &ctrl);
 658         base = cxld->hpa_range.start;
 659         size = range_len(&cxld->hpa_range);
 660
 661         writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
 662         writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
 663         writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
 664         writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
 665
 666         if (is_switch_decoder(&cxld->dev)) {
 667                 struct cxl_switch_decoder *cxlsd =
 668                         to_cxl_switch_decoder(&cxld->dev);
 669                 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
 670                 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
 671                 u64 targets;
 672
 673                 rc = cxlsd_set_targets(cxlsd, &targets);
 674                 if (rc) {
 675                         dev_dbg(&port->dev, "%s: target configuration error\n",
 676                                 dev_name(&cxld->dev));
 677                         goto err;
 678                 }
 679
 680                 writel(upper_32_bits(targets), tl_hi);
 681                 writel(lower_32_bits(targets), tl_lo);
 682         } else {
 683                 struct cxl_endpoint_decoder *cxled =
 684                         to_cxl_endpoint_decoder(&cxld->dev);
 685                 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
 686                 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
 687
 688                 writel(upper_32_bits(cxled->skip), sk_hi);
 689                 writel(lower_32_bits(cxled->skip), sk_lo);
 690         }
 691
 692         writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 693         up_read(&cxl_dpa_rwsem);
 694
 695         port->commit_end++;
 696         rc = cxld_await_commit(hdm, cxld->id);
 697 err:
 698         if (rc) {
 699                 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
 700                         dev_name(&cxld->dev), rc);
 701                 cxld->reset(cxld);
 702                 return rc;
 703         }
 704         cxld->flags |= CXL_DECODER_F_ENABLE;
 705
 706         return 0;
 707 }
 708
 709 static int cxl_decoder_reset(struct cxl_decoder *cxld)
 710 {
 711         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 712         struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
 713         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 714         int id = cxld->id;
 715         u32 ctrl;
 716
 717         if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
 718                 return 0;
 719
 720         if (port->commit_end != id) {
 721                 dev_dbg(&port->dev,
 722                         "%s: out of order reset, expected decoder%d.%d\n",
 723                         dev_name(&cxld->dev), port->id, port->commit_end);
 724                 return -EBUSY;
 725         }
 726
 727         down_read(&cxl_dpa_rwsem);
 728         ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 729         ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
 730         writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 731
 732         writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
 733         writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
 734         writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
 735         writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
 736         up_read(&cxl_dpa_rwsem);
 737
 738         port->commit_end--;
 739         cxld->flags &= ~CXL_DECODER_F_ENABLE;
 740
 741         /* Userspace is now responsible for reconfiguring this decoder */
 742         if (is_endpoint_decoder(&cxld->dev)) {
 743                 struct cxl_endpoint_decoder *cxled;
 744
 745                 cxled = to_cxl_endpoint_decoder(&cxld->dev);
 746                 cxled->state = CXL_DECODER_STATE_MANUAL;
 747         }
 748
 749         return 0;
 750 }
 751
 752 static int cxl_setup_hdm_decoder_from_dvsec(
 753         struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
 754         int which, struct cxl_endpoint_dvsec_info *info)
 755 {
 756         struct cxl_endpoint_decoder *cxled;
 757         u64 len;
 758         int rc;
 759
 760         if (!is_cxl_endpoint(port))
 761                 return -EOPNOTSUPP;
 762
 763         cxled = to_cxl_endpoint_decoder(&cxld->dev);
 764         len = range_len(&info->dvsec_range[which]);
 765         if (!len)
 766                 return -ENOENT;
 767
 768         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 769         cxld->commit = NULL;
 770         cxld->reset = NULL;
 771         cxld->hpa_range = info->dvsec_range[which];
 772
 773         /*
 774          * Set the emulated decoder as locked pending additional support to
 775          * change the range registers at run time.
 776          */
 777         cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
 778         port->commit_end = cxld->id;
 779
 780         rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
 781         if (rc) {
 782                 dev_err(&port->dev,
 783                         "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
 784                         port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
 785                 return rc;
 786         }
 787         *dpa_base += len;
 788         cxled->state = CXL_DECODER_STATE_AUTO;
 789
 790         return 0;
 791 }
 792
 793 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
 794                             int *target_map, void __iomem *hdm, int which,
 795                             u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
 796 {
 797         struct cxl_endpoint_decoder *cxled = NULL;
 798         u64 size, base, skip, dpa_size, lo, hi;
 799         bool committed;
 800         u32 remainder;
 801         int i, rc;
 802         u32 ctrl;
 803         union {
 804                 u64 value;
 805                 unsigned char target_id[8];
 806         } target_list;
 807
 808         if (should_emulate_decoders(info))
 809                 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
 810                                                         which, info);
 811
 812         ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
 813         lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
 814         hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
 815         base = (hi << 32) + lo;
 816         lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
 817         hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
 818         size = (hi << 32) + lo;
 819         committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
 820         cxld->commit = cxl_decoder_commit;
 821         cxld->reset = cxl_decoder_reset;
 822
 823         if (!committed)
 824                 size = 0;
 825         if (base == U64_MAX || size == U64_MAX) {
 826                 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
 827                          port->id, cxld->id);
 828                 return -ENXIO;
 829         }
 830
 831         if (info)
 832                 cxled = to_cxl_endpoint_decoder(&cxld->dev);
 833         cxld->hpa_range = (struct range) {
 834                 .start = base,
 835                 .end = base + size - 1,
 836         };
 837
 838         /* decoders are enabled if committed */
 839         if (committed) {
 840                 cxld->flags |= CXL_DECODER_F_ENABLE;
 841                 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
 842                         cxld->flags |= CXL_DECODER_F_LOCK;
 843                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
 844                         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 845                 else
 846                         cxld->target_type = CXL_DECODER_DEVMEM;
 847                 if (cxld->id != port->commit_end + 1) {
 848                         dev_warn(&port->dev,
 849                                  "decoder%d.%d: Committed out of order\n",
 850                                  port->id, cxld->id);
 851                         return -ENXIO;
 852                 }
 853
 854                 if (size == 0) {
 855                         dev_warn(&port->dev,
 856                                  "decoder%d.%d: Committed with zero size\n",
 857                                  port->id, cxld->id);
 858                         return -ENXIO;
 859                 }
 860                 port->commit_end = cxld->id;
 861         } else {
 862                 if (cxled) {
 863                         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 864                         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 865
 866                         /*
 867                          * Default by devtype until a device arrives that needs
 868                          * more precision.
 869                          */
 870                         if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
 871                                 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 872                         else
 873                                 cxld->target_type = CXL_DECODER_DEVMEM;
 874                 } else {
 875                         /* To be overridden by region type at commit time */
 876                         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 877                 }
 878
 879                 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
 880                     cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
 881                         ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
 882                         writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
 883                 }
 884         }
 885         rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
 886                           &cxld->interleave_ways);
 887         if (rc) {
 888                 dev_warn(&port->dev,
 889                          "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
 890                          port->id, cxld->id, ctrl);
 891                 return rc;
 892         }
 893         rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
 894                                  &cxld->interleave_granularity);
 895         if (rc)
 896                 return rc;
 897
 898         dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
 899                 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
 900                 cxld->interleave_ways, cxld->interleave_granularity);
 901
 902         if (!cxled) {
 903                 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
 904                 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
 905                 target_list.value = (hi << 32) + lo;
 906                 for (i = 0; i < cxld->interleave_ways; i++)
 907                         target_map[i] = target_list.target_id[i];
 908
 909                 return 0;
 910         }
 911
 912         if (!committed)
 913                 return 0;
 914
 915         dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
 916         if (remainder) {
 917                 dev_err(&port->dev,
 918                         "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
 919                         port->id, cxld->id, size, cxld->interleave_ways);
 920                 return -ENXIO;
 921         }
 922         lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
 923         hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
 924         skip = (hi << 32) + lo;
 925         rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
 926         if (rc) {
 927                 dev_err(&port->dev,
 928                         "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
 929                         port->id, cxld->id, *dpa_base,
 930                         *dpa_base + dpa_size + skip - 1, rc);
 931                 return rc;
 932         }
 933         *dpa_base += dpa_size + skip;
 934
 935         cxled->state = CXL_DECODER_STATE_AUTO;
 936
 937         return 0;
 938 }
 939
 940 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
 941 {
 942         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 943         int committed, i;
 944         u32 ctrl;
 945
 946         if (!hdm)
 947                 return;
 948
 949         /*
 950          * Since the register resource was recently claimed via request_region()
 951          * be careful about trusting the "not-committed" status until the commit
 952          * timeout has elapsed.  The commit timeout is 10ms (CXL 2.0
 953          * 8.2.5.12.20), but double it to be tolerant of any clock skew between
 954          * host and target.
 955          */
 956         for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
 957                 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
 958                 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
 959                         committed++;
 960         }
 961
 962         /* ensure that future checks of committed can be trusted */
 963         if (committed != cxlhdm->decoder_count)
 964                 msleep(20);
 965 }
 966
 967 /**
 968  * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
 969  * @cxlhdm: Structure to populate with HDM capabilities
 970  * @info: cached DVSEC range register info
 971  */
 972 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
 973                                 struct cxl_endpoint_dvsec_info *info)
 974 {
 975         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 976         struct cxl_port *port = cxlhdm->port;
 977         int i;
 978         u64 dpa_base = 0;
 979
 980         cxl_settle_decoders(cxlhdm);
 981
 982         for (i = 0; i < cxlhdm->decoder_count; i++) {
 983                 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
 984                 int rc, target_count = cxlhdm->target_count;
 985                 struct cxl_decoder *cxld;
 986
 987                 if (is_cxl_endpoint(port)) {
 988                         struct cxl_endpoint_decoder *cxled;
 989
 990                         cxled = cxl_endpoint_decoder_alloc(port);
 991                         if (IS_ERR(cxled)) {
 992                                 dev_warn(&port->dev,
 993                                          "Failed to allocate decoder%d.%d\n",
 994                                          port->id, i);
 995                                 return PTR_ERR(cxled);
 996                         }
 997                         cxld = &cxled->cxld;
 998                 } else {
 999                         struct cxl_switch_decoder *cxlsd;
1000
1001                         cxlsd = cxl_switch_decoder_alloc(port, target_count);
1002                         if (IS_ERR(cxlsd)) {
1003                                 dev_warn(&port->dev,
1004                                          "Failed to allocate decoder%d.%d\n",
1005                                          port->id, i);
1006                                 return PTR_ERR(cxlsd);
1007                         }
1008                         cxld = &cxlsd->cxld;
1009                 }
1010
1011                 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,
1012                                       &dpa_base, info);
1013                 if (rc) {
1014                         dev_warn(&port->dev,
1015                                  "Failed to initialize decoder%d.%d\n",
1016                                  port->id, i);
1017                         put_device(&cxld->dev);
1018                         return rc;
1019                 }
1020                 rc = add_hdm_decoder(port, cxld, target_map);
1021                 if (rc) {
1022                         dev_warn(&port->dev,
1023                                  "Failed to add decoder%d.%d\n", port->id, i);
1024                         return rc;
1025                 }
1026         }
1027
1028         return 0;
1029 }
1030 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);