git.itanic.dy.fi Git - linux-stable/blob - drivers/cxl/core/hdm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
   3 #include <linux/seq_file.h>
   4 #include <linux/device.h>
   5 #include <linux/delay.h>
   6
   7 #include "cxlmem.h"
   8 #include "core.h"
   9
  10 /**
  11  * DOC: cxl core hdm
  12  *
  13  * Compute Express Link Host Managed Device Memory, starting with the
  14  * CXL 2.0 specification, is managed by an array of HDM Decoder register
  15  * instances per CXL port and per CXL endpoint. Define common helpers
  16  * for enumerating these registers and capabilities.
  17  */
  18
  19 DECLARE_RWSEM(cxl_dpa_rwsem);
  20
  21 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
  22                            int *target_map)
  23 {
  24         int rc;
  25
  26         rc = cxl_decoder_add_locked(cxld, target_map);
  27         if (rc) {
  28                 put_device(&cxld->dev);
  29                 dev_err(&port->dev, "Failed to add decoder\n");
  30                 return rc;
  31         }
  32
  33         rc = cxl_decoder_autoremove(&port->dev, cxld);
  34         if (rc)
  35                 return rc;
  36
  37         dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
  38
  39         return 0;
  40 }
  41
  42 /*
  43  * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
  44  * single ported host-bridges need not publish a decoder capability when a
  45  * passthrough decode can be assumed, i.e. all transactions that the uport sees
  46  * are claimed and passed to the single dport. Disable the range until the first
  47  * CXL region is enumerated / activated.
  48  */
  49 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
  50 {
  51         struct cxl_switch_decoder *cxlsd;
  52         struct cxl_dport *dport = NULL;
  53         int single_port_map[1];
  54         unsigned long index;
  55
  56         cxlsd = cxl_switch_decoder_alloc(port, 1);
  57         if (IS_ERR(cxlsd))
  58                 return PTR_ERR(cxlsd);
  59
  60         device_lock_assert(&port->dev);
  61
  62         xa_for_each(&port->dports, index, dport)
  63                 break;
  64         single_port_map[0] = dport->port_id;
  65
  66         return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
  67 }
  68 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
  69
  70 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
  71 {
  72         u32 hdm_cap;
  73
  74         hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
  75         cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
  76         cxlhdm->target_count =
  77                 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
  78         if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
  79                 cxlhdm->interleave_mask |= GENMASK(11, 8);
  80         if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
  81                 cxlhdm->interleave_mask |= GENMASK(14, 12);
  82 }
  83
  84 static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb,
  85                                 struct cxl_component_regs *regs)
  86 {
  87         struct cxl_register_map map = {
  88                 .resource = port->component_reg_phys,
  89                 .base = crb,
  90                 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
  91         };
  92
  93         cxl_probe_component_regs(&port->dev, crb, &map.component_map);
  94         if (!map.component_map.hdm_decoder.valid) {
  95                 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
  96                 /* unique error code to indicate no HDM decoder capability */
  97                 return -ENODEV;
  98         }
  99
 100         return cxl_map_component_regs(&port->dev, regs, &map,
 101                                       BIT(CXL_CM_CAP_CAP_ID_HDM));
 102 }
 103
 104 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
 105 {
 106         struct cxl_hdm *cxlhdm;
 107         void __iomem *hdm;
 108         u32 ctrl;
 109         int i;
 110
 111         if (!info)
 112                 return false;
 113
 114         cxlhdm = dev_get_drvdata(&info->port->dev);
 115         hdm = cxlhdm->regs.hdm_decoder;
 116
 117         if (!hdm)
 118                 return true;
 119
 120         /*
 121          * If HDM decoders are present and the driver is in control of
 122          * Mem_Enable skip DVSEC based emulation
 123          */
 124         if (!info->mem_enabled)
 125                 return false;
 126
 127         /*
 128          * If any decoders are committed already, there should not be any
 129          * emulated DVSEC decoders.
 130          */
 131         for (i = 0; i < cxlhdm->decoder_count; i++) {
 132                 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
 133                 dev_dbg(&info->port->dev,
 134                         "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
 135                         info->port->id, i,
 136                         FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
 137                         readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
 138                         readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
 139                         readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
 140                         readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
 141                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
 142                         return false;
 143         }
 144
 145         return true;
 146 }
 147
 148 /**
 149  * devm_cxl_setup_hdm - map HDM decoder component registers
 150  * @port: cxl_port to map
 151  * @info: cached DVSEC range register info
 152  */
 153 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
 154                                    struct cxl_endpoint_dvsec_info *info)
 155 {
 156         struct device *dev = &port->dev;
 157         struct cxl_hdm *cxlhdm;
 158         void __iomem *crb;
 159         int rc;
 160
 161         cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
 162         if (!cxlhdm)
 163                 return ERR_PTR(-ENOMEM);
 164         cxlhdm->port = port;
 165         dev_set_drvdata(dev, cxlhdm);
 166
 167         crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE);
 168         if (!crb && info && info->mem_enabled) {
 169                 cxlhdm->decoder_count = info->ranges;
 170                 return cxlhdm;
 171         } else if (!crb) {
 172                 dev_err(dev, "No component registers mapped\n");
 173                 return ERR_PTR(-ENXIO);
 174         }
 175
 176         rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs);
 177         iounmap(crb);
 178         if (rc)
 179                 return ERR_PTR(rc);
 180
 181         parse_hdm_decoder_caps(cxlhdm);
 182         if (cxlhdm->decoder_count == 0) {
 183                 dev_err(dev, "Spec violation. Caps invalid\n");
 184                 return ERR_PTR(-ENXIO);
 185         }
 186
 187         /*
 188          * Now that the hdm capability is parsed, decide if range
 189          * register emulation is needed and fixup cxlhdm accordingly.
 190          */
 191         if (should_emulate_decoders(info)) {
 192                 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
 193                         info->ranges > 1 ? "s" : "");
 194                 cxlhdm->decoder_count = info->ranges;
 195         }
 196
 197         return cxlhdm;
 198 }
 199 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
 200
 201 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
 202 {
 203         unsigned long long start = r->start, end = r->end;
 204
 205         seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
 206                    r->name);
 207 }
 208
 209 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
 210 {
 211         struct resource *p1, *p2;
 212
 213         down_read(&cxl_dpa_rwsem);
 214         for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
 215                 __cxl_dpa_debug(file, p1, 0);
 216                 for (p2 = p1->child; p2; p2 = p2->sibling)
 217                         __cxl_dpa_debug(file, p2, 1);
 218         }
 219         up_read(&cxl_dpa_rwsem);
 220 }
 221 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
 222
 223 /*
 224  * Must be called in a context that synchronizes against this decoder's
 225  * port ->remove() callback (like an endpoint decoder sysfs attribute)
 226  */
 227 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
 228 {
 229         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 230         struct cxl_port *port = cxled_to_port(cxled);
 231         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 232         struct resource *res = cxled->dpa_res;
 233         resource_size_t skip_start;
 234
 235         lockdep_assert_held_write(&cxl_dpa_rwsem);
 236
 237         /* save @skip_start, before @res is released */
 238         skip_start = res->start - cxled->skip;
 239         __release_region(&cxlds->dpa_res, res->start, resource_size(res));
 240         if (cxled->skip)
 241                 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
 242         cxled->skip = 0;
 243         cxled->dpa_res = NULL;
 244         put_device(&cxled->cxld.dev);
 245         port->hdm_end--;
 246 }
 247
 248 static void cxl_dpa_release(void *cxled)
 249 {
 250         down_write(&cxl_dpa_rwsem);
 251         __cxl_dpa_release(cxled);
 252         up_write(&cxl_dpa_rwsem);
 253 }
 254
 255 /*
 256  * Must be called from context that will not race port device
 257  * unregistration, like decoder sysfs attribute methods
 258  */
 259 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
 260 {
 261         struct cxl_port *port = cxled_to_port(cxled);
 262
 263         lockdep_assert_held_write(&cxl_dpa_rwsem);
 264         devm_remove_action(&port->dev, cxl_dpa_release, cxled);
 265         __cxl_dpa_release(cxled);
 266 }
 267
 268 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 269                              resource_size_t base, resource_size_t len,
 270                              resource_size_t skipped)
 271 {
 272         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 273         struct cxl_port *port = cxled_to_port(cxled);
 274         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 275         struct device *dev = &port->dev;
 276         struct resource *res;
 277
 278         lockdep_assert_held_write(&cxl_dpa_rwsem);
 279
 280         if (!len) {
 281                 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
 282                          port->id, cxled->cxld.id);
 283                 return -EINVAL;
 284         }
 285
 286         if (cxled->dpa_res) {
 287                 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
 288                         port->id, cxled->cxld.id, cxled->dpa_res);
 289                 return -EBUSY;
 290         }
 291
 292         if (port->hdm_end + 1 != cxled->cxld.id) {
 293                 /*
 294                  * Assumes alloc and commit order is always in hardware instance
 295                  * order per expectations from 8.2.5.12.20 Committing Decoder
 296                  * Programming that enforce decoder[m] committed before
 297                  * decoder[m+1] commit start.
 298                  */
 299                 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
 300                         cxled->cxld.id, port->id, port->hdm_end + 1);
 301                 return -EBUSY;
 302         }
 303
 304         if (skipped) {
 305                 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
 306                                        dev_name(&cxled->cxld.dev), 0);
 307                 if (!res) {
 308                         dev_dbg(dev,
 309                                 "decoder%d.%d: failed to reserve skipped space\n",
 310                                 port->id, cxled->cxld.id);
 311                         return -EBUSY;
 312                 }
 313         }
 314         res = __request_region(&cxlds->dpa_res, base, len,
 315                                dev_name(&cxled->cxld.dev), 0);
 316         if (!res) {
 317                 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
 318                         port->id, cxled->cxld.id);
 319                 if (skipped)
 320                         __release_region(&cxlds->dpa_res, base - skipped,
 321                                          skipped);
 322                 return -EBUSY;
 323         }
 324         cxled->dpa_res = res;
 325         cxled->skip = skipped;
 326
 327         if (resource_contains(&cxlds->pmem_res, res))
 328                 cxled->mode = CXL_DECODER_PMEM;
 329         else if (resource_contains(&cxlds->ram_res, res))
 330                 cxled->mode = CXL_DECODER_RAM;
 331         else {
 332                 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id,
 333                         cxled->cxld.id, cxled->dpa_res);
 334                 cxled->mode = CXL_DECODER_MIXED;
 335         }
 336
 337         port->hdm_end++;
 338         get_device(&cxled->cxld.dev);
 339         return 0;
 340 }
 341
 342 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 343                                 resource_size_t base, resource_size_t len,
 344                                 resource_size_t skipped)
 345 {
 346         struct cxl_port *port = cxled_to_port(cxled);
 347         int rc;
 348
 349         down_write(&cxl_dpa_rwsem);
 350         rc = __cxl_dpa_reserve(cxled, base, len, skipped);
 351         up_write(&cxl_dpa_rwsem);
 352
 353         if (rc)
 354                 return rc;
 355
 356         return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
 357 }
 358 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL);
 359
 360 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
 361 {
 362         resource_size_t size = 0;
 363
 364         down_read(&cxl_dpa_rwsem);
 365         if (cxled->dpa_res)
 366                 size = resource_size(cxled->dpa_res);
 367         up_read(&cxl_dpa_rwsem);
 368
 369         return size;
 370 }
 371
 372 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
 373 {
 374         resource_size_t base = -1;
 375
 376         down_read(&cxl_dpa_rwsem);
 377         if (cxled->dpa_res)
 378                 base = cxled->dpa_res->start;
 379         up_read(&cxl_dpa_rwsem);
 380
 381         return base;
 382 }
 383
 384 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
 385 {
 386         struct cxl_port *port = cxled_to_port(cxled);
 387         struct device *dev = &cxled->cxld.dev;
 388         int rc;
 389
 390         down_write(&cxl_dpa_rwsem);
 391         if (!cxled->dpa_res) {
 392                 rc = 0;
 393                 goto out;
 394         }
 395         if (cxled->cxld.region) {
 396                 dev_dbg(dev, "decoder assigned to: %s\n",
 397                         dev_name(&cxled->cxld.region->dev));
 398                 rc = -EBUSY;
 399                 goto out;
 400         }
 401         if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
 402                 dev_dbg(dev, "decoder enabled\n");
 403                 rc = -EBUSY;
 404                 goto out;
 405         }
 406         if (cxled->cxld.id != port->hdm_end) {
 407                 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
 408                         port->hdm_end);
 409                 rc = -EBUSY;
 410                 goto out;
 411         }
 412         devm_cxl_dpa_release(cxled);
 413         rc = 0;
 414 out:
 415         up_write(&cxl_dpa_rwsem);
 416         return rc;
 417 }
 418
 419 int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
 420                      enum cxl_decoder_mode mode)
 421 {
 422         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 423         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 424         struct device *dev = &cxled->cxld.dev;
 425         int rc;
 426
 427         switch (mode) {
 428         case CXL_DECODER_RAM:
 429         case CXL_DECODER_PMEM:
 430                 break;
 431         default:
 432                 dev_dbg(dev, "unsupported mode: %d\n", mode);
 433                 return -EINVAL;
 434         }
 435
 436         down_write(&cxl_dpa_rwsem);
 437         if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
 438                 rc = -EBUSY;
 439                 goto out;
 440         }
 441
 442         /*
 443          * Only allow modes that are supported by the current partition
 444          * configuration
 445          */
 446         if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
 447                 dev_dbg(dev, "no available pmem capacity\n");
 448                 rc = -ENXIO;
 449                 goto out;
 450         }
 451         if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
 452                 dev_dbg(dev, "no available ram capacity\n");
 453                 rc = -ENXIO;
 454                 goto out;
 455         }
 456
 457         cxled->mode = mode;
 458         rc = 0;
 459 out:
 460         up_write(&cxl_dpa_rwsem);
 461
 462         return rc;
 463 }
 464
 465 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
 466 {
 467         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 468         resource_size_t free_ram_start, free_pmem_start;
 469         struct cxl_port *port = cxled_to_port(cxled);
 470         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 471         struct device *dev = &cxled->cxld.dev;
 472         resource_size_t start, avail, skip;
 473         struct resource *p, *last;
 474         int rc;
 475
 476         down_write(&cxl_dpa_rwsem);
 477         if (cxled->cxld.region) {
 478                 dev_dbg(dev, "decoder attached to %s\n",
 479                         dev_name(&cxled->cxld.region->dev));
 480                 rc = -EBUSY;
 481                 goto out;
 482         }
 483
 484         if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
 485                 dev_dbg(dev, "decoder enabled\n");
 486                 rc = -EBUSY;
 487                 goto out;
 488         }
 489
 490         for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
 491                 last = p;
 492         if (last)
 493                 free_ram_start = last->end + 1;
 494         else
 495                 free_ram_start = cxlds->ram_res.start;
 496
 497         for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
 498                 last = p;
 499         if (last)
 500                 free_pmem_start = last->end + 1;
 501         else
 502                 free_pmem_start = cxlds->pmem_res.start;
 503
 504         if (cxled->mode == CXL_DECODER_RAM) {
 505                 start = free_ram_start;
 506                 avail = cxlds->ram_res.end - start + 1;
 507                 skip = 0;
 508         } else if (cxled->mode == CXL_DECODER_PMEM) {
 509                 resource_size_t skip_start, skip_end;
 510
 511                 start = free_pmem_start;
 512                 avail = cxlds->pmem_res.end - start + 1;
 513                 skip_start = free_ram_start;
 514
 515                 /*
 516                  * If some pmem is already allocated, then that allocation
 517                  * already handled the skip.
 518                  */
 519                 if (cxlds->pmem_res.child &&
 520                     skip_start == cxlds->pmem_res.child->start)
 521                         skip_end = skip_start - 1;
 522                 else
 523                         skip_end = start - 1;
 524                 skip = skip_end - skip_start + 1;
 525         } else {
 526                 dev_dbg(dev, "mode not set\n");
 527                 rc = -EINVAL;
 528                 goto out;
 529         }
 530
 531         if (size > avail) {
 532                 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
 533                         cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
 534                         &avail);
 535                 rc = -ENOSPC;
 536                 goto out;
 537         }
 538
 539         rc = __cxl_dpa_reserve(cxled, start, size, skip);
 540 out:
 541         up_write(&cxl_dpa_rwsem);
 542
 543         if (rc)
 544                 return rc;
 545
 546         return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
 547 }
 548
 549 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
 550 {
 551         u16 eig;
 552         u8 eiw;
 553
 554         /*
 555          * Input validation ensures these warns never fire, but otherwise
 556          * suppress unititalized variable usage warnings.
 557          */
 558         if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
 559                       "invalid interleave_ways: %d\n", cxld->interleave_ways))
 560                 return;
 561         if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
 562                       "invalid interleave_granularity: %d\n",
 563                       cxld->interleave_granularity))
 564                 return;
 565
 566         u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
 567         u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
 568         *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
 569 }
 570
 571 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
 572 {
 573         u32p_replace_bits(ctrl, !!(cxld->target_type == 3),
 574                           CXL_HDM_DECODER0_CTRL_TYPE);
 575 }
 576
 577 static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
 578 {
 579         struct cxl_dport **t = &cxlsd->target[0];
 580         int ways = cxlsd->cxld.interleave_ways;
 581
 582         if (dev_WARN_ONCE(&cxlsd->cxld.dev,
 583                           ways > 8 || ways > cxlsd->nr_targets,
 584                           "ways: %d overflows targets: %d\n", ways,
 585                           cxlsd->nr_targets))
 586                 return -ENXIO;
 587
 588         *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
 589         if (ways > 1)
 590                 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
 591         if (ways > 2)
 592                 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
 593         if (ways > 3)
 594                 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
 595         if (ways > 4)
 596                 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
 597         if (ways > 5)
 598                 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
 599         if (ways > 6)
 600                 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
 601         if (ways > 7)
 602                 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
 603
 604         return 0;
 605 }
 606
 607 /*
 608  * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
 609  * committed or error within 10ms, but just be generous with 20ms to account for
 610  * clock skew and other marginal behavior
 611  */
 612 #define COMMIT_TIMEOUT_MS 20
 613 static int cxld_await_commit(void __iomem *hdm, int id)
 614 {
 615         u32 ctrl;
 616         int i;
 617
 618         for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
 619                 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 620                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
 621                         ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
 622                         writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 623                         return -EIO;
 624                 }
 625                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
 626                         return 0;
 627                 fsleep(1000);
 628         }
 629
 630         return -ETIMEDOUT;
 631 }
 632
 633 static int cxl_decoder_commit(struct cxl_decoder *cxld)
 634 {
 635         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 636         struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
 637         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 638         int id = cxld->id, rc;
 639         u64 base, size;
 640         u32 ctrl;
 641
 642         if (cxld->flags & CXL_DECODER_F_ENABLE)
 643                 return 0;
 644
 645         if (port->commit_end + 1 != id) {
 646                 dev_dbg(&port->dev,
 647                         "%s: out of order commit, expected decoder%d.%d\n",
 648                         dev_name(&cxld->dev), port->id, port->commit_end + 1);
 649                 return -EBUSY;
 650         }
 651
 652         down_read(&cxl_dpa_rwsem);
 653         /* common decoder settings */
 654         ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
 655         cxld_set_interleave(cxld, &ctrl);
 656         cxld_set_type(cxld, &ctrl);
 657         base = cxld->hpa_range.start;
 658         size = range_len(&cxld->hpa_range);
 659
 660         writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
 661         writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
 662         writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
 663         writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
 664
 665         if (is_switch_decoder(&cxld->dev)) {
 666                 struct cxl_switch_decoder *cxlsd =
 667                         to_cxl_switch_decoder(&cxld->dev);
 668                 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
 669                 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
 670                 u64 targets;
 671
 672                 rc = cxlsd_set_targets(cxlsd, &targets);
 673                 if (rc) {
 674                         dev_dbg(&port->dev, "%s: target configuration error\n",
 675                                 dev_name(&cxld->dev));
 676                         goto err;
 677                 }
 678
 679                 writel(upper_32_bits(targets), tl_hi);
 680                 writel(lower_32_bits(targets), tl_lo);
 681         } else {
 682                 struct cxl_endpoint_decoder *cxled =
 683                         to_cxl_endpoint_decoder(&cxld->dev);
 684                 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
 685                 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
 686
 687                 writel(upper_32_bits(cxled->skip), sk_hi);
 688                 writel(lower_32_bits(cxled->skip), sk_lo);
 689         }
 690
 691         writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 692         up_read(&cxl_dpa_rwsem);
 693
 694         port->commit_end++;
 695         rc = cxld_await_commit(hdm, cxld->id);
 696 err:
 697         if (rc) {
 698                 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
 699                         dev_name(&cxld->dev), rc);
 700                 cxld->reset(cxld);
 701                 return rc;
 702         }
 703         cxld->flags |= CXL_DECODER_F_ENABLE;
 704
 705         return 0;
 706 }
 707
 708 static int cxl_decoder_reset(struct cxl_decoder *cxld)
 709 {
 710         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 711         struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
 712         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 713         int id = cxld->id;
 714         u32 ctrl;
 715
 716         if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
 717                 return 0;
 718
 719         if (port->commit_end != id) {
 720                 dev_dbg(&port->dev,
 721                         "%s: out of order reset, expected decoder%d.%d\n",
 722                         dev_name(&cxld->dev), port->id, port->commit_end);
 723                 return -EBUSY;
 724         }
 725
 726         down_read(&cxl_dpa_rwsem);
 727         ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 728         ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
 729         writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
 730
 731         writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
 732         writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
 733         writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
 734         writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
 735         up_read(&cxl_dpa_rwsem);
 736
 737         port->commit_end--;
 738         cxld->flags &= ~CXL_DECODER_F_ENABLE;
 739
 740         /* Userspace is now responsible for reconfiguring this decoder */
 741         if (is_endpoint_decoder(&cxld->dev)) {
 742                 struct cxl_endpoint_decoder *cxled;
 743
 744                 cxled = to_cxl_endpoint_decoder(&cxld->dev);
 745                 cxled->state = CXL_DECODER_STATE_MANUAL;
 746         }
 747
 748         return 0;
 749 }
 750
 751 static int cxl_setup_hdm_decoder_from_dvsec(
 752         struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
 753         int which, struct cxl_endpoint_dvsec_info *info)
 754 {
 755         struct cxl_endpoint_decoder *cxled;
 756         u64 len;
 757         int rc;
 758
 759         if (!is_cxl_endpoint(port))
 760                 return -EOPNOTSUPP;
 761
 762         cxled = to_cxl_endpoint_decoder(&cxld->dev);
 763         len = range_len(&info->dvsec_range[which]);
 764         if (!len)
 765                 return -ENOENT;
 766
 767         cxld->target_type = CXL_DECODER_EXPANDER;
 768         cxld->commit = NULL;
 769         cxld->reset = NULL;
 770         cxld->hpa_range = info->dvsec_range[which];
 771
 772         /*
 773          * Set the emulated decoder as locked pending additional support to
 774          * change the range registers at run time.
 775          */
 776         cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
 777         port->commit_end = cxld->id;
 778
 779         rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
 780         if (rc) {
 781                 dev_err(&port->dev,
 782                         "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
 783                         port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
 784                 return rc;
 785         }
 786         *dpa_base += len;
 787         cxled->state = CXL_DECODER_STATE_AUTO;
 788
 789         return 0;
 790 }
 791
 792 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
 793                             int *target_map, void __iomem *hdm, int which,
 794                             u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
 795 {
 796         u64 size, base, skip, dpa_size, lo, hi;
 797         struct cxl_endpoint_decoder *cxled;
 798         bool committed;
 799         u32 remainder;
 800         int i, rc;
 801         u32 ctrl;
 802         union {
 803                 u64 value;
 804                 unsigned char target_id[8];
 805         } target_list;
 806
 807         if (should_emulate_decoders(info))
 808                 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
 809                                                         which, info);
 810
 811         ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
 812         lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
 813         hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
 814         base = (hi << 32) + lo;
 815         lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
 816         hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
 817         size = (hi << 32) + lo;
 818         committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
 819         cxld->commit = cxl_decoder_commit;
 820         cxld->reset = cxl_decoder_reset;
 821
 822         if (!committed)
 823                 size = 0;
 824         if (base == U64_MAX || size == U64_MAX) {
 825                 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
 826                          port->id, cxld->id);
 827                 return -ENXIO;
 828         }
 829
 830         cxld->hpa_range = (struct range) {
 831                 .start = base,
 832                 .end = base + size - 1,
 833         };
 834
 835         /* decoders are enabled if committed */
 836         if (committed) {
 837                 cxld->flags |= CXL_DECODER_F_ENABLE;
 838                 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
 839                         cxld->flags |= CXL_DECODER_F_LOCK;
 840                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
 841                         cxld->target_type = CXL_DECODER_EXPANDER;
 842                 else
 843                         cxld->target_type = CXL_DECODER_ACCELERATOR;
 844                 if (cxld->id != port->commit_end + 1) {
 845                         dev_warn(&port->dev,
 846                                  "decoder%d.%d: Committed out of order\n",
 847                                  port->id, cxld->id);
 848                         return -ENXIO;
 849                 }
 850
 851                 if (size == 0) {
 852                         dev_warn(&port->dev,
 853                                  "decoder%d.%d: Committed with zero size\n",
 854                                  port->id, cxld->id);
 855                         return -ENXIO;
 856                 }
 857                 port->commit_end = cxld->id;
 858         } else {
 859                 /* unless / until type-2 drivers arrive, assume type-3 */
 860                 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl) == 0) {
 861                         ctrl |= CXL_HDM_DECODER0_CTRL_TYPE;
 862                         writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
 863                 }
 864                 cxld->target_type = CXL_DECODER_EXPANDER;
 865         }
 866         rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
 867                           &cxld->interleave_ways);
 868         if (rc) {
 869                 dev_warn(&port->dev,
 870                          "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
 871                          port->id, cxld->id, ctrl);
 872                 return rc;
 873         }
 874         rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
 875                                  &cxld->interleave_granularity);
 876         if (rc)
 877                 return rc;
 878
 879         dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
 880                 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
 881                 cxld->interleave_ways, cxld->interleave_granularity);
 882
 883         if (!info) {
 884                 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
 885                 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
 886                 target_list.value = (hi << 32) + lo;
 887                 for (i = 0; i < cxld->interleave_ways; i++)
 888                         target_map[i] = target_list.target_id[i];
 889
 890                 return 0;
 891         }
 892
 893         if (!committed)
 894                 return 0;
 895
 896         dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
 897         if (remainder) {
 898                 dev_err(&port->dev,
 899                         "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
 900                         port->id, cxld->id, size, cxld->interleave_ways);
 901                 return -ENXIO;
 902         }
 903         lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
 904         hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
 905         skip = (hi << 32) + lo;
 906         cxled = to_cxl_endpoint_decoder(&cxld->dev);
 907         rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
 908         if (rc) {
 909                 dev_err(&port->dev,
 910                         "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
 911                         port->id, cxld->id, *dpa_base,
 912                         *dpa_base + dpa_size + skip - 1, rc);
 913                 return rc;
 914         }
 915         *dpa_base += dpa_size + skip;
 916
 917         cxled->state = CXL_DECODER_STATE_AUTO;
 918
 919         return 0;
 920 }
 921
 922 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
 923 {
 924         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 925         int committed, i;
 926         u32 ctrl;
 927
 928         if (!hdm)
 929                 return;
 930
 931         /*
 932          * Since the register resource was recently claimed via request_region()
 933          * be careful about trusting the "not-committed" status until the commit
 934          * timeout has elapsed.  The commit timeout is 10ms (CXL 2.0
 935          * 8.2.5.12.20), but double it to be tolerant of any clock skew between
 936          * host and target.
 937          */
 938         for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
 939                 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
 940                 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
 941                         committed++;
 942         }
 943
 944         /* ensure that future checks of committed can be trusted */
 945         if (committed != cxlhdm->decoder_count)
 946                 msleep(20);
 947 }
 948
 949 /**
 950  * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
 951  * @cxlhdm: Structure to populate with HDM capabilities
 952  * @info: cached DVSEC range register info
 953  */
 954 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
 955                                 struct cxl_endpoint_dvsec_info *info)
 956 {
 957         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 958         struct cxl_port *port = cxlhdm->port;
 959         int i;
 960         u64 dpa_base = 0;
 961
 962         cxl_settle_decoders(cxlhdm);
 963
 964         for (i = 0; i < cxlhdm->decoder_count; i++) {
 965                 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
 966                 int rc, target_count = cxlhdm->target_count;
 967                 struct cxl_decoder *cxld;
 968
 969                 if (is_cxl_endpoint(port)) {
 970                         struct cxl_endpoint_decoder *cxled;
 971
 972                         cxled = cxl_endpoint_decoder_alloc(port);
 973                         if (IS_ERR(cxled)) {
 974                                 dev_warn(&port->dev,
 975                                          "Failed to allocate decoder%d.%d\n",
 976                                          port->id, i);
 977                                 return PTR_ERR(cxled);
 978                         }
 979                         cxld = &cxled->cxld;
 980                 } else {
 981                         struct cxl_switch_decoder *cxlsd;
 982
 983                         cxlsd = cxl_switch_decoder_alloc(port, target_count);
 984                         if (IS_ERR(cxlsd)) {
 985                                 dev_warn(&port->dev,
 986                                          "Failed to allocate decoder%d.%d\n",
 987                                          port->id, i);
 988                                 return PTR_ERR(cxlsd);
 989                         }
 990                         cxld = &cxlsd->cxld;
 991                 }
 992
 993                 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,
 994                                       &dpa_base, info);
 995                 if (rc) {
 996                         dev_warn(&port->dev,
 997                                  "Failed to initialize decoder%d.%d\n",
 998                                  port->id, i);
 999                         put_device(&cxld->dev);
1000                         return rc;
1001                 }
1002                 rc = add_hdm_decoder(port, cxld, target_map);
1003                 if (rc) {
1004                         dev_warn(&port->dev,
1005                                  "Failed to add decoder%d.%d\n", port->id, i);
1006                         return rc;
1007                 }
1008         }
1009
1010         return 0;
1011 }
1012 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);