1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2018 Intel Corporation 3 4 #include <linux/device.h> 5 #include <linux/iopoll.h> 6 #include <linux/slab.h> 7 8 #include "ipu3.h" 9 #include "ipu3-css.h" 10 #include "ipu3-css-fw.h" 11 #include "ipu3-css-params.h" 12 #include "ipu3-dmamap.h" 13 #include "ipu3-tables.h" 14 15 /* IRQ configuration */ 16 #define IMGU_IRQCTRL_IRQ_MASK (IMGU_IRQCTRL_IRQ_SP1 | \ 17 IMGU_IRQCTRL_IRQ_SP2 | \ 18 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \ 19 IMGU_IRQCTRL_IRQ_SW_PIN(1)) 20 21 #define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */ 22 23 /* Some sane limits for resolutions */ 24 #define IPU3_CSS_MIN_RES 32 25 #define IPU3_CSS_MAX_H 3136 26 #define IPU3_CSS_MAX_W 4224 27 28 /* minimal envelope size(GDC in - out) should be 4 */ 29 #define MIN_ENVELOPE 4 30 31 /* 32 * pre-allocated buffer size for CSS ABI, auxiliary frames 33 * after BDS and before GDC. Those values should be tuned 34 * to big enough to avoid buffer re-allocation when 35 * streaming to lower streaming latency. 36 */ 37 #define CSS_ABI_SIZE 136 38 #define CSS_BDS_SIZE (4480 * 3200 * 3) 39 #define CSS_GDC_SIZE (4224 * 3200 * 12 / 8) 40 41 #define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q)) 42 #define IPU3_CSS_FORMAT_FL_IN \ 43 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN) 44 #define IPU3_CSS_FORMAT_FL_OUT \ 45 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT) 46 #define IPU3_CSS_FORMAT_FL_VF \ 47 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF) 48 49 /* Formats supported by IPU3 Camera Sub System */ 50 static const struct imgu_css_format imgu_css_formats[] = { 51 { 52 .pixelformat = V4L2_PIX_FMT_NV12, 53 .colorspace = V4L2_COLORSPACE_SRGB, 54 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12, 55 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12, 56 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE, 57 .chroma_decim = 4, 58 .width_align = IPU3_UAPI_ISP_VEC_ELEMS, 59 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF, 60 }, { 61 /* Each 32 bytes contains 25 10-bit pixels */ 62 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10, 63 .colorspace = V4L2_COLORSPACE_RAW, 64 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 65 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR, 66 .bit_depth = 10, 67 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 68 .flags = IPU3_CSS_FORMAT_FL_IN, 69 }, { 70 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10, 71 .colorspace = V4L2_COLORSPACE_RAW, 72 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 73 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG, 74 .bit_depth = 10, 75 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 76 .flags = IPU3_CSS_FORMAT_FL_IN, 77 }, { 78 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10, 79 .colorspace = V4L2_COLORSPACE_RAW, 80 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 81 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG, 82 .bit_depth = 10, 83 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 84 .flags = IPU3_CSS_FORMAT_FL_IN, 85 }, { 86 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10, 87 .colorspace = V4L2_COLORSPACE_RAW, 88 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 89 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB, 90 .bit_depth = 10, 91 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 92 .flags = IPU3_CSS_FORMAT_FL_IN, 93 }, 94 }; 95 96 static const struct { 97 enum imgu_abi_queue_id qid; 98 size_t ptr_ofs; 99 } imgu_css_queues[IPU3_CSS_QUEUES] = { 100 [IPU3_CSS_QUEUE_IN] = { 101 IMGU_ABI_QUEUE_C_ID, 102 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 103 }, 104 [IPU3_CSS_QUEUE_OUT] = { 105 IMGU_ABI_QUEUE_D_ID, 106 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 107 }, 108 [IPU3_CSS_QUEUE_VF] = { 109 IMGU_ABI_QUEUE_E_ID, 110 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 111 }, 112 [IPU3_CSS_QUEUE_STAT_3A] = { 113 IMGU_ABI_QUEUE_F_ID, 114 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr) 115 }, 116 }; 117 118 /* Initialize queue based on given format, adjust format as needed */ 119 static int imgu_css_queue_init(struct imgu_css_queue *queue, 120 struct v4l2_pix_format_mplane *fmt, u32 flags) 121 { 122 struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix; 123 unsigned int i; 124 u32 sizeimage; 125 126 INIT_LIST_HEAD(&queue->bufs); 127 128 queue->css_fmt = NULL; /* Disable */ 129 if (!fmt) 130 return 0; 131 132 for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) { 133 if (!(imgu_css_formats[i].flags & flags)) 134 continue; 135 queue->css_fmt = &imgu_css_formats[i]; 136 if (imgu_css_formats[i].pixelformat == fmt->pixelformat) 137 break; 138 } 139 if (!queue->css_fmt) 140 return -EINVAL; /* Could not find any suitable format */ 141 142 queue->fmt.mpix = *fmt; 143 144 f->width = ALIGN(clamp_t(u32, f->width, 145 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2); 146 f->height = ALIGN(clamp_t(u32, f->height, 147 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2); 148 queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align); 149 f->plane_fmt[0].bytesperline = 150 imgu_bytesperline(f->width, queue->css_fmt->frame_format); 151 sizeimage = f->height * f->plane_fmt[0].bytesperline; 152 if (queue->css_fmt->chroma_decim) 153 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim; 154 155 f->plane_fmt[0].sizeimage = sizeimage; 156 f->field = V4L2_FIELD_NONE; 157 f->num_planes = 1; 158 f->colorspace = queue->css_fmt->colorspace; 159 f->flags = 0; 160 f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; 161 f->quantization = V4L2_QUANTIZATION_DEFAULT; 162 f->xfer_func = V4L2_XFER_FUNC_DEFAULT; 163 memset(f->reserved, 0, sizeof(f->reserved)); 164 165 return 0; 166 } 167 168 static bool imgu_css_queue_enabled(struct imgu_css_queue *q) 169 { 170 return q->css_fmt; 171 } 172 173 /******************* css hw *******************/ 174 175 /* In the style of writesl() defined in include/asm-generic/io.h */ 176 static inline void writes(const void *mem, ssize_t count, void __iomem *addr) 177 { 178 if (count >= 4) { 179 const u32 *buf = mem; 180 181 count /= 4; 182 do { 183 writel(*buf++, addr); 184 addr += 4; 185 } while (--count); 186 } 187 } 188 189 /* Wait until register `reg', masked with `mask', becomes `cmp' */ 190 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp) 191 { 192 u32 val; 193 194 return readl_poll_timeout(base + reg, val, (val & mask) == cmp, 195 1000, 100 * 1000); 196 } 197 198 /* Initialize the IPU3 CSS hardware and associated h/w blocks */ 199 200 int imgu_css_set_powerup(struct device *dev, void __iomem *base, 201 unsigned int freq) 202 { 203 u32 pm_ctrl, state, val; 204 205 dev_dbg(dev, "%s with freq %u\n", __func__, freq); 206 /* Clear the CSS busy signal */ 207 readl(base + IMGU_REG_GP_BUSY); 208 writel(0, base + IMGU_REG_GP_BUSY); 209 210 /* Wait for idle signal */ 211 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 212 IMGU_STATE_IDLE_STS)) { 213 dev_err(dev, "failed to set CSS idle\n"); 214 goto fail; 215 } 216 217 /* Reset the css */ 218 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET, 219 base + IMGU_REG_PM_CTRL); 220 221 usleep_range(200, 300); 222 223 /** Prepare CSS */ 224 225 pm_ctrl = readl(base + IMGU_REG_PM_CTRL); 226 state = readl(base + IMGU_REG_STATE); 227 228 dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n", 229 pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up"); 230 231 /* Power up CSS using wrapper */ 232 if (state & IMGU_STATE_POWER_DOWN) { 233 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START, 234 base + IMGU_REG_PM_CTRL); 235 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, 236 IMGU_PM_CTRL_START, 0)) { 237 dev_err(dev, "failed to power up CSS\n"); 238 goto fail; 239 } 240 usleep_range(2000, 3000); 241 } else { 242 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL); 243 } 244 245 /* Set the busy bit */ 246 writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY); 247 248 /* Set CSS clock frequency */ 249 pm_ctrl = readl(base + IMGU_REG_PM_CTRL); 250 val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 251 writel(val, base + IMGU_REG_PM_CTRL); 252 writel(0, base + IMGU_REG_GP_BUSY); 253 if (imgu_hw_wait(base, IMGU_REG_STATE, 254 IMGU_STATE_PWRDNM_FSM_MASK, 0)) { 255 dev_err(dev, "failed to pwrdn CSS\n"); 256 goto fail; 257 } 258 val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK; 259 writel(val, base + IMGU_REG_SYSTEM_REQ); 260 writel(1, base + IMGU_REG_GP_BUSY); 261 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT, 262 base + IMGU_REG_PM_CTRL); 263 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS, 264 IMGU_STATE_HALT_STS)) { 265 dev_err(dev, "failed to halt CSS\n"); 266 goto fail; 267 } 268 269 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START, 270 base + IMGU_REG_PM_CTRL); 271 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) { 272 dev_err(dev, "failed to start CSS\n"); 273 goto fail; 274 } 275 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT, 276 base + IMGU_REG_PM_CTRL); 277 278 val = readl(base + IMGU_REG_PM_CTRL); /* get pm_ctrl */ 279 val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 280 val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 281 writel(val, base + IMGU_REG_PM_CTRL); 282 283 return 0; 284 285 fail: 286 imgu_css_set_powerdown(dev, base); 287 return -EIO; 288 } 289 290 void imgu_css_set_powerdown(struct device *dev, void __iomem *base) 291 { 292 dev_dbg(dev, "%s\n", __func__); 293 /* wait for cio idle signal */ 294 if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE, 295 IMGU_CIO_GATE_BURST_MASK, 0)) 296 dev_warn(dev, "wait cio gate idle timeout"); 297 298 /* wait for css idle signal */ 299 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 300 IMGU_STATE_IDLE_STS)) 301 dev_warn(dev, "wait css idle timeout\n"); 302 303 /* do halt-halted handshake with css */ 304 writel(1, base + IMGU_REG_GP_HALT); 305 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS, 306 IMGU_STATE_HALT_STS)) 307 dev_warn(dev, "failed to halt css"); 308 309 /* de-assert the busy bit */ 310 writel(0, base + IMGU_REG_GP_BUSY); 311 } 312 313 static void imgu_css_hw_enable_irq(struct imgu_css *css) 314 { 315 void __iomem *const base = css->base; 316 u32 val, i; 317 318 /* Set up interrupts */ 319 320 /* 321 * Enable IRQ on the SP which signals that SP goes to idle 322 * (aka ready state) and set trigger to pulse 323 */ 324 val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY; 325 writel(val, base + IMGU_REG_SP_CTRL(0)); 326 writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0)); 327 328 /* Enable IRQs from the IMGU wrapper */ 329 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE); 330 /* Clear */ 331 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS); 332 333 /* Enable IRQs from main IRQ controller */ 334 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN)); 335 writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN)); 336 writel(IMGU_IRQCTRL_IRQ_MASK, 337 base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN)); 338 writel(IMGU_IRQCTRL_IRQ_MASK, 339 base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN)); 340 writel(IMGU_IRQCTRL_IRQ_MASK, 341 base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN)); 342 writel(IMGU_IRQCTRL_IRQ_MASK, 343 base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN)); 344 /* Wait for write complete */ 345 readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN)); 346 347 /* Enable IRQs from SP0 and SP1 controllers */ 348 for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) { 349 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i)); 350 writel(0, base + IMGU_REG_IRQCTRL_MASK(i)); 351 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i)); 352 writel(IMGU_IRQCTRL_IRQ_MASK, 353 base + IMGU_REG_IRQCTRL_ENABLE(i)); 354 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i)); 355 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i)); 356 /* Wait for write complete */ 357 readl(base + IMGU_REG_IRQCTRL_ENABLE(i)); 358 } 359 } 360 361 static int imgu_css_hw_init(struct imgu_css *css) 362 { 363 /* For checking that streaming monitor statuses are valid */ 364 static const struct { 365 u32 reg; 366 u32 mask; 367 const char *name; 368 } stream_monitors[] = { 369 { 370 IMGU_REG_GP_SP1_STRMON_STAT, 371 IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP, 372 "ISP0 to SP0" 373 }, { 374 IMGU_REG_GP_ISP_STRMON_STAT, 375 IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1, 376 "SP0 to ISP0" 377 }, { 378 IMGU_REG_GP_MOD_STRMON_STAT, 379 IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA, 380 "ISP0 to DMA0" 381 }, { 382 IMGU_REG_GP_ISP_STRMON_STAT, 383 IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP, 384 "DMA0 to ISP0" 385 }, { 386 IMGU_REG_GP_MOD_STRMON_STAT, 387 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC, 388 "ISP0 to GDC0" 389 }, { 390 IMGU_REG_GP_MOD_STRMON_STAT, 391 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS, 392 "GDC0 to ISP0" 393 }, { 394 IMGU_REG_GP_MOD_STRMON_STAT, 395 IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA, 396 "SP0 to DMA0" 397 }, { 398 IMGU_REG_GP_SP1_STRMON_STAT, 399 IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1, 400 "DMA0 to SP0" 401 }, { 402 IMGU_REG_GP_MOD_STRMON_STAT, 403 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC, 404 "SP0 to GDC0" 405 }, { 406 IMGU_REG_GP_MOD_STRMON_STAT, 407 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS, 408 "GDC0 to SP0" 409 }, 410 }; 411 412 struct device *dev = css->dev; 413 void __iomem *const base = css->base; 414 u32 val, i; 415 416 /* Set instruction cache address and inv bit for ISP, SP, and SP1 */ 417 for (i = 0; i < IMGU_NUM_SP; i++) { 418 struct imgu_fw_info *bi = 419 &css->fwp->binary_header[css->fw_sp[i]]; 420 421 writel(css->binary[css->fw_sp[i]].daddr, 422 base + IMGU_REG_SP_ICACHE_ADDR(bi->type)); 423 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) | 424 IMGU_CTRL_ICACHE_INV, 425 base + IMGU_REG_SP_CTRL(bi->type)); 426 } 427 writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR); 428 writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV, 429 base + IMGU_REG_ISP_CTRL); 430 431 /* Check that IMGU hardware is ready */ 432 433 if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) { 434 dev_err(dev, "SP is not idle\n"); 435 return -EIO; 436 } 437 if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) { 438 dev_err(dev, "ISP is not idle\n"); 439 return -EIO; 440 } 441 442 for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) { 443 val = readl(base + stream_monitors[i].reg); 444 if (val & stream_monitors[i].mask) { 445 dev_err(dev, "error: Stream monitor %s is valid\n", 446 stream_monitors[i].name); 447 return -EIO; 448 } 449 } 450 451 /* Initialize GDC with default values */ 452 453 for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) { 454 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK; 455 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK; 456 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK; 457 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK; 458 459 writel(val0 | (val1 << 16), 460 base + IMGU_REG_GDC_LUT_BASE + i * 8); 461 writel(val2 | (val3 << 16), 462 base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4); 463 } 464 465 return 0; 466 } 467 468 /* Boot the given IPU3 CSS SP */ 469 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp) 470 { 471 void __iomem *const base = css->base; 472 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 473 struct imgu_abi_sp_init_dmem_cfg dmem_cfg = { 474 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr 475 + bi->blob.data_source, 476 .dmem_data_addr = bi->blob.data_target, 477 .dmem_bss_addr = bi->blob.bss_target, 478 .data_size = bi->blob.data_size, 479 .bss_size = bi->blob.bss_size, 480 .sp_id = sp, 481 }; 482 483 writes(&dmem_cfg, sizeof(dmem_cfg), base + 484 IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data); 485 486 writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp)); 487 488 writel(readl(base + IMGU_REG_SP_CTRL(sp)) 489 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp)); 490 491 if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp) 492 + bi->info.sp.sw_state, 493 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED)) 494 return -EIO; 495 496 return 0; 497 } 498 499 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */ 500 static int imgu_css_hw_start(struct imgu_css *css) 501 { 502 static const u32 event_mask = 503 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) | 504 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) | 505 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) | 506 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) | 507 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) | 508 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) | 509 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) | 510 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) | 511 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) | 512 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) | 513 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE)) 514 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT; 515 516 void __iomem *const base = css->base; 517 struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl]; 518 unsigned int i; 519 520 writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE); 521 522 /* Start bootloader */ 523 524 writel(IMGU_ABI_BL_SWSTATE_BUSY, 525 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state); 526 writel(IMGU_NUM_SP, 527 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds); 528 529 for (i = 0; i < IMGU_NUM_SP; i++) { 530 int j = IMGU_NUM_SP - i - 1; /* load sp1 first, then sp0 */ 531 struct imgu_fw_info *sp = 532 &css->fwp->binary_header[css->fw_sp[j]]; 533 struct imgu_abi_bl_dma_cmd_entry dma_cmd = { 534 .src_addr = css->binary[css->fw_sp[j]].daddr 535 + sp->blob.text_source, 536 .size = sp->blob.text_size, 537 .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM, 538 .dst_addr = IMGU_SP_PMEM_BASE(j), 539 }; 540 541 writes(&dma_cmd, sizeof(dma_cmd), 542 base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) + 543 bl->info.bl.dma_cmd_list); 544 } 545 546 writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR); 547 548 writel(readl(base + IMGU_REG_ISP_CTRL) 549 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL); 550 if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE 551 + bl->info.bl.sw_state, ~0, 552 IMGU_ABI_BL_SWSTATE_OK)) { 553 dev_err(css->dev, "failed to start bootloader\n"); 554 return -EIO; 555 } 556 557 /* Start ISP */ 558 559 memset(css->xmem_sp_group_ptrs.vaddr, 0, 560 sizeof(struct imgu_abi_sp_group)); 561 562 bi = &css->fwp->binary_header[css->fw_sp[0]]; 563 564 writel(css->xmem_sp_group_ptrs.daddr, 565 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data); 566 567 writel(IMGU_ABI_SP_SWSTATE_TERMINATED, 568 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state); 569 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb); 570 571 if (imgu_css_hw_start_sp(css, 0)) 572 return -EIO; 573 574 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started); 575 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + 576 bi->info.sp.host_sp_queues_initialized); 577 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode); 578 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb); 579 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0) 580 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 581 582 /* Enable all events for all queues */ 583 584 for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++) 585 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0) 586 + bi->info.sp.host_sp_com 587 + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i)); 588 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + 589 bi->info.sp.host_sp_queues_initialized); 590 591 /* Start SP1 */ 592 593 bi = &css->fwp->binary_header[css->fw_sp[1]]; 594 595 writel(IMGU_ABI_SP_SWSTATE_TERMINATED, 596 base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state); 597 598 if (imgu_css_hw_start_sp(css, 1)) 599 return -EIO; 600 601 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1) 602 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 603 604 return 0; 605 } 606 607 static void imgu_css_hw_stop(struct imgu_css *css) 608 { 609 void __iomem *const base = css->base; 610 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]]; 611 612 /* Stop fw */ 613 writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE, 614 base + IMGU_REG_SP_DMEM_BASE(0) + 615 bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 616 if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0), 617 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE)) 618 dev_err(css->dev, "wait sp0 idle timeout.\n"); 619 if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) != 620 IMGU_ABI_SP_SWSTATE_TERMINATED) 621 dev_err(css->dev, "sp0 is not terminated.\n"); 622 if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL, 623 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE)) 624 dev_err(css->dev, "wait isp idle timeout\n"); 625 } 626 627 static void imgu_css_hw_cleanup(struct imgu_css *css) 628 { 629 void __iomem *const base = css->base; 630 631 /** Reset CSS **/ 632 633 /* Clear the CSS busy signal */ 634 readl(base + IMGU_REG_GP_BUSY); 635 writel(0, base + IMGU_REG_GP_BUSY); 636 637 /* Wait for idle signal */ 638 if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 639 IMGU_STATE_IDLE_STS)) 640 dev_err(css->dev, "failed to shut down hw cleanly\n"); 641 642 /* Reset the css */ 643 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET, 644 base + IMGU_REG_PM_CTRL); 645 646 usleep_range(200, 300); 647 } 648 649 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe) 650 { 651 struct imgu_device *imgu = dev_get_drvdata(css->dev); 652 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 653 unsigned int i; 654 655 imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info); 656 imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc); 657 imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc); 658 imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid); 659 660 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 661 imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]); 662 } 663 664 /* 665 * This function initializes various stages of the 666 * IPU3 CSS ISP pipeline 667 */ 668 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe) 669 { 670 static const int BYPC = 2; /* Bytes per component */ 671 static const struct imgu_abi_buffer_sp buffer_sp_init = { 672 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID}, 673 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID, 674 }; 675 676 struct imgu_abi_isp_iterator_config *cfg_iter; 677 struct imgu_abi_isp_ref_config *cfg_ref; 678 struct imgu_abi_isp_dvs_config *cfg_dvs; 679 struct imgu_abi_isp_tnr3_config *cfg_tnr; 680 struct imgu_abi_isp_ref_dmem_state *cfg_ref_state; 681 struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state; 682 683 const int stage = 0; 684 unsigned int i, j; 685 686 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 687 struct imgu_css_queue *css_queue_in = 688 &css_pipe->queue[IPU3_CSS_QUEUE_IN]; 689 struct imgu_css_queue *css_queue_out = 690 &css_pipe->queue[IPU3_CSS_QUEUE_OUT]; 691 struct imgu_css_queue *css_queue_vf = 692 &css_pipe->queue[IPU3_CSS_QUEUE_VF]; 693 const struct imgu_fw_info *bi = 694 &css->fwp->binary_header[css_pipe->bindex]; 695 const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes; 696 697 struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp + 698 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG]; 699 struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp + 700 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE]; 701 702 struct imgu_abi_isp_stage *isp_stage; 703 struct imgu_abi_sp_stage *sp_stage; 704 struct imgu_abi_sp_group *sp_group; 705 struct imgu_abi_frames_sp *frames_sp; 706 struct imgu_abi_frame_sp *frame_sp; 707 struct imgu_abi_frame_sp_info *frame_sp_info; 708 709 const unsigned int bds_width_pad = 710 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, 711 2 * IPU3_UAPI_ISP_VEC_ELEMS); 712 713 const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0; 714 enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG; 715 void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr; 716 717 struct imgu_device *imgu = dev_get_drvdata(css->dev); 718 719 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe); 720 721 /* Configure iterator */ 722 723 cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 724 &cofs->dmem.iterator, 725 sizeof(*cfg_iter), vaddr); 726 if (!cfg_iter) 727 goto bad_firmware; 728 729 frame_sp_info = &cfg_iter->input_info; 730 frame_sp_info->res.width = css_queue_in->fmt.mpix.width; 731 frame_sp_info->res.height = css_queue_in->fmt.mpix.height; 732 frame_sp_info->padded_width = css_queue_in->width_pad; 733 frame_sp_info->format = css_queue_in->css_fmt->frame_format; 734 frame_sp_info->raw_bit_depth = css_queue_in->css_fmt->bit_depth; 735 frame_sp_info->raw_bayer_order = css_queue_in->css_fmt->bayer_order; 736 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 737 738 frame_sp_info = &cfg_iter->internal_info; 739 frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width; 740 frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height; 741 frame_sp_info->padded_width = bds_width_pad; 742 frame_sp_info->format = css_queue_out->css_fmt->frame_format; 743 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth; 744 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order; 745 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 746 747 frame_sp_info = &cfg_iter->output_info; 748 frame_sp_info->res.width = css_queue_out->fmt.mpix.width; 749 frame_sp_info->res.height = css_queue_out->fmt.mpix.height; 750 frame_sp_info->padded_width = css_queue_out->width_pad; 751 frame_sp_info->format = css_queue_out->css_fmt->frame_format; 752 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth; 753 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order; 754 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 755 756 frame_sp_info = &cfg_iter->vf_info; 757 frame_sp_info->res.width = css_queue_vf->fmt.mpix.width; 758 frame_sp_info->res.height = css_queue_vf->fmt.mpix.height; 759 frame_sp_info->padded_width = css_queue_vf->width_pad; 760 frame_sp_info->format = css_queue_vf->css_fmt->frame_format; 761 frame_sp_info->raw_bit_depth = css_queue_vf->css_fmt->bit_depth; 762 frame_sp_info->raw_bayer_order = css_queue_vf->css_fmt->bayer_order; 763 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 764 765 cfg_iter->dvs_envelope.width = 766 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width; 767 cfg_iter->dvs_envelope.height = 768 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height; 769 770 /* Configure reference (delay) frames */ 771 772 cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 773 &cofs->dmem.ref, 774 sizeof(*cfg_ref), vaddr); 775 if (!cfg_ref) 776 goto bad_firmware; 777 778 cfg_ref->port_b.crop = 0; 779 cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC; 780 cfg_ref->port_b.width = 781 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width; 782 cfg_ref->port_b.stride = 783 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline; 784 cfg_ref->width_a_over_b = 785 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems; 786 cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; 787 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) { 788 cfg_ref->ref_frame_addr_y[i] = 789 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr; 790 cfg_ref->ref_frame_addr_c[i] = 791 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr + 792 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline * 793 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; 794 } 795 for (; i < IMGU_ABI_FRAMES_REF; i++) { 796 cfg_ref->ref_frame_addr_y[i] = 0; 797 cfg_ref->ref_frame_addr_c[i] = 0; 798 } 799 800 /* Configure DVS (digital video stabilization) */ 801 802 cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 803 &cofs->dmem.dvs, sizeof(*cfg_dvs), 804 vaddr); 805 if (!cfg_dvs) 806 goto bad_firmware; 807 808 cfg_dvs->num_horizontal_blocks = 809 ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width, 810 IMGU_DVS_BLOCK_W), 2); 811 cfg_dvs->num_vertical_blocks = 812 DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height, 813 IMGU_DVS_BLOCK_H); 814 815 /* Configure TNR (temporal noise reduction) */ 816 817 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { 818 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 819 &cofs->dmem.tnr3, 820 sizeof(*cfg_tnr), 821 vaddr); 822 if (!cfg_tnr) 823 goto bad_firmware; 824 825 cfg_tnr->port_b.crop = 0; 826 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES; 827 cfg_tnr->port_b.width = 828 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; 829 cfg_tnr->port_b.stride = 830 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline; 831 cfg_tnr->width_a_over_b = 832 IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems; 833 cfg_tnr->frame_height = 834 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; 835 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1; 836 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 837 cfg_tnr->frame_addr[i] = 838 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR] 839 .mem[i].daddr; 840 for (; i < IMGU_ABI_FRAMES_TNR; i++) 841 cfg_tnr->frame_addr[i] = 0; 842 } 843 844 /* Configure ref dmem state parameters */ 845 846 cfg = IMGU_ABI_PARAM_CLASS_STATE; 847 vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr; 848 849 cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 850 &sofs->dmem.ref, 851 sizeof(*cfg_ref_state), 852 vaddr); 853 if (!cfg_ref_state) 854 goto bad_firmware; 855 856 cfg_ref_state->ref_in_buf_idx = 0; 857 cfg_ref_state->ref_out_buf_idx = 1; 858 859 /* Configure tnr dmem state parameters */ 860 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { 861 cfg_tnr_state = 862 imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 863 &sofs->dmem.tnr3, 864 sizeof(*cfg_tnr_state), 865 vaddr); 866 if (!cfg_tnr_state) 867 goto bad_firmware; 868 869 cfg_tnr_state->in_bufidx = 0; 870 cfg_tnr_state->out_bufidx = 1; 871 cfg_tnr_state->bypass_filter = 0; 872 cfg_tnr_state->total_frame_counter = 0; 873 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++) 874 cfg_tnr_state->buffer_frame_counter[i] = 0; 875 } 876 877 /* Configure ISP stage */ 878 879 isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr; 880 memset(isp_stage, 0, sizeof(*isp_stage)); 881 isp_stage->blob_info = bi->blob; 882 isp_stage->binary_info = bi->info.isp.sp; 883 strscpy(isp_stage->binary_name, 884 (char *)css->fwp + bi->blob.prog_name_offset, 885 sizeof(isp_stage->binary_name)); 886 isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers; 887 for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++) 888 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++) 889 isp_stage->mem_initializers.params[i][j].address = 890 css_pipe->binary_params_cs[i - 1][j].daddr; 891 892 /* Configure SP stage */ 893 894 sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr; 895 memset(sp_stage, 0, sizeof(*sp_stage)); 896 897 frames_sp = &sp_stage->frames; 898 frames_sp->in.buf_attr = buffer_sp_init; 899 for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++) 900 frames_sp->out[i].buf_attr = buffer_sp_init; 901 frames_sp->out_vf.buf_attr = buffer_sp_init; 902 frames_sp->s3a_buf = buffer_sp_init; 903 frames_sp->dvs_buf = buffer_sp_init; 904 905 sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP; 906 sp_stage->num = stage; 907 sp_stage->isp_online = 0; 908 sp_stage->isp_copy_vf = 0; 909 sp_stage->isp_copy_output = 0; 910 911 sp_stage->enable.vf_output = css_pipe->vf_output_en; 912 913 frames_sp->effective_in_res.width = 914 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width; 915 frames_sp->effective_in_res.height = 916 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height; 917 918 frame_sp = &frames_sp->in; 919 frame_sp->info.res.width = css_queue_in->fmt.mpix.width; 920 frame_sp->info.res.height = css_queue_in->fmt.mpix.height; 921 frame_sp->info.padded_width = css_queue_in->width_pad; 922 frame_sp->info.format = css_queue_in->css_fmt->frame_format; 923 frame_sp->info.raw_bit_depth = css_queue_in->css_fmt->bit_depth; 924 frame_sp->info.raw_bayer_order = css_queue_in->css_fmt->bayer_order; 925 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 926 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID; 927 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_INPUT_FRAME; 928 929 frame_sp = &frames_sp->out[0]; 930 frame_sp->info.res.width = css_queue_out->fmt.mpix.width; 931 frame_sp->info.res.height = css_queue_out->fmt.mpix.height; 932 frame_sp->info.padded_width = css_queue_out->width_pad; 933 frame_sp->info.format = css_queue_out->css_fmt->frame_format; 934 frame_sp->info.raw_bit_depth = css_queue_out->css_fmt->bit_depth; 935 frame_sp->info.raw_bayer_order = css_queue_out->css_fmt->bayer_order; 936 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 937 frame_sp->planes.nv.uv.offset = css_queue_out->width_pad * 938 css_queue_out->fmt.mpix.height; 939 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID; 940 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME; 941 942 frame_sp = &frames_sp->out[1]; 943 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID; 944 945 frame_sp_info = &frames_sp->internal_frame_info; 946 frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width; 947 frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height; 948 frame_sp_info->padded_width = bds_width_pad; 949 frame_sp_info->format = css_queue_out->css_fmt->frame_format; 950 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth; 951 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order; 952 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER; 953 954 frame_sp = &frames_sp->out_vf; 955 frame_sp->info.res.width = css_queue_vf->fmt.mpix.width; 956 frame_sp->info.res.height = css_queue_vf->fmt.mpix.height; 957 frame_sp->info.padded_width = css_queue_vf->width_pad; 958 frame_sp->info.format = css_queue_vf->css_fmt->frame_format; 959 frame_sp->info.raw_bit_depth = css_queue_vf->css_fmt->bit_depth; 960 frame_sp->info.raw_bayer_order = css_queue_vf->css_fmt->bayer_order; 961 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 962 frame_sp->planes.yuv.u.offset = css_queue_vf->width_pad * 963 css_queue_vf->fmt.mpix.height; 964 frame_sp->planes.yuv.v.offset = css_queue_vf->width_pad * 965 css_queue_vf->fmt.mpix.height * 5 / 4; 966 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID; 967 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME; 968 969 frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID; 970 frames_sp->s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS; 971 972 frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID; 973 frames_sp->dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS; 974 975 sp_stage->dvs_envelope.width = 976 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width; 977 sp_stage->dvs_envelope.height = 978 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height; 979 980 sp_stage->isp_pipe_version = 981 bi->info.isp.sp.pipeline.isp_pipe_version; 982 sp_stage->isp_deci_log_factor = 983 clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width / 984 IMGU_MAX_BQ_GRID_WIDTH), 985 fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height / 986 IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5); 987 sp_stage->isp_vf_downscale_bits = 0; 988 sp_stage->if_config_index = 255; 989 sp_stage->sp_enable_xnr = 0; 990 sp_stage->num_stripes = stripes; 991 sp_stage->enable.s3a = 1; 992 sp_stage->enable.dvs_stats = 0; 993 994 sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr; 995 sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr; 996 sp_stage->isp_stage_addr = 997 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr; 998 999 /* Configure SP group */ 1000 1001 sp_group = css->xmem_sp_group_ptrs.vaddr; 1002 memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline)); 1003 1004 sp_group->pipe[pipe].num_stages = 1; 1005 sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id; 1006 sp_group->pipe[pipe].thread_id = pipe; 1007 sp_group->pipe[pipe].pipe_num = pipe; 1008 sp_group->pipe[pipe].num_execs = -1; 1009 sp_group->pipe[pipe].pipe_qos_config = -1; 1010 sp_group->pipe[pipe].required_bds_factor = 0; 1011 sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; 1012 sp_group->pipe[pipe].inout_port_config = 1013 IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST | 1014 IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST; 1015 sp_group->pipe[pipe].scaler_pp_lut = 0; 1016 sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0; 1017 sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0; 1018 sp_group->pipe[pipe].sp_stage_addr[stage] = 1019 css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr; 1020 sp_group->pipe[pipe].pipe_config = 1021 bi->info.isp.sp.enable.params ? (1 << pipe) : 0; 1022 sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP; 1023 1024 /* Initialize parameter pools */ 1025 1026 if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info, 1027 sizeof(struct imgu_abi_parameter_set_info)) || 1028 imgu_css_pool_init(imgu, &css_pipe->pool.acc, 1029 sizeof(struct imgu_abi_acc_param)) || 1030 imgu_css_pool_init(imgu, &css_pipe->pool.gdc, 1031 sizeof(struct imgu_abi_gdc_warp_param) * 1032 3 * cfg_dvs->num_horizontal_blocks / 2 * 1033 cfg_dvs->num_vertical_blocks) || 1034 imgu_css_pool_init(imgu, &css_pipe->pool.obgrid, 1035 imgu_css_fw_obgrid_size( 1036 &css->fwp->binary_header[css_pipe->bindex]))) 1037 goto out_of_memory; 1038 1039 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1040 if (imgu_css_pool_init(imgu, 1041 &css_pipe->pool.binary_params_p[i], 1042 bi->info.isp.sp.mem_initializers.params 1043 [IMGU_ABI_PARAM_CLASS_PARAM][i].size)) 1044 goto out_of_memory; 1045 1046 return 0; 1047 1048 bad_firmware: 1049 imgu_css_pipeline_cleanup(css, pipe); 1050 return -EPROTO; 1051 1052 out_of_memory: 1053 imgu_css_pipeline_cleanup(css, pipe); 1054 return -ENOMEM; 1055 } 1056 1057 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread) 1058 { 1059 static const unsigned int sp; 1060 void __iomem *const base = css->base; 1061 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1062 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1063 bi->info.sp.host_sp_queue; 1064 1065 return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) : 1066 readb(&q->host2sp_evtq_info.end); 1067 } 1068 1069 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */ 1070 static int imgu_css_queue_data(struct imgu_css *css, 1071 int queue, int thread, u32 data) 1072 { 1073 static const unsigned int sp; 1074 void __iomem *const base = css->base; 1075 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1076 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1077 bi->info.sp.host_sp_queue; 1078 u8 size, start, end, end2; 1079 1080 if (queue >= 0) { 1081 size = readb(&q->host2sp_bufq_info[thread][queue].size); 1082 start = readb(&q->host2sp_bufq_info[thread][queue].start); 1083 end = readb(&q->host2sp_bufq_info[thread][queue].end); 1084 } else { 1085 size = readb(&q->host2sp_evtq_info.size); 1086 start = readb(&q->host2sp_evtq_info.start); 1087 end = readb(&q->host2sp_evtq_info.end); 1088 } 1089 1090 if (size == 0) 1091 return -EIO; 1092 1093 end2 = (end + 1) % size; 1094 if (end2 == start) 1095 return -EBUSY; /* Queue full */ 1096 1097 if (queue >= 0) { 1098 writel(data, &q->host2sp_bufq[thread][queue][end]); 1099 writeb(end2, &q->host2sp_bufq_info[thread][queue].end); 1100 } else { 1101 writel(data, &q->host2sp_evtq[end]); 1102 writeb(end2, &q->host2sp_evtq_info.end); 1103 } 1104 1105 return 0; 1106 } 1107 1108 /* Receive data using given buffer queue, or if queue < 0, event queue. */ 1109 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data) 1110 { 1111 static const unsigned int sp; 1112 void __iomem *const base = css->base; 1113 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1114 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1115 bi->info.sp.host_sp_queue; 1116 u8 size, start, end, start2; 1117 1118 if (queue >= 0) { 1119 size = readb(&q->sp2host_bufq_info[queue].size); 1120 start = readb(&q->sp2host_bufq_info[queue].start); 1121 end = readb(&q->sp2host_bufq_info[queue].end); 1122 } else { 1123 size = readb(&q->sp2host_evtq_info.size); 1124 start = readb(&q->sp2host_evtq_info.start); 1125 end = readb(&q->sp2host_evtq_info.end); 1126 } 1127 1128 if (size == 0) 1129 return -EIO; 1130 1131 if (end == start) 1132 return -EBUSY; /* Queue empty */ 1133 1134 start2 = (start + 1) % size; 1135 1136 if (queue >= 0) { 1137 *data = readl(&q->sp2host_bufq[queue][start]); 1138 writeb(start2, &q->sp2host_bufq_info[queue].start); 1139 } else { 1140 int r; 1141 1142 *data = readl(&q->sp2host_evtq[start]); 1143 writeb(start2, &q->sp2host_evtq_info.start); 1144 1145 /* Acknowledge events dequeued from event queue */ 1146 r = imgu_css_queue_data(css, queue, 0, 1147 IMGU_ABI_EVENT_EVENT_DEQUEUED); 1148 if (r < 0) 1149 return r; 1150 } 1151 1152 return 0; 1153 } 1154 1155 /* Free binary-specific resources */ 1156 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe) 1157 { 1158 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1159 unsigned int i, j; 1160 1161 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1162 1163 for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++) 1164 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1165 imgu_dmamap_free(imgu, 1166 &css_pipe->binary_params_cs[j][i]); 1167 1168 j = IPU3_CSS_AUX_FRAME_REF; 1169 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1170 imgu_dmamap_free(imgu, 1171 &css_pipe->aux_frames[j].mem[i]); 1172 1173 j = IPU3_CSS_AUX_FRAME_TNR; 1174 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1175 imgu_dmamap_free(imgu, 1176 &css_pipe->aux_frames[j].mem[i]); 1177 } 1178 1179 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe) 1180 { 1181 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1182 unsigned int i, j; 1183 1184 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1185 1186 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; 1187 j < IMGU_ABI_PARAM_CLASS_NUM; j++) 1188 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1189 if (!imgu_dmamap_alloc(imgu, 1190 &css_pipe->binary_params_cs[j - 1][i], 1191 CSS_ABI_SIZE)) 1192 goto out_of_memory; 1193 1194 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1195 if (!imgu_dmamap_alloc(imgu, 1196 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF]. 1197 mem[i], CSS_BDS_SIZE)) 1198 goto out_of_memory; 1199 1200 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1201 if (!imgu_dmamap_alloc(imgu, 1202 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]. 1203 mem[i], CSS_GDC_SIZE)) 1204 goto out_of_memory; 1205 1206 return 0; 1207 1208 out_of_memory: 1209 imgu_css_binary_cleanup(css, pipe); 1210 return -ENOMEM; 1211 } 1212 1213 /* allocate binary-specific resources */ 1214 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe) 1215 { 1216 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1217 struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex]; 1218 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1219 int i, j, size; 1220 static const int BYPC = 2; /* Bytes per component */ 1221 unsigned int w, h; 1222 1223 /* Allocate parameter memory blocks for this binary */ 1224 1225 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++) 1226 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) { 1227 if (imgu_css_dma_buffer_resize( 1228 imgu, 1229 &css_pipe->binary_params_cs[j - 1][i], 1230 bi->info.isp.sp.mem_initializers.params[j][i].size)) 1231 goto out_of_memory; 1232 } 1233 1234 /* Allocate internal frame buffers */ 1235 1236 /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */ 1237 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC; 1238 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width = 1239 css_pipe->rect[IPU3_CSS_RECT_BDS].width; 1240 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height = 1241 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height, 1242 IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y; 1243 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; 1244 w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, 1245 2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X; 1246 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline = 1247 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w; 1248 size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2; 1249 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1250 if (imgu_css_dma_buffer_resize( 1251 imgu, 1252 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i], 1253 size)) 1254 goto out_of_memory; 1255 1256 /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */ 1257 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1; 1258 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width = 1259 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width, 1260 bi->info.isp.sp.block.block_width * 1261 IPU3_UAPI_ISP_VEC_ELEMS); 1262 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height = 1263 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height, 1264 bi->info.isp.sp.block.output_block_height); 1265 1266 w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; 1267 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w; 1268 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; 1269 size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */ 1270 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1271 if (imgu_css_dma_buffer_resize( 1272 imgu, 1273 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i], 1274 size)) 1275 goto out_of_memory; 1276 1277 return 0; 1278 1279 out_of_memory: 1280 imgu_css_binary_cleanup(css, pipe); 1281 return -ENOMEM; 1282 } 1283 1284 int imgu_css_start_streaming(struct imgu_css *css) 1285 { 1286 u32 data; 1287 int r, pipe; 1288 1289 if (css->streaming) 1290 return -EPROTO; 1291 1292 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1293 r = imgu_css_binary_setup(css, pipe); 1294 if (r < 0) 1295 return r; 1296 } 1297 1298 r = imgu_css_hw_init(css); 1299 if (r < 0) 1300 return r; 1301 1302 r = imgu_css_hw_start(css); 1303 if (r < 0) 1304 goto fail; 1305 1306 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1307 r = imgu_css_pipeline_init(css, pipe); 1308 if (r < 0) 1309 goto fail; 1310 } 1311 1312 css->streaming = true; 1313 1314 imgu_css_hw_enable_irq(css); 1315 1316 /* Initialize parameters to default */ 1317 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1318 r = imgu_css_set_parameters(css, pipe, NULL); 1319 if (r < 0) 1320 goto fail; 1321 } 1322 1323 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data))) 1324 ; 1325 if (r != -EBUSY) 1326 goto fail; 1327 1328 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data))) 1329 ; 1330 if (r != -EBUSY) 1331 goto fail; 1332 1333 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1334 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 1335 IMGU_ABI_EVENT_START_STREAM | 1336 pipe << 16); 1337 if (r < 0) 1338 goto fail; 1339 } 1340 1341 return 0; 1342 1343 fail: 1344 css->streaming = false; 1345 imgu_css_hw_cleanup(css); 1346 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1347 imgu_css_pipeline_cleanup(css, pipe); 1348 imgu_css_binary_cleanup(css, pipe); 1349 } 1350 1351 return r; 1352 } 1353 1354 void imgu_css_stop_streaming(struct imgu_css *css) 1355 { 1356 struct imgu_css_buffer *b, *b0; 1357 int q, r, pipe; 1358 1359 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1360 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 1361 IMGU_ABI_EVENT_STOP_STREAM); 1362 if (r < 0) 1363 dev_warn(css->dev, "failed on stop stream event\n"); 1364 } 1365 1366 if (!css->streaming) 1367 return; 1368 1369 imgu_css_hw_stop(css); 1370 1371 imgu_css_hw_cleanup(css); 1372 1373 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1374 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1375 1376 imgu_css_pipeline_cleanup(css, pipe); 1377 1378 spin_lock(&css_pipe->qlock); 1379 for (q = 0; q < IPU3_CSS_QUEUES; q++) 1380 list_for_each_entry_safe(b, b0, 1381 &css_pipe->queue[q].bufs, 1382 list) { 1383 b->state = IPU3_CSS_BUFFER_FAILED; 1384 list_del(&b->list); 1385 } 1386 spin_unlock(&css_pipe->qlock); 1387 } 1388 1389 css->streaming = false; 1390 } 1391 1392 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe) 1393 { 1394 int q; 1395 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1396 1397 spin_lock(&css_pipe->qlock); 1398 for (q = 0; q < IPU3_CSS_QUEUES; q++) 1399 if (!list_empty(&css_pipe->queue[q].bufs)) 1400 break; 1401 spin_unlock(&css_pipe->qlock); 1402 return (q == IPU3_CSS_QUEUES); 1403 } 1404 1405 bool imgu_css_queue_empty(struct imgu_css *css) 1406 { 1407 unsigned int pipe; 1408 bool ret = false; 1409 1410 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) 1411 ret &= imgu_css_pipe_queue_empty(css, pipe); 1412 1413 return ret; 1414 } 1415 1416 bool imgu_css_is_streaming(struct imgu_css *css) 1417 { 1418 return css->streaming; 1419 } 1420 1421 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe) 1422 { 1423 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1424 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1425 unsigned int p, q, i; 1426 1427 /* Allocate and map common structures with imgu hardware */ 1428 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) 1429 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { 1430 if (!imgu_dmamap_alloc(imgu, 1431 &css_pipe-> 1432 xmem_sp_stage_ptrs[p][i], 1433 sizeof(struct imgu_abi_sp_stage))) 1434 return -ENOMEM; 1435 if (!imgu_dmamap_alloc(imgu, 1436 &css_pipe-> 1437 xmem_isp_stage_ptrs[p][i], 1438 sizeof(struct imgu_abi_isp_stage))) 1439 return -ENOMEM; 1440 } 1441 1442 if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs, 1443 ALIGN(sizeof(struct imgu_abi_ddr_address_map), 1444 IMGU_ABI_ISP_DDR_WORD_BYTES))) 1445 return -ENOMEM; 1446 1447 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1448 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]); 1449 1450 for (i = 0; i < abi_buf_num; i++) 1451 if (!imgu_dmamap_alloc(imgu, 1452 &css_pipe->abi_buffers[q][i], 1453 sizeof(struct imgu_abi_buffer))) 1454 return -ENOMEM; 1455 } 1456 1457 if (imgu_css_binary_preallocate(css, pipe)) { 1458 imgu_css_binary_cleanup(css, pipe); 1459 return -ENOMEM; 1460 } 1461 1462 return 0; 1463 } 1464 1465 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe) 1466 { 1467 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1468 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1469 unsigned int p, q, i, abi_buf_num; 1470 1471 imgu_css_binary_cleanup(css, pipe); 1472 1473 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1474 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]); 1475 for (i = 0; i < abi_buf_num; i++) 1476 imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]); 1477 } 1478 1479 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) 1480 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { 1481 imgu_dmamap_free(imgu, 1482 &css_pipe->xmem_sp_stage_ptrs[p][i]); 1483 imgu_dmamap_free(imgu, 1484 &css_pipe->xmem_isp_stage_ptrs[p][i]); 1485 } 1486 1487 imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs); 1488 } 1489 1490 void imgu_css_cleanup(struct imgu_css *css) 1491 { 1492 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1493 unsigned int pipe; 1494 1495 imgu_css_stop_streaming(css); 1496 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) 1497 imgu_css_pipe_cleanup(css, pipe); 1498 imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs); 1499 imgu_css_fw_cleanup(css); 1500 } 1501 1502 int imgu_css_init(struct device *dev, struct imgu_css *css, 1503 void __iomem *base, int length) 1504 { 1505 struct imgu_device *imgu = dev_get_drvdata(dev); 1506 int r, q, pipe; 1507 1508 /* Initialize main data structure */ 1509 css->dev = dev; 1510 css->base = base; 1511 css->iomem_length = length; 1512 1513 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) { 1514 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1515 1516 css_pipe->vf_output_en = false; 1517 spin_lock_init(&css_pipe->qlock); 1518 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY; 1519 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO; 1520 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1521 r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0); 1522 if (r) 1523 return r; 1524 } 1525 r = imgu_css_map_init(css, pipe); 1526 if (r) { 1527 imgu_css_cleanup(css); 1528 return r; 1529 } 1530 } 1531 if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs, 1532 sizeof(struct imgu_abi_sp_group))) 1533 return -ENOMEM; 1534 1535 r = imgu_css_fw_init(css); 1536 if (r) 1537 return r; 1538 1539 return 0; 1540 } 1541 1542 static u32 imgu_css_adjust(u32 res, u32 align) 1543 { 1544 u32 val = max_t(u32, IPU3_CSS_MIN_RES, res); 1545 1546 return DIV_ROUND_CLOSEST(val, align) * align; 1547 } 1548 1549 /* Select a binary matching the required resolutions and formats */ 1550 static int imgu_css_find_binary(struct imgu_css *css, 1551 unsigned int pipe, 1552 struct imgu_css_queue queue[IPU3_CSS_QUEUES], 1553 struct v4l2_rect rects[IPU3_CSS_RECTS]) 1554 { 1555 const int binary_nr = css->fwp->file_header.binary_nr; 1556 unsigned int binary_mode = 1557 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ? 1558 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO; 1559 const struct v4l2_pix_format_mplane *in = 1560 &queue[IPU3_CSS_QUEUE_IN].fmt.mpix; 1561 const struct v4l2_pix_format_mplane *out = 1562 &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix; 1563 const struct v4l2_pix_format_mplane *vf = 1564 &queue[IPU3_CSS_QUEUE_VF].fmt.mpix; 1565 u32 stripe_w = 0, stripe_h = 0; 1566 const char *name; 1567 int i, j; 1568 1569 if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN])) 1570 return -EINVAL; 1571 1572 /* Find out the strip size boundary */ 1573 for (i = 0; i < binary_nr; i++) { 1574 struct imgu_fw_info *bi = &css->fwp->binary_header[i]; 1575 1576 u32 max_width = bi->info.isp.sp.output.max_width; 1577 u32 max_height = bi->info.isp.sp.output.max_height; 1578 1579 if (bi->info.isp.sp.iterator.num_stripes <= 1) { 1580 stripe_w = stripe_w ? 1581 min(stripe_w, max_width) : max_width; 1582 stripe_h = stripe_h ? 1583 min(stripe_h, max_height) : max_height; 1584 } 1585 } 1586 1587 for (i = 0; i < binary_nr; i++) { 1588 struct imgu_fw_info *bi = &css->fwp->binary_header[i]; 1589 enum imgu_abi_frame_format q_fmt; 1590 1591 name = (void *)css->fwp + bi->blob.prog_name_offset; 1592 1593 /* Check that binary supports memory-to-memory processing */ 1594 if (bi->info.isp.sp.input.source != 1595 IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY) 1596 continue; 1597 1598 /* Check that binary supports raw10 input */ 1599 if (!bi->info.isp.sp.enable.input_feeder && 1600 !bi->info.isp.sp.enable.input_raw) 1601 continue; 1602 1603 /* Check binary mode */ 1604 if (bi->info.isp.sp.pipeline.mode != binary_mode) 1605 continue; 1606 1607 /* Since input is RGGB bayer, need to process colors */ 1608 if (bi->info.isp.sp.enable.luma_only) 1609 continue; 1610 1611 if (in->width < bi->info.isp.sp.input.min_width || 1612 in->width > bi->info.isp.sp.input.max_width || 1613 in->height < bi->info.isp.sp.input.min_height || 1614 in->height > bi->info.isp.sp.input.max_height) 1615 continue; 1616 1617 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) { 1618 if (bi->info.isp.num_output_pins <= 0) 1619 continue; 1620 1621 q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 1622 for (j = 0; j < bi->info.isp.num_output_formats; j++) 1623 if (bi->info.isp.output_formats[j] == q_fmt) 1624 break; 1625 if (j >= bi->info.isp.num_output_formats) 1626 continue; 1627 1628 if (out->width < bi->info.isp.sp.output.min_width || 1629 out->width > bi->info.isp.sp.output.max_width || 1630 out->height < bi->info.isp.sp.output.min_height || 1631 out->height > bi->info.isp.sp.output.max_height) 1632 continue; 1633 1634 if (out->width > bi->info.isp.sp.internal.max_width || 1635 out->height > bi->info.isp.sp.internal.max_height) 1636 continue; 1637 } 1638 1639 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) { 1640 if (bi->info.isp.num_output_pins <= 1) 1641 continue; 1642 1643 q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; 1644 for (j = 0; j < bi->info.isp.num_output_formats; j++) 1645 if (bi->info.isp.output_formats[j] == q_fmt) 1646 break; 1647 if (j >= bi->info.isp.num_output_formats) 1648 continue; 1649 1650 if (vf->width < bi->info.isp.sp.output.min_width || 1651 vf->width > bi->info.isp.sp.output.max_width || 1652 vf->height < bi->info.isp.sp.output.min_height || 1653 vf->height > bi->info.isp.sp.output.max_height) 1654 continue; 1655 } 1656 1657 /* All checks passed, select the binary */ 1658 dev_dbg(css->dev, "using binary %s id = %u\n", name, 1659 bi->info.isp.sp.id); 1660 return i; 1661 } 1662 1663 /* Can not find suitable binary for these parameters */ 1664 return -EINVAL; 1665 } 1666 1667 /* 1668 * Check that there is a binary matching requirements. Parameters may be 1669 * NULL indicating disabled input/output. Return negative if given 1670 * parameters can not be supported or on error, zero or positive indicating 1671 * found binary number. May modify the given parameters if not exact match 1672 * is found. 1673 */ 1674 int imgu_css_fmt_try(struct imgu_css *css, 1675 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES], 1676 struct v4l2_rect *rects[IPU3_CSS_RECTS], 1677 unsigned int pipe) 1678 { 1679 static const u32 EFF_ALIGN_W = 2; 1680 static const u32 BDS_ALIGN_W = 4; 1681 static const u32 OUT_ALIGN_W = 8; 1682 static const u32 OUT_ALIGN_H = 4; 1683 static const u32 VF_ALIGN_W = 2; 1684 static const char *qnames[IPU3_CSS_QUEUES] = { 1685 [IPU3_CSS_QUEUE_IN] = "in", 1686 [IPU3_CSS_QUEUE_PARAMS] = "params", 1687 [IPU3_CSS_QUEUE_OUT] = "out", 1688 [IPU3_CSS_QUEUE_VF] = "vf", 1689 [IPU3_CSS_QUEUE_STAT_3A] = "3a", 1690 }; 1691 static const char *rnames[IPU3_CSS_RECTS] = { 1692 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution", 1693 [IPU3_CSS_RECT_BDS] = "bayer-domain scaled resolution", 1694 [IPU3_CSS_RECT_ENVELOPE] = "DVS envelope size", 1695 [IPU3_CSS_RECT_GDC] = "GDC output res", 1696 }; 1697 struct v4l2_rect r[IPU3_CSS_RECTS] = { }; 1698 struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE]; 1699 struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS]; 1700 struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE]; 1701 struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC]; 1702 struct imgu_css_queue *q; 1703 struct v4l2_pix_format_mplane *in, *out, *vf; 1704 int i, s, ret; 1705 1706 q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL); 1707 if (!q) 1708 return -ENOMEM; 1709 1710 in = &q[IPU3_CSS_QUEUE_IN].fmt.mpix; 1711 out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix; 1712 vf = &q[IPU3_CSS_QUEUE_VF].fmt.mpix; 1713 1714 /* Adjust all formats, get statistics buffer sizes and formats */ 1715 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 1716 if (fmts[i]) 1717 dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__, 1718 qnames[i], fmts[i]->width, fmts[i]->height, 1719 fmts[i]->pixelformat); 1720 else 1721 dev_dbg(css->dev, "%s %s: (not set)\n", __func__, 1722 qnames[i]); 1723 if (imgu_css_queue_init(&q[i], fmts[i], 1724 IPU3_CSS_QUEUE_TO_FLAGS(i))) { 1725 dev_notice(css->dev, "can not initialize queue %s\n", 1726 qnames[i]); 1727 ret = -EINVAL; 1728 goto out; 1729 } 1730 } 1731 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1732 if (rects[i]) { 1733 dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__, 1734 rnames[i], rects[i]->width, rects[i]->height); 1735 r[i].width = rects[i]->width; 1736 r[i].height = rects[i]->height; 1737 } else { 1738 dev_dbg(css->dev, "%s %s: (not set)\n", __func__, 1739 rnames[i]); 1740 } 1741 /* For now, force known good resolutions */ 1742 r[i].left = 0; 1743 r[i].top = 0; 1744 } 1745 1746 /* Always require one input and vf only if out is also enabled */ 1747 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) || 1748 !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) { 1749 dev_warn(css->dev, "required queues are disabled\n"); 1750 ret = -EINVAL; 1751 goto out; 1752 } 1753 1754 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) { 1755 out->width = in->width; 1756 out->height = in->height; 1757 } 1758 if (eff->width <= 0 || eff->height <= 0) { 1759 eff->width = in->width; 1760 eff->height = in->height; 1761 } 1762 if (bds->width <= 0 || bds->height <= 0) { 1763 bds->width = out->width; 1764 bds->height = out->height; 1765 } 1766 if (gdc->width <= 0 || gdc->height <= 0) { 1767 gdc->width = out->width; 1768 gdc->height = out->height; 1769 } 1770 1771 in->width = imgu_css_adjust(in->width, 1); 1772 in->height = imgu_css_adjust(in->height, 1); 1773 eff->width = imgu_css_adjust(eff->width, EFF_ALIGN_W); 1774 eff->height = imgu_css_adjust(eff->height, 1); 1775 bds->width = imgu_css_adjust(bds->width, BDS_ALIGN_W); 1776 bds->height = imgu_css_adjust(bds->height, 1); 1777 gdc->width = imgu_css_adjust(gdc->width, OUT_ALIGN_W); 1778 gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H); 1779 out->width = imgu_css_adjust(out->width, OUT_ALIGN_W); 1780 out->height = imgu_css_adjust(out->height, OUT_ALIGN_H); 1781 vf->width = imgu_css_adjust(vf->width, VF_ALIGN_W); 1782 vf->height = imgu_css_adjust(vf->height, 1); 1783 1784 s = (bds->width - gdc->width) / 2; 1785 env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s; 1786 s = (bds->height - gdc->height) / 2; 1787 env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s; 1788 1789 ret = imgu_css_find_binary(css, pipe, q, r); 1790 if (ret < 0) { 1791 dev_err(css->dev, "failed to find suitable binary\n"); 1792 ret = -EINVAL; 1793 goto out; 1794 } 1795 css->pipes[pipe].bindex = ret; 1796 1797 dev_dbg(css->dev, "Binary index %d for pipe %d found.", 1798 css->pipes[pipe].bindex, pipe); 1799 1800 /* Final adjustment and set back the queried formats */ 1801 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 1802 if (fmts[i]) { 1803 if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix, 1804 IPU3_CSS_QUEUE_TO_FLAGS(i))) { 1805 dev_err(css->dev, 1806 "final resolution adjustment failed\n"); 1807 ret = -EINVAL; 1808 goto out; 1809 } 1810 *fmts[i] = q[i].fmt.mpix; 1811 } 1812 } 1813 1814 for (i = 0; i < IPU3_CSS_RECTS; i++) 1815 if (rects[i]) 1816 *rects[i] = r[i]; 1817 1818 dev_dbg(css->dev, 1819 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)", 1820 in->width, in->height, eff->width, eff->height, 1821 bds->width, bds->height, gdc->width, gdc->height, 1822 out->width, out->height, vf->width, vf->height); 1823 1824 ret = 0; 1825 out: 1826 kfree(q); 1827 return ret; 1828 } 1829 1830 int imgu_css_fmt_set(struct imgu_css *css, 1831 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES], 1832 struct v4l2_rect *rects[IPU3_CSS_RECTS], 1833 unsigned int pipe) 1834 { 1835 struct v4l2_rect rect_data[IPU3_CSS_RECTS]; 1836 struct v4l2_rect *all_rects[IPU3_CSS_RECTS]; 1837 int i, r; 1838 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1839 1840 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1841 if (rects[i]) 1842 rect_data[i] = *rects[i]; 1843 else 1844 memset(&rect_data[i], 0, sizeof(rect_data[i])); 1845 all_rects[i] = &rect_data[i]; 1846 } 1847 r = imgu_css_fmt_try(css, fmts, all_rects, pipe); 1848 if (r < 0) 1849 return r; 1850 1851 for (i = 0; i < IPU3_CSS_QUEUES; i++) 1852 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i], 1853 IPU3_CSS_QUEUE_TO_FLAGS(i))) 1854 return -EINVAL; 1855 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1856 css_pipe->rect[i] = rect_data[i]; 1857 if (rects[i]) 1858 *rects[i] = rect_data[i]; 1859 } 1860 1861 return 0; 1862 } 1863 1864 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt) 1865 { 1866 switch (fmt->dataformat) { 1867 case V4L2_META_FMT_IPU3_PARAMS: 1868 fmt->buffersize = sizeof(struct ipu3_uapi_params); 1869 1870 /* 1871 * Sanity check for the parameter struct size. This must 1872 * not change! 1873 */ 1874 BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328); 1875 1876 break; 1877 case V4L2_META_FMT_IPU3_STAT_3A: 1878 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a); 1879 break; 1880 default: 1881 return -EINVAL; 1882 } 1883 1884 return 0; 1885 } 1886 1887 /* 1888 * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first 1889 * called for the buffer. May be called from interrupt context. 1890 * Returns 0 on success, -EBUSY if the buffer queue is full, or some other 1891 * code on error conditions. 1892 */ 1893 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe, 1894 struct imgu_css_buffer *b) 1895 { 1896 struct imgu_abi_buffer *abi_buf; 1897 struct imgu_addr_t *buf_addr; 1898 u32 data; 1899 int r; 1900 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1901 1902 if (!css->streaming) 1903 return -EPROTO; /* CSS or buffer in wrong state */ 1904 1905 if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid) 1906 return -EINVAL; 1907 1908 b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid, 1909 pipe); 1910 1911 if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue])) 1912 return -EIO; 1913 abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr; 1914 1915 /* Fill struct abi_buffer for firmware */ 1916 memset(abi_buf, 0, sizeof(*abi_buf)); 1917 1918 buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs; 1919 *(imgu_addr_t *)buf_addr = b->daddr; 1920 1921 if (b->queue == IPU3_CSS_QUEUE_STAT_3A) 1922 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr; 1923 1924 if (b->queue == IPU3_CSS_QUEUE_OUT) 1925 abi_buf->payload.frame.padded_width = 1926 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad; 1927 1928 if (b->queue == IPU3_CSS_QUEUE_VF) 1929 abi_buf->payload.frame.padded_width = 1930 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad; 1931 1932 spin_lock(&css_pipe->qlock); 1933 list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs); 1934 spin_unlock(&css_pipe->qlock); 1935 b->state = IPU3_CSS_BUFFER_QUEUED; 1936 1937 data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr; 1938 r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid, 1939 pipe, data); 1940 if (r < 0) 1941 goto queueing_failed; 1942 1943 data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe, 1944 imgu_css_queues[b->queue].qid); 1945 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data); 1946 if (r < 0) 1947 goto queueing_failed; 1948 1949 dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n", 1950 b, b->queue, pipe); 1951 1952 return 0; 1953 1954 queueing_failed: 1955 b->state = (r == -EBUSY || r == -EAGAIN) ? 1956 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED; 1957 list_del(&b->list); 1958 1959 return r; 1960 } 1961 1962 /* 1963 * Get next ready CSS buffer. Returns -EAGAIN in which case the function 1964 * should be called again, or -EBUSY which means that there are no more 1965 * buffers available. May be called from interrupt context. 1966 */ 1967 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css) 1968 { 1969 static const unsigned char evtype_to_queue[] = { 1970 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN, 1971 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT, 1972 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF, 1973 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A, 1974 }; 1975 struct imgu_css_buffer *b = ERR_PTR(-EAGAIN); 1976 u32 event, daddr; 1977 int evtype, pipe, pipeid, queue, qid, r; 1978 struct imgu_css_pipe *css_pipe; 1979 1980 if (!css->streaming) 1981 return ERR_PTR(-EPROTO); 1982 1983 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event); 1984 if (r < 0) 1985 return ERR_PTR(r); 1986 1987 evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >> 1988 IMGU_ABI_EVTTYPE_EVENT_SHIFT; 1989 1990 switch (evtype) { 1991 case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE: 1992 case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE: 1993 case IMGU_ABI_EVTTYPE_3A_STATS_DONE: 1994 case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE: 1995 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >> 1996 IMGU_ABI_EVTTYPE_PIPE_SHIFT; 1997 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >> 1998 IMGU_ABI_EVTTYPE_PIPEID_SHIFT; 1999 queue = evtype_to_queue[evtype]; 2000 qid = imgu_css_queues[queue].qid; 2001 2002 if (pipe >= IMGU_MAX_PIPE_NUM) { 2003 dev_err(css->dev, "Invalid pipe: %i\n", pipe); 2004 return ERR_PTR(-EIO); 2005 } 2006 2007 if (qid >= IMGU_ABI_QUEUE_NUM) { 2008 dev_err(css->dev, "Invalid qid: %i\n", qid); 2009 return ERR_PTR(-EIO); 2010 } 2011 css_pipe = &css->pipes[pipe]; 2012 dev_dbg(css->dev, 2013 "event: buffer done 0x%x queue %i pipe %i pipeid %i\n", 2014 event, queue, pipe, pipeid); 2015 2016 r = imgu_css_dequeue_data(css, qid, &daddr); 2017 if (r < 0) { 2018 dev_err(css->dev, "failed to dequeue buffer\n"); 2019 /* Force real error, not -EBUSY */ 2020 return ERR_PTR(-EIO); 2021 } 2022 2023 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2024 IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid)); 2025 if (r < 0) { 2026 dev_err(css->dev, "failed to queue event\n"); 2027 return ERR_PTR(-EIO); 2028 } 2029 2030 spin_lock(&css_pipe->qlock); 2031 if (list_empty(&css_pipe->queue[queue].bufs)) { 2032 spin_unlock(&css_pipe->qlock); 2033 dev_err(css->dev, "event on empty queue\n"); 2034 return ERR_PTR(-EIO); 2035 } 2036 b = list_first_entry(&css_pipe->queue[queue].bufs, 2037 struct imgu_css_buffer, list); 2038 if (queue != b->queue || 2039 daddr != css_pipe->abi_buffers 2040 [b->queue][b->queue_pos].daddr) { 2041 spin_unlock(&css_pipe->qlock); 2042 dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr); 2043 return ERR_PTR(-EIO); 2044 } 2045 2046 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe); 2047 b->pipe = pipe; 2048 b->state = IPU3_CSS_BUFFER_DONE; 2049 list_del(&b->list); 2050 spin_unlock(&css_pipe->qlock); 2051 break; 2052 case IMGU_ABI_EVTTYPE_PIPELINE_DONE: 2053 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >> 2054 IMGU_ABI_EVTTYPE_PIPE_SHIFT; 2055 if (pipe >= IMGU_MAX_PIPE_NUM) { 2056 dev_err(css->dev, "Invalid pipe: %i\n", pipe); 2057 return ERR_PTR(-EIO); 2058 } 2059 2060 css_pipe = &css->pipes[pipe]; 2061 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n", 2062 event, pipe); 2063 break; 2064 case IMGU_ABI_EVTTYPE_TIMER: 2065 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event); 2066 if (r < 0) 2067 return ERR_PTR(r); 2068 2069 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >> 2070 IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER) 2071 dev_dbg(css->dev, "event: timer\n"); 2072 else 2073 dev_warn(css->dev, "half of timer event missing\n"); 2074 break; 2075 case IMGU_ABI_EVTTYPE_FW_WARNING: 2076 dev_warn(css->dev, "event: firmware warning 0x%x\n", event); 2077 break; 2078 case IMGU_ABI_EVTTYPE_FW_ASSERT: 2079 dev_err(css->dev, 2080 "event: firmware assert 0x%x module_id %i line_no %i\n", 2081 event, 2082 (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >> 2083 IMGU_ABI_EVTTYPE_MODULEID_SHIFT, 2084 swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >> 2085 IMGU_ABI_EVTTYPE_LINENO_SHIFT)); 2086 break; 2087 default: 2088 dev_warn(css->dev, "received unknown event 0x%x\n", event); 2089 } 2090 2091 return b; 2092 } 2093 2094 /* 2095 * Get a new set of parameters from pool and initialize them based on 2096 * the parameters params, gdc, and obgrid. Any of these may be NULL, 2097 * in which case the previously set parameters are used. 2098 * If parameters haven't been set previously, initialize from scratch. 2099 * 2100 * Return index to css->parameter_set_info which has the newly created 2101 * parameters or negative value on error. 2102 */ 2103 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe, 2104 struct ipu3_uapi_params *set_params) 2105 { 2106 static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID; 2107 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 2108 const int stage = 0; 2109 const struct imgu_fw_info *bi; 2110 int obgrid_size; 2111 unsigned int stripes, i; 2112 struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL; 2113 2114 /* Destination buffers which are filled here */ 2115 struct imgu_abi_parameter_set_info *param_set; 2116 struct imgu_abi_acc_param *acc = NULL; 2117 struct imgu_abi_gdc_warp_param *gdc = NULL; 2118 struct ipu3_uapi_obgrid_param *obgrid = NULL; 2119 const struct imgu_css_map *map; 2120 void *vmem0 = NULL; 2121 void *dmem0 = NULL; 2122 2123 enum imgu_abi_memories m; 2124 int r = -EBUSY; 2125 2126 if (!css->streaming) 2127 return -EPROTO; 2128 2129 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe); 2130 2131 bi = &css->fwp->binary_header[css_pipe->bindex]; 2132 obgrid_size = imgu_css_fw_obgrid_size(bi); 2133 stripes = bi->info.isp.sp.iterator.num_stripes ? : 1; 2134 2135 imgu_css_pool_get(&css_pipe->pool.parameter_set_info); 2136 param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 2137 0)->vaddr; 2138 2139 /* Get a new acc only if new parameters given, or none yet */ 2140 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2141 if (set_params || !map->vaddr) { 2142 imgu_css_pool_get(&css_pipe->pool.acc); 2143 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2144 acc = map->vaddr; 2145 } 2146 2147 /* Get new VMEM0 only if needed, or none yet */ 2148 m = IMGU_ABI_MEM_ISP_VMEM0; 2149 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2150 if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params || 2151 set_params->use.tnr3_vmem_params || 2152 set_params->use.xnr3_vmem_params))) { 2153 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]); 2154 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2155 vmem0 = map->vaddr; 2156 } 2157 2158 /* Get new DMEM0 only if needed, or none yet */ 2159 m = IMGU_ABI_MEM_ISP_DMEM0; 2160 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2161 if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params || 2162 set_params->use.xnr3_dmem_params))) { 2163 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]); 2164 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2165 dmem0 = map->vaddr; 2166 } 2167 2168 /* Configure acc parameter cluster */ 2169 if (acc) { 2170 /* get acc_old */ 2171 map = imgu_css_pool_last(&css_pipe->pool.acc, 1); 2172 /* user acc */ 2173 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr, 2174 set_params ? &set_params->acc_param : NULL); 2175 if (r < 0) 2176 goto fail; 2177 } 2178 2179 /* Configure late binding parameters */ 2180 if (vmem0) { 2181 m = IMGU_ABI_MEM_ISP_VMEM0; 2182 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1); 2183 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0, 2184 map->vaddr, set_params); 2185 if (r < 0) 2186 goto fail; 2187 } 2188 2189 if (dmem0) { 2190 m = IMGU_ABI_MEM_ISP_DMEM0; 2191 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1); 2192 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0, 2193 map->vaddr, set_params); 2194 if (r < 0) 2195 goto fail; 2196 } 2197 2198 /* Get a new gdc only if a new gdc is given, or none yet */ 2199 if (bi->info.isp.sp.enable.dvs_6axis) { 2200 unsigned int a = IPU3_CSS_AUX_FRAME_REF; 2201 unsigned int g = IPU3_CSS_RECT_GDC; 2202 unsigned int e = IPU3_CSS_RECT_ENVELOPE; 2203 2204 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2205 if (!map->vaddr) { 2206 imgu_css_pool_get(&css_pipe->pool.gdc); 2207 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2208 gdc = map->vaddr; 2209 imgu_css_cfg_gdc_table(map->vaddr, 2210 css_pipe->aux_frames[a].bytesperline / 2211 css_pipe->aux_frames[a].bytesperpixel, 2212 css_pipe->aux_frames[a].height, 2213 css_pipe->rect[g].width, 2214 css_pipe->rect[g].height, 2215 css_pipe->rect[e].width, 2216 css_pipe->rect[e].height); 2217 } 2218 } 2219 2220 /* Get a new obgrid only if a new obgrid is given, or none yet */ 2221 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2222 if (!map->vaddr || (set_params && set_params->use.obgrid_param)) { 2223 imgu_css_pool_get(&css_pipe->pool.obgrid); 2224 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2225 obgrid = map->vaddr; 2226 2227 /* Configure optical black level grid (obgrid) */ 2228 if (set_params && set_params->use.obgrid_param) 2229 for (i = 0; i < obgrid_size / sizeof(*obgrid); i++) 2230 obgrid[i] = set_params->obgrid_param; 2231 else 2232 memset(obgrid, 0, obgrid_size); 2233 } 2234 2235 /* Configure parameter set info, queued to `queue_id' */ 2236 2237 memset(param_set, 0, sizeof(*param_set)); 2238 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2239 param_set->mem_map.acc_cluster_params_for_sp = map->daddr; 2240 2241 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2242 param_set->mem_map.dvs_6axis_params_y = map->daddr; 2243 2244 for (i = 0; i < stripes; i++) { 2245 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2246 param_set->mem_map.obgrid_tbl[i] = 2247 map->daddr + (obgrid_size / stripes) * i; 2248 } 2249 2250 for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) { 2251 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2252 param_set->mem_map.isp_mem_param[stage][m] = map->daddr; 2253 } 2254 2255 /* Then queue the new parameter buffer */ 2256 map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0); 2257 r = imgu_css_queue_data(css, queue_id, pipe, map->daddr); 2258 if (r < 0) 2259 goto fail; 2260 2261 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2262 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe, 2263 queue_id)); 2264 if (r < 0) 2265 goto fail_no_put; 2266 2267 /* Finally dequeue all old parameter buffers */ 2268 2269 do { 2270 u32 daddr; 2271 2272 r = imgu_css_dequeue_data(css, queue_id, &daddr); 2273 if (r == -EBUSY) 2274 break; 2275 if (r) 2276 goto fail_no_put; 2277 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2278 IMGU_ABI_EVENT_BUFFER_DEQUEUED 2279 (queue_id)); 2280 if (r < 0) { 2281 dev_err(css->dev, "failed to queue parameter event\n"); 2282 goto fail_no_put; 2283 } 2284 } while (1); 2285 2286 return 0; 2287 2288 fail: 2289 /* 2290 * A failure, most likely the parameter queue was full. 2291 * Return error but continue streaming. User can try submitting new 2292 * parameters again later. 2293 */ 2294 2295 imgu_css_pool_put(&css_pipe->pool.parameter_set_info); 2296 if (acc) 2297 imgu_css_pool_put(&css_pipe->pool.acc); 2298 if (gdc) 2299 imgu_css_pool_put(&css_pipe->pool.gdc); 2300 if (obgrid) 2301 imgu_css_pool_put(&css_pipe->pool.obgrid); 2302 if (vmem0) 2303 imgu_css_pool_put( 2304 &css_pipe->pool.binary_params_p 2305 [IMGU_ABI_MEM_ISP_VMEM0]); 2306 if (dmem0) 2307 imgu_css_pool_put( 2308 &css_pipe->pool.binary_params_p 2309 [IMGU_ABI_MEM_ISP_DMEM0]); 2310 2311 fail_no_put: 2312 return r; 2313 } 2314 2315 int imgu_css_irq_ack(struct imgu_css *css) 2316 { 2317 static const int NUM_SWIRQS = 3; 2318 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]]; 2319 void __iomem *const base = css->base; 2320 u32 irq_status[IMGU_IRQCTRL_NUM]; 2321 int i; 2322 2323 u32 imgu_status = readl(base + IMGU_REG_INT_STATUS); 2324 2325 writel(imgu_status, base + IMGU_REG_INT_STATUS); 2326 for (i = 0; i < IMGU_IRQCTRL_NUM; i++) 2327 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i)); 2328 2329 for (i = 0; i < NUM_SWIRQS; i++) { 2330 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) { 2331 /* SP SW interrupt */ 2332 u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) + 2333 bi->info.sp.output); 2334 u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) + 2335 bi->info.sp.output + 4 + 4 * i); 2336 2337 dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n", 2338 __func__, i, cnt, val); 2339 } 2340 } 2341 2342 for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--) 2343 if (irq_status[i]) { 2344 writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i)); 2345 /* Wait for write to complete */ 2346 readl(base + IMGU_REG_IRQCTRL_ENABLE(i)); 2347 } 2348 2349 dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n", 2350 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN], 2351 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]); 2352 2353 if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN]) 2354 return -ENOMSG; 2355 2356 return 0; 2357 } 2358