1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2018 Intel Corporation 3 4 #include <linux/device.h> 5 #include <linux/iopoll.h> 6 #include <linux/slab.h> 7 8 #include "ipu3-css.h" 9 #include "ipu3-css-fw.h" 10 #include "ipu3-css-params.h" 11 #include "ipu3-dmamap.h" 12 #include "ipu3-tables.h" 13 14 /* IRQ configuration */ 15 #define IMGU_IRQCTRL_IRQ_MASK (IMGU_IRQCTRL_IRQ_SP1 | \ 16 IMGU_IRQCTRL_IRQ_SP2 | \ 17 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \ 18 IMGU_IRQCTRL_IRQ_SW_PIN(1)) 19 20 #define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */ 21 22 /* Some sane limits for resolutions */ 23 #define IPU3_CSS_MIN_RES 32 24 #define IPU3_CSS_MAX_H 3136 25 #define IPU3_CSS_MAX_W 4224 26 27 /* minimal envelope size(GDC in - out) should be 4 */ 28 #define MIN_ENVELOPE 4 29 30 /* 31 * pre-allocated buffer size for CSS ABI, auxiliary frames 32 * after BDS and before GDC. Those values should be tuned 33 * to big enough to avoid buffer re-allocation when 34 * streaming to lower streaming latency. 35 */ 36 #define CSS_ABI_SIZE 136 37 #define CSS_BDS_SIZE (4480 * 3200 * 3) 38 #define CSS_GDC_SIZE (4224 * 3200 * 12 / 8) 39 40 #define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q)) 41 #define IPU3_CSS_FORMAT_FL_IN \ 42 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN) 43 #define IPU3_CSS_FORMAT_FL_OUT \ 44 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT) 45 #define IPU3_CSS_FORMAT_FL_VF \ 46 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF) 47 48 /* Formats supported by IPU3 Camera Sub System */ 49 static const struct imgu_css_format imgu_css_formats[] = { 50 { 51 .pixelformat = V4L2_PIX_FMT_NV12, 52 .colorspace = V4L2_COLORSPACE_SRGB, 53 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12, 54 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12, 55 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE, 56 .bytesperpixel_num = 1 * IPU3_CSS_FORMAT_BPP_DEN, 57 .chroma_decim = 4, 58 .width_align = IPU3_UAPI_ISP_VEC_ELEMS, 59 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF, 60 }, { 61 /* Each 32 bytes contains 25 10-bit pixels */ 62 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10, 63 .colorspace = V4L2_COLORSPACE_RAW, 64 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 65 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR, 66 .bit_depth = 10, 67 .bytesperpixel_num = 64, 68 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 69 .flags = IPU3_CSS_FORMAT_FL_IN, 70 }, { 71 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10, 72 .colorspace = V4L2_COLORSPACE_RAW, 73 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 74 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG, 75 .bit_depth = 10, 76 .bytesperpixel_num = 64, 77 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 78 .flags = IPU3_CSS_FORMAT_FL_IN, 79 }, { 80 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10, 81 .colorspace = V4L2_COLORSPACE_RAW, 82 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 83 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG, 84 .bit_depth = 10, 85 .bytesperpixel_num = 64, 86 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 87 .flags = IPU3_CSS_FORMAT_FL_IN, 88 }, { 89 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10, 90 .colorspace = V4L2_COLORSPACE_RAW, 91 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED, 92 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB, 93 .bit_depth = 10, 94 .bytesperpixel_num = 64, 95 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS, 96 .flags = IPU3_CSS_FORMAT_FL_IN, 97 }, 98 }; 99 100 static const struct { 101 enum imgu_abi_queue_id qid; 102 size_t ptr_ofs; 103 } imgu_css_queues[IPU3_CSS_QUEUES] = { 104 [IPU3_CSS_QUEUE_IN] = { 105 IMGU_ABI_QUEUE_C_ID, 106 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 107 }, 108 [IPU3_CSS_QUEUE_OUT] = { 109 IMGU_ABI_QUEUE_D_ID, 110 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 111 }, 112 [IPU3_CSS_QUEUE_VF] = { 113 IMGU_ABI_QUEUE_E_ID, 114 offsetof(struct imgu_abi_buffer, payload.frame.frame_data) 115 }, 116 [IPU3_CSS_QUEUE_STAT_3A] = { 117 IMGU_ABI_QUEUE_F_ID, 118 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr) 119 }, 120 }; 121 122 /* Initialize queue based on given format, adjust format as needed */ 123 static int imgu_css_queue_init(struct imgu_css_queue *queue, 124 struct v4l2_pix_format_mplane *fmt, u32 flags) 125 { 126 struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix; 127 unsigned int i; 128 u32 sizeimage; 129 130 INIT_LIST_HEAD(&queue->bufs); 131 132 queue->css_fmt = NULL; /* Disable */ 133 if (!fmt) 134 return 0; 135 136 for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) { 137 if (!(imgu_css_formats[i].flags & flags)) 138 continue; 139 queue->css_fmt = &imgu_css_formats[i]; 140 if (imgu_css_formats[i].pixelformat == fmt->pixelformat) 141 break; 142 } 143 if (!queue->css_fmt) 144 return -EINVAL; /* Could not find any suitable format */ 145 146 queue->fmt.mpix = *fmt; 147 148 f->width = ALIGN(clamp_t(u32, f->width, 149 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2); 150 f->height = ALIGN(clamp_t(u32, f->height, 151 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2); 152 queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align); 153 if (queue->css_fmt->frame_format != IMGU_ABI_FRAME_FORMAT_RAW_PACKED) 154 f->plane_fmt[0].bytesperline = DIV_ROUND_UP(queue->width_pad * 155 queue->css_fmt->bytesperpixel_num, 156 IPU3_CSS_FORMAT_BPP_DEN); 157 else 158 /* For packed raw, alignment for bpl is by 50 to the width */ 159 f->plane_fmt[0].bytesperline = 160 DIV_ROUND_UP(f->width, 161 IPU3_CSS_FORMAT_BPP_DEN) * 162 queue->css_fmt->bytesperpixel_num; 163 164 sizeimage = f->height * f->plane_fmt[0].bytesperline; 165 if (queue->css_fmt->chroma_decim) 166 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim; 167 168 f->plane_fmt[0].sizeimage = sizeimage; 169 f->field = V4L2_FIELD_NONE; 170 f->num_planes = 1; 171 f->colorspace = queue->css_fmt->colorspace; 172 f->flags = 0; 173 f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; 174 f->quantization = V4L2_QUANTIZATION_DEFAULT; 175 f->xfer_func = V4L2_XFER_FUNC_DEFAULT; 176 memset(f->reserved, 0, sizeof(f->reserved)); 177 178 return 0; 179 } 180 181 static bool imgu_css_queue_enabled(struct imgu_css_queue *q) 182 { 183 return q->css_fmt; 184 } 185 186 /******************* css hw *******************/ 187 188 /* In the style of writesl() defined in include/asm-generic/io.h */ 189 static inline void writes(const void *mem, ssize_t count, void __iomem *addr) 190 { 191 if (count >= 4) { 192 const u32 *buf = mem; 193 194 count /= 4; 195 do { 196 writel(*buf++, addr); 197 addr += 4; 198 } while (--count); 199 } 200 } 201 202 /* Wait until register `reg', masked with `mask', becomes `cmp' */ 203 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp) 204 { 205 u32 val; 206 207 return readl_poll_timeout(base + reg, val, (val & mask) == cmp, 208 1000, 100 * 1000); 209 } 210 211 /* Initialize the IPU3 CSS hardware and associated h/w blocks */ 212 213 int imgu_css_set_powerup(struct device *dev, void __iomem *base) 214 { 215 static const unsigned int freq = 450; 216 u32 pm_ctrl, state, val; 217 218 dev_dbg(dev, "%s\n", __func__); 219 /* Clear the CSS busy signal */ 220 readl(base + IMGU_REG_GP_BUSY); 221 writel(0, base + IMGU_REG_GP_BUSY); 222 223 /* Wait for idle signal */ 224 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 225 IMGU_STATE_IDLE_STS)) { 226 dev_err(dev, "failed to set CSS idle\n"); 227 goto fail; 228 } 229 230 /* Reset the css */ 231 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET, 232 base + IMGU_REG_PM_CTRL); 233 234 usleep_range(200, 300); 235 236 /** Prepare CSS */ 237 238 pm_ctrl = readl(base + IMGU_REG_PM_CTRL); 239 state = readl(base + IMGU_REG_STATE); 240 241 dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n", 242 pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up"); 243 244 /* Power up CSS using wrapper */ 245 if (state & IMGU_STATE_POWER_DOWN) { 246 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START, 247 base + IMGU_REG_PM_CTRL); 248 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, 249 IMGU_PM_CTRL_START, 0)) { 250 dev_err(dev, "failed to power up CSS\n"); 251 goto fail; 252 } 253 usleep_range(2000, 3000); 254 } else { 255 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL); 256 } 257 258 /* Set the busy bit */ 259 writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY); 260 261 /* Set CSS clock frequency */ 262 pm_ctrl = readl(base + IMGU_REG_PM_CTRL); 263 val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 264 writel(val, base + IMGU_REG_PM_CTRL); 265 writel(0, base + IMGU_REG_GP_BUSY); 266 if (imgu_hw_wait(base, IMGU_REG_STATE, 267 IMGU_STATE_PWRDNM_FSM_MASK, 0)) { 268 dev_err(dev, "failed to pwrdn CSS\n"); 269 goto fail; 270 } 271 val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK; 272 writel(val, base + IMGU_REG_SYSTEM_REQ); 273 writel(1, base + IMGU_REG_GP_BUSY); 274 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT, 275 base + IMGU_REG_PM_CTRL); 276 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS, 277 IMGU_STATE_HALT_STS)) { 278 dev_err(dev, "failed to halt CSS\n"); 279 goto fail; 280 } 281 282 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START, 283 base + IMGU_REG_PM_CTRL); 284 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) { 285 dev_err(dev, "failed to start CSS\n"); 286 goto fail; 287 } 288 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT, 289 base + IMGU_REG_PM_CTRL); 290 291 val = readl(base + IMGU_REG_PM_CTRL); /* get pm_ctrl */ 292 val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 293 val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF); 294 writel(val, base + IMGU_REG_PM_CTRL); 295 296 return 0; 297 298 fail: 299 imgu_css_set_powerdown(dev, base); 300 return -EIO; 301 } 302 303 void imgu_css_set_powerdown(struct device *dev, void __iomem *base) 304 { 305 dev_dbg(dev, "%s\n", __func__); 306 /* wait for cio idle signal */ 307 if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE, 308 IMGU_CIO_GATE_BURST_MASK, 0)) 309 dev_warn(dev, "wait cio gate idle timeout"); 310 311 /* wait for css idle signal */ 312 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 313 IMGU_STATE_IDLE_STS)) 314 dev_warn(dev, "wait css idle timeout\n"); 315 316 /* do halt-halted handshake with css */ 317 writel(1, base + IMGU_REG_GP_HALT); 318 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS, 319 IMGU_STATE_HALT_STS)) 320 dev_warn(dev, "failed to halt css"); 321 322 /* de-assert the busy bit */ 323 writel(0, base + IMGU_REG_GP_BUSY); 324 } 325 326 static void imgu_css_hw_enable_irq(struct imgu_css *css) 327 { 328 void __iomem *const base = css->base; 329 u32 val, i; 330 331 /* Set up interrupts */ 332 333 /* 334 * Enable IRQ on the SP which signals that SP goes to idle 335 * (aka ready state) and set trigger to pulse 336 */ 337 val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY; 338 writel(val, base + IMGU_REG_SP_CTRL(0)); 339 writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0)); 340 341 /* Enable IRQs from the IMGU wrapper */ 342 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE); 343 /* Clear */ 344 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS); 345 346 /* Enable IRQs from main IRQ controller */ 347 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN)); 348 writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN)); 349 writel(IMGU_IRQCTRL_IRQ_MASK, 350 base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN)); 351 writel(IMGU_IRQCTRL_IRQ_MASK, 352 base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN)); 353 writel(IMGU_IRQCTRL_IRQ_MASK, 354 base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN)); 355 writel(IMGU_IRQCTRL_IRQ_MASK, 356 base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN)); 357 /* Wait for write complete */ 358 readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN)); 359 360 /* Enable IRQs from SP0 and SP1 controllers */ 361 for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) { 362 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i)); 363 writel(0, base + IMGU_REG_IRQCTRL_MASK(i)); 364 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i)); 365 writel(IMGU_IRQCTRL_IRQ_MASK, 366 base + IMGU_REG_IRQCTRL_ENABLE(i)); 367 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i)); 368 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i)); 369 /* Wait for write complete */ 370 readl(base + IMGU_REG_IRQCTRL_ENABLE(i)); 371 } 372 } 373 374 static int imgu_css_hw_init(struct imgu_css *css) 375 { 376 /* For checking that streaming monitor statuses are valid */ 377 static const struct { 378 u32 reg; 379 u32 mask; 380 const char *name; 381 } stream_monitors[] = { 382 { 383 IMGU_REG_GP_SP1_STRMON_STAT, 384 IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP, 385 "ISP0 to SP0" 386 }, { 387 IMGU_REG_GP_ISP_STRMON_STAT, 388 IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1, 389 "SP0 to ISP0" 390 }, { 391 IMGU_REG_GP_MOD_STRMON_STAT, 392 IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA, 393 "ISP0 to DMA0" 394 }, { 395 IMGU_REG_GP_ISP_STRMON_STAT, 396 IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP, 397 "DMA0 to ISP0" 398 }, { 399 IMGU_REG_GP_MOD_STRMON_STAT, 400 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC, 401 "ISP0 to GDC0" 402 }, { 403 IMGU_REG_GP_MOD_STRMON_STAT, 404 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS, 405 "GDC0 to ISP0" 406 }, { 407 IMGU_REG_GP_MOD_STRMON_STAT, 408 IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA, 409 "SP0 to DMA0" 410 }, { 411 IMGU_REG_GP_SP1_STRMON_STAT, 412 IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1, 413 "DMA0 to SP0" 414 }, { 415 IMGU_REG_GP_MOD_STRMON_STAT, 416 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC, 417 "SP0 to GDC0" 418 }, { 419 IMGU_REG_GP_MOD_STRMON_STAT, 420 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS, 421 "GDC0 to SP0" 422 }, 423 }; 424 425 struct device *dev = css->dev; 426 void __iomem *const base = css->base; 427 u32 val, i; 428 429 /* Set instruction cache address and inv bit for ISP, SP, and SP1 */ 430 for (i = 0; i < IMGU_NUM_SP; i++) { 431 struct imgu_fw_info *bi = 432 &css->fwp->binary_header[css->fw_sp[i]]; 433 434 writel(css->binary[css->fw_sp[i]].daddr, 435 base + IMGU_REG_SP_ICACHE_ADDR(bi->type)); 436 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) | 437 IMGU_CTRL_ICACHE_INV, 438 base + IMGU_REG_SP_CTRL(bi->type)); 439 } 440 writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR); 441 writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV, 442 base + IMGU_REG_ISP_CTRL); 443 444 /* Check that IMGU hardware is ready */ 445 446 if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) { 447 dev_err(dev, "SP is not idle\n"); 448 return -EIO; 449 } 450 if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) { 451 dev_err(dev, "ISP is not idle\n"); 452 return -EIO; 453 } 454 455 for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) { 456 val = readl(base + stream_monitors[i].reg); 457 if (val & stream_monitors[i].mask) { 458 dev_err(dev, "error: Stream monitor %s is valid\n", 459 stream_monitors[i].name); 460 return -EIO; 461 } 462 } 463 464 /* Initialize GDC with default values */ 465 466 for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) { 467 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK; 468 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK; 469 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK; 470 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK; 471 472 writel(val0 | (val1 << 16), 473 base + IMGU_REG_GDC_LUT_BASE + i * 8); 474 writel(val2 | (val3 << 16), 475 base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4); 476 } 477 478 return 0; 479 } 480 481 /* Boot the given IPU3 CSS SP */ 482 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp) 483 { 484 void __iomem *const base = css->base; 485 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 486 struct imgu_abi_sp_init_dmem_cfg dmem_cfg = { 487 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr 488 + bi->blob.data_source, 489 .dmem_data_addr = bi->blob.data_target, 490 .dmem_bss_addr = bi->blob.bss_target, 491 .data_size = bi->blob.data_size, 492 .bss_size = bi->blob.bss_size, 493 .sp_id = sp, 494 }; 495 496 writes(&dmem_cfg, sizeof(dmem_cfg), base + 497 IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data); 498 499 writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp)); 500 501 writel(readl(base + IMGU_REG_SP_CTRL(sp)) 502 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp)); 503 504 if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp) 505 + bi->info.sp.sw_state, 506 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED)) 507 return -EIO; 508 509 return 0; 510 } 511 512 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */ 513 static int imgu_css_hw_start(struct imgu_css *css) 514 { 515 static const u32 event_mask = 516 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) | 517 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) | 518 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) | 519 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) | 520 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) | 521 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) | 522 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) | 523 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) | 524 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) | 525 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) | 526 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE)) 527 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT; 528 529 void __iomem *const base = css->base; 530 struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl]; 531 unsigned int i; 532 533 writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE); 534 535 /* Start bootloader */ 536 537 writel(IMGU_ABI_BL_SWSTATE_BUSY, 538 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state); 539 writel(IMGU_NUM_SP, 540 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds); 541 542 for (i = 0; i < IMGU_NUM_SP; i++) { 543 int j = IMGU_NUM_SP - i - 1; /* load sp1 first, then sp0 */ 544 struct imgu_fw_info *sp = 545 &css->fwp->binary_header[css->fw_sp[j]]; 546 struct imgu_abi_bl_dma_cmd_entry dma_cmd = { 547 .src_addr = css->binary[css->fw_sp[j]].daddr 548 + sp->blob.text_source, 549 .size = sp->blob.text_size, 550 .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM, 551 .dst_addr = IMGU_SP_PMEM_BASE(j), 552 }; 553 554 writes(&dma_cmd, sizeof(dma_cmd), 555 base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) + 556 bl->info.bl.dma_cmd_list); 557 } 558 559 writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR); 560 561 writel(readl(base + IMGU_REG_ISP_CTRL) 562 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL); 563 if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE 564 + bl->info.bl.sw_state, ~0, 565 IMGU_ABI_BL_SWSTATE_OK)) { 566 dev_err(css->dev, "failed to start bootloader\n"); 567 return -EIO; 568 } 569 570 /* Start ISP */ 571 572 memset(css->xmem_sp_group_ptrs.vaddr, 0, 573 sizeof(struct imgu_abi_sp_group)); 574 575 bi = &css->fwp->binary_header[css->fw_sp[0]]; 576 577 writel(css->xmem_sp_group_ptrs.daddr, 578 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data); 579 580 writel(IMGU_ABI_SP_SWSTATE_TERMINATED, 581 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state); 582 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb); 583 584 if (imgu_css_hw_start_sp(css, 0)) 585 return -EIO; 586 587 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started); 588 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + 589 bi->info.sp.host_sp_queues_initialized); 590 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode); 591 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb); 592 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0) 593 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 594 595 /* Enable all events for all queues */ 596 597 for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++) 598 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0) 599 + bi->info.sp.host_sp_com 600 + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i)); 601 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + 602 bi->info.sp.host_sp_queues_initialized); 603 604 /* Start SP1 */ 605 606 bi = &css->fwp->binary_header[css->fw_sp[1]]; 607 608 writel(IMGU_ABI_SP_SWSTATE_TERMINATED, 609 base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state); 610 611 if (imgu_css_hw_start_sp(css, 1)) 612 return -EIO; 613 614 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1) 615 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 616 617 return 0; 618 } 619 620 static void imgu_css_hw_stop(struct imgu_css *css) 621 { 622 void __iomem *const base = css->base; 623 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]]; 624 625 /* Stop fw */ 626 writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE, 627 base + IMGU_REG_SP_DMEM_BASE(0) + 628 bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND); 629 if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0), 630 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE)) 631 dev_err(css->dev, "wait sp0 idle timeout.\n"); 632 if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) != 633 IMGU_ABI_SP_SWSTATE_TERMINATED) 634 dev_err(css->dev, "sp0 is not terminated.\n"); 635 if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL, 636 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE)) 637 dev_err(css->dev, "wait isp idle timeout\n"); 638 } 639 640 static void imgu_css_hw_cleanup(struct imgu_css *css) 641 { 642 void __iomem *const base = css->base; 643 644 /** Reset CSS **/ 645 646 /* Clear the CSS busy signal */ 647 readl(base + IMGU_REG_GP_BUSY); 648 writel(0, base + IMGU_REG_GP_BUSY); 649 650 /* Wait for idle signal */ 651 if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS, 652 IMGU_STATE_IDLE_STS)) 653 dev_err(css->dev, "failed to shut down hw cleanly\n"); 654 655 /* Reset the css */ 656 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET, 657 base + IMGU_REG_PM_CTRL); 658 659 usleep_range(200, 300); 660 } 661 662 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe) 663 { 664 struct imgu_device *imgu = dev_get_drvdata(css->dev); 665 unsigned int i; 666 667 imgu_css_pool_cleanup(imgu, 668 &css->pipes[pipe].pool.parameter_set_info); 669 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.acc); 670 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.gdc); 671 imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.obgrid); 672 673 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 674 imgu_css_pool_cleanup(imgu, 675 &css->pipes[pipe].pool.binary_params_p[i]); 676 } 677 678 /* 679 * This function initializes various stages of the 680 * IPU3 CSS ISP pipeline 681 */ 682 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe) 683 { 684 static const int BYPC = 2; /* Bytes per component */ 685 static const struct imgu_abi_buffer_sp buffer_sp_init = { 686 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID}, 687 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID, 688 }; 689 690 struct imgu_abi_isp_iterator_config *cfg_iter; 691 struct imgu_abi_isp_ref_config *cfg_ref; 692 struct imgu_abi_isp_dvs_config *cfg_dvs; 693 struct imgu_abi_isp_tnr3_config *cfg_tnr; 694 struct imgu_abi_isp_ref_dmem_state *cfg_ref_state; 695 struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state; 696 697 const int stage = 0; 698 unsigned int i, j; 699 700 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 701 const struct imgu_fw_info *bi = 702 &css->fwp->binary_header[css_pipe->bindex]; 703 const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes; 704 705 struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp + 706 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG]; 707 struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp + 708 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE]; 709 710 struct imgu_abi_isp_stage *isp_stage; 711 struct imgu_abi_sp_stage *sp_stage; 712 struct imgu_abi_sp_group *sp_group; 713 714 const unsigned int bds_width_pad = 715 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, 716 2 * IPU3_UAPI_ISP_VEC_ELEMS); 717 718 const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0; 719 enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG; 720 void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr; 721 722 struct imgu_device *imgu = dev_get_drvdata(css->dev); 723 724 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe); 725 726 /* Configure iterator */ 727 728 cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 729 &cofs->dmem.iterator, 730 sizeof(*cfg_iter), vaddr); 731 if (!cfg_iter) 732 goto bad_firmware; 733 734 cfg_iter->input_info.res.width = 735 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width; 736 cfg_iter->input_info.res.height = 737 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height; 738 cfg_iter->input_info.padded_width = 739 css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad; 740 cfg_iter->input_info.format = 741 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format; 742 cfg_iter->input_info.raw_bit_depth = 743 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth; 744 cfg_iter->input_info.raw_bayer_order = 745 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order; 746 cfg_iter->input_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 747 748 cfg_iter->internal_info.res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width; 749 cfg_iter->internal_info.res.height = 750 css_pipe->rect[IPU3_CSS_RECT_BDS].height; 751 cfg_iter->internal_info.padded_width = bds_width_pad; 752 cfg_iter->internal_info.format = 753 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 754 cfg_iter->internal_info.raw_bit_depth = 755 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; 756 cfg_iter->internal_info.raw_bayer_order = 757 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; 758 cfg_iter->internal_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 759 760 cfg_iter->output_info.res.width = 761 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width; 762 cfg_iter->output_info.res.height = 763 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; 764 cfg_iter->output_info.padded_width = 765 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad; 766 cfg_iter->output_info.format = 767 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 768 cfg_iter->output_info.raw_bit_depth = 769 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; 770 cfg_iter->output_info.raw_bayer_order = 771 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; 772 cfg_iter->output_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 773 774 cfg_iter->vf_info.res.width = 775 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width; 776 cfg_iter->vf_info.res.height = 777 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; 778 cfg_iter->vf_info.padded_width = 779 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad; 780 cfg_iter->vf_info.format = 781 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; 782 cfg_iter->vf_info.raw_bit_depth = 783 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth; 784 cfg_iter->vf_info.raw_bayer_order = 785 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order; 786 cfg_iter->vf_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 787 788 cfg_iter->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width; 789 cfg_iter->dvs_envelope.height = 790 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height; 791 792 /* Configure reference (delay) frames */ 793 794 cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 795 &cofs->dmem.ref, 796 sizeof(*cfg_ref), vaddr); 797 if (!cfg_ref) 798 goto bad_firmware; 799 800 cfg_ref->port_b.crop = 0; 801 cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC; 802 cfg_ref->port_b.width = 803 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width; 804 cfg_ref->port_b.stride = 805 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline; 806 cfg_ref->width_a_over_b = 807 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems; 808 cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; 809 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) { 810 cfg_ref->ref_frame_addr_y[i] = 811 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr; 812 cfg_ref->ref_frame_addr_c[i] = 813 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr + 814 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline * 815 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; 816 } 817 for (; i < IMGU_ABI_FRAMES_REF; i++) { 818 cfg_ref->ref_frame_addr_y[i] = 0; 819 cfg_ref->ref_frame_addr_c[i] = 0; 820 } 821 822 /* Configure DVS (digital video stabilization) */ 823 824 cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 825 &cofs->dmem.dvs, sizeof(*cfg_dvs), 826 vaddr); 827 if (!cfg_dvs) 828 goto bad_firmware; 829 830 cfg_dvs->num_horizontal_blocks = 831 ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width, 832 IMGU_DVS_BLOCK_W), 2); 833 cfg_dvs->num_vertical_blocks = 834 DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height, 835 IMGU_DVS_BLOCK_H); 836 837 /* Configure TNR (temporal noise reduction) */ 838 839 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { 840 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 841 &cofs->dmem.tnr3, 842 sizeof(*cfg_tnr), 843 vaddr); 844 if (!cfg_tnr) 845 goto bad_firmware; 846 847 cfg_tnr->port_b.crop = 0; 848 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES; 849 cfg_tnr->port_b.width = 850 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; 851 cfg_tnr->port_b.stride = 852 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline; 853 cfg_tnr->width_a_over_b = 854 IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems; 855 cfg_tnr->frame_height = 856 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; 857 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1; 858 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 859 cfg_tnr->frame_addr[i] = 860 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR] 861 .mem[i].daddr; 862 for (; i < IMGU_ABI_FRAMES_TNR; i++) 863 cfg_tnr->frame_addr[i] = 0; 864 } 865 866 /* Configure ref dmem state parameters */ 867 868 cfg = IMGU_ABI_PARAM_CLASS_STATE; 869 vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr; 870 871 cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 872 &sofs->dmem.ref, 873 sizeof(*cfg_ref_state), 874 vaddr); 875 if (!cfg_ref_state) 876 goto bad_firmware; 877 878 cfg_ref_state->ref_in_buf_idx = 0; 879 cfg_ref_state->ref_out_buf_idx = 1; 880 881 /* Configure tnr dmem state parameters */ 882 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) { 883 cfg_tnr_state = 884 imgu_css_fw_pipeline_params(css, pipe, cfg, m0, 885 &sofs->dmem.tnr3, 886 sizeof(*cfg_tnr_state), 887 vaddr); 888 if (!cfg_tnr_state) 889 goto bad_firmware; 890 891 cfg_tnr_state->in_bufidx = 0; 892 cfg_tnr_state->out_bufidx = 1; 893 cfg_tnr_state->bypass_filter = 0; 894 cfg_tnr_state->total_frame_counter = 0; 895 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++) 896 cfg_tnr_state->buffer_frame_counter[i] = 0; 897 } 898 899 /* Configure ISP stage */ 900 901 isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr; 902 memset(isp_stage, 0, sizeof(*isp_stage)); 903 isp_stage->blob_info = bi->blob; 904 isp_stage->binary_info = bi->info.isp.sp; 905 strscpy(isp_stage->binary_name, 906 (char *)css->fwp + bi->blob.prog_name_offset, 907 sizeof(isp_stage->binary_name)); 908 isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers; 909 for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++) 910 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++) 911 isp_stage->mem_initializers.params[i][j].address = 912 css_pipe->binary_params_cs[i - 1][j].daddr; 913 914 /* Configure SP stage */ 915 916 sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr; 917 memset(sp_stage, 0, sizeof(*sp_stage)); 918 919 sp_stage->frames.in.buf_attr = buffer_sp_init; 920 for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++) 921 sp_stage->frames.out[i].buf_attr = buffer_sp_init; 922 sp_stage->frames.out_vf.buf_attr = buffer_sp_init; 923 sp_stage->frames.s3a_buf = buffer_sp_init; 924 sp_stage->frames.dvs_buf = buffer_sp_init; 925 926 sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP; 927 sp_stage->num = stage; 928 sp_stage->isp_online = 0; 929 sp_stage->isp_copy_vf = 0; 930 sp_stage->isp_copy_output = 0; 931 932 sp_stage->enable.vf_output = css_pipe->vf_output_en; 933 934 sp_stage->frames.effective_in_res.width = 935 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width; 936 sp_stage->frames.effective_in_res.height = 937 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height; 938 sp_stage->frames.in.info.res.width = 939 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width; 940 sp_stage->frames.in.info.res.height = 941 css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height; 942 sp_stage->frames.in.info.padded_width = 943 css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad; 944 sp_stage->frames.in.info.format = 945 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format; 946 sp_stage->frames.in.info.raw_bit_depth = 947 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth; 948 sp_stage->frames.in.info.raw_bayer_order = 949 css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order; 950 sp_stage->frames.in.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 951 sp_stage->frames.in.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID; 952 sp_stage->frames.in.buf_attr.buf_type = 953 IMGU_ABI_BUFFER_TYPE_INPUT_FRAME; 954 955 sp_stage->frames.out[0].info.res.width = 956 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width; 957 sp_stage->frames.out[0].info.res.height = 958 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; 959 sp_stage->frames.out[0].info.padded_width = 960 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad; 961 sp_stage->frames.out[0].info.format = 962 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 963 sp_stage->frames.out[0].info.raw_bit_depth = 964 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; 965 sp_stage->frames.out[0].info.raw_bayer_order = 966 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; 967 sp_stage->frames.out[0].info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 968 sp_stage->frames.out[0].planes.nv.uv.offset = 969 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad * 970 css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height; 971 sp_stage->frames.out[0].buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID; 972 sp_stage->frames.out[0].buf_attr.buf_type = 973 IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME; 974 975 sp_stage->frames.out[1].buf_attr.buf_src.queue_id = 976 IMGU_ABI_QUEUE_EVENT_ID; 977 978 sp_stage->frames.internal_frame_info.res.width = 979 css_pipe->rect[IPU3_CSS_RECT_BDS].width; 980 sp_stage->frames.internal_frame_info.res.height = 981 css_pipe->rect[IPU3_CSS_RECT_BDS].height; 982 sp_stage->frames.internal_frame_info.padded_width = bds_width_pad; 983 984 sp_stage->frames.internal_frame_info.format = 985 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 986 sp_stage->frames.internal_frame_info.raw_bit_depth = 987 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth; 988 sp_stage->frames.internal_frame_info.raw_bayer_order = 989 css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order; 990 sp_stage->frames.internal_frame_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 991 992 sp_stage->frames.out_vf.info.res.width = 993 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width; 994 sp_stage->frames.out_vf.info.res.height = 995 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; 996 sp_stage->frames.out_vf.info.padded_width = 997 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad; 998 sp_stage->frames.out_vf.info.format = 999 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; 1000 sp_stage->frames.out_vf.info.raw_bit_depth = 1001 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth; 1002 sp_stage->frames.out_vf.info.raw_bayer_order = 1003 css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order; 1004 sp_stage->frames.out_vf.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER; 1005 sp_stage->frames.out_vf.planes.yuv.u.offset = 1006 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad * 1007 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height; 1008 sp_stage->frames.out_vf.planes.yuv.v.offset = 1009 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad * 1010 css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height * 5 / 4; 1011 sp_stage->frames.out_vf.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID; 1012 sp_stage->frames.out_vf.buf_attr.buf_type = 1013 IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME; 1014 1015 sp_stage->frames.s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID; 1016 sp_stage->frames.s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS; 1017 1018 sp_stage->frames.dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID; 1019 sp_stage->frames.dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS; 1020 1021 sp_stage->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width; 1022 sp_stage->dvs_envelope.height = 1023 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height; 1024 1025 sp_stage->isp_pipe_version = 1026 bi->info.isp.sp.pipeline.isp_pipe_version; 1027 sp_stage->isp_deci_log_factor = 1028 clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width / 1029 IMGU_MAX_BQ_GRID_WIDTH), 1030 fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height / 1031 IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5); 1032 sp_stage->isp_vf_downscale_bits = 0; 1033 sp_stage->if_config_index = 255; 1034 sp_stage->sp_enable_xnr = 0; 1035 sp_stage->num_stripes = stripes; 1036 sp_stage->enable.s3a = 1; 1037 sp_stage->enable.dvs_stats = 0; 1038 1039 sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr; 1040 sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr; 1041 sp_stage->isp_stage_addr = 1042 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr; 1043 1044 /* Configure SP group */ 1045 1046 sp_group = css->xmem_sp_group_ptrs.vaddr; 1047 memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline)); 1048 1049 sp_group->pipe[pipe].num_stages = 1; 1050 sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id; 1051 sp_group->pipe[pipe].thread_id = pipe; 1052 sp_group->pipe[pipe].pipe_num = pipe; 1053 sp_group->pipe[pipe].num_execs = -1; 1054 sp_group->pipe[pipe].pipe_qos_config = -1; 1055 sp_group->pipe[pipe].required_bds_factor = 0; 1056 sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1; 1057 sp_group->pipe[pipe].inout_port_config = 1058 IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST | 1059 IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST; 1060 sp_group->pipe[pipe].scaler_pp_lut = 0; 1061 sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0; 1062 sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0; 1063 sp_group->pipe[pipe].sp_stage_addr[stage] = 1064 css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr; 1065 sp_group->pipe[pipe].pipe_config = 1066 bi->info.isp.sp.enable.params ? (1 << pipe) : 0; 1067 sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP; 1068 1069 /* Initialize parameter pools */ 1070 1071 if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info, 1072 sizeof(struct imgu_abi_parameter_set_info)) || 1073 imgu_css_pool_init(imgu, &css_pipe->pool.acc, 1074 sizeof(struct imgu_abi_acc_param)) || 1075 imgu_css_pool_init(imgu, &css_pipe->pool.gdc, 1076 sizeof(struct imgu_abi_gdc_warp_param) * 1077 3 * cfg_dvs->num_horizontal_blocks / 2 * 1078 cfg_dvs->num_vertical_blocks) || 1079 imgu_css_pool_init(imgu, &css_pipe->pool.obgrid, 1080 imgu_css_fw_obgrid_size( 1081 &css->fwp->binary_header[css_pipe->bindex]))) 1082 goto out_of_memory; 1083 1084 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1085 if (imgu_css_pool_init(imgu, 1086 &css_pipe->pool.binary_params_p[i], 1087 bi->info.isp.sp.mem_initializers.params 1088 [IMGU_ABI_PARAM_CLASS_PARAM][i].size)) 1089 goto out_of_memory; 1090 1091 return 0; 1092 1093 bad_firmware: 1094 imgu_css_pipeline_cleanup(css, pipe); 1095 return -EPROTO; 1096 1097 out_of_memory: 1098 imgu_css_pipeline_cleanup(css, pipe); 1099 return -ENOMEM; 1100 } 1101 1102 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread) 1103 { 1104 static const unsigned int sp; 1105 void __iomem *const base = css->base; 1106 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1107 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1108 bi->info.sp.host_sp_queue; 1109 1110 return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) : 1111 readb(&q->host2sp_evtq_info.end); 1112 } 1113 1114 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */ 1115 static int imgu_css_queue_data(struct imgu_css *css, 1116 int queue, int thread, u32 data) 1117 { 1118 static const unsigned int sp; 1119 void __iomem *const base = css->base; 1120 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1121 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1122 bi->info.sp.host_sp_queue; 1123 u8 size, start, end, end2; 1124 1125 if (queue >= 0) { 1126 size = readb(&q->host2sp_bufq_info[thread][queue].size); 1127 start = readb(&q->host2sp_bufq_info[thread][queue].start); 1128 end = readb(&q->host2sp_bufq_info[thread][queue].end); 1129 } else { 1130 size = readb(&q->host2sp_evtq_info.size); 1131 start = readb(&q->host2sp_evtq_info.start); 1132 end = readb(&q->host2sp_evtq_info.end); 1133 } 1134 1135 if (size == 0) 1136 return -EIO; 1137 1138 end2 = (end + 1) % size; 1139 if (end2 == start) 1140 return -EBUSY; /* Queue full */ 1141 1142 if (queue >= 0) { 1143 writel(data, &q->host2sp_bufq[thread][queue][end]); 1144 writeb(end2, &q->host2sp_bufq_info[thread][queue].end); 1145 } else { 1146 writel(data, &q->host2sp_evtq[end]); 1147 writeb(end2, &q->host2sp_evtq_info.end); 1148 } 1149 1150 return 0; 1151 } 1152 1153 /* Receive data using given buffer queue, or if queue < 0, event queue. */ 1154 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data) 1155 { 1156 static const unsigned int sp; 1157 void __iomem *const base = css->base; 1158 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]]; 1159 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) + 1160 bi->info.sp.host_sp_queue; 1161 u8 size, start, end, start2; 1162 1163 if (queue >= 0) { 1164 size = readb(&q->sp2host_bufq_info[queue].size); 1165 start = readb(&q->sp2host_bufq_info[queue].start); 1166 end = readb(&q->sp2host_bufq_info[queue].end); 1167 } else { 1168 size = readb(&q->sp2host_evtq_info.size); 1169 start = readb(&q->sp2host_evtq_info.start); 1170 end = readb(&q->sp2host_evtq_info.end); 1171 } 1172 1173 if (size == 0) 1174 return -EIO; 1175 1176 if (end == start) 1177 return -EBUSY; /* Queue empty */ 1178 1179 start2 = (start + 1) % size; 1180 1181 if (queue >= 0) { 1182 *data = readl(&q->sp2host_bufq[queue][start]); 1183 writeb(start2, &q->sp2host_bufq_info[queue].start); 1184 } else { 1185 int r; 1186 1187 *data = readl(&q->sp2host_evtq[start]); 1188 writeb(start2, &q->sp2host_evtq_info.start); 1189 1190 /* Acknowledge events dequeued from event queue */ 1191 r = imgu_css_queue_data(css, queue, 0, 1192 IMGU_ABI_EVENT_EVENT_DEQUEUED); 1193 if (r < 0) 1194 return r; 1195 } 1196 1197 return 0; 1198 } 1199 1200 /* Free binary-specific resources */ 1201 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe) 1202 { 1203 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1204 unsigned int i, j; 1205 1206 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1207 1208 for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++) 1209 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1210 imgu_dmamap_free(imgu, 1211 &css_pipe->binary_params_cs[j][i]); 1212 1213 j = IPU3_CSS_AUX_FRAME_REF; 1214 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1215 imgu_dmamap_free(imgu, 1216 &css_pipe->aux_frames[j].mem[i]); 1217 1218 j = IPU3_CSS_AUX_FRAME_TNR; 1219 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1220 imgu_dmamap_free(imgu, 1221 &css_pipe->aux_frames[j].mem[i]); 1222 } 1223 1224 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe) 1225 { 1226 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1227 unsigned int i, j; 1228 1229 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1230 1231 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; 1232 j < IMGU_ABI_PARAM_CLASS_NUM; j++) 1233 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) 1234 if (!imgu_dmamap_alloc(imgu, 1235 &css_pipe->binary_params_cs[j - 1][i], 1236 CSS_ABI_SIZE)) 1237 goto out_of_memory; 1238 1239 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1240 if (!imgu_dmamap_alloc(imgu, 1241 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF]. 1242 mem[i], CSS_BDS_SIZE)) 1243 goto out_of_memory; 1244 1245 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1246 if (!imgu_dmamap_alloc(imgu, 1247 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]. 1248 mem[i], CSS_GDC_SIZE)) 1249 goto out_of_memory; 1250 1251 return 0; 1252 1253 out_of_memory: 1254 imgu_css_binary_cleanup(css, pipe); 1255 return -ENOMEM; 1256 } 1257 1258 /* allocate binary-specific resources */ 1259 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe) 1260 { 1261 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1262 struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex]; 1263 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1264 int i, j, size; 1265 static const int BYPC = 2; /* Bytes per component */ 1266 unsigned int w, h; 1267 1268 /* Allocate parameter memory blocks for this binary */ 1269 1270 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++) 1271 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) { 1272 if (imgu_css_dma_buffer_resize( 1273 imgu, 1274 &css_pipe->binary_params_cs[j - 1][i], 1275 bi->info.isp.sp.mem_initializers.params[j][i].size)) 1276 goto out_of_memory; 1277 } 1278 1279 /* Allocate internal frame buffers */ 1280 1281 /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */ 1282 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC; 1283 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width = 1284 css_pipe->rect[IPU3_CSS_RECT_BDS].width; 1285 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height = 1286 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height, 1287 IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y; 1288 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height; 1289 w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, 1290 2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X; 1291 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline = 1292 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w; 1293 size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2; 1294 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1295 if (imgu_css_dma_buffer_resize( 1296 imgu, 1297 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i], 1298 size)) 1299 goto out_of_memory; 1300 1301 /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */ 1302 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1; 1303 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width = 1304 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width, 1305 bi->info.isp.sp.block.block_width * 1306 IPU3_UAPI_ISP_VEC_ELEMS); 1307 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height = 1308 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height, 1309 bi->info.isp.sp.block.output_block_height); 1310 1311 w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width; 1312 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w; 1313 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height; 1314 size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */ 1315 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) 1316 if (imgu_css_dma_buffer_resize( 1317 imgu, 1318 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i], 1319 size)) 1320 goto out_of_memory; 1321 1322 return 0; 1323 1324 out_of_memory: 1325 imgu_css_binary_cleanup(css, pipe); 1326 return -ENOMEM; 1327 } 1328 1329 int imgu_css_start_streaming(struct imgu_css *css) 1330 { 1331 u32 data; 1332 int r, pipe; 1333 1334 if (css->streaming) 1335 return -EPROTO; 1336 1337 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1338 r = imgu_css_binary_setup(css, pipe); 1339 if (r < 0) 1340 return r; 1341 } 1342 1343 r = imgu_css_hw_init(css); 1344 if (r < 0) 1345 return r; 1346 1347 r = imgu_css_hw_start(css); 1348 if (r < 0) 1349 goto fail; 1350 1351 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1352 r = imgu_css_pipeline_init(css, pipe); 1353 if (r < 0) 1354 goto fail; 1355 } 1356 1357 css->streaming = true; 1358 1359 imgu_css_hw_enable_irq(css); 1360 1361 /* Initialize parameters to default */ 1362 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1363 r = imgu_css_set_parameters(css, pipe, NULL); 1364 if (r < 0) 1365 goto fail; 1366 } 1367 1368 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data))) 1369 ; 1370 if (r != -EBUSY) 1371 goto fail; 1372 1373 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data))) 1374 ; 1375 if (r != -EBUSY) 1376 goto fail; 1377 1378 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1379 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 1380 IMGU_ABI_EVENT_START_STREAM | 1381 pipe << 16); 1382 if (r < 0) 1383 goto fail; 1384 } 1385 1386 return 0; 1387 1388 fail: 1389 css->streaming = false; 1390 imgu_css_hw_cleanup(css); 1391 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1392 imgu_css_pipeline_cleanup(css, pipe); 1393 imgu_css_binary_cleanup(css, pipe); 1394 } 1395 1396 return r; 1397 } 1398 1399 void imgu_css_stop_streaming(struct imgu_css *css) 1400 { 1401 struct imgu_css_buffer *b, *b0; 1402 int q, r, pipe; 1403 1404 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1405 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 1406 IMGU_ABI_EVENT_STOP_STREAM); 1407 if (r < 0) 1408 dev_warn(css->dev, "failed on stop stream event\n"); 1409 } 1410 1411 if (!css->streaming) 1412 return; 1413 1414 imgu_css_hw_stop(css); 1415 1416 imgu_css_hw_cleanup(css); 1417 1418 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) { 1419 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1420 1421 imgu_css_pipeline_cleanup(css, pipe); 1422 1423 spin_lock(&css_pipe->qlock); 1424 for (q = 0; q < IPU3_CSS_QUEUES; q++) 1425 list_for_each_entry_safe(b, b0, 1426 &css_pipe->queue[q].bufs, 1427 list) { 1428 b->state = IPU3_CSS_BUFFER_FAILED; 1429 list_del(&b->list); 1430 } 1431 spin_unlock(&css_pipe->qlock); 1432 } 1433 1434 css->streaming = false; 1435 } 1436 1437 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe) 1438 { 1439 int q; 1440 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1441 1442 spin_lock(&css_pipe->qlock); 1443 for (q = 0; q < IPU3_CSS_QUEUES; q++) 1444 if (!list_empty(&css_pipe->queue[q].bufs)) 1445 break; 1446 spin_unlock(&css_pipe->qlock); 1447 return (q == IPU3_CSS_QUEUES); 1448 } 1449 1450 bool imgu_css_queue_empty(struct imgu_css *css) 1451 { 1452 unsigned int pipe; 1453 bool ret = false; 1454 1455 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) 1456 ret &= imgu_css_pipe_queue_empty(css, pipe); 1457 1458 return ret; 1459 } 1460 1461 bool imgu_css_is_streaming(struct imgu_css *css) 1462 { 1463 return css->streaming; 1464 } 1465 1466 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe) 1467 { 1468 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1469 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1470 unsigned int p, q, i; 1471 1472 /* Allocate and map common structures with imgu hardware */ 1473 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) 1474 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { 1475 if (!imgu_dmamap_alloc(imgu, 1476 &css_pipe-> 1477 xmem_sp_stage_ptrs[p][i], 1478 sizeof(struct imgu_abi_sp_stage))) 1479 return -ENOMEM; 1480 if (!imgu_dmamap_alloc(imgu, 1481 &css_pipe-> 1482 xmem_isp_stage_ptrs[p][i], 1483 sizeof(struct imgu_abi_isp_stage))) 1484 return -ENOMEM; 1485 } 1486 1487 if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs, 1488 ALIGN(sizeof(struct imgu_abi_ddr_address_map), 1489 IMGU_ABI_ISP_DDR_WORD_BYTES))) 1490 return -ENOMEM; 1491 1492 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1493 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]); 1494 1495 for (i = 0; i < abi_buf_num; i++) 1496 if (!imgu_dmamap_alloc(imgu, 1497 &css_pipe->abi_buffers[q][i], 1498 sizeof(struct imgu_abi_buffer))) 1499 return -ENOMEM; 1500 } 1501 1502 if (imgu_css_binary_preallocate(css, pipe)) { 1503 imgu_css_binary_cleanup(css, pipe); 1504 return -ENOMEM; 1505 } 1506 1507 return 0; 1508 } 1509 1510 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe) 1511 { 1512 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1513 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1514 unsigned int p, q, i, abi_buf_num; 1515 1516 imgu_css_binary_cleanup(css, pipe); 1517 1518 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1519 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]); 1520 for (i = 0; i < abi_buf_num; i++) 1521 imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]); 1522 } 1523 1524 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++) 1525 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) { 1526 imgu_dmamap_free(imgu, 1527 &css_pipe->xmem_sp_stage_ptrs[p][i]); 1528 imgu_dmamap_free(imgu, 1529 &css_pipe->xmem_isp_stage_ptrs[p][i]); 1530 } 1531 1532 imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs); 1533 } 1534 1535 void imgu_css_cleanup(struct imgu_css *css) 1536 { 1537 struct imgu_device *imgu = dev_get_drvdata(css->dev); 1538 unsigned int pipe; 1539 1540 imgu_css_stop_streaming(css); 1541 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) 1542 imgu_css_pipe_cleanup(css, pipe); 1543 imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs); 1544 imgu_css_fw_cleanup(css); 1545 } 1546 1547 int imgu_css_init(struct device *dev, struct imgu_css *css, 1548 void __iomem *base, int length) 1549 { 1550 struct imgu_device *imgu = dev_get_drvdata(dev); 1551 int r, q, pipe; 1552 1553 /* Initialize main data structure */ 1554 css->dev = dev; 1555 css->base = base; 1556 css->iomem_length = length; 1557 1558 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) { 1559 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1560 1561 css_pipe->vf_output_en = false; 1562 spin_lock_init(&css_pipe->qlock); 1563 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY; 1564 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO; 1565 for (q = 0; q < IPU3_CSS_QUEUES; q++) { 1566 r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0); 1567 if (r) 1568 return r; 1569 } 1570 r = imgu_css_map_init(css, pipe); 1571 if (r) { 1572 imgu_css_cleanup(css); 1573 return r; 1574 } 1575 } 1576 if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs, 1577 sizeof(struct imgu_abi_sp_group))) 1578 return -ENOMEM; 1579 1580 r = imgu_css_fw_init(css); 1581 if (r) 1582 return r; 1583 1584 return 0; 1585 } 1586 1587 static u32 imgu_css_adjust(u32 res, u32 align) 1588 { 1589 u32 val = max_t(u32, IPU3_CSS_MIN_RES, res); 1590 1591 return DIV_ROUND_CLOSEST(val, align) * align; 1592 } 1593 1594 /* Select a binary matching the required resolutions and formats */ 1595 static int imgu_css_find_binary(struct imgu_css *css, 1596 unsigned int pipe, 1597 struct imgu_css_queue queue[IPU3_CSS_QUEUES], 1598 struct v4l2_rect rects[IPU3_CSS_RECTS]) 1599 { 1600 const int binary_nr = css->fwp->file_header.binary_nr; 1601 unsigned int binary_mode = 1602 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ? 1603 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO; 1604 const struct v4l2_pix_format_mplane *in = 1605 &queue[IPU3_CSS_QUEUE_IN].fmt.mpix; 1606 const struct v4l2_pix_format_mplane *out = 1607 &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix; 1608 const struct v4l2_pix_format_mplane *vf = 1609 &queue[IPU3_CSS_QUEUE_VF].fmt.mpix; 1610 u32 stripe_w = 0, stripe_h = 0; 1611 const char *name; 1612 int i, j; 1613 1614 if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN])) 1615 return -EINVAL; 1616 1617 /* Find out the strip size boundary */ 1618 for (i = 0; i < binary_nr; i++) { 1619 struct imgu_fw_info *bi = &css->fwp->binary_header[i]; 1620 1621 u32 max_width = bi->info.isp.sp.output.max_width; 1622 u32 max_height = bi->info.isp.sp.output.max_height; 1623 1624 if (bi->info.isp.sp.iterator.num_stripes <= 1) { 1625 stripe_w = stripe_w ? 1626 min(stripe_w, max_width) : max_width; 1627 stripe_h = stripe_h ? 1628 min(stripe_h, max_height) : max_height; 1629 } 1630 } 1631 1632 for (i = 0; i < binary_nr; i++) { 1633 struct imgu_fw_info *bi = &css->fwp->binary_header[i]; 1634 enum imgu_abi_frame_format q_fmt; 1635 1636 name = (void *)css->fwp + bi->blob.prog_name_offset; 1637 1638 /* Check that binary supports memory-to-memory processing */ 1639 if (bi->info.isp.sp.input.source != 1640 IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY) 1641 continue; 1642 1643 /* Check that binary supports raw10 input */ 1644 if (!bi->info.isp.sp.enable.input_feeder && 1645 !bi->info.isp.sp.enable.input_raw) 1646 continue; 1647 1648 /* Check binary mode */ 1649 if (bi->info.isp.sp.pipeline.mode != binary_mode) 1650 continue; 1651 1652 /* Since input is RGGB bayer, need to process colors */ 1653 if (bi->info.isp.sp.enable.luma_only) 1654 continue; 1655 1656 if (in->width < bi->info.isp.sp.input.min_width || 1657 in->width > bi->info.isp.sp.input.max_width || 1658 in->height < bi->info.isp.sp.input.min_height || 1659 in->height > bi->info.isp.sp.input.max_height) 1660 continue; 1661 1662 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) { 1663 if (bi->info.isp.num_output_pins <= 0) 1664 continue; 1665 1666 q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format; 1667 for (j = 0; j < bi->info.isp.num_output_formats; j++) 1668 if (bi->info.isp.output_formats[j] == q_fmt) 1669 break; 1670 if (j >= bi->info.isp.num_output_formats) 1671 continue; 1672 1673 if (out->width < bi->info.isp.sp.output.min_width || 1674 out->width > bi->info.isp.sp.output.max_width || 1675 out->height < bi->info.isp.sp.output.min_height || 1676 out->height > bi->info.isp.sp.output.max_height) 1677 continue; 1678 1679 if (out->width > bi->info.isp.sp.internal.max_width || 1680 out->height > bi->info.isp.sp.internal.max_height) 1681 continue; 1682 } 1683 1684 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) { 1685 if (bi->info.isp.num_output_pins <= 1) 1686 continue; 1687 1688 q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format; 1689 for (j = 0; j < bi->info.isp.num_output_formats; j++) 1690 if (bi->info.isp.output_formats[j] == q_fmt) 1691 break; 1692 if (j >= bi->info.isp.num_output_formats) 1693 continue; 1694 1695 if (vf->width < bi->info.isp.sp.output.min_width || 1696 vf->width > bi->info.isp.sp.output.max_width || 1697 vf->height < bi->info.isp.sp.output.min_height || 1698 vf->height > bi->info.isp.sp.output.max_height) 1699 continue; 1700 } 1701 1702 /* All checks passed, select the binary */ 1703 dev_dbg(css->dev, "using binary %s id = %u\n", name, 1704 bi->info.isp.sp.id); 1705 return i; 1706 } 1707 1708 /* Can not find suitable binary for these parameters */ 1709 return -EINVAL; 1710 } 1711 1712 /* 1713 * Check that there is a binary matching requirements. Parameters may be 1714 * NULL indicating disabled input/output. Return negative if given 1715 * parameters can not be supported or on error, zero or positive indicating 1716 * found binary number. May modify the given parameters if not exact match 1717 * is found. 1718 */ 1719 int imgu_css_fmt_try(struct imgu_css *css, 1720 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES], 1721 struct v4l2_rect *rects[IPU3_CSS_RECTS], 1722 unsigned int pipe) 1723 { 1724 static const u32 EFF_ALIGN_W = 2; 1725 static const u32 BDS_ALIGN_W = 4; 1726 static const u32 OUT_ALIGN_W = 8; 1727 static const u32 OUT_ALIGN_H = 4; 1728 static const u32 VF_ALIGN_W = 2; 1729 static const char *qnames[IPU3_CSS_QUEUES] = { 1730 [IPU3_CSS_QUEUE_IN] = "in", 1731 [IPU3_CSS_QUEUE_PARAMS] = "params", 1732 [IPU3_CSS_QUEUE_OUT] = "out", 1733 [IPU3_CSS_QUEUE_VF] = "vf", 1734 [IPU3_CSS_QUEUE_STAT_3A] = "3a", 1735 }; 1736 static const char *rnames[IPU3_CSS_RECTS] = { 1737 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution", 1738 [IPU3_CSS_RECT_BDS] = "bayer-domain scaled resolution", 1739 [IPU3_CSS_RECT_ENVELOPE] = "DVS envelope size", 1740 [IPU3_CSS_RECT_GDC] = "GDC output res", 1741 }; 1742 struct v4l2_rect r[IPU3_CSS_RECTS] = { }; 1743 struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE]; 1744 struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS]; 1745 struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE]; 1746 struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC]; 1747 struct imgu_css_queue *q; 1748 struct v4l2_pix_format_mplane *in, *out, *vf; 1749 int i, s, ret; 1750 1751 q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL); 1752 if (!q) 1753 return -ENOMEM; 1754 1755 in = &q[IPU3_CSS_QUEUE_IN].fmt.mpix; 1756 out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix; 1757 vf = &q[IPU3_CSS_QUEUE_VF].fmt.mpix; 1758 1759 /* Adjust all formats, get statistics buffer sizes and formats */ 1760 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 1761 if (fmts[i]) 1762 dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__, 1763 qnames[i], fmts[i]->width, fmts[i]->height, 1764 fmts[i]->pixelformat); 1765 else 1766 dev_dbg(css->dev, "%s %s: (not set)\n", __func__, 1767 qnames[i]); 1768 if (imgu_css_queue_init(&q[i], fmts[i], 1769 IPU3_CSS_QUEUE_TO_FLAGS(i))) { 1770 dev_notice(css->dev, "can not initialize queue %s\n", 1771 qnames[i]); 1772 ret = -EINVAL; 1773 goto out; 1774 } 1775 } 1776 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1777 if (rects[i]) { 1778 dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__, 1779 rnames[i], rects[i]->width, rects[i]->height); 1780 r[i].width = rects[i]->width; 1781 r[i].height = rects[i]->height; 1782 } else { 1783 dev_dbg(css->dev, "%s %s: (not set)\n", __func__, 1784 rnames[i]); 1785 } 1786 /* For now, force known good resolutions */ 1787 r[i].left = 0; 1788 r[i].top = 0; 1789 } 1790 1791 /* Always require one input and vf only if out is also enabled */ 1792 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) || 1793 !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) { 1794 dev_warn(css->dev, "required queues are disabled\n"); 1795 ret = -EINVAL; 1796 goto out; 1797 } 1798 1799 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) { 1800 out->width = in->width; 1801 out->height = in->height; 1802 } 1803 if (eff->width <= 0 || eff->height <= 0) { 1804 eff->width = in->width; 1805 eff->height = in->height; 1806 } 1807 if (bds->width <= 0 || bds->height <= 0) { 1808 bds->width = out->width; 1809 bds->height = out->height; 1810 } 1811 if (gdc->width <= 0 || gdc->height <= 0) { 1812 gdc->width = out->width; 1813 gdc->height = out->height; 1814 } 1815 1816 in->width = imgu_css_adjust(in->width, 1); 1817 in->height = imgu_css_adjust(in->height, 1); 1818 eff->width = imgu_css_adjust(eff->width, EFF_ALIGN_W); 1819 eff->height = imgu_css_adjust(eff->height, 1); 1820 bds->width = imgu_css_adjust(bds->width, BDS_ALIGN_W); 1821 bds->height = imgu_css_adjust(bds->height, 1); 1822 gdc->width = imgu_css_adjust(gdc->width, OUT_ALIGN_W); 1823 gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H); 1824 out->width = imgu_css_adjust(out->width, OUT_ALIGN_W); 1825 out->height = imgu_css_adjust(out->height, OUT_ALIGN_H); 1826 vf->width = imgu_css_adjust(vf->width, VF_ALIGN_W); 1827 vf->height = imgu_css_adjust(vf->height, 1); 1828 1829 s = (bds->width - gdc->width) / 2; 1830 env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s; 1831 s = (bds->height - gdc->height) / 2; 1832 env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s; 1833 1834 ret = imgu_css_find_binary(css, pipe, q, r); 1835 if (ret < 0) { 1836 dev_err(css->dev, "failed to find suitable binary\n"); 1837 ret = -EINVAL; 1838 goto out; 1839 } 1840 css->pipes[pipe].bindex = ret; 1841 1842 dev_dbg(css->dev, "Binary index %d for pipe %d found.", 1843 css->pipes[pipe].bindex, pipe); 1844 1845 /* Final adjustment and set back the queried formats */ 1846 for (i = 0; i < IPU3_CSS_QUEUES; i++) { 1847 if (fmts[i]) { 1848 if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix, 1849 IPU3_CSS_QUEUE_TO_FLAGS(i))) { 1850 dev_err(css->dev, 1851 "final resolution adjustment failed\n"); 1852 ret = -EINVAL; 1853 goto out; 1854 } 1855 *fmts[i] = q[i].fmt.mpix; 1856 } 1857 } 1858 1859 for (i = 0; i < IPU3_CSS_RECTS; i++) 1860 if (rects[i]) 1861 *rects[i] = r[i]; 1862 1863 dev_dbg(css->dev, 1864 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)", 1865 in->width, in->height, eff->width, eff->height, 1866 bds->width, bds->height, gdc->width, gdc->height, 1867 out->width, out->height, vf->width, vf->height); 1868 1869 ret = 0; 1870 out: 1871 kfree(q); 1872 return ret; 1873 } 1874 1875 int imgu_css_fmt_set(struct imgu_css *css, 1876 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES], 1877 struct v4l2_rect *rects[IPU3_CSS_RECTS], 1878 unsigned int pipe) 1879 { 1880 struct v4l2_rect rect_data[IPU3_CSS_RECTS]; 1881 struct v4l2_rect *all_rects[IPU3_CSS_RECTS]; 1882 int i, r; 1883 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1884 1885 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1886 if (rects[i]) 1887 rect_data[i] = *rects[i]; 1888 else 1889 memset(&rect_data[i], 0, sizeof(rect_data[i])); 1890 all_rects[i] = &rect_data[i]; 1891 } 1892 r = imgu_css_fmt_try(css, fmts, all_rects, pipe); 1893 if (r < 0) 1894 return r; 1895 1896 for (i = 0; i < IPU3_CSS_QUEUES; i++) 1897 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i], 1898 IPU3_CSS_QUEUE_TO_FLAGS(i))) 1899 return -EINVAL; 1900 for (i = 0; i < IPU3_CSS_RECTS; i++) { 1901 css_pipe->rect[i] = rect_data[i]; 1902 if (rects[i]) 1903 *rects[i] = rect_data[i]; 1904 } 1905 1906 return 0; 1907 } 1908 1909 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt) 1910 { 1911 switch (fmt->dataformat) { 1912 case V4L2_META_FMT_IPU3_PARAMS: 1913 fmt->buffersize = sizeof(struct ipu3_uapi_params); 1914 break; 1915 case V4L2_META_FMT_IPU3_STAT_3A: 1916 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a); 1917 break; 1918 default: 1919 return -EINVAL; 1920 } 1921 1922 return 0; 1923 } 1924 1925 /* 1926 * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first 1927 * called for the buffer. May be called from interrupt context. 1928 * Returns 0 on success, -EBUSY if the buffer queue is full, or some other 1929 * code on error conditions. 1930 */ 1931 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe, 1932 struct imgu_css_buffer *b) 1933 { 1934 struct imgu_abi_buffer *abi_buf; 1935 struct imgu_addr_t *buf_addr; 1936 u32 data; 1937 int r; 1938 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 1939 1940 if (!css->streaming) 1941 return -EPROTO; /* CSS or buffer in wrong state */ 1942 1943 if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid) 1944 return -EINVAL; 1945 1946 b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid, 1947 pipe); 1948 1949 if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue])) 1950 return -EIO; 1951 abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr; 1952 1953 /* Fill struct abi_buffer for firmware */ 1954 memset(abi_buf, 0, sizeof(*abi_buf)); 1955 1956 buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs; 1957 *(imgu_addr_t *)buf_addr = b->daddr; 1958 1959 if (b->queue == IPU3_CSS_QUEUE_STAT_3A) 1960 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr; 1961 1962 if (b->queue == IPU3_CSS_QUEUE_OUT) 1963 abi_buf->payload.frame.padded_width = 1964 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad; 1965 1966 if (b->queue == IPU3_CSS_QUEUE_VF) 1967 abi_buf->payload.frame.padded_width = 1968 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad; 1969 1970 spin_lock(&css_pipe->qlock); 1971 list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs); 1972 spin_unlock(&css_pipe->qlock); 1973 b->state = IPU3_CSS_BUFFER_QUEUED; 1974 1975 data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr; 1976 r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid, 1977 pipe, data); 1978 if (r < 0) 1979 goto queueing_failed; 1980 1981 data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe, 1982 imgu_css_queues[b->queue].qid); 1983 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data); 1984 if (r < 0) 1985 goto queueing_failed; 1986 1987 dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n", 1988 b, b->queue, pipe); 1989 1990 return 0; 1991 1992 queueing_failed: 1993 b->state = (r == -EBUSY || r == -EAGAIN) ? 1994 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED; 1995 list_del(&b->list); 1996 1997 return r; 1998 } 1999 2000 /* 2001 * Get next ready CSS buffer. Returns -EAGAIN in which case the function 2002 * should be called again, or -EBUSY which means that there are no more 2003 * buffers available. May be called from interrupt context. 2004 */ 2005 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css) 2006 { 2007 static const unsigned char evtype_to_queue[] = { 2008 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN, 2009 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT, 2010 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF, 2011 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A, 2012 }; 2013 struct imgu_css_buffer *b = ERR_PTR(-EAGAIN); 2014 u32 event, daddr; 2015 int evtype, pipe, pipeid, queue, qid, r; 2016 struct imgu_css_pipe *css_pipe; 2017 2018 if (!css->streaming) 2019 return ERR_PTR(-EPROTO); 2020 2021 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event); 2022 if (r < 0) 2023 return ERR_PTR(r); 2024 2025 evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >> 2026 IMGU_ABI_EVTTYPE_EVENT_SHIFT; 2027 2028 switch (evtype) { 2029 case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE: 2030 case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE: 2031 case IMGU_ABI_EVTTYPE_3A_STATS_DONE: 2032 case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE: 2033 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >> 2034 IMGU_ABI_EVTTYPE_PIPE_SHIFT; 2035 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >> 2036 IMGU_ABI_EVTTYPE_PIPEID_SHIFT; 2037 queue = evtype_to_queue[evtype]; 2038 qid = imgu_css_queues[queue].qid; 2039 2040 if (pipe >= IMGU_MAX_PIPE_NUM) { 2041 dev_err(css->dev, "Invalid pipe: %i\n", pipe); 2042 return ERR_PTR(-EIO); 2043 } 2044 2045 if (qid >= IMGU_ABI_QUEUE_NUM) { 2046 dev_err(css->dev, "Invalid qid: %i\n", qid); 2047 return ERR_PTR(-EIO); 2048 } 2049 css_pipe = &css->pipes[pipe]; 2050 dev_dbg(css->dev, 2051 "event: buffer done 0x%x queue %i pipe %i pipeid %i\n", 2052 event, queue, pipe, pipeid); 2053 2054 r = imgu_css_dequeue_data(css, qid, &daddr); 2055 if (r < 0) { 2056 dev_err(css->dev, "failed to dequeue buffer\n"); 2057 /* Force real error, not -EBUSY */ 2058 return ERR_PTR(-EIO); 2059 } 2060 2061 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2062 IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid)); 2063 if (r < 0) { 2064 dev_err(css->dev, "failed to queue event\n"); 2065 return ERR_PTR(-EIO); 2066 } 2067 2068 spin_lock(&css_pipe->qlock); 2069 if (list_empty(&css_pipe->queue[queue].bufs)) { 2070 spin_unlock(&css_pipe->qlock); 2071 dev_err(css->dev, "event on empty queue\n"); 2072 return ERR_PTR(-EIO); 2073 } 2074 b = list_first_entry(&css_pipe->queue[queue].bufs, 2075 struct imgu_css_buffer, list); 2076 if (queue != b->queue || 2077 daddr != css_pipe->abi_buffers 2078 [b->queue][b->queue_pos].daddr) { 2079 spin_unlock(&css_pipe->qlock); 2080 dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr); 2081 return ERR_PTR(-EIO); 2082 } 2083 2084 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe); 2085 b->pipe = pipe; 2086 b->state = IPU3_CSS_BUFFER_DONE; 2087 list_del(&b->list); 2088 spin_unlock(&css_pipe->qlock); 2089 break; 2090 case IMGU_ABI_EVTTYPE_PIPELINE_DONE: 2091 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >> 2092 IMGU_ABI_EVTTYPE_PIPE_SHIFT; 2093 if (pipe >= IMGU_MAX_PIPE_NUM) { 2094 dev_err(css->dev, "Invalid pipe: %i\n", pipe); 2095 return ERR_PTR(-EIO); 2096 } 2097 2098 css_pipe = &css->pipes[pipe]; 2099 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n", 2100 event, pipe); 2101 break; 2102 case IMGU_ABI_EVTTYPE_TIMER: 2103 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event); 2104 if (r < 0) 2105 return ERR_PTR(r); 2106 2107 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >> 2108 IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER) 2109 dev_dbg(css->dev, "event: timer\n"); 2110 else 2111 dev_warn(css->dev, "half of timer event missing\n"); 2112 break; 2113 case IMGU_ABI_EVTTYPE_FW_WARNING: 2114 dev_warn(css->dev, "event: firmware warning 0x%x\n", event); 2115 break; 2116 case IMGU_ABI_EVTTYPE_FW_ASSERT: 2117 dev_err(css->dev, 2118 "event: firmware assert 0x%x module_id %i line_no %i\n", 2119 event, 2120 (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >> 2121 IMGU_ABI_EVTTYPE_MODULEID_SHIFT, 2122 swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >> 2123 IMGU_ABI_EVTTYPE_LINENO_SHIFT)); 2124 break; 2125 default: 2126 dev_warn(css->dev, "received unknown event 0x%x\n", event); 2127 } 2128 2129 return b; 2130 } 2131 2132 /* 2133 * Get a new set of parameters from pool and initialize them based on 2134 * the parameters params, gdc, and obgrid. Any of these may be NULL, 2135 * in which case the previously set parameters are used. 2136 * If parameters haven't been set previously, initialize from scratch. 2137 * 2138 * Return index to css->parameter_set_info which has the newly created 2139 * parameters or negative value on error. 2140 */ 2141 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe, 2142 struct ipu3_uapi_params *set_params) 2143 { 2144 static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID; 2145 struct imgu_css_pipe *css_pipe = &css->pipes[pipe]; 2146 const int stage = 0; 2147 const struct imgu_fw_info *bi; 2148 int obgrid_size; 2149 unsigned int stripes, i; 2150 struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL; 2151 2152 /* Destination buffers which are filled here */ 2153 struct imgu_abi_parameter_set_info *param_set; 2154 struct imgu_abi_acc_param *acc = NULL; 2155 struct imgu_abi_gdc_warp_param *gdc = NULL; 2156 struct ipu3_uapi_obgrid_param *obgrid = NULL; 2157 const struct imgu_css_map *map; 2158 void *vmem0 = NULL; 2159 void *dmem0 = NULL; 2160 2161 enum imgu_abi_memories m; 2162 int r = -EBUSY; 2163 2164 if (!css->streaming) 2165 return -EPROTO; 2166 2167 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe); 2168 2169 bi = &css->fwp->binary_header[css_pipe->bindex]; 2170 obgrid_size = imgu_css_fw_obgrid_size(bi); 2171 stripes = bi->info.isp.sp.iterator.num_stripes ? : 1; 2172 2173 imgu_css_pool_get(&css_pipe->pool.parameter_set_info); 2174 param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 2175 0)->vaddr; 2176 2177 /* Get a new acc only if new parameters given, or none yet */ 2178 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2179 if (set_params || !map->vaddr) { 2180 imgu_css_pool_get(&css_pipe->pool.acc); 2181 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2182 acc = map->vaddr; 2183 } 2184 2185 /* Get new VMEM0 only if needed, or none yet */ 2186 m = IMGU_ABI_MEM_ISP_VMEM0; 2187 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2188 if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params || 2189 set_params->use.tnr3_vmem_params || 2190 set_params->use.xnr3_vmem_params))) { 2191 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]); 2192 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2193 vmem0 = map->vaddr; 2194 } 2195 2196 /* Get new DMEM0 only if needed, or none yet */ 2197 m = IMGU_ABI_MEM_ISP_DMEM0; 2198 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2199 if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params || 2200 set_params->use.xnr3_dmem_params))) { 2201 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]); 2202 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2203 dmem0 = map->vaddr; 2204 } 2205 2206 /* Configure acc parameter cluster */ 2207 if (acc) { 2208 /* get acc_old */ 2209 map = imgu_css_pool_last(&css_pipe->pool.acc, 1); 2210 /* user acc */ 2211 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr, 2212 set_params ? &set_params->acc_param : NULL); 2213 if (r < 0) 2214 goto fail; 2215 } 2216 2217 /* Configure late binding parameters */ 2218 if (vmem0) { 2219 m = IMGU_ABI_MEM_ISP_VMEM0; 2220 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1); 2221 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0, 2222 map->vaddr, set_params); 2223 if (r < 0) 2224 goto fail; 2225 } 2226 2227 if (dmem0) { 2228 m = IMGU_ABI_MEM_ISP_DMEM0; 2229 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1); 2230 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0, 2231 map->vaddr, set_params); 2232 if (r < 0) 2233 goto fail; 2234 } 2235 2236 /* Get a new gdc only if a new gdc is given, or none yet */ 2237 if (bi->info.isp.sp.enable.dvs_6axis) { 2238 unsigned int a = IPU3_CSS_AUX_FRAME_REF; 2239 unsigned int g = IPU3_CSS_RECT_GDC; 2240 unsigned int e = IPU3_CSS_RECT_ENVELOPE; 2241 2242 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2243 if (!map->vaddr) { 2244 imgu_css_pool_get(&css_pipe->pool.gdc); 2245 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2246 gdc = map->vaddr; 2247 imgu_css_cfg_gdc_table(map->vaddr, 2248 css_pipe->aux_frames[a].bytesperline / 2249 css_pipe->aux_frames[a].bytesperpixel, 2250 css_pipe->aux_frames[a].height, 2251 css_pipe->rect[g].width, 2252 css_pipe->rect[g].height, 2253 css_pipe->rect[e].width, 2254 css_pipe->rect[e].height); 2255 } 2256 } 2257 2258 /* Get a new obgrid only if a new obgrid is given, or none yet */ 2259 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2260 if (!map->vaddr || (set_params && set_params->use.obgrid_param)) { 2261 imgu_css_pool_get(&css_pipe->pool.obgrid); 2262 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2263 obgrid = map->vaddr; 2264 2265 /* Configure optical black level grid (obgrid) */ 2266 if (set_params && set_params->use.obgrid_param) 2267 for (i = 0; i < obgrid_size / sizeof(*obgrid); i++) 2268 obgrid[i] = set_params->obgrid_param; 2269 else 2270 memset(obgrid, 0, obgrid_size); 2271 } 2272 2273 /* Configure parameter set info, queued to `queue_id' */ 2274 2275 memset(param_set, 0, sizeof(*param_set)); 2276 map = imgu_css_pool_last(&css_pipe->pool.acc, 0); 2277 param_set->mem_map.acc_cluster_params_for_sp = map->daddr; 2278 2279 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0); 2280 param_set->mem_map.dvs_6axis_params_y = map->daddr; 2281 2282 for (i = 0; i < stripes; i++) { 2283 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0); 2284 param_set->mem_map.obgrid_tbl[i] = 2285 map->daddr + (obgrid_size / stripes) * i; 2286 } 2287 2288 for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) { 2289 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0); 2290 param_set->mem_map.isp_mem_param[stage][m] = map->daddr; 2291 } 2292 2293 /* Then queue the new parameter buffer */ 2294 map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0); 2295 r = imgu_css_queue_data(css, queue_id, pipe, map->daddr); 2296 if (r < 0) 2297 goto fail; 2298 2299 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2300 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe, 2301 queue_id)); 2302 if (r < 0) 2303 goto fail_no_put; 2304 2305 /* Finally dequeue all old parameter buffers */ 2306 2307 do { 2308 u32 daddr; 2309 2310 r = imgu_css_dequeue_data(css, queue_id, &daddr); 2311 if (r == -EBUSY) 2312 break; 2313 if (r) 2314 goto fail_no_put; 2315 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, 2316 IMGU_ABI_EVENT_BUFFER_DEQUEUED 2317 (queue_id)); 2318 if (r < 0) { 2319 dev_err(css->dev, "failed to queue parameter event\n"); 2320 goto fail_no_put; 2321 } 2322 } while (1); 2323 2324 return 0; 2325 2326 fail: 2327 /* 2328 * A failure, most likely the parameter queue was full. 2329 * Return error but continue streaming. User can try submitting new 2330 * parameters again later. 2331 */ 2332 2333 imgu_css_pool_put(&css_pipe->pool.parameter_set_info); 2334 if (acc) 2335 imgu_css_pool_put(&css_pipe->pool.acc); 2336 if (gdc) 2337 imgu_css_pool_put(&css_pipe->pool.gdc); 2338 if (obgrid) 2339 imgu_css_pool_put(&css_pipe->pool.obgrid); 2340 if (vmem0) 2341 imgu_css_pool_put( 2342 &css_pipe->pool.binary_params_p 2343 [IMGU_ABI_MEM_ISP_VMEM0]); 2344 if (dmem0) 2345 imgu_css_pool_put( 2346 &css_pipe->pool.binary_params_p 2347 [IMGU_ABI_MEM_ISP_DMEM0]); 2348 2349 fail_no_put: 2350 return r; 2351 } 2352 2353 int imgu_css_irq_ack(struct imgu_css *css) 2354 { 2355 static const int NUM_SWIRQS = 3; 2356 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]]; 2357 void __iomem *const base = css->base; 2358 u32 irq_status[IMGU_IRQCTRL_NUM]; 2359 int i; 2360 2361 u32 imgu_status = readl(base + IMGU_REG_INT_STATUS); 2362 2363 writel(imgu_status, base + IMGU_REG_INT_STATUS); 2364 for (i = 0; i < IMGU_IRQCTRL_NUM; i++) 2365 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i)); 2366 2367 for (i = 0; i < NUM_SWIRQS; i++) { 2368 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) { 2369 /* SP SW interrupt */ 2370 u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) + 2371 bi->info.sp.output); 2372 u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) + 2373 bi->info.sp.output + 4 + 4 * i); 2374 2375 dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n", 2376 __func__, i, cnt, val); 2377 } 2378 } 2379 2380 for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--) 2381 if (irq_status[i]) { 2382 writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i)); 2383 /* Wait for write to complete */ 2384 readl(base + IMGU_REG_IRQCTRL_ENABLE(i)); 2385 } 2386 2387 dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n", 2388 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN], 2389 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]); 2390 2391 if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN]) 2392 return -ENOMSG; 2393 2394 return 0; 2395 } 2396