1 #include "IR.h"
2
3 #include "IRMutator.h"
4 #include "IRPrinter.h"
5 #include "IRVisitor.h"
6 #include <utility>
7
8 namespace Halide {
9 namespace Internal {
10
make(Type t,Expr v)11 Expr Cast::make(Type t, Expr v) {
12 internal_assert(v.defined()) << "Cast of undefined\n";
13 internal_assert(t.lanes() == v.type().lanes()) << "Cast may not change vector widths\n";
14
15 Cast *node = new Cast;
16 node->type = t;
17 node->value = std::move(v);
18 return node;
19 }
20
make(Expr a,Expr b)21 Expr Add::make(Expr a, Expr b) {
22 internal_assert(a.defined()) << "Add of undefined\n";
23 internal_assert(b.defined()) << "Add of undefined\n";
24 internal_assert(a.type() == b.type()) << "Add of mismatched types\n";
25
26 Add *node = new Add;
27 node->type = a.type();
28 node->a = std::move(a);
29 node->b = std::move(b);
30 return node;
31 }
32
make(Expr a,Expr b)33 Expr Sub::make(Expr a, Expr b) {
34 internal_assert(a.defined()) << "Sub of undefined\n";
35 internal_assert(b.defined()) << "Sub of undefined\n";
36 internal_assert(a.type() == b.type()) << "Sub of mismatched types\n";
37
38 Sub *node = new Sub;
39 node->type = a.type();
40 node->a = std::move(a);
41 node->b = std::move(b);
42 return node;
43 }
44
make(Expr a,Expr b)45 Expr Mul::make(Expr a, Expr b) {
46 internal_assert(a.defined()) << "Mul of undefined\n";
47 internal_assert(b.defined()) << "Mul of undefined\n";
48 internal_assert(a.type() == b.type()) << "Mul of mismatched types\n";
49
50 Mul *node = new Mul;
51 node->type = a.type();
52 node->a = std::move(a);
53 node->b = std::move(b);
54 return node;
55 }
56
make(Expr a,Expr b)57 Expr Div::make(Expr a, Expr b) {
58 internal_assert(a.defined()) << "Div of undefined\n";
59 internal_assert(b.defined()) << "Div of undefined\n";
60 internal_assert(a.type() == b.type()) << "Div of mismatched types\n";
61
62 Div *node = new Div;
63 node->type = a.type();
64 node->a = std::move(a);
65 node->b = std::move(b);
66 return node;
67 }
68
make(Expr a,Expr b)69 Expr Mod::make(Expr a, Expr b) {
70 internal_assert(a.defined()) << "Mod of undefined\n";
71 internal_assert(b.defined()) << "Mod of undefined\n";
72 internal_assert(a.type() == b.type()) << "Mod of mismatched types\n";
73
74 Mod *node = new Mod;
75 node->type = a.type();
76 node->a = std::move(a);
77 node->b = std::move(b);
78 return node;
79 }
80
make(Expr a,Expr b)81 Expr Min::make(Expr a, Expr b) {
82 internal_assert(a.defined()) << "Min of undefined\n";
83 internal_assert(b.defined()) << "Min of undefined\n";
84 internal_assert(a.type() == b.type()) << "Min of mismatched types\n";
85
86 Min *node = new Min;
87 node->type = a.type();
88 node->a = std::move(a);
89 node->b = std::move(b);
90 return node;
91 }
92
make(Expr a,Expr b)93 Expr Max::make(Expr a, Expr b) {
94 internal_assert(a.defined()) << "Max of undefined\n";
95 internal_assert(b.defined()) << "Max of undefined\n";
96 internal_assert(a.type() == b.type()) << "Max of mismatched types\n";
97
98 Max *node = new Max;
99 node->type = a.type();
100 node->a = std::move(a);
101 node->b = std::move(b);
102 return node;
103 }
104
make(Expr a,Expr b)105 Expr EQ::make(Expr a, Expr b) {
106 internal_assert(a.defined()) << "EQ of undefined\n";
107 internal_assert(b.defined()) << "EQ of undefined\n";
108 internal_assert(a.type() == b.type()) << "EQ of mismatched types\n";
109
110 EQ *node = new EQ;
111 node->type = Bool(a.type().lanes());
112 node->a = std::move(a);
113 node->b = std::move(b);
114 return node;
115 }
116
make(Expr a,Expr b)117 Expr NE::make(Expr a, Expr b) {
118 internal_assert(a.defined()) << "NE of undefined\n";
119 internal_assert(b.defined()) << "NE of undefined\n";
120 internal_assert(a.type() == b.type()) << "NE of mismatched types\n";
121
122 NE *node = new NE;
123 node->type = Bool(a.type().lanes());
124 node->a = std::move(a);
125 node->b = std::move(b);
126 return node;
127 }
128
make(Expr a,Expr b)129 Expr LT::make(Expr a, Expr b) {
130 internal_assert(a.defined()) << "LT of undefined\n";
131 internal_assert(b.defined()) << "LT of undefined\n";
132 internal_assert(a.type() == b.type()) << "LT of mismatched types\n";
133
134 LT *node = new LT;
135 node->type = Bool(a.type().lanes());
136 node->a = std::move(a);
137 node->b = std::move(b);
138 return node;
139 }
140
make(Expr a,Expr b)141 Expr LE::make(Expr a, Expr b) {
142 internal_assert(a.defined()) << "LE of undefined\n";
143 internal_assert(b.defined()) << "LE of undefined\n";
144 internal_assert(a.type() == b.type()) << "LE of mismatched types\n";
145
146 LE *node = new LE;
147 node->type = Bool(a.type().lanes());
148 node->a = std::move(a);
149 node->b = std::move(b);
150 return node;
151 }
152
make(Expr a,Expr b)153 Expr GT::make(Expr a, Expr b) {
154 internal_assert(a.defined()) << "GT of undefined\n";
155 internal_assert(b.defined()) << "GT of undefined\n";
156 internal_assert(a.type() == b.type()) << "GT of mismatched types\n";
157
158 GT *node = new GT;
159 node->type = Bool(a.type().lanes());
160 node->a = std::move(a);
161 node->b = std::move(b);
162 return node;
163 }
164
make(Expr a,Expr b)165 Expr GE::make(Expr a, Expr b) {
166 internal_assert(a.defined()) << "GE of undefined\n";
167 internal_assert(b.defined()) << "GE of undefined\n";
168 internal_assert(a.type() == b.type()) << "GE of mismatched types\n";
169
170 GE *node = new GE;
171 node->type = Bool(a.type().lanes());
172 node->a = std::move(a);
173 node->b = std::move(b);
174 return node;
175 }
176
make(Expr a,Expr b)177 Expr And::make(Expr a, Expr b) {
178 internal_assert(a.defined()) << "And of undefined\n";
179 internal_assert(b.defined()) << "And of undefined\n";
180 internal_assert(a.type().is_bool()) << "lhs of And is not a bool\n";
181 internal_assert(b.type().is_bool()) << "rhs of And is not a bool\n";
182 internal_assert(a.type() == b.type()) << "And of mismatched types\n";
183
184 And *node = new And;
185 node->type = Bool(a.type().lanes());
186 node->a = std::move(a);
187 node->b = std::move(b);
188 return node;
189 }
190
make(Expr a,Expr b)191 Expr Or::make(Expr a, Expr b) {
192 internal_assert(a.defined()) << "Or of undefined\n";
193 internal_assert(b.defined()) << "Or of undefined\n";
194 internal_assert(a.type().is_bool()) << "lhs of Or is not a bool\n";
195 internal_assert(b.type().is_bool()) << "rhs of Or is not a bool\n";
196 internal_assert(a.type() == b.type()) << "Or of mismatched types\n";
197
198 Or *node = new Or;
199 node->type = Bool(a.type().lanes());
200 node->a = std::move(a);
201 node->b = std::move(b);
202 return node;
203 }
204
make(Expr a)205 Expr Not::make(Expr a) {
206 internal_assert(a.defined()) << "Not of undefined\n";
207 internal_assert(a.type().is_bool()) << "argument of Not is not a bool\n";
208
209 Not *node = new Not;
210 node->type = Bool(a.type().lanes());
211 node->a = std::move(a);
212 return node;
213 }
214
make(Expr condition,Expr true_value,Expr false_value)215 Expr Select::make(Expr condition, Expr true_value, Expr false_value) {
216 internal_assert(condition.defined()) << "Select of undefined\n";
217 internal_assert(true_value.defined()) << "Select of undefined\n";
218 internal_assert(false_value.defined()) << "Select of undefined\n";
219 internal_assert(condition.type().is_bool()) << "First argument to Select is not a bool: " << condition.type() << "\n";
220 internal_assert(false_value.type() == true_value.type()) << "Select of mismatched types\n";
221 internal_assert(condition.type().is_scalar() ||
222 condition.type().lanes() == true_value.type().lanes())
223 << "In Select, vector lanes of condition must either be 1, or equal to vector lanes of arguments\n";
224
225 Select *node = new Select;
226 node->type = true_value.type();
227 node->condition = std::move(condition);
228 node->true_value = std::move(true_value);
229 node->false_value = std::move(false_value);
230 return node;
231 }
232
make(Type type,const std::string & name,Expr index,Buffer<> image,Parameter param,Expr predicate,ModulusRemainder alignment)233 Expr Load::make(Type type, const std::string &name, Expr index, Buffer<> image, Parameter param, Expr predicate, ModulusRemainder alignment) {
234 internal_assert(predicate.defined()) << "Load with undefined predicate\n";
235 internal_assert(index.defined()) << "Load of undefined\n";
236 internal_assert(type.lanes() == index.type().lanes()) << "Vector lanes of Load must match vector lanes of index\n";
237 internal_assert(type.lanes() == predicate.type().lanes())
238 << "Vector lanes of Load must match vector lanes of predicate\n";
239
240 Load *node = new Load;
241 node->type = type;
242 node->name = name;
243 node->predicate = std::move(predicate);
244 node->index = std::move(index);
245 node->image = std::move(image);
246 node->param = std::move(param);
247 node->alignment = alignment;
248 return node;
249 }
250
make(Expr base,Expr stride,int lanes)251 Expr Ramp::make(Expr base, Expr stride, int lanes) {
252 internal_assert(base.defined()) << "Ramp of undefined\n";
253 internal_assert(stride.defined()) << "Ramp of undefined\n";
254 internal_assert(base.type().is_scalar()) << "Ramp with vector base\n";
255 internal_assert(stride.type().is_scalar()) << "Ramp with vector stride\n";
256 internal_assert(lanes > 1) << "Ramp of lanes <= 1\n";
257 internal_assert(stride.type() == base.type()) << "Ramp of mismatched types\n";
258
259 Ramp *node = new Ramp;
260 node->type = base.type().with_lanes(lanes);
261 node->base = std::move(base);
262 node->stride = std::move(stride);
263 node->lanes = lanes;
264 return node;
265 }
266
make(Expr value,int lanes)267 Expr Broadcast::make(Expr value, int lanes) {
268 internal_assert(value.defined()) << "Broadcast of undefined\n";
269 internal_assert(value.type().is_scalar()) << "Broadcast of vector\n";
270 internal_assert(lanes != 1) << "Broadcast of lanes 1\n";
271
272 Broadcast *node = new Broadcast;
273 node->type = value.type().with_lanes(lanes);
274 node->value = std::move(value);
275 node->lanes = lanes;
276 return node;
277 }
278
make(const std::string & name,Expr value,Expr body)279 Expr Let::make(const std::string &name, Expr value, Expr body) {
280 internal_assert(value.defined()) << "Let of undefined\n";
281 internal_assert(body.defined()) << "Let of undefined\n";
282
283 Let *node = new Let;
284 node->type = body.type();
285 node->name = name;
286 node->value = std::move(value);
287 node->body = std::move(body);
288 return node;
289 }
290
make(const std::string & name,Expr value,Stmt body)291 Stmt LetStmt::make(const std::string &name, Expr value, Stmt body) {
292 internal_assert(value.defined()) << "Let of undefined\n";
293 internal_assert(body.defined()) << "Let of undefined\n";
294
295 LetStmt *node = new LetStmt;
296 node->name = name;
297 node->value = std::move(value);
298 node->body = std::move(body);
299 return node;
300 }
301
make(Expr condition,Expr message)302 Stmt AssertStmt::make(Expr condition, Expr message) {
303 internal_assert(condition.defined()) << "AssertStmt of undefined\n";
304 internal_assert(message.type() == Int(32)) << "AssertStmt message must be an int:" << message << "\n";
305
306 AssertStmt *node = new AssertStmt;
307 node->condition = std::move(condition);
308 node->message = std::move(message);
309 return node;
310 }
311
make(const std::string & name,bool is_producer,Stmt body)312 Stmt ProducerConsumer::make(const std::string &name, bool is_producer, Stmt body) {
313 internal_assert(body.defined()) << "ProducerConsumer of undefined\n";
314
315 ProducerConsumer *node = new ProducerConsumer;
316 node->name = name;
317 node->is_producer = is_producer;
318 node->body = std::move(body);
319 return node;
320 }
321
make_produce(const std::string & name,Stmt body)322 Stmt ProducerConsumer::make_produce(const std::string &name, Stmt body) {
323 return ProducerConsumer::make(name, true, std::move(body));
324 }
325
make_consume(const std::string & name,Stmt body)326 Stmt ProducerConsumer::make_consume(const std::string &name, Stmt body) {
327 return ProducerConsumer::make(name, false, std::move(body));
328 }
329
make(const std::string & name,Expr min,Expr extent,ForType for_type,DeviceAPI device_api,Stmt body)330 Stmt For::make(const std::string &name, Expr min, Expr extent, ForType for_type, DeviceAPI device_api, Stmt body) {
331 internal_assert(min.defined()) << "For of undefined\n";
332 internal_assert(extent.defined()) << "For of undefined\n";
333 internal_assert(min.type() == Int(32)) << "For with non-integer min\n";
334 internal_assert(extent.type() == Int(32)) << "For with non-integer extent\n";
335 internal_assert(body.defined()) << "For of undefined\n";
336
337 For *node = new For;
338 node->name = name;
339 node->min = std::move(min);
340 node->extent = std::move(extent);
341 node->for_type = for_type;
342 node->device_api = device_api;
343 node->body = std::move(body);
344 return node;
345 }
346
make(Expr semaphore,Expr count,Stmt body)347 Stmt Acquire::make(Expr semaphore, Expr count, Stmt body) {
348 internal_assert(semaphore.defined()) << "Acquire with undefined semaphore\n";
349 internal_assert(body.defined()) << "Acquire with undefined body\n";
350
351 Acquire *node = new Acquire;
352 node->semaphore = std::move(semaphore);
353 node->count = std::move(count);
354 node->body = std::move(body);
355 return node;
356 }
357
make(const std::string & name,Expr value,Expr index,Parameter param,Expr predicate,ModulusRemainder alignment)358 Stmt Store::make(const std::string &name, Expr value, Expr index, Parameter param, Expr predicate, ModulusRemainder alignment) {
359 internal_assert(predicate.defined()) << "Store with undefined predicate\n";
360 internal_assert(value.defined()) << "Store of undefined\n";
361 internal_assert(index.defined()) << "Store of undefined\n";
362 internal_assert(value.type().lanes() == index.type().lanes()) << "Vector lanes of Store must match vector lanes of index\n";
363 internal_assert(value.type().lanes() == predicate.type().lanes())
364 << "Vector lanes of Store must match vector lanes of predicate\n";
365
366 Store *node = new Store;
367 node->name = name;
368 node->predicate = std::move(predicate);
369 node->value = std::move(value);
370 node->index = std::move(index);
371 node->param = std::move(param);
372 node->alignment = alignment;
373 return node;
374 }
375
make(const std::string & name,const std::vector<Expr> & values,const std::vector<Expr> & args)376 Stmt Provide::make(const std::string &name, const std::vector<Expr> &values, const std::vector<Expr> &args) {
377 internal_assert(!values.empty()) << "Provide of no values\n";
378 for (size_t i = 0; i < values.size(); i++) {
379 internal_assert(values[i].defined()) << "Provide of undefined value\n";
380 }
381 for (size_t i = 0; i < args.size(); i++) {
382 internal_assert(args[i].defined()) << "Provide to undefined location\n";
383 }
384
385 Provide *node = new Provide;
386 node->name = name;
387 node->values = values;
388 node->args = args;
389 return node;
390 }
391
make(const std::string & name,Type type,MemoryType memory_type,const std::vector<Expr> & extents,Expr condition,Stmt body,Expr new_expr,const std::string & free_function)392 Stmt Allocate::make(const std::string &name, Type type, MemoryType memory_type,
393 const std::vector<Expr> &extents,
394 Expr condition, Stmt body,
395 Expr new_expr, const std::string &free_function) {
396 for (size_t i = 0; i < extents.size(); i++) {
397 internal_assert(extents[i].defined()) << "Allocate of undefined extent\n";
398 internal_assert(extents[i].type().is_scalar() == 1) << "Allocate of vector extent\n";
399 }
400 internal_assert(body.defined()) << "Allocate of undefined\n";
401 internal_assert(condition.defined()) << "Allocate with undefined condition\n";
402 internal_assert(condition.type().is_bool()) << "Allocate condition is not boolean\n";
403
404 Allocate *node = new Allocate;
405 node->name = name;
406 node->type = type;
407 node->memory_type = memory_type;
408 node->extents = extents;
409 node->new_expr = std::move(new_expr);
410 node->free_function = free_function;
411 node->condition = std::move(condition);
412 node->body = std::move(body);
413 return node;
414 }
415
constant_allocation_size(const std::vector<Expr> & extents,const std::string & name)416 int32_t Allocate::constant_allocation_size(const std::vector<Expr> &extents, const std::string &name) {
417 int64_t result = 1;
418
419 for (size_t i = 0; i < extents.size(); i++) {
420 if (const IntImm *int_size = extents[i].as<IntImm>()) {
421 // Check if the individual dimension is > 2^31 - 1. Not
422 // currently necessary because it's an int32_t, which is
423 // always smaller than 2^31 - 1. If we ever upgrade the
424 // type of IntImm but not the maximum allocation size, we
425 // should re-enable this.
426 /*
427 if ((int64_t)int_size->value > (((int64_t)(1)<<31) - 1)) {
428 user_error
429 << "Dimension " << i << " for allocation " << name << " has size " <<
430 int_size->value << " which is greater than 2^31 - 1.";
431 }
432 */
433 result *= int_size->value;
434 if (result > (static_cast<int64_t>(1) << 31) - 1) {
435 user_error
436 << "Total size for allocation " << name
437 << " is constant but exceeds 2^31 - 1.\n";
438 }
439 } else {
440 return 0;
441 }
442 }
443
444 return static_cast<int32_t>(result);
445 }
446
constant_allocation_size() const447 int32_t Allocate::constant_allocation_size() const {
448 return Allocate::constant_allocation_size(extents, name);
449 }
450
make(const std::string & name)451 Stmt Free::make(const std::string &name) {
452 Free *node = new Free;
453 node->name = name;
454 return node;
455 }
456
make(const std::string & name,const std::vector<Type> & types,MemoryType memory_type,const Region & bounds,Expr condition,Stmt body)457 Stmt Realize::make(const std::string &name, const std::vector<Type> &types, MemoryType memory_type, const Region &bounds, Expr condition, Stmt body) {
458 for (size_t i = 0; i < bounds.size(); i++) {
459 internal_assert(bounds[i].min.defined()) << "Realize of undefined\n";
460 internal_assert(bounds[i].extent.defined()) << "Realize of undefined\n";
461 internal_assert(bounds[i].min.type().is_scalar()) << "Realize of vector size\n";
462 internal_assert(bounds[i].extent.type().is_scalar()) << "Realize of vector size\n";
463 }
464 internal_assert(body.defined()) << "Realize of undefined\n";
465 internal_assert(!types.empty()) << "Realize has empty type\n";
466 internal_assert(condition.defined()) << "Realize with undefined condition\n";
467 internal_assert(condition.type().is_bool()) << "Realize condition is not boolean\n";
468
469 Realize *node = new Realize;
470 node->name = name;
471 node->types = types;
472 node->memory_type = memory_type;
473 node->bounds = bounds;
474 node->condition = std::move(condition);
475 node->body = std::move(body);
476 return node;
477 }
478
make(const std::string & name,const std::vector<Type> & types,const Region & bounds,const PrefetchDirective & prefetch,Expr condition,Stmt body)479 Stmt Prefetch::make(const std::string &name, const std::vector<Type> &types,
480 const Region &bounds,
481 const PrefetchDirective &prefetch,
482 Expr condition, Stmt body) {
483 for (size_t i = 0; i < bounds.size(); i++) {
484 internal_assert(bounds[i].min.defined()) << "Prefetch of undefined\n";
485 internal_assert(bounds[i].extent.defined()) << "Prefetch of undefined\n";
486 internal_assert(bounds[i].min.type().is_scalar()) << "Prefetch of vector size\n";
487 internal_assert(bounds[i].extent.type().is_scalar()) << "Prefetch of vector size\n";
488 }
489 internal_assert(!types.empty()) << "Prefetch has empty type\n";
490 internal_assert(body.defined()) << "Prefetch of undefined\n";
491 internal_assert(condition.defined()) << "Prefetch with undefined condition\n";
492 internal_assert(condition.type().is_bool()) << "Prefetch condition is not boolean\n";
493
494 Prefetch *node = new Prefetch;
495 node->name = name;
496 node->types = types;
497 node->bounds = bounds;
498 node->prefetch = prefetch;
499 node->condition = std::move(condition);
500 node->body = std::move(body);
501 return node;
502 }
503
make(Stmt first,Stmt rest)504 Stmt Block::make(Stmt first, Stmt rest) {
505 internal_assert(first.defined()) << "Block of undefined\n";
506 internal_assert(rest.defined()) << "Block of undefined\n";
507
508 Block *node = new Block;
509
510 if (const Block *b = first.as<Block>()) {
511 // Use a canonical block nesting order
512 node->first = b->first;
513 node->rest = Block::make(b->rest, std::move(rest));
514 } else {
515 node->first = std::move(first);
516 node->rest = std::move(rest);
517 }
518
519 return node;
520 }
521
make(const std::vector<Stmt> & stmts)522 Stmt Block::make(const std::vector<Stmt> &stmts) {
523 if (stmts.empty()) {
524 return Stmt();
525 }
526 Stmt result = stmts.back();
527 for (size_t i = stmts.size() - 1; i > 0; i--) {
528 result = Block::make(stmts[i - 1], result);
529 }
530 return result;
531 }
532
make(Stmt first,Stmt rest)533 Stmt Fork::make(Stmt first, Stmt rest) {
534 internal_assert(first.defined()) << "Fork of undefined\n";
535 internal_assert(rest.defined()) << "Fork of undefined\n";
536
537 Fork *node = new Fork;
538
539 if (const Fork *b = first.as<Fork>()) {
540 // Use a canonical fork nesting order
541 node->first = b->first;
542 node->rest = Fork::make(b->rest, std::move(rest));
543 } else {
544 node->first = std::move(first);
545 node->rest = std::move(rest);
546 }
547
548 return node;
549 }
550
make(Expr condition,Stmt then_case,Stmt else_case)551 Stmt IfThenElse::make(Expr condition, Stmt then_case, Stmt else_case) {
552 internal_assert(condition.defined() && then_case.defined()) << "IfThenElse of undefined\n";
553 // else_case may be null.
554
555 IfThenElse *node = new IfThenElse;
556 node->condition = std::move(condition);
557 node->then_case = std::move(then_case);
558 node->else_case = std::move(else_case);
559 return node;
560 }
561
make(Expr v)562 Stmt Evaluate::make(Expr v) {
563 internal_assert(v.defined()) << "Evaluate of undefined\n";
564
565 Evaluate *node = new Evaluate;
566 node->value = std::move(v);
567 return node;
568 }
569
make(const Function & func,const std::vector<Expr> & args,int idx)570 Expr Call::make(const Function &func, const std::vector<Expr> &args, int idx) {
571 internal_assert(idx >= 0 &&
572 idx < func.outputs())
573 << "Value index out of range in call to halide function\n";
574 internal_assert(func.has_pure_definition() || func.has_extern_definition())
575 << "Call to undefined halide function\n";
576 return make(func.output_types()[(size_t)idx], func.name(), args, Halide,
577 func.get_contents(), idx, Buffer<>(), Parameter());
578 }
579
580 namespace {
581
582 const char *const intrinsic_op_names[] = {
583 "abs",
584 "absd",
585 "add_image_checks_marker",
586 "alloca",
587 "bitwise_and",
588 "bitwise_not",
589 "bitwise_or",
590 "bitwise_xor",
591 "bool_to_mask",
592 "bundle",
593 "call_cached_indirect_function",
594 "cast_mask",
595 "count_leading_zeros",
596 "count_trailing_zeros",
597 "declare_box_touched",
598 "debug_to_file",
599 "div_round_to_zero",
600 "dynamic_shuffle",
601 "extract_mask_element",
602 "gather",
603 "glsl_texture_load",
604 "glsl_texture_store",
605 "glsl_varying",
606 "gpu_thread_barrier",
607 "if_then_else",
608 "if_then_else_mask",
609 "image_load",
610 "image_store",
611 "lerp",
612 "likely",
613 "likely_if_innermost",
614 "make_struct",
615 "memoize_expr",
616 "mod_round_to_zero",
617 "mulhi_shr",
618 "popcount",
619 "prefetch",
620 "promise_clamped",
621 "random",
622 "register_destructor",
623 "reinterpret",
624 "require",
625 "require_mask",
626 "return_second",
627 "rewrite_buffer",
628 "scatter",
629 "scatter_acc",
630 "scatter_release",
631 "select_mask",
632 "shift_left",
633 "shift_right",
634 "signed_integer_overflow",
635 "size_of_halide_buffer_t",
636 "sorted_avg",
637 "strict_float",
638 "stringify",
639 "undef",
640 "unsafe_promise_clamped",
641 };
642
643 static_assert(sizeof(intrinsic_op_names) / sizeof(intrinsic_op_names[0]) == Call::IntrinsicOpCount,
644 "intrinsic_op_names needs attention");
645
646 } // namespace
647
get_intrinsic_name(IntrinsicOp op)648 const char *Call::get_intrinsic_name(IntrinsicOp op) {
649 return intrinsic_op_names[op];
650 }
651
make(Type type,Call::IntrinsicOp op,const std::vector<Expr> & args,CallType call_type,FunctionPtr func,int value_index,const Buffer<> & image,Parameter param)652 Expr Call::make(Type type, Call::IntrinsicOp op, const std::vector<Expr> &args, CallType call_type,
653 FunctionPtr func, int value_index,
654 const Buffer<> &image, Parameter param) {
655 internal_assert(call_type == Call::Intrinsic || call_type == Call::PureIntrinsic);
656 return Call::make(type, intrinsic_op_names[op], args, call_type, std::move(func), value_index, image, std::move(param));
657 }
658
make(Type type,const std::string & name,const std::vector<Expr> & args,CallType call_type,FunctionPtr func,int value_index,Buffer<> image,Parameter param)659 Expr Call::make(Type type, const std::string &name, const std::vector<Expr> &args, CallType call_type,
660 FunctionPtr func, int value_index,
661 Buffer<> image, Parameter param) {
662 if (name == intrinsic_op_names[Call::prefetch] && call_type == Call::Intrinsic) {
663 internal_assert(args.size() % 2 == 0)
664 << "Number of args to a prefetch call should be even: {base, offset, extent0, stride0, extent1, stride1, ...}\n";
665 }
666 for (size_t i = 0; i < args.size(); i++) {
667 internal_assert(args[i].defined()) << "Call of " << name << " with argument " << i << " undefined.\n";
668 }
669 if (call_type == Halide) {
670 for (size_t i = 0; i < args.size(); i++) {
671 internal_assert(args[i].type() == Int(32))
672 << "Args to call to halide function must be type Int(32)\n";
673 }
674 } else if (call_type == Image) {
675 internal_assert((param.defined() || image.defined()))
676 << "Call node to undefined image\n";
677 for (size_t i = 0; i < args.size(); i++) {
678 internal_assert(args[i].type() == Int(32))
679 << "Args to load from image must be type Int(32)\n";
680 }
681 }
682
683 Call *node = new Call;
684 node->type = type;
685 node->name = name;
686 node->args = args;
687 node->call_type = call_type;
688 node->func = std::move(func);
689 node->value_index = value_index;
690 node->image = std::move(image);
691 node->param = std::move(param);
692 return node;
693 }
694
make(Type type,const std::string & name,Buffer<> image,Parameter param,ReductionDomain reduction_domain)695 Expr Variable::make(Type type, const std::string &name, Buffer<> image, Parameter param, ReductionDomain reduction_domain) {
696 internal_assert(!name.empty());
697 Variable *node = new Variable;
698 node->type = type;
699 node->name = name;
700 node->image = std::move(image);
701 node->param = std::move(param);
702 node->reduction_domain = std::move(reduction_domain);
703 return node;
704 }
705
make(const std::vector<Expr> & vectors,const std::vector<int> & indices)706 Expr Shuffle::make(const std::vector<Expr> &vectors,
707 const std::vector<int> &indices) {
708 internal_assert(!vectors.empty()) << "Shuffle of zero vectors.\n";
709 internal_assert(!indices.empty()) << "Shufle with zero indices.\n";
710 Type element_ty = vectors.front().type().element_of();
711 int input_lanes = 0;
712 for (Expr i : vectors) {
713 internal_assert(i.type().element_of() == element_ty) << "Shuffle of vectors of mismatched types.\n";
714 input_lanes += i.type().lanes();
715 }
716 for (int i : indices) {
717 internal_assert(0 <= i && i < input_lanes) << "Shuffle vector index out of range: " << i << "\n";
718 }
719
720 Shuffle *node = new Shuffle;
721 node->type = element_ty.with_lanes((int)indices.size());
722 node->vectors = vectors;
723 node->indices = indices;
724 return node;
725 }
726
make_interleave(const std::vector<Expr> & vectors)727 Expr Shuffle::make_interleave(const std::vector<Expr> &vectors) {
728 internal_assert(!vectors.empty()) << "Interleave of zero vectors.\n";
729
730 if (vectors.size() == 1) {
731 return vectors.front();
732 }
733
734 int lanes = vectors.front().type().lanes();
735
736 for (Expr i : vectors) {
737 internal_assert(i.type().lanes() == lanes)
738 << "Interleave of vectors with different sizes.\n";
739 }
740
741 std::vector<int> indices;
742 for (int i = 0; i < lanes; i++) {
743 for (int j = 0; j < (int)vectors.size(); j++) {
744 indices.push_back(j * lanes + i);
745 }
746 }
747
748 return make(vectors, indices);
749 }
750
make_concat(const std::vector<Expr> & vectors)751 Expr Shuffle::make_concat(const std::vector<Expr> &vectors) {
752 internal_assert(!vectors.empty()) << "Concat of zero vectors.\n";
753
754 if (vectors.size() == 1) {
755 return vectors.front();
756 }
757
758 std::vector<int> indices;
759 int lane = 0;
760 for (int i = 0; i < (int)vectors.size(); i++) {
761 for (int j = 0; j < vectors[i].type().lanes(); j++) {
762 indices.push_back(lane++);
763 }
764 }
765
766 return make(vectors, indices);
767 }
768
make_slice(Expr vector,int begin,int stride,int size)769 Expr Shuffle::make_slice(Expr vector, int begin, int stride, int size) {
770 if (begin == 0 && size == vector.type().lanes() && stride == 1) {
771 return vector;
772 }
773
774 std::vector<int> indices;
775 for (int i = 0; i < size; i++) {
776 indices.push_back(begin + i * stride);
777 }
778
779 return make({std::move(vector)}, indices);
780 }
781
make_extract_element(Expr vector,int i)782 Expr Shuffle::make_extract_element(Expr vector, int i) {
783 return make_slice(std::move(vector), i, 1, 1);
784 }
785
is_interleave() const786 bool Shuffle::is_interleave() const {
787 int lanes = vectors.front().type().lanes();
788
789 // Don't consider concat of scalars as an interleave.
790 if (lanes == 1) {
791 return false;
792 }
793
794 for (Expr i : vectors) {
795 if (i.type().lanes() != lanes) {
796 return false;
797 }
798 }
799
800 // Require that we are a complete interleaving.
801 if (lanes * vectors.size() != indices.size()) {
802 return false;
803 }
804
805 for (int i = 0; i < (int)vectors.size(); i++) {
806 for (int j = 0; j < lanes; j++) {
807 if (indices[j * (int)vectors.size() + i] != i * lanes + j) {
808 return false;
809 }
810 }
811 }
812
813 return true;
814 }
815
make(const std::string & producer_name,const std::string & mutex_name,Stmt body)816 Stmt Atomic::make(const std::string &producer_name,
817 const std::string &mutex_name,
818 Stmt body) {
819 Atomic *node = new Atomic;
820 node->producer_name = producer_name;
821 node->mutex_name = mutex_name;
822 internal_assert(body.defined()) << "Atomic must have a body statement.\n";
823 node->body = std::move(body);
824 return node;
825 }
826
make(VectorReduce::Operator op,Expr vec,int lanes)827 Expr VectorReduce::make(VectorReduce::Operator op,
828 Expr vec,
829 int lanes) {
830 if (vec.type().is_bool()) {
831 internal_assert(op == VectorReduce::And || op == VectorReduce::Or)
832 << "The only legal operators for VectorReduce on a Bool"
833 << "vector are VectorReduce::And and VectorReduce::Or\n";
834 }
835 internal_assert(!vec.type().is_handle()) << "VectorReduce of handle type";
836 // Check the output lanes is a factor of the input lanes. They can
837 // also both be zero if we're constructing a wildcard expression.
838 internal_assert((lanes == 0 && vec.type().lanes() == 0) ||
839 (lanes != 0 && (vec.type().lanes() % lanes == 0)))
840 << "Vector reduce output lanes must be a divisor of the number of lanes in the argument "
841 << lanes << " " << vec.type().lanes() << "\n";
842 VectorReduce *node = new VectorReduce;
843 node->type = vec.type().with_lanes(lanes);
844 node->op = op;
845 node->value = std::move(vec);
846 return node;
847 }
848
849 namespace {
850
851 // Helper function to determine if a sequence of indices is a
852 // contiguous ramp.
is_ramp(const std::vector<int> & indices,int stride=1)853 bool is_ramp(const std::vector<int> &indices, int stride = 1) {
854 for (size_t i = 0; i + 1 < indices.size(); i++) {
855 if (indices[i + 1] != indices[i] + stride) {
856 return false;
857 }
858 }
859 return true;
860 }
861
862 } // namespace
863
is_concat() const864 bool Shuffle::is_concat() const {
865 size_t input_lanes = 0;
866 for (Expr i : vectors) {
867 input_lanes += i.type().lanes();
868 }
869
870 // A concat is a ramp where the output has the same number of
871 // lanes as the input.
872 return indices.size() == input_lanes && is_ramp(indices);
873 }
874
is_slice() const875 bool Shuffle::is_slice() const {
876 size_t input_lanes = 0;
877 for (Expr i : vectors) {
878 input_lanes += i.type().lanes();
879 }
880
881 // A slice is a ramp where the output does not contain all of the
882 // lanes of the input.
883 return indices.size() < input_lanes && is_ramp(indices, slice_stride());
884 }
885
is_extract_element() const886 bool Shuffle::is_extract_element() const {
887 return indices.size() == 1;
888 }
889
890 template<>
accept(IRVisitor * v) const891 void ExprNode<IntImm>::accept(IRVisitor *v) const {
892 v->visit((const IntImm *)this);
893 }
894 template<>
accept(IRVisitor * v) const895 void ExprNode<UIntImm>::accept(IRVisitor *v) const {
896 v->visit((const UIntImm *)this);
897 }
898 template<>
accept(IRVisitor * v) const899 void ExprNode<FloatImm>::accept(IRVisitor *v) const {
900 v->visit((const FloatImm *)this);
901 }
902 template<>
accept(IRVisitor * v) const903 void ExprNode<StringImm>::accept(IRVisitor *v) const {
904 v->visit((const StringImm *)this);
905 }
906 template<>
accept(IRVisitor * v) const907 void ExprNode<Cast>::accept(IRVisitor *v) const {
908 v->visit((const Cast *)this);
909 }
910 template<>
accept(IRVisitor * v) const911 void ExprNode<Variable>::accept(IRVisitor *v) const {
912 v->visit((const Variable *)this);
913 }
914 template<>
accept(IRVisitor * v) const915 void ExprNode<Add>::accept(IRVisitor *v) const {
916 v->visit((const Add *)this);
917 }
918 template<>
accept(IRVisitor * v) const919 void ExprNode<Sub>::accept(IRVisitor *v) const {
920 v->visit((const Sub *)this);
921 }
922 template<>
accept(IRVisitor * v) const923 void ExprNode<Mul>::accept(IRVisitor *v) const {
924 v->visit((const Mul *)this);
925 }
926 template<>
accept(IRVisitor * v) const927 void ExprNode<Div>::accept(IRVisitor *v) const {
928 v->visit((const Div *)this);
929 }
930 template<>
accept(IRVisitor * v) const931 void ExprNode<Mod>::accept(IRVisitor *v) const {
932 v->visit((const Mod *)this);
933 }
934 template<>
accept(IRVisitor * v) const935 void ExprNode<Min>::accept(IRVisitor *v) const {
936 v->visit((const Min *)this);
937 }
938 template<>
accept(IRVisitor * v) const939 void ExprNode<Max>::accept(IRVisitor *v) const {
940 v->visit((const Max *)this);
941 }
942 template<>
accept(IRVisitor * v) const943 void ExprNode<EQ>::accept(IRVisitor *v) const {
944 v->visit((const EQ *)this);
945 }
946 template<>
accept(IRVisitor * v) const947 void ExprNode<NE>::accept(IRVisitor *v) const {
948 v->visit((const NE *)this);
949 }
950 template<>
accept(IRVisitor * v) const951 void ExprNode<LT>::accept(IRVisitor *v) const {
952 v->visit((const LT *)this);
953 }
954 template<>
accept(IRVisitor * v) const955 void ExprNode<LE>::accept(IRVisitor *v) const {
956 v->visit((const LE *)this);
957 }
958 template<>
accept(IRVisitor * v) const959 void ExprNode<GT>::accept(IRVisitor *v) const {
960 v->visit((const GT *)this);
961 }
962 template<>
accept(IRVisitor * v) const963 void ExprNode<GE>::accept(IRVisitor *v) const {
964 v->visit((const GE *)this);
965 }
966 template<>
accept(IRVisitor * v) const967 void ExprNode<And>::accept(IRVisitor *v) const {
968 v->visit((const And *)this);
969 }
970 template<>
accept(IRVisitor * v) const971 void ExprNode<Or>::accept(IRVisitor *v) const {
972 v->visit((const Or *)this);
973 }
974 template<>
accept(IRVisitor * v) const975 void ExprNode<Not>::accept(IRVisitor *v) const {
976 v->visit((const Not *)this);
977 }
978 template<>
accept(IRVisitor * v) const979 void ExprNode<Select>::accept(IRVisitor *v) const {
980 v->visit((const Select *)this);
981 }
982 template<>
accept(IRVisitor * v) const983 void ExprNode<Load>::accept(IRVisitor *v) const {
984 v->visit((const Load *)this);
985 }
986 template<>
accept(IRVisitor * v) const987 void ExprNode<Ramp>::accept(IRVisitor *v) const {
988 v->visit((const Ramp *)this);
989 }
990 template<>
accept(IRVisitor * v) const991 void ExprNode<Broadcast>::accept(IRVisitor *v) const {
992 v->visit((const Broadcast *)this);
993 }
994 template<>
accept(IRVisitor * v) const995 void ExprNode<Call>::accept(IRVisitor *v) const {
996 v->visit((const Call *)this);
997 }
998 template<>
accept(IRVisitor * v) const999 void ExprNode<Shuffle>::accept(IRVisitor *v) const {
1000 v->visit((const Shuffle *)this);
1001 }
1002 template<>
accept(IRVisitor * v) const1003 void ExprNode<VectorReduce>::accept(IRVisitor *v) const {
1004 v->visit((const VectorReduce *)this);
1005 }
1006 template<>
accept(IRVisitor * v) const1007 void ExprNode<Let>::accept(IRVisitor *v) const {
1008 v->visit((const Let *)this);
1009 }
1010 template<>
accept(IRVisitor * v) const1011 void StmtNode<LetStmt>::accept(IRVisitor *v) const {
1012 v->visit((const LetStmt *)this);
1013 }
1014 template<>
accept(IRVisitor * v) const1015 void StmtNode<AssertStmt>::accept(IRVisitor *v) const {
1016 v->visit((const AssertStmt *)this);
1017 }
1018 template<>
accept(IRVisitor * v) const1019 void StmtNode<ProducerConsumer>::accept(IRVisitor *v) const {
1020 v->visit((const ProducerConsumer *)this);
1021 }
1022 template<>
accept(IRVisitor * v) const1023 void StmtNode<For>::accept(IRVisitor *v) const {
1024 v->visit((const For *)this);
1025 }
1026 template<>
accept(IRVisitor * v) const1027 void StmtNode<Store>::accept(IRVisitor *v) const {
1028 v->visit((const Store *)this);
1029 }
1030 template<>
accept(IRVisitor * v) const1031 void StmtNode<Provide>::accept(IRVisitor *v) const {
1032 v->visit((const Provide *)this);
1033 }
1034 template<>
accept(IRVisitor * v) const1035 void StmtNode<Allocate>::accept(IRVisitor *v) const {
1036 v->visit((const Allocate *)this);
1037 }
1038 template<>
accept(IRVisitor * v) const1039 void StmtNode<Free>::accept(IRVisitor *v) const {
1040 v->visit((const Free *)this);
1041 }
1042 template<>
accept(IRVisitor * v) const1043 void StmtNode<Realize>::accept(IRVisitor *v) const {
1044 v->visit((const Realize *)this);
1045 }
1046 template<>
accept(IRVisitor * v) const1047 void StmtNode<Block>::accept(IRVisitor *v) const {
1048 v->visit((const Block *)this);
1049 }
1050 template<>
accept(IRVisitor * v) const1051 void StmtNode<IfThenElse>::accept(IRVisitor *v) const {
1052 v->visit((const IfThenElse *)this);
1053 }
1054 template<>
accept(IRVisitor * v) const1055 void StmtNode<Evaluate>::accept(IRVisitor *v) const {
1056 v->visit((const Evaluate *)this);
1057 }
1058 template<>
accept(IRVisitor * v) const1059 void StmtNode<Prefetch>::accept(IRVisitor *v) const {
1060 v->visit((const Prefetch *)this);
1061 }
1062 template<>
accept(IRVisitor * v) const1063 void StmtNode<Acquire>::accept(IRVisitor *v) const {
1064 v->visit((const Acquire *)this);
1065 }
1066 template<>
accept(IRVisitor * v) const1067 void StmtNode<Fork>::accept(IRVisitor *v) const {
1068 v->visit((const Fork *)this);
1069 }
1070 template<>
accept(IRVisitor * v) const1071 void StmtNode<Atomic>::accept(IRVisitor *v) const {
1072 v->visit((const Atomic *)this);
1073 }
1074
1075 template<>
mutate_expr(IRMutator * v) const1076 Expr ExprNode<IntImm>::mutate_expr(IRMutator *v) const {
1077 return v->visit((const IntImm *)this);
1078 }
1079 template<>
mutate_expr(IRMutator * v) const1080 Expr ExprNode<UIntImm>::mutate_expr(IRMutator *v) const {
1081 return v->visit((const UIntImm *)this);
1082 }
1083 template<>
mutate_expr(IRMutator * v) const1084 Expr ExprNode<FloatImm>::mutate_expr(IRMutator *v) const {
1085 return v->visit((const FloatImm *)this);
1086 }
1087 template<>
mutate_expr(IRMutator * v) const1088 Expr ExprNode<StringImm>::mutate_expr(IRMutator *v) const {
1089 return v->visit((const StringImm *)this);
1090 }
1091 template<>
mutate_expr(IRMutator * v) const1092 Expr ExprNode<Cast>::mutate_expr(IRMutator *v) const {
1093 return v->visit((const Cast *)this);
1094 }
1095 template<>
mutate_expr(IRMutator * v) const1096 Expr ExprNode<Variable>::mutate_expr(IRMutator *v) const {
1097 return v->visit((const Variable *)this);
1098 }
1099 template<>
mutate_expr(IRMutator * v) const1100 Expr ExprNode<Add>::mutate_expr(IRMutator *v) const {
1101 return v->visit((const Add *)this);
1102 }
1103 template<>
mutate_expr(IRMutator * v) const1104 Expr ExprNode<Sub>::mutate_expr(IRMutator *v) const {
1105 return v->visit((const Sub *)this);
1106 }
1107 template<>
mutate_expr(IRMutator * v) const1108 Expr ExprNode<Mul>::mutate_expr(IRMutator *v) const {
1109 return v->visit((const Mul *)this);
1110 }
1111 template<>
mutate_expr(IRMutator * v) const1112 Expr ExprNode<Div>::mutate_expr(IRMutator *v) const {
1113 return v->visit((const Div *)this);
1114 }
1115 template<>
mutate_expr(IRMutator * v) const1116 Expr ExprNode<Mod>::mutate_expr(IRMutator *v) const {
1117 return v->visit((const Mod *)this);
1118 }
1119 template<>
mutate_expr(IRMutator * v) const1120 Expr ExprNode<Min>::mutate_expr(IRMutator *v) const {
1121 return v->visit((const Min *)this);
1122 }
1123 template<>
mutate_expr(IRMutator * v) const1124 Expr ExprNode<Max>::mutate_expr(IRMutator *v) const {
1125 return v->visit((const Max *)this);
1126 }
1127 template<>
mutate_expr(IRMutator * v) const1128 Expr ExprNode<EQ>::mutate_expr(IRMutator *v) const {
1129 return v->visit((const EQ *)this);
1130 }
1131 template<>
mutate_expr(IRMutator * v) const1132 Expr ExprNode<NE>::mutate_expr(IRMutator *v) const {
1133 return v->visit((const NE *)this);
1134 }
1135 template<>
mutate_expr(IRMutator * v) const1136 Expr ExprNode<LT>::mutate_expr(IRMutator *v) const {
1137 return v->visit((const LT *)this);
1138 }
1139 template<>
mutate_expr(IRMutator * v) const1140 Expr ExprNode<LE>::mutate_expr(IRMutator *v) const {
1141 return v->visit((const LE *)this);
1142 }
1143 template<>
mutate_expr(IRMutator * v) const1144 Expr ExprNode<GT>::mutate_expr(IRMutator *v) const {
1145 return v->visit((const GT *)this);
1146 }
1147 template<>
mutate_expr(IRMutator * v) const1148 Expr ExprNode<GE>::mutate_expr(IRMutator *v) const {
1149 return v->visit((const GE *)this);
1150 }
1151 template<>
mutate_expr(IRMutator * v) const1152 Expr ExprNode<And>::mutate_expr(IRMutator *v) const {
1153 return v->visit((const And *)this);
1154 }
1155 template<>
mutate_expr(IRMutator * v) const1156 Expr ExprNode<Or>::mutate_expr(IRMutator *v) const {
1157 return v->visit((const Or *)this);
1158 }
1159 template<>
mutate_expr(IRMutator * v) const1160 Expr ExprNode<Not>::mutate_expr(IRMutator *v) const {
1161 return v->visit((const Not *)this);
1162 }
1163 template<>
mutate_expr(IRMutator * v) const1164 Expr ExprNode<Select>::mutate_expr(IRMutator *v) const {
1165 return v->visit((const Select *)this);
1166 }
1167 template<>
mutate_expr(IRMutator * v) const1168 Expr ExprNode<Load>::mutate_expr(IRMutator *v) const {
1169 return v->visit((const Load *)this);
1170 }
1171 template<>
mutate_expr(IRMutator * v) const1172 Expr ExprNode<Ramp>::mutate_expr(IRMutator *v) const {
1173 return v->visit((const Ramp *)this);
1174 }
1175 template<>
mutate_expr(IRMutator * v) const1176 Expr ExprNode<Broadcast>::mutate_expr(IRMutator *v) const {
1177 return v->visit((const Broadcast *)this);
1178 }
1179 template<>
mutate_expr(IRMutator * v) const1180 Expr ExprNode<Call>::mutate_expr(IRMutator *v) const {
1181 return v->visit((const Call *)this);
1182 }
1183 template<>
mutate_expr(IRMutator * v) const1184 Expr ExprNode<Shuffle>::mutate_expr(IRMutator *v) const {
1185 return v->visit((const Shuffle *)this);
1186 }
1187 template<>
mutate_expr(IRMutator * v) const1188 Expr ExprNode<VectorReduce>::mutate_expr(IRMutator *v) const {
1189 return v->visit((const VectorReduce *)this);
1190 }
1191 template<>
mutate_expr(IRMutator * v) const1192 Expr ExprNode<Let>::mutate_expr(IRMutator *v) const {
1193 return v->visit((const Let *)this);
1194 }
1195
1196 template<>
mutate_stmt(IRMutator * v) const1197 Stmt StmtNode<LetStmt>::mutate_stmt(IRMutator *v) const {
1198 return v->visit((const LetStmt *)this);
1199 }
1200 template<>
mutate_stmt(IRMutator * v) const1201 Stmt StmtNode<AssertStmt>::mutate_stmt(IRMutator *v) const {
1202 return v->visit((const AssertStmt *)this);
1203 }
1204 template<>
mutate_stmt(IRMutator * v) const1205 Stmt StmtNode<ProducerConsumer>::mutate_stmt(IRMutator *v) const {
1206 return v->visit((const ProducerConsumer *)this);
1207 }
1208 template<>
mutate_stmt(IRMutator * v) const1209 Stmt StmtNode<For>::mutate_stmt(IRMutator *v) const {
1210 return v->visit((const For *)this);
1211 }
1212 template<>
mutate_stmt(IRMutator * v) const1213 Stmt StmtNode<Store>::mutate_stmt(IRMutator *v) const {
1214 return v->visit((const Store *)this);
1215 }
1216 template<>
mutate_stmt(IRMutator * v) const1217 Stmt StmtNode<Provide>::mutate_stmt(IRMutator *v) const {
1218 return v->visit((const Provide *)this);
1219 }
1220 template<>
mutate_stmt(IRMutator * v) const1221 Stmt StmtNode<Allocate>::mutate_stmt(IRMutator *v) const {
1222 return v->visit((const Allocate *)this);
1223 }
1224 template<>
mutate_stmt(IRMutator * v) const1225 Stmt StmtNode<Free>::mutate_stmt(IRMutator *v) const {
1226 return v->visit((const Free *)this);
1227 }
1228 template<>
mutate_stmt(IRMutator * v) const1229 Stmt StmtNode<Realize>::mutate_stmt(IRMutator *v) const {
1230 return v->visit((const Realize *)this);
1231 }
1232 template<>
mutate_stmt(IRMutator * v) const1233 Stmt StmtNode<Block>::mutate_stmt(IRMutator *v) const {
1234 return v->visit((const Block *)this);
1235 }
1236 template<>
mutate_stmt(IRMutator * v) const1237 Stmt StmtNode<IfThenElse>::mutate_stmt(IRMutator *v) const {
1238 return v->visit((const IfThenElse *)this);
1239 }
1240 template<>
mutate_stmt(IRMutator * v) const1241 Stmt StmtNode<Evaluate>::mutate_stmt(IRMutator *v) const {
1242 return v->visit((const Evaluate *)this);
1243 }
1244 template<>
mutate_stmt(IRMutator * v) const1245 Stmt StmtNode<Prefetch>::mutate_stmt(IRMutator *v) const {
1246 return v->visit((const Prefetch *)this);
1247 }
1248 template<>
mutate_stmt(IRMutator * v) const1249 Stmt StmtNode<Acquire>::mutate_stmt(IRMutator *v) const {
1250 return v->visit((const Acquire *)this);
1251 }
1252 template<>
mutate_stmt(IRMutator * v) const1253 Stmt StmtNode<Fork>::mutate_stmt(IRMutator *v) const {
1254 return v->visit((const Fork *)this);
1255 }
1256 template<>
mutate_stmt(IRMutator * v) const1257 Stmt StmtNode<Atomic>::mutate_stmt(IRMutator *v) const {
1258 return v->visit((const Atomic *)this);
1259 }
1260
1261 Call::ConstString Call::buffer_get_dimensions = "_halide_buffer_get_dimensions";
1262 Call::ConstString Call::buffer_get_min = "_halide_buffer_get_min";
1263 Call::ConstString Call::buffer_get_extent = "_halide_buffer_get_extent";
1264 Call::ConstString Call::buffer_get_stride = "_halide_buffer_get_stride";
1265 Call::ConstString Call::buffer_get_max = "_halide_buffer_get_max";
1266 Call::ConstString Call::buffer_get_host = "_halide_buffer_get_host";
1267 Call::ConstString Call::buffer_get_device = "_halide_buffer_get_device";
1268 Call::ConstString Call::buffer_get_device_interface = "_halide_buffer_get_device_interface";
1269 Call::ConstString Call::buffer_get_shape = "_halide_buffer_get_shape";
1270 Call::ConstString Call::buffer_get_host_dirty = "_halide_buffer_get_host_dirty";
1271 Call::ConstString Call::buffer_get_device_dirty = "_halide_buffer_get_device_dirty";
1272 Call::ConstString Call::buffer_get_type = "_halide_buffer_get_type";
1273 Call::ConstString Call::buffer_set_host_dirty = "_halide_buffer_set_host_dirty";
1274 Call::ConstString Call::buffer_set_device_dirty = "_halide_buffer_set_device_dirty";
1275 Call::ConstString Call::buffer_is_bounds_query = "_halide_buffer_is_bounds_query";
1276 Call::ConstString Call::buffer_init = "_halide_buffer_init";
1277 Call::ConstString Call::buffer_init_from_buffer = "_halide_buffer_init_from_buffer";
1278 Call::ConstString Call::buffer_crop = "_halide_buffer_crop";
1279 Call::ConstString Call::buffer_set_bounds = "_halide_buffer_set_bounds";
1280 Call::ConstString Call::trace = "halide_trace_helper";
1281
1282 } // namespace Internal
1283 } // namespace Halide
1284